jDR UddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZddlZddlmZddlmZddlmZmZddlmZdd lmZdd lmZmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.m/Z/m0Z0m1Z1dd l2m3Z3ddl4m5Z5ddl6m7Z7ej8e9Z:ej;eGdde'Z?Gdde?Z@ d.de eAe fdedeBde&fdZCde e&de e&fd ZDde eAe fd!eBde)fd"ZE d.de e)d#eBdee e)effd$ZFde!defd%ZGd&ed'edeBfd(ZHd)ejId*edeBfd+ZJd,e e)dedeBfd-ZKdS)/N)ChainMap)reduce)AnycastDictListOptionalTupleUnion)narrow_tensor_by_index)dedup_save_plans)FLATTEN_MAPPINGflatten_state_dict)_flatten_sharded_tensors) set_element)BytesStorageMetadataChunkStorageMetadataMetadata MetadataIndexSTATE_DICT_TYPE STORAGE_TYPES StorageMetaTensorStorageMetadata)LoadPlan LoadPlannerReadItemSavePlan SavePlanner WriteItem WriteItemType)"_create_default_metadata_only_plan_create_read_items_create_write_items_init_state_dict)find_state_dict_object)DTensor)_versionlogger)DefaultSavePlannerDefaultLoadPlannercreate_default_local_load_plancreate_default_global_load_plancreate_default_local_save_plancreate_default_global_save_planc eZdZUeed< ddededeeded df d Z dd ed ee d ed dfdZ d e fdZ de e d ee e effdZde d e fdZded eejejffdZded efdZdedefdZdS)r*mappingsTNFrflatten_sharded_tensorsdedup_replicated_tensorsdedup_save_to_lowest_rankreturncz||_||_i|_||_|tddSdS)NzDefaultSavePlanner's `dedup_replicated_tensors` argument is being deprecated, and no longer has any effect. Please remove this argument from your call.)rr2r1r4r)warning)selfrr2r3r4s ]C:\Projects\Historical\venv\Lib\site-packages\torch/distributed/checkpoint/default_planner.py__init__zDefaultSavePlanner.__init__EsV#5'>$ )B& # / NN"      0 / state_dict storage_metais_coordinatorc|jrt|\}|_|jrt|}||_||_dSN)rr1r2rr<r>)r8r<r=r>s r9set_up_plannerz!DefaultSavePlanner.set_up_plannerWsQ  " G(::(F(F %J  ' >1*==J$,r;ct|j|j}|jrt j||j}||_|jS)N planner_data)r.r<r>r dataclassesreplacer1plan)r8rGs r9create_local_planz$DefaultSavePlanner.create_local_plandsH-dot?RSS  " I&t$-HHHD yr; all_planscDt||j}t|\}}|jr8d|D}t t |}t j||}t||std||_ ||_ |j |j fS)Ncg|] }|j SrC).0ps r9 z9DefaultSavePlanner.create_global_plan..xs E E EA E E Er;rCzFailed to validate global plan) r r4r/rdictrrErF_validate_global_plan ValueError global_planmetadata)r8rIrSrTplanner_data_dictmerged_mappingss r9create_global_planz%DefaultSavePlanner.create_global_planls%Y0NOO ? J J X  " S !F E E E E "8->#?@@O"*8/RRRH$[(;; ?=>> >&  ..r;new_planc||_|Sr@)rGr8rXs r9 finish_planzDefaultSavePlanner.finish_plans r; write_itemcb||j}|||Sr@) lookup_objectindextransform_object)r8r\objects r9 resolve_datazDefaultSavePlanner.resolve_datas.##J$455$$Z888r;r_c,t|j|SzSExtension from the planner interface to make it easy to extend the default planner.r%r<r8r_s r9r^z DefaultSavePlanner.lookup_object%dou===r;rac|jtjkr*tj}t j|||}|Srd)typer BYTE_IOioBytesIOtorchsave)r8r\rabytess r9r`z#DefaultSavePlanner.transform_objects9 ?m3 3 3JLLE Jvu % % %F r;)TTNFNF)__name__ __module__ __qualname__r__annotations__boolr r:rrrArrHrr rrWr[rr rmTensorrkrlrbrrr^r`rLr;r9r*r*Bs$((,37*/  "&#+4.  $(   */3$ - -# -{+ - -  - - - -8/h/ tH~x' (////0H9y9U5<;S5T9999>=>S>>>>9cr;r*c 2eZdZUdZeed<eed< ddededed d fd Z dd ed e e ded d fdZ d e fdZ dee d ee fdZde d e fdZdedejd d fdZdefdZdedejd d fdZded ejfdZdedejfdZd S) r+ak DefaultLoadPlanner that adds multiple features on top of LoadPlanner. In particular it adds the following: flatten_state_dict: Handle state_dict with nested dicts flatten_sharded_tensors: For FSDP in 2D parallel mode allow_partial_load: If False, will raise a runtime error if a key is present in state_dict, but not in the checkpoint. original_state_dictr1TFrr2allow_partial_loadr5NcL||_||_i|_i|_||_dSr@)rr2rxr1ry)r8rr2rys r9r:zDefaultLoadPlanner.__init__s1 #5'>$#%  "4r;r<rTr>ct|||_|jrt|}|jrt |\}|_||_||_||_dSr@) r$rxr2rrr1r<rTr>)r8r<rTr>s r9rAz!DefaultLoadPlanner.set_up_plannersn $$$#-  ' >1*==J  " G(::(F(F %J $  ,r;c|jJ|jrt|j}t|jj}||z }|rddt _t|j\}}t|}||zr||c|_|_ dt _t|j|j|j S)N2_3) rTrsetr<keysstate_dict_metadatar(_derived_versionrxr1r,ry)r8 current_keys load_keys missing_keysold_state_dict old_mappingsold_keyss r9rHz$DefaultLoadPlanner.create_local_plans}(((  " 1$t335566LDM=BBDDEEI$|3L 1,1)/A,00, ~224455l*R5C\2DOT]-1)- OT]0G,G   r;rSc t|Sr@)r-)r8rSs r9rWz%DefaultLoadPlanner.create_global_plans.{;;;r;rXc|Sr@rLrZs r9r[zDefaultLoadPlanner.finish_plansr; read_itemvaluec|jrAt|j|j|jjt j|ddSt j|d|j|jj<dS)NF) weights_only) rrrxr1 dest_indexfqnrmloadr<)r8rrs r9 load_byteszDefaultLoadPlanner.load_bytess  "  ( i267 5u555      9> E999DOI04 5 5 5r;cb||j}|||Sr@) lookup_tensorrtransform_tensorr8rtensors r9resolve_tensorz!DefaultLoadPlanner.resolve_tensors.##I$899$$Y777r;rcdSr@rLrs r9 commit_tensorz DefaultLoadPlanner.commit_tensors r;r_c,t|j|Srdrerfs r9rz DefaultLoadPlanner.lookup_tensorrgr;c8t||j|jSrd)r dest_offsetslengthsrs r9rz#DefaultLoadPlanner.transform_tensor s%fi.DiFWXXXr;)TTFrp)rqrrrs__doc__rrtrrur:r rrArrHrrWr[rrkrlrrrmrvrrrrrLr;r9r+r+s)((($((,#( 5 5  5"& 5! 5  5 5 5 5(,$ --#-8$- -  ----&% 8% % % % N=>U\>>>>Y(YELYYYYYYr;r+c feZdZdZd fd ZdededefdZ d d e de ed eddffd Z xZ S)_EmptyStateDictLoadPlannera Extension of DefaultLoadPlanner, which rebuilds state_dict from the saved metadata. Useful for loading in state_dict without first initializing a model, such as when converting a DCP checkpoint into a Torch save file. . N.B. `state_dict` must be an empty dictionary when used with this LoadPlanner .. warning:: Because the entire state dict is initialized, It's recommended to only utilize this LoadPlanner on a single rank or process to avoid OOM. NcH||_tj|i|dSr@)rsuperr:)r8rargskwargs __class__s r9r:z#_EmptyStateDictLoadPlanner.__init__s, $)&)))))r;keyrTr5c TjdS|jvr g}|j|}|D]W}|r>|d|dt |gB||Xt fd|DrdSdS)NT.c3*K|] }|jvVdSr@)r)rMunflattened_keyr8s r9 zA_EmptyStateDictLoadPlanner._should_include_key..3s*TT$)+TTTTTTr;F)rrDgetappendjoinstrany)r8rrTunflattened_keysrDrs` r9_should_include_keyz._EmptyStateDictLoadPlanner._should_include_key!s 9 4 $)   &(,0055 + 9 9O 9 ''HH.r2C4H4HIJJ !''8888 TTTTCSTTT T T 4ur;Fr<r>c|rJ|J|jD]\}}|||st|tr%t j|j|jj }||j vrt||j ||||||<t |||dS)N)dtype)ritemsr isinstancerrmemptysize propertiesrrDrrrA)r8r<rTr>kvrs r9rAz)_EmptyStateDictLoadPlanner.set_up_planner8s ~###06688 " "DAq++Ax88 !233 BKal.@AAAH)))J(=a(@!DDDD ! 1  z8^DDDDDr;r@rp) rqrrrsrr:rrrurrr rA __classcell__)rs@r9rrs  ******sh44(,$ EE#E8$E E  EEEEEEEEEEr;rTr<rTstrictr5c Dg} |D]\}}||jvr|rtd|d$|j|}t|tr]t |ddL|j|kr/td|jd|d|t|tr.|j |t|||z }|t|||z }t|S)Nz&Missing key in checkpoint state_dict: rrzSize mismatch between saved z and current: z for ) rr RuntimeErrorrrgetattrrrRr& device_meshget_coordinater"r)r<rTrrequestsrobjmds r9r,r,PsHH$$&&99S h2 2 2 "#RC#R#R#RSSS  )# . r0 1 1 VT**6388::%%\rw\\chhjj\\WZ\\  c7 # # 9--//;.sB<<< *3C88 8HH H  r;rIc|S)z Create global load plan used by DefaultLoadPlanner. The default load behavior involved no global coordination and this function currently doesn't change the local plans. rL)rIs r9r-r-zs  r;r>cg}|D]Z\}}t|tr-|j|t ||z }G|t ||z }[t |S)a Create the ``SavePlan`` used by DefaultSavePlanner. On non-coordinator ranks, this function ignores tensors and non-tensor objects, only producing writes for ShardedTensor objects. On the coordinator rank, produce writes for all values. )rrr&rrr#r)r<r>rrrs r9r.r.sH$$&& 6 6S c7 # # 6--//;/S999 +C55 5HH H  r;rewrite_index_hintsc i}g}|D]}g}|jD]r}|jtjks|jj|vsJ|jtjkr1t||jj<||n|j Jtt| |jjt|j j |j jg}|}|rCtj|jt#|j} tj|| }|||j jJd|jjd|j|j jt|tj|||t)|fS)a6 Create the global plan and metadata used by DefaultSavePlanner. Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans. The only global planning change is to update index hints in all ``MetadataIndex`` objects if ``rewrite_index_hints`` is True. N)rrchunks)r_zZ Cannot create MD for tensor without bounds. FQN: z )r)rrir SHARDr_rrjrr tensor_datarr setdefaultrrrErFlenrchunkr) rIrr new_plansrG new_itemsitem tensor_mdnew_item new_indexs r9r/r/s$&BI%E%E J" @" @D9 333z~R////yM111%9%;%;4:>"  &&&&'333 )MM -'+'7'B!%!1!6#%    &J + 3 #i.>*?*?!!!I +24yIIIH  ***$*66*.766  ''(8(>????,TCCCDDDD x|| $$r;cJt|}t|g\}}|S)zTReturn the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``.)r!r/)r<rG_rs r9_create_default_local_metadatars' -j 9 9D +TF 3 3EAr Ir;box0box1ct|j}t|D]\}|j||j||j|zkrdS|j||j||j|zkrdS]dS)z9Check if two boxes overlap. Tuples are (offset, lengths).FT)roffsetsrangesizes)rrndimsis r9_check_box_overlaprs   E 5\\ <?dl1o 1 = = =55 <?dl1o 1 = = =55 > 4r;outer_box_size inner_boxctt|D]R}|j|dkrdS|j|dkrdS|j||j|z||krdSSdS)NrFT)rrrr)rrrs r9_check_box_boundsrs3~&& ' '  Q ! # #55 ?1  ! !55  Q )/!"4 4~a7H H H55 I 4r;rSc d}|jD]6\}}t|trt |jdkr5d}t |jD]\}}t|j|s$t d||j|d}|ttj |jdz }|j|dzdD]1}t||rt d|||d}2ttj |jd} || krt d|| |d}8|S)NTrz~ key:%s has out of bounds chunk: tensor-size:%s chunk: %s Fr'z$key:%s has overlapping chunks: %s %szq key:%s invalid fill tensor-volume: %s chunks-volume: %s )rrrrrr enumeraterrr)r7roperatormulrr) rSrTall_goodrr chunks_volume chunk_idxchunk0chunk1 tensor_volumes r9rQrQsH288::)) U e1 2 2   uz??a    !*5VV %H  %x|UZ;; M ) ) NN   H Or;)T)LrErkloggingr collectionsr functoolsrtypingrrrrr r r rmtorch.distributed._shard._utilsr .torch.distributed.checkpoint._dedup_save_plansr )torch.distributed.checkpoint._nested_dictrr2torch.distributed.checkpoint._sharded_tensor_utilsr&torch.distributed.checkpoint._traverser%torch.distributed.checkpoint.metadatarrrrrrrr$torch.distributed.checkpoint.plannerrrrrrrr ,torch.distributed.checkpoint.planner_helpersr!r"r#r$"torch.distributed.checkpoint.utilsr%torch.distributed.tensorr&r( getLoggerrqr)Loggerrt__all__r*r+rrrur,r-r.r/rrSizerrQrLr;r9rs  @@@@@@@@@@@@@@@@@@ BBBBBBKKKKKKXWWWWW>>>>>>                     FEEEEE,,,,,,+*844444   TTTTTTTTnsYsYsYsYsYsYsYsYl>E>E>E>E>E!3>E>E>EDDH''S#X'*2'<@' ''''T H~  (^    S#X04 :!%4%4%H~4%4% 4>8 #$4%4%4%4%n8 1 9M RV     J +?      -tH~--d------r;