§
    Ÿ²Åj¼4  ã                   ó’  — d dl Z d dlmZmZmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZm Z m!Z!mZm"Z" d dl#m$Z$m%Z% d dl&m'Z'm(Z( d dl)m*Z* d dl+m,Z, d dl-m.Z.m/Z/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8 ee9eeee:                  ee:         f         f         Z;dgZ<d*de:de9de9fd„Z=	 d+deej>                 defd„Z?de	j@        deAfd„ZB	 d*dedee:         de9de	j@        fd „ZCd!e!dee;eej>                 f         fd"„ZD G d#„ d$e¦  «        ZE	 d+d%e!d&e9d'e,d(ee%         de!f
d)„ZFdS ),é    N)ÚcastÚDictÚListÚOptionalÚSequenceÚTupleÚUnion)Ú_get_device_module)ÚShardedTensor)ÚTensorProperties)ÚShard)ÚChunkShardingSpec)Úunflatten_state_dict)ÚDefaultLoadPlanner)ÚBytesStorageMetadataÚChunkStorageMetadataÚMetadataÚMetadataIndexÚSTATE_DICT_TYPEr   ÚTensorStorageMetadata)ÚLoadPlanÚLoadPlanner)Ú_create_read_itemsÚ create_read_items_for_chunk_list)Úload_state_dict)ÚStorageReader)Ú_element_wise_addÚ_element_wise_subÚ_normalize_device_info)Ú_get_default_group)Ú_create_chunk_sharded_tensor)Ú_remote_device)ÚDTensorÚ!load_sharded_optimizer_state_dictÚcudaÚglobal_rankÚdevice_typeÚreturnc                 ó¦   — |dk    rdS t          |¦  «        }|                     ¦   «         r%t          || |                     ¦   «         z  ¦  «        S dS )NÚcpu)r
   Úis_availabler   Údevice_count)r&   r'   Údevice_modules      úWC:\Projects\Historical\venv\Lib\site-packages\torch/distributed/checkpoint/optimizer.pyÚ_gen_rank_devicer/   5   sb   € Ø�eÒÐØˆuÝ& {Ñ3Ô3€MØ×!Ò!Ñ#Ô#ð 
Ý%Ø˜ }×'AÒ'AÑ'CÔ'CÑCñ
ô 
ð 	
ð ˆ5ó    Úpgc           	      óŠ  ‡ ‡— t           j                             ‰ ¦  «        j        Š‰ €-ˆfd„t	          t          j        ¦   «         ¦  «        D ¦   «         }n.ˆ ˆfd„t	          ‰                      ¦   «         ¦  «        D ¦   «         }t          dt          t          t          t          t          f                  |¦  «        ¬¦  «        S )Nc           	      ó<   •— g | ]}d |› dt          |‰¦  «        › �‘ŒS ©úrank:ú/)r/   )Ú.0ÚidxÚpg_device_types     €r.   ú
<listcomp>z(_create_colwise_spec.<locals>.<listcomp>E   sE   ø€ ð 
ð 
ð 
àð B�CÐAÐAÕ*¨3°Ñ?Ô?ÐAÐAð
ð 
ð 
r0   c                 ób   •— g | ]+}d |› dt          t          j        ‰|¦  «        ‰¦  «        › �‘Œ,S r4   )r/   ÚdistÚget_global_rank)r7   r8   r1   r9   s     €€r.   r:   z(_create_colwise_spec.<locals>.<listcomp>J   sR   ø€ ð 
ð 
ð 
àð \�CÐ[Ð[Õ*­4Ô+?ÀÀCÑ+HÔ+HÈ.ÑYÔYÐ[Ð[ð
ð 
ð 
r0   r   ©ÚdimÚ
placements)r<   Údistributed_c10dÚ_get_pg_default_deviceÚtypeÚrangeÚget_world_sizeÚsizer   r   r   r	   r"   Ústr)r1   r@   r9   s   ` @r.   Ú_create_colwise_specrH   @   sÔ   øø€ õ Ô*×AÒAÀ"ÑEÔEÔJ€NØ	€zð
ð 
ð 
ð 
å�TÔ0Ñ2Ô2Ñ3Ô3ð
ñ 
ô 
ˆ
ˆ
ð

ð 
ð 
ð 
ð 
å˜RŸWšW™YœYÑ'Ô'ð
ñ 
ô 
ˆ
õ ØÝ��U¥>µ3Ð#6Ô7Ô8¸*ÑEÔEðñ ô ð r0   Úvalc                 ó&  — t          | ¦  «        t          u rŸt          |                      ¦   «         ¦  «        dk    rdS t          |                      ¦   «         d         j        ¦  «        t          u rdS t          |                      ¦   «         d         j        ¦  «        t
          u rt          d¦  «        ‚n[t          | ¦  «        t
          u rEt          | j        ¦  «        t
          u st          | j        ¦  «        t          u rt          d¦  «        ‚dS )Nr   FTz2Cannot handle DTensor nested insided ShardedTensorzCannot handle nested DTensor)rC   r   ÚlenÚlocal_shardsÚtensorr#   Ú
ValueErrorÚ_local_tensor)rI   s    r.   Ú_is_nested_tensorrP   T   sî   € ÝˆC�y„y•MÐ!Ð!Ýˆs×ÒÑ!Ô!Ñ"Ô" aÒ'Ð'Ø�5Ý�× Ò Ñ"Ô" 1Ô%Ô,Ñ-Ô-µÐ>Ð>Ø�4Ý�× Ò Ñ"Ô" 1Ô%Ô,Ñ-Ô-µÐ8Ð8ÝÐQÑRÔRÐRð 9å	ˆc‰Œ•gÐ	Ð	ÝˆSÔÑÔ¥7Ð*Ð*­d°3Ô3DÑ.EÔ.EÍÐ.VÐ.VåÐ7Ñ8Ô8Ð8Øˆ5r0   ÚpropsrF   c                 óF  — |dk    r:t          t          j        t          |¦  «                             ¦   «         ¦  «        }n4t          j        |t          |¦  «                             ¦   «         ¦  «        }t          j        || j        | j        | j        | j	        |¬¦  «        S )Nr*   )rF   ÚdtypeÚlayoutÚrequires_gradÚ
pin_memoryÚdevice)
r   ÚtorchrW   r
   Úcurrent_deviceÚemptyrS   rT   rU   rV   )rQ   rF   r'   rW   s       r.   Ú_alloc_tensorr[   c   s™   € ð �eÒÐÝ•e”lÕ$6°{Ñ$CÔ$C×$RÒ$RÑ$TÔ$TÑUÔUˆˆå”ØÕ+¨KÑ8Ô8×GÒGÑIÔIñ
ô 
ˆõ Œ;ØØŒkØŒ|ØÔ)ØÔ#Øðñ ô ð r0   Ú
state_dictc                 ó´  — i }d}|                       ¦   «         D ]¼\  }}d|                     ¦   «         f||<   t          |¦  «        r�t          |                     ¦   «         ¦  «        dk    s
J d¦   «         ‚t          |t          ¦  «        s
J d¦   «         ‚|                     ¦   «         d         }|j        j        |j        j	        f||<   |j
        j        }Œ½||fS )a+  
    Load the right TP slice of the optimizer state.

    This is not easy since the per-tensor slicing can't be inferred from checkpoint metadata.
    We take advantage of the model state_dict producing a sliced ST to figure out what we need to load.
    This is pretty fragile and it might be easier for FSDP to compute this info for us.
    Returns a dictionary where keys are the same of the state_dict and the value is a tuple of
    (offset, size) for the current rank TP slice.
    N.B. The state_dict *MUST* come from FSDP.sharded_state_dict.
    Né   z%Cannot handle ST with multiple shardsz$Can only handle nested ShardedTensorr   )ÚitemsrF   rP   rK   rL   Ú
isinstancer   ÚmetadataÚshard_offsetsÚshard_sizesrM   Ú_process_group)r\   ÚspecsÚdp_pgÚkeyÚvalueÚshards         r.   Ú_get_state_dict_2d_layoutrj   w   s  € ð #%€EØ)-€EØ ×&Ò&Ñ(Ô(ð 0ð 0‰
ˆˆUØ˜EŸJšJ™LœLÐ)ˆˆc‰
Ý˜UÑ#Ô#ð 	0å�E×&Ò&Ñ(Ô(Ñ)Ô)¨QÒ.Ð.Ð.Ø6ñ /Ô.Ð.åØ•}ñô ð 6ð 6à5ñ6ô 6ð ð ×&Ò&Ñ(Ô(¨Ô+ˆEà”Ô,Ø”Ô*ðˆE�#‰Jð ”LÔ/ˆEøð 	Øðð r0   c                   óž   ‡ — e Zd ZU eeef         ed<   eed<   eed<   deee	e
         f         ddfˆ fd„Zdefd„Zd	edej        fˆ fd
„Zˆ xZS )Ú_ReaderWithOffsetÚtranslationr\   ra   Úfqn_to_offsetr(   Nc                 óš   •— t          ¦   «                              ¦   «          || _        t          i ¦  «        | _        i | _        i | _        d S ©N)ÚsuperÚ__init__rn   r   ra   r\   rm   )Úselfrn   Ú	__class__s     €r.   rr   z_ReaderWithOffset.__init__¡   sC   ø€ Ý‰Œ×ÒÑÔÐØ*ˆÔÝ  ™œˆŒØˆŒØˆÔÐÐr0   c           	      ó˜  — g }i | _         | j                             ¦   «         D �]˜\  }}| j        j        |         }t          |t          ¦  «        s|t          |||¦  «        z  }ŒB|| j        vr|t          |||¦  «        z  }Œ`| j        |         }t          | 
                    ¦   «         ¦  «        dk    sJ ‚| 
                    ¦   «         d         }t          t          j        t          |j        j        |¦  «        ¦  «        t          j        |j        j        ¦  «        ¬¦  «        g}t#          |t%          t&          |¦  «        |¦  «        }|D ]f}	|	j        j        €J ‚t-          |	j        j        |¦  «        }
t/          j        |	j        t          j        |
¦  «        ¬¦  «        }|| j         |	j        <   Œg||z  }�Œšt3          |¦  «        S )Nr^   r   )ÚoffsetsÚsizes)Úoffset)rm   r\   r_   ra   Ústate_dict_metadatar`   r   r   rn   rK   rL   r   rX   ÚSizer   rb   rc   r   r   r   Ú
dest_indexrx   r   ÚdataclassesÚreplacer   )rs   ÚrequestsÚfqnÚobjÚmdrx   Úoriginal_shardÚlocal_chunksÚreqsÚriÚoriginal_offsetÚoriginal_indexs               r.   Úcreate_local_planz#_ReaderWithOffset.create_local_plan¨   sÞ  € ØˆØˆÔØœ×-Ò-Ñ/Ô/ð $	ñ $	‰HˆC�Ø”Ô2°3Ô7ˆBÝ˜c¥=Ñ1Ô1ð ØÕ.¨s°B¸Ñ<Ô<Ñ<�Øà˜$Ô,Ð,Ð,ØÕ.¨s°B¸Ñ<Ô<Ñ<�ØàÔ'¨Ô,ˆFå�s×'Ò'Ñ)Ô)Ñ*Ô*¨aÒ/Ð/Ð/Ð/Ø ×-Ò-Ñ/Ô/°Ô2ˆNå$Ý!œJÝ)¨.Ô*AÔ*OÐQWÑXÔXñô õ  œ* ^Ô%<Ô%HÑIÔIð	ñ ô ðˆLõ 4Ø•TÕ/°Ñ4Ô4°lñô ˆDð
 ð Að A�Ø”}Ô+Ð7Ð7Ð7Ý"3°B´MÔ4HÈ&Ñ"QÔ"Q�Ý!,Ô!4Ø”M­%¬*°_Ñ*EÔ*Eð"ñ "ô "�ð 3A�Ô  ¤Ñ/Ð/à˜ÑˆH‰HÝ˜Ñ!Ô!Ð!r0   Úindexc                 óx   •— t          ¦   «                              | j                             ||¦  «        ¦  «        S rp   )rq   Úlookup_tensorrm   Úget)rs   r‰   rt   s     €r.   r‹   z_ReaderWithOffset.lookup_tensorÒ   s.   ø€ Ý‰wŒw×$Ò$ TÔ%5×%9Ò%9¸%ÀÑ%GÔ%GÑHÔHÐHr0   )Ú__name__Ú
__module__Ú__qualname__r   r   Ú__annotations__r   r   rG   r   Úintrr   r   rˆ   rX   ÚTensorr‹   Ú__classcell__)rt   s   @r.   rl   rl   œ   sß   ø€ € € € € € Ø�m ]Ð2Ô3Ð3Ð3Ñ3ØÐÐÑØÐÐÑð d¨3°¸´Ð+=Ô&>ð À4ð ð ð ð ð ð ð(" 8ð ("ð ("ð ("ð ("ðTI =ð I°U´\ð Ið Ið Ið Ið Ið Ið Ið Ið Ið Ir0   rl   Úmodel_state_dictÚoptimizer_keyÚstorage_readerÚplannerc                 óR  — |                      ¦   «         }t          | ¦  «        \  }}t          j                             |¦  «        j        }t          |¦  «        }|€wg }	t          t          j        ¦   «         ¦  «        D ]B}
t          ||
| 
                    ¦   «         z  ¦  «        }|	                     d|
› d|› �¦  «         ŒCt          d|	¬¦  «        }nt          |¦  «        }i }i }|j                             ¦   «         D �]n\  }}|j        |         }|d         |k    rŒ t#          |t$          ¦  «        rd||<   Œ;|j                             ¦   «         dk    rt+          |j        |j        |¦  «        ||<   Œw|€qt/          t+          |j        |j        |¦  «        t          j        ¦   «         t          j        ¦   «         | 
                    ¦   «         t3          ¦   «         ¬¦  «        ||<   Œê|d	         }|                     |d|j        f¦  «        d         }t7          |j        j        |j        j        |j        j        |j        j        |j        j         ¬
¦  «        }| !                    tE          j#        |¦  «        |¦  «        }g }t          j        |¦  «        }|j$        D ]p}tK          tL          |j'        ¦  «         (                    ¦   «         |k    rŒ3|                     tS          t+          |j        |j*        |¦  «        |¬¦  «        ¦  «         ŒqtW          j,        |||¬¦  «        }||v r=||         d         �/tK          tZ          t\                   ||         d         ¦  «        ||<   |||<   �Œpt_          |||�ta          |¦  «        n|¬¦  «         tc          ||j        ¦  «        }|S )aç  
    Load a state_dict in conjunction with FSDP sharded optimizer state.

    This is the current recommended way to checkpoint FSDP.
    >>> # xdoctest: +SKIP
    >>> import torch.distributed.checkpoint as dist_cp
    >>> # Save
    >>> model: torch.nn.Model
    >>> optim_params = model.parameters()
    >>> optim = torch.optim.SGD(optim_params, lr=0.01)
    >>> # Save
    >>> with FSDP.state_dict_type(model, StateDictType.SHARDED_STATE_DICT):
    >>>     state_dict = {
    >>>         "optimizer": FSDP.optim_state_dict(model, optim),
    >>>         "model": model.state_dict()
    >>>     }
    >>>     dist_cp.save_state_dict(
    >>>         state_dict=optim_state,
    >>>         storage_writer=dist_cp.FileSystemWriter("checkpoint"),
    >>>         planner=dist_cp.DefaultSavePlanner(),
    >>>     )
    >>>
    >>> # Load
    >>> with FSDP.state_dict_type(model_tp, StateDictType.SHARDED_STATE_DICT):
    >>>     model_state_dict = model_tp.state_dict()
    >>>     checkpoint = {
    >>>         "model": model_state_dict
    >>>     }
    >>>     dist_cp.load_state_dict(
    >>>         state_dict=checkpoint,
    >>>         storage_reader=dist_cp.FileSystemReader(checkpoint_file),
    >>>         planner=dist_cp.DefaultLoadPlanner(),
    >>>     )
    >>>     model.load_state_dict(checkpoint["model_state"])
    >>>
    >>>     optim_state = dist_cp.load_sharded_optimizer_state_dict(
    >>>         model_state_dict,
    >>>         optimizer_key="optimizer",
    >>>         storage_reader=dist_cp.FileSystemReader("checkpoint"),
    >>>     )
    >>>
    >>>     flattened_osd = FSDP.optim_state_dict_to_load(
    >>>        model, optim, optim_state["optimizer"]
    >>>     )
    >>>
    >>>     optim.load_state_dict(flattened_osd)
    Nr5   r6   r   r>   z
<bytes_io>r^   )ÚrankÚ
world_sizeÚnum_devices_per_noder1   é   )rS   rT   rU   Úmemory_formatrV   )rM   ra   )Úprocess_group)r\   r–   r—   )2Úread_metadatarj   r<   rA   rB   rC   r
   rD   rE   r   r,   Úappendr   rH   ry   r_   Úplanner_datar`   r   rF   Únumelr[   Ú
propertiesr!   Úget_rankr    rŒ   ÚShardTensorPropertiesrS   rT   rU   r�   rV   Úbuild_metadatarX   rz   Úshards_metadatar   r"   Ú	placementr™   r   rc   r   Ú+_init_from_local_shards_and_global_metadatar   r‘   r   rl   r   )r”   r•   r–   r—   ra   Úlayout_specsrf   Údp_pg_device_typer-   r@   ÚiÚdevice_infoÚsharding_specr\   rn   rg   rh   Úkey_pathÚspec_keyÚ
alloc_sizer£   Úst_mdrL   Úcurrent_rankÚshard_mdÚsts                             r.   r$   r$   Ö   sÚ  € ðj ×+Ò+Ñ-Ô-€Hå3Ð4DÑEÔEÑ€L�%ÝÔ-×DÒDÀUÑKÔKÔPÐÝ&Ð'8Ñ9Ô9€Mà€}Øˆ
Ý•tÔ*Ñ,Ô,Ñ-Ô-ð 	9ð 	9ˆAÝ0Ø! 1 }×'AÒ'AÑ'CÔ'CÑ#Cñô ˆKð ×ÒÐ7 aÐ7Ð7¨+Ð7Ð7Ñ8Ô8Ð8Ð8Ý)¨a¸JÐGÑGÔGˆˆå,¨UÑ3Ô3ˆð #%€Jà.0€MØÔ2×8Ò8Ñ:Ô:ð 8!ñ 8!‰
ˆˆUØÔ(¨Ô-ˆØ�AŒ;˜-Ò'Ð'Øå�eÕ1Ñ2Ô2ð 	Ø*ˆJ�s‰OØð Œ:×ÒÑÔ Ò"Ð"Ý+ØÔ  %¤*Ð.?ñô ˆJ�s‰OˆOð ˆ]Ý:Ý˜eÔ.°´
Ð<MÑNÔNÝ”]‘_”_ÝÔ.Ñ0Ô0Ø%2×%?Ò%?Ñ%AÔ%AÝ%Ñ'Ô'ðñ ô ˆJ�s‰OˆOð   ”{ˆHØ%×)Ò)¨(°T¸5¼:Ð4FÑGÔGÈÔJˆJå.ØÔ&Ô,ØÔ'Ô.Ø#Ô.Ô<Ø#Ô.Ô<Ø Ô+Ô6ðñ ô ˆJð "×0Ò0µ´¸JÑ1GÔ1GÈÑTÔTˆEØˆLÝœ=¨Ñ/Ô/ˆLØ!Ô1ð 
ð 
�Ý�¨Ô(:Ñ;Ô;×@Ò@ÑBÔBÀlÒRÐRØØ×#Ò#ÝÝ,Ø!Ô,¨hÔ.BÐDUñ ô  ð "*ð	ñ ô ñô ð ð õ ÔJØ˜e°5ðñ ô ˆBð ˜<Ð'Ð'¨L¸Ô,BÀ1Ô,EÐ,QÝ%)­(µ3¬-¸ÀhÔ9OÐPQÔ9RÑ%SÔ%S�˜cÑ"à ˆJ�s‰O‰Oõ ØØ%à49Ð4EÕ! -Ñ0Ô0Ð0È7ð	ñ ô ð õ & j°(Ô2GÑHÔH€JàÐr0   )r%   rp   )Gr|   Útypingr   r   r   r   r   r   r	   rX   Útorch.distributedÚdistributedr<   Útorch._utilsr
   Ú+torch.distributed._shard.sharded_tensor.apir   Ú0torch.distributed._shard.sharded_tensor.metadatar   r¥   Ú-torch.distributed._shard.sharded_tensor.shardr   Ú:torch.distributed._shard.sharding_spec.chunk_sharding_specr   Ú)torch.distributed.checkpoint._nested_dictr   Ú,torch.distributed.checkpoint.default_plannerr   Ú%torch.distributed.checkpoint.metadatar   r   r   r   r   r   Ú$torch.distributed.checkpoint.plannerr   r   Ú,torch.distributed.checkpoint.planner_helpersr   r   Ú.torch.distributed.checkpoint.state_dict_loaderr   Ú$torch.distributed.checkpoint.storager   Ú"torch.distributed.checkpoint.utilsr   r   r   Ú"torch.distributed.distributed_c10dr    Ú#torch.distributed.fsdp._shard_utilsr!   Útorch.distributed.remote_devicer"   Útorch.distributed.tensorr#   rG   r‘   ÚSTATE_DICT_2D_LAYOUTÚ__all__r/   ÚProcessGrouprH   r’   ÚboolrP   r[   rj   rl   r$   © r0   r.   ú<module>rÏ      s  ðð Ð Ð Ð Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eà €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø +Ð +Ð +Ð +Ð +Ð +Ø EÐ EÐ EÐ EÐ EÐ Eðð ð ð ð ð ð @Ð ?Ð ?Ð ?Ð ?Ð ?Ø XÐ XÐ XÐ XÐ XÐ XØ JÐ JÐ JÐ JÐ JÐ JØ KÐ KÐ KÐ KÐ KÐ Kðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ Fðð ð ð ð ð ð ð ð KÐ JÐ JÐ JÐ JÐ JØ >Ð >Ð >Ð >Ð >Ð >ðð ð ð ð ð ð ð ð ð ð
 BÐ AÐ AÐ AÐ AÐ AØ LÐ LÐ LÐ LÐ LÐ LØ :Ð :Ð :Ð :Ð :Ð :Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ð ˜C  x°¸´Ô'>ÀÈÄÐ'MÔ!NÐNÔOÐ ð
 (ð€ð
ð  #ð °Cð ÀSð ð ð ð ð '+ðð Ø�Ô"Ô#ðàðð ð ð ð(˜5œ<ð ¨Dð ð ð ð ð  FLðð ØðØ#+¨C¤=ðØ?Bðà
„\ðð ð ð ð("Øð"à
Ð ¨$Ô*;Ô!<Ð<Ô=ð"ð "ð "ð "ðJ7Ið 7Ið 7Ið 7Ið 7IÐ*ñ 7Iô 7Ið 7Ið| &*ð	Nð NØ%ðNàðNð "ðNð �kÔ"ð	Nð
 ðNð Nð Nð Nð Nð Nr0   