§
    N�’je  ã                   ó  — d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZmZmZmZ d dlmZ d dlmZmZmZmZ d„ Z	 ddej        d	ed
ededej        dej        dz  defd„Zdej        d	ededefd„Zdededz  dej        fd„ZdS )é    N)Ú_get_device_module)Údistributed_c10d)ÚShardÚShardedTensorÚShardedTensorMetadataÚTensorProperties©ÚShardMetadata)Ú
DeviceMeshÚDTensorÚ	Replicater   c                 óâ   — |                      ¦   «         dk    rd| › d|› �S |                      ¦   «         dk    r*d| › d|› dt          |¦  «                             ¦   «         › �S d| › d|› d| |z  › �S )NÚcpuzrank:ú/Úhpuú:)Úlowerr   Úcurrent_device)ÚrankÚdevice_typeÚnum_devices_per_nodes      úQC:\Projects\DSCons\.venv\Lib\site-packages\torch\distributed\fsdp\_shard_utils.pyÚ_get_remote_device_strr      s�   € Ø×ÒÑÔ˜eÒ#Ð#Ø+�tÐ+Ð+˜kÐ+Ð+Ð+Ø	×	Ò	Ñ	Ô	 Ò	%Ð	%Ø^�tÐ^Ð^˜kÐ^Ð^Õ,>¸{Ñ,KÔ,K×,ZÒ,ZÑ,\Ô,\Ð^Ð^Ð^àI�tÐIÐI˜kÐIÐI¨DÐ3GÑ,GÐIÐIÐIó    Útensorr   Ú
world_sizer   ÚpgÚdeviceÚreturnc                 óø  ‡‡‡‡— |                       |d¬¦  «        }t          |¦  «        |k    r…||                              ¦   «         }d„ |                      ¦   «         D ¦   «         Št	          j        |                      ¦   «         d         |z  ¦  «        |z  ‰d<   t          j        |‰|¦  «        g}ng }d„ |D ¦   «         }	dgt          t          j
        d„ |	D ¦   «         ¦  «        ¦  «        dd…         z   }
dgt          |	d         ¦  «        dz
  z  Šˆfd	„|
D ¦   «         }|€t          j        ‰¦  «        j        n|j        Šˆˆˆfd
„t          t          |	¦  «        ¦  «        D ¦   «         }t          |	¦  «        t          |¦  «        k    s t          |	¦  «        t          |¦  «        k    r?t          dt          |	¦  «        › dt          |¦  «        › dt          |¦  «        › �¦  «        ‚d„ t!          ||	|¦  «        D ¦   «         }t#          ||                      ¦   «         t%          | j        | j        dt*          j        |                      ¦   «         ¬¦  «        ¬¦  «        }t1          j        ||‰¬¦  «        S )zŸ
    Shard a tensor to chunks along the first dimension. The local rank will get its
    corresponding chunk as the local shard to create a ShardedTensor.
    r   )Údimc                 ó   — g | ]}d ‘ŒS ©r   © ©Ú.0Ú_s     r   ú
<listcomp>z0_create_chunk_sharded_tensor.<locals>.<listcomp>,   s   € Ð,Ð,Ð,˜�1Ð,Ð,Ð,r   c                 óP   — g | ]#}t          |                     ¦   «         ¦  «        ‘Œ$S r$   )ÚlistÚsize)r&   Úchunks     r   r(   z0_create_chunk_sharded_tensor.<locals>.<listcomp>3   s(   € Ð:Ð:Ð:¨%•4˜Ÿ
š
™œÑ%Ô%Ð:Ð:Ð:r   c                 ó   — g | ]
}|d          ‘ŒS r#   r$   )r&   Ú
chunk_sizes     r   r(   z0_create_chunk_sharded_tensor.<locals>.<listcomp>5   s   € ÐJÐJÐJ°
˜j¨œmÐJÐJÐJr   Néÿÿÿÿé   c                 ó   •— g | ]}|g‰z   ‘Œ	S r$   r$   )r&   Úd0Úoffsetss     €r   r(   z0_create_chunk_sharded_tensor.<locals>.<listcomp>8   s   ø€ Ð;Ð;Ð;¨�b�T˜G‘^Ð;Ð;Ð;r   c                 óX   •— g | ]&}t          t          j        ‰|¦  «        ‰‰¦  «        ‘Œ'S r$   )r   ÚdistÚget_global_rank)r&   Úrr   r   r   s     €€€r   r(   z0_create_chunk_sharded_tensor.<locals>.<listcomp>>   sK   ø€ ð ð ð ð õ 	ÝÔ   QÑ'Ô'ØØ ñ	
ô 	
ðð ð r   zQExpected chunk_sizes, chunk_offsets, and placements to have the same length, got z, c                 ó8   — g | ]\  }}}t          |||¦  «        ‘ŒS r$   r	   )r&   Úoffsetr+   Ú	placements       r   r(   z0_create_chunk_sharded_tensor.<locals>.<listcomp>K   s:   € ð ð ð á#ˆF�D˜)õ 	�f˜d IÑ.Ô.ðð ð r   F)ÚdtypeÚlayoutÚrequires_gradÚmemory_formatÚ
pin_memory)Úshards_metadatar+   Útensor_properties)Úsharded_tensor_metadataÚprocess_group)r,   ÚlenÚcloner+   ÚmathÚceilr   Úfrom_tensor_and_offsetsr*   Ú	itertoolsÚ
accumulater   Ú_get_pg_default_deviceÚtypeÚrangeÚAssertionErrorÚzipr   r   r;   r<   ÚtorchÚcontiguous_formatÚ	is_pinnedr   Ú+_init_from_local_shards_and_global_metadata)r   r   r   r   r   r   ÚchunksÚlocal_shardÚlocal_shardsÚchunk_sizesÚdim0_offsetsÚchunk_offsetsÚ
placementsÚshard_metadatarB   r   r3   s      ``          @@r   Ú_create_chunk_sharded_tensorr\      s×  øøøø€ ð �\Š\˜*¨!ˆ\Ñ,Ô,€FÝ
ˆ6�{„{�TÒÐØ˜T”l×(Ò(Ñ*Ô*ˆØ,Ð,˜fŸkšk™mœmÐ,Ñ,Ô,ˆÝ”Y˜vŸ{š{™}œ}¨QÔ/°*Ñ<Ñ=Ô=ÀÑDˆ�‰
ÝÔ5°kÀ7ÈDÑQÔQÐRˆˆàˆð ;Ð:°6Ð:Ñ:Ô:€KØ�3�ÝÔÐJÐJ¸kÐJÑJÔJÑKÔKñô à	€r€côñ €Lð ˆc•S˜ QœÑ(Ô(¨1Ñ,Ñ-€GØ;Ð;Ð;Ð;¨lÐ;Ñ;Ô;€Mð ˆ>õ 	Ô/°Ñ3Ô3Ô8Ð8àŒ[ð ð
ð ð ð ð ð õ •s˜;Ñ'Ô'Ñ(Ô(ðñ ô €Jõ ˆ;ÑÔ�3˜}Ñ-Ô-Ò-Ð-µ°[Ñ1AÔ1AÅSÈÁ_Ä_Ò1TÐ1TÝðOÝ�{Ñ#Ô#ðOð OÝ'*¨=Ñ'9Ô'9ðOð OÝ=@À¹_¼_ðOð Oñ
ô 
ð 	
ðð å'*¨=¸+ÀzÑ'RÔ'Rðñ ô €Nõ 4Ø&Ø�[Š[‰]Œ]Ý*Ø”,Ø”=ØÝÔ1Ø×'Ò'Ñ)Ô)ð
ñ 
ô 
ð
ñ 
ô 
Ðõ ÔDØÐ.EÐUWðñ ô ð r   Údevice_meshc                 óB  — |                       ¦   «                              ¦   «         } d„ t          |j        ¦  «        D ¦   «         }d„ t          |j        ¦  «        D ¦   «         }t	          d¦  «        |d<   t          j        | ||d¬¦  «                             |¬¦  «        S )zš
    Shard a tensor to chunks along the first dimension. The local rank will get its
    corresponding chunk as the local tensor to create a DTensor.
    c                 ó*   — g | ]}t          ¦   «         ‘ŒS r$   ©r   r%   s     r   r(   z)_create_chunk_dtensor.<locals>.<listcomp>m   s   € ÐIÐIÐI¨A�I™KœKÐIÐIÐIr   c                 ó*   — g | ]}t          ¦   «         ‘ŒS r$   r`   r%   s     r   r(   z)_create_chunk_dtensor.<locals>.<listcomp>n   s   € ÐEÐEÐE¨�	™œÐEÐEÐEr   r   r/   F)Ú	run_check)rZ   )ÚdetachrE   rM   ÚndimÚDShardr   Ú
from_localÚredistribute)r   r   r]   Úreplicate_placementsÚshard_placementss        r   Ú_create_chunk_dtensorrj   _   sª   € ð �]Š]‰_Œ_×"Ò"Ñ$Ô$€Fð JÐIµ°{Ô7GÑ1HÔ1HÐIÑIÔIÐØEÐE­U°;Ô3CÑ-DÔ-DÐEÑEÔEÐÝ! !™9œ9Ð�RÑåÔØ�Ð1¸Uðñ ô ç‚lØ#ð ñ ô ðr   Ú	root_meshc                 ó  — || j         k    rt          d¦  «        ‚t          t          j        | j        ¦  «        ¦  «        }t          ¦   «         |d<   |                      | j         |¬¦  «        } |                      ¦   «         S )zT
    All gather a DTensor in its sharded dimension and return the local tensor.
    z2The device mesh of a tensor should be a root mesh.r/   )r]   rZ   )	r]   rN   r*   ÚcopyÚdeepcopyrZ   r   rg   Úto_local)r   rk   rZ   s      r   Ú_all_gather_dtensorrp   x   s   € ð �FÔ&Ò&Ð&ÝÐQÑRÔRÐRå•d”m FÔ$5Ñ6Ô6Ñ7Ô7€Jõ ‘[”[€Jˆr�NØ× Ò ØÔ&Øð !ñ ô €Fð
 �?Š?ÑÔÐr   )N)rm   rI   rF   rP   Útorch.distributedÚdistributedr5   Útorch._utilsr   r   Ú'torch.distributed._shard.sharded_tensorr   r   r   r   Ú&torch.distributed._shard.sharding_specr
   Útorch.distributed.tensorr   r   r   re   r   ÚTensorÚintÚProcessGroupr   r\   rj   rp   r$   r   r   ú<module>rz      sÏ  ðà €€€Ø Ð Ð Ð Ø €€€à €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø +Ð +Ð +Ð +Ð +Ð +Ø .Ð .Ð .Ð .Ð .Ð .ðð ð ð ð ð ð ð ð ð ð ð ð AÐ @Ð @Ð @Ð @Ð @Ø TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ TðJð Jð Jð #'ð?ð ?ØŒLð?à
ð?ð ð?ð ð	?ð
 	Ôð?ð ŒL˜4Ñð?ð ð?ð ?ð ?ð ?ðDØŒLðà
ðð ðð ð	ð ð ð ð2Øðà˜DÑ ðð „\ðð ð ð ð ð r   