§
    O�’jk8  ã                   óì  — d dl Z d dlmZmZ d dlZd dlmZ d dlmc m	c m
Z d dlmc mZ d dlmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lm Z m!Z!m"Z"mZ# d dl$m%Z%m&Z& dgZ'de!de(ej)        ej)        f         fd„Z*de!de+de(ej)        ej)        f         fd„Z,de!de(ej)        ej)        f         fd„Z-de!de+defd„Z.de!dej/        defd„Z0de!dej/        fd„Z1dej2        dej3        de+dej2        fd„Z4dej3        de+de+de+dej/        dej3        fd„Z5dej3        de+d e de!fd!„Z6dej3        de(ej3        e7e         f         fd"„Z8de!d#e dz  dej3        fd$„Z9 G d%„ de¦  «        Z:dS )&é    N)ÚAnyÚcast)ÚShardÚShardedTensorÚShardedTensorMetadataÚTensorProperties)ÚShardMetadata)ÚChunkShardingSpec)Ú_set_fsdp_flattened)ÚFSDPExtensions)Ú_create_chunk_sharded_tensor)Ú_remote_device)Ú
DeviceMeshÚDTensorÚ	Replicater   )Ú_flatten_tensorÚ_unflatten_tensorÚDTensorExtensionsÚtensorÚreturnc                 óÚ  — | j         }|j        dk    rt          d¦  «        ‚| j        d         }dgt	          |                      ¦   «         ¦  «        z  }|                     d¬¦  «        }| j        d                              ¦   «         r7t          t          |¦  «        j	        }|                      |¦  «        |z  }|||<   t          j        |¦  «        | j                             ¦   «         fS )Né   ú&Only 1D DeviceMeshes currently handledr   )Úmesh_dim)Údevice_meshÚndimÚAssertionErrorÚ
placementsÚlenÚsizeÚis_shardr   ÚDShardÚdimÚtorchÚSizeÚ_local_tensor)r   r   Ú	placementÚoffsetsÚ
num_chunksÚ	shard_dimÚ
chunk_sizes          úTC:\Projects\DSCons\.venv\Lib\site-packages\torch\distributed\tensor\parallel\fsdp.pyÚ_get_boxr-      sÚ   € ØÔ$€KØÔ˜1ÒÐÝÐEÑFÔFÐFàÔ! !Ô$€IØˆc•C˜Ÿš™œÑ&Ô&Ñ&€GØ×!Ò!¨1Ð!Ñ-Ô-€Jð Ô˜Ô×$Ò$Ñ&Ô&ð (Ý� Ñ+Ô+Ô/ˆ	Ø—[’[ Ñ+Ô+¨zÑ9ˆ
Ø'ˆ�	ÑåŒJ�wÑÔ Ô!5×!:Ò!:Ñ!<Ô!<Ð=Ð=ó    Úidxc                 ól   ‡— t          | ¦  «        \  }}t          j        ˆfd„|D ¦   «         ¦  «        |fS )Nc                 ó   •— g | ]}|‰z  ‘ŒS © r2   )Ú.0Úvalr/   s     €r,   ú
<listcomp>z _get_box_for.<locals>.<listcomp>3   s   ø€ Ð5Ð5Ð5 c˜˜c™	Ð5Ð5Ð5r.   )r-   r$   r%   )r   r/   r(   r    s    `  r,   Ú_get_box_forr6   1   s>   ø€ Ý˜VÑ$Ô$�M€GˆTÝŒJÐ5Ð5Ð5Ð5¨WÐ5Ñ5Ô5Ñ6Ô6¸Ð=Ð=r.   c                 óv   — | j         }|                     ¦   «         }|€t          ‚t          | |d         ¦  «        S )Nr   )r   Úget_coordinater   r6   )r   r   Úcoords      r,   Ú_get_local_boxr:   6   s:   € ØÔ$€KØ×&Ò&Ñ(Ô(€EØ€}ÝÐÝ˜  a¤Ñ)Ô)Ð)r.   ÚdtÚcurrent_rankc                 óà   — | j         }|j        dk    rt          d¦  «        ‚t          | ¦  «        \  }}t	          t          |¦  «        t          |¦  «        d|› d| j        j        › �¬¦  «        S )Nr   r   úrank:ú/©Úshard_offsetsÚshard_sizesr'   )r   r   r   r:   r	   Úlistr&   Údevice)r;   r<   Úmeshr(   Úsizess        r,   Ú_create_shard_md_from_dtrG   >   sv   € ØŒ>€DØ„y�A‚~€~ÝÐEÑFÔFÐFå# BÑ'Ô'�N€GˆUÝÝ˜7‘m”mÝ˜‘K”KØB˜,ÐBÐB¨Ô)9Ô)@ÐBÐBðñ ô ð r.   Údt_pgc                 ó,  — g }t          j        |¦  «        }|dk    rdnd}| j        d                              ¦   «         r|                     ¦   «         }nd}t          |¦  «        D ]l}t          | |¦  «        \  }}|                     t          t          |¦  «        t          |¦  «        d|dk    r|n|› d| j
        j        › �¬¦  «        ¦  «         Œmt          ||                      ¦   «         t          | j        | j        | j        ¬¦  «        ¬¦  «        S )Nr   r   r>   r?   r@   )ÚdtypeÚlayoutÚrequires_grad)Úshards_metadatar    Útensor_properties)ÚdistÚget_rankr   r!   r    Úranger6   Úappendr	   rC   r&   rD   r   r   rJ   rK   rL   )	r;   rH   Ú	shards_mdÚmy_rankÚscapegoat_rankÚshard_countÚir(   rF   s	            r,   Ú!_create_sharded_tensor_md_from_dtrX   K   s3  € ð €IÝŒm˜EÑ"Ô"€GØ! Aš+˜+�Q�Q¨1€Nð 
„}�QÔ× Ò Ñ"Ô"ð Ø—j’j‘l”lˆˆàˆå�;ÑÔð 

ð 

ˆÝ% b¨!Ñ,Ô,‰ˆ�Ø×ÒÝÝ" 7™mœmÝ  ™KœKà\¨a°!ªe¨e˜N˜N¸Ð\Ð\À2ÔCSÔCZÐ\Ð\ð	ñ ô ñ	
ô 	
ð 	
ð 	
õ !Ø!Ø�WŠW‰YŒYÝ*Ø”(Ø”9ØÔ*ð
ñ 
ô 
ð	ñ 	ô 	ð 	r.   c                 ól   — | j         }|j        dk    rt          d¦  «        ‚|                     ¦   «         S )Nr   r   )r   r   r   Ú	get_group)r;   rE   s     r,   Ú
_get_dt_pgr[   s   s3   € ØŒ>€DØ„y�A‚~€~ÝÐEÑFÔFÐFØ�>Š>ÑÔÐr.   ÚspecÚrankc                 ó   — t          | t          ¦  «        s| S d}| j        D ]P}t          t          |¦  «        }|                     ¦   «         |k    r!|                     ¦   «         |j        k    rd} nŒQ|ršt          j        | ¦  «        } t          | j        ¦  «        D ]q\  }}t          t          |¦  «        }|                     ¦   «         |k    r?|                     ¦   «         |j        k    r"t	          d|› d|j        › �¦  «        | j        |<   Œr| S )zØ
    Rewrite ``spec`` to match the device of ``tensor``.

    FSDP.sharded_optim_state_dict sneakly ships optimizer state to CPU so if the original ShardingSpec
    produces CUDA metadata, ST construction bombs.
    FTr>   r?   )
Ú
isinstancer
   r   r   r   r]   rD   ÚcopyÚdeepcopyÚ	enumerate)r\   r   r]   ÚrewriteÚprW   r'   s          r,   Ú_rewrite_spec_if_neededre   z   s  € õ �dÕ-Ñ.Ô.ð Øˆð €GØŒ_ð ð ˆÝ• Ñ#Ô#ˆØ�6Š6‰8Œ8�tÒÐ §¢¡
¤
¨f¬mÒ ;Ð ;ØˆGØˆEøØð TÝŒ}˜TÑ"Ô"ˆå% d¤oÑ6Ô6ð 	Tð 	T‰LˆAˆyÝ�^¨YÑ7Ô7ˆIØ�~Š~ÑÔ 4Ò'Ð'¨I×,<Ò,<Ñ,>Ô,>À&Ä-Ò,OÐ,Oå%3Ð4R¸DÐ4RÐ4RÀ6Ä=Ð4RÐ4RÑ%SÔ%S�” Ñ"øà€Kr.   Ú
world_sizeÚnum_devices_per_nodeÚpgc           	      óæ  — t          | ¦  «        t          u ræt          |                      ¦   «         ¦  «        dk    rt          ‚|                      ¦   «         }t          |||||¦  «        }|                      ¦   «         d         }t          |t          j	        |j
        ¦  «        ¦  «        g}t          j	        |  
                    ¦   «         ¦  «        }	d|	j        _        t          j        ||	| j        d¬¦  «        }
|
S t          | ¦  «        t          u rÍ| j        }|j        dk    rt	          d¦  «        ‚| j        }t          |||t&          j                             ¦   «         |¦  «        }t-          | ¦  «        }t          |t/          | t1          j        |¦  «        ¦  «        ¦  «        g}t5          | |¦  «        }	d|	j        _        t          j        ||	|d¬¦  «        }
|
S t          | ||||¦  «        S )Nr   r   F)Úsharded_tensor_metadataÚprocess_groupÚ
init_rrefsr   )Útyper   r   Úlocal_shardsr   Úlocal_tensorr   r   r`   ra   ÚmetadatarN   rL   Ú+_init_from_local_shards_and_global_metadataÚ_process_groupr   r   r   r&   r$   ÚacceleratorÚdevice_countr[   rG   rO   rP   rX   )r   r]   rf   rg   rh   Úinner_paramÚinner_stÚouter_local_shardÚshardsÚst_metaÚst_outerr   rH   s                r,   Ú_chunk_tensorr{   ™   s  € õ ˆF�|„|•}Ð$Ð$Ýˆv×"Ò"Ñ$Ô$Ñ%Ô%¨Ò*Ð*Ý Ð à×)Ò)Ñ+Ô+ˆÝ/ØØØØ Øñ
ô 
ˆð #×/Ò/Ñ1Ô1°!Ô4Ðå�(�DœMÐ*;Ô*DÑEÔEÑFÔFð
ˆõ ”- §¢Ñ 1Ô 1Ñ2Ô2ˆØ27ˆÔ!Ô/å ÔLØØ$+Ø Ô/Øð	
ñ 
ô 
ˆð ˆÝ	ˆf‰Œ�Ð	 Ð	 ØÔ(ˆØÔ˜qÒ Ð Ý Ð!IÑJÔJÐJàÔ*ˆå/ØØØÝÔ×*Ò*Ñ,Ô,Øñ
ô 
ˆõ ˜6Ñ"Ô"ˆõ �(Õ4°V½T¼]È5Ñ=QÔ=QÑRÔRÑSÔSð
ˆõ 4°F¸EÑBÔBˆØ27ˆÔ!Ô/å ÔLØØ$+ØØð	
ñ 
ô 
ˆð ˆå+ØØØØ Øñ
ô 
ð 	
r.   r   c                 óŒ  — |�|                      ¦   «         nd}|€t          d¦  «        ‚|j        dk     rt          d|j        › d�d¦  «        ‚|                      ¦   «                              ¦   «         } t          | t          j        ¦  «        r�t          | t          ¦  «        s{d„ t          |j        ¦  «        D ¦   «         }d„ t          |j        ¦  «        D ¦   «         }t          d	¦  «        |d	<   t          j        | ||d
¬¦  «                             ||¬¦  «        S | j        }|d	         }|                      ¦   «         } d„ t          |j        ¦  «        D ¦   «         }||d<   d„ t          |j        ¦  «        D ¦   «         }t          d	¦  «        |d<   ||d<   t          j        | ||d
¬¦  «                             ||¬¦  «        S )zœ
    Shard a tensor to chunks along the first dimension.

    The local rank will gets its corresponding chunk as the local tensor to create a DTensor.
    Nz4No parent device_mesh is found for FSDP device_mesh.é   z!Found parent device_mesh of ndim=ú,zbut meshes must be at least 2D.c                 ó*   — g | ]}t          ¦   «         ‘ŒS r2   ©r   ©r3   Ú_s     r,   r5   z"_chunk_dtensor.<locals>.<listcomp>  ó   € ÐKÐKÐK°¥	¡¤ÐKÐKÐKr.   c                 ó*   — g | ]}t          ¦   «         ‘ŒS r2   r€   r�   s     r,   r5   z"_chunk_dtensor.<locals>.<listcomp>  ó   € ÐGÐGÐG¨A�I™KœKÐGÐGÐGr.   r   F)Ú	run_check©r   r   c                 ó*   — g | ]}t          ¦   «         ‘ŒS r2   r€   r�   s     r,   r5   z"_chunk_dtensor.<locals>.<listcomp>  rƒ   r.   éÿÿÿÿc                 ó*   — g | ]}t          ¦   «         ‘ŒS r2   r€   )r3   rW   s     r,   r5   z"_chunk_dtensor.<locals>.<listcomp>  r…   r.   éþÿÿÿ)Ú_get_root_meshÚRuntimeErrorr   ÚdetachÚcloner_   r$   ÚTensorr   rQ   r"   Ú
from_localÚredistributer   Úto_local)r   r]   r   Ú	root_meshÚreplicate_placementsÚshard_placementsÚtp_placementsÚtp_placements           r,   Ú_chunk_dtensorr™   å   sý  € ð 1<Ð0G�×*Ò*Ñ,Ô,Ð,ÈT€IØÐÝÐQÑRÔRÐRØ„~˜ÒÐÝØA°	´ÐAÐAÐAØ-ñ
ô 
ð 	
ð �]Š]‰_Œ_×"Ò"Ñ$Ô$€Fõ
 �&�%œ,Ñ'Ô'ð %
µ
¸6Å7Ñ0KÔ0Kð %
ð  LÐKµU¸9¼>Ñ5JÔ5JÐKÑKÔKÐØGÐGµ°y´~Ñ1FÔ1FÐGÑGÔGÐÝ$ Q™iœiÐ˜ÑåÔ!Ø�IÐ3¸uð
ñ 
ô 
ç
Š,Ø!Ø'ð ñ 
ô 
ð	
ð Ô)ˆØ$ QÔ'ˆà—’Ñ"Ô"ˆð  LÐKµU¸9¼>Ñ5JÔ5JÐKÑKÔKÐØ#/Ð˜RÑ ØGÐGµ°y´~Ñ1FÔ1FÐGÑGÔGÐÝ% a™yœyÐ˜ÑØ+Ð˜ÑåÔ!Ø�IÐ3¸uð
ñ 
ô 
ç
Š,Ø!Ø'ð ñ 
ô 
ð	
r.   c                 ó0  — t          t          | ¦  «                             ¦   «         }t          |¦  «        dk    rDt	          |d         j        ¦  «        t          u r#|d         j        }|                     ¦   «         }|} | t          |¦  «        dk    r|ng fS )Nr   r   )r   r   rn   r   rm   r   )r   rx   Úinner_tensors      r,   Ú_pre_load_state_dictrœ   &  sˆ   € õ •- Ñ(Ô(×5Ò5Ñ7Ô7€FÝ
ˆ6�{„{�aÒÐ�D ¨¤Ô!1Ñ2Ô2µmÐCÐCØ˜a”yÔ'ˆØ×*Ò*Ñ,Ô,ˆØˆà�c &™kœk¨Ašo˜o�F�F°2Ð6Ð6r.   Úparent_meshc                 ó8  — || j         k    rt          ‚t          t          j        | j        ¦  «        ¦  «        }t          t          |¦  «        dz
  ¦  «        D ]}t          ¦   «         ||<   Œ|  	                    | j         |¬¦  «        } |  
                    ¦   «         S )zGAll gather a DTensor in its FSDP dimension and return the local tensor.r   r‡   )r   r   rC   r`   ra   r   rQ   r   r   r’   r“   )r   r�   r   rW   s       r,   Ú_all_gather_dtensorrŸ   2  sš   € ð
 �fÔ(Ò(Ð(ÝÐå•d”m FÔ$5Ñ6Ô6Ñ7Ô7€Jõ •3�z‘?”? QÑ&Ñ'Ô'ð $ð $ˆÝ!™œˆ
�1‰ˆØ× Ò ØÔ&Øð !ñ ô €Fð
 �?Š?ÑÔÐr.   c                   ól  ‡ — e Zd ZdZdˆ fd„Zdej        deej        edz  f         fd„Z	dej        dedej        fd„Z
	 ddej        d	ed
ededej        dej        dz  dej        fd„Zdej        d	ededej        fd„Zdej        deej        ee         f         fd„Zdededz  dej        fd„Zˆ xZS )r   zî
    DTensorExtension is the TensorFlattener extension needed for 2D FSDP + TP.

    This is the implementation for FSDPExtensions defined in
    https://github.com/pytorch/pytorch/blob/main/torch/distributed/fsdp/_fsdp_extensions.py
    r   Nc                 ó¶   •— t          ¦   «                              ¦   «          d | _        || _        t          j                             | j        ¦  «        | _        d S ©N)ÚsuperÚ__init__Úcompute_streamÚdevice_handler$   Ú_dynamoÚdisableÚpost_unflatten_transform)Úselfr¦   Ú	__class__s     €r,   r¤   zDTensorExtensions.__init__O  sQ   ø€ Ý‰Œ×ÒÑÔÐØ"ˆÔØ*ˆÔõ ).¬×(=Ò(=ØÔ)ñ)
ô )
ˆÔ%Ð%Ð%r.   r   c                 ó    — t          |¦  «        S r¢   )r   ©rª   r   s     r,   Úpre_flatten_transformz'DTensorExtensions.pre_flatten_transformY  s   € õ ˜vÑ&Ô&Ð&r.   Úparam_extensionc                 ó  — | j         p| j                             ¦   «         }| j                             |¦  «        5  t	          ||| j        | j         ¬¦  «        }t          |¦  «         |cd d d ¦  «         S # 1 swxY w Y   d S )N)r¦   r¥   )r¥   r¦   Úcurrent_streamÚstreamr   r   )rª   r   r¯   r²   Úresults        r,   r©   z*DTensorExtensions.post_unflatten_transform_  sÖ   € ð Ô$ÐK¨Ô(:×(IÒ(IÑ(KÔ(KˆØÔ×&Ò& vÑ.Ô.ð 	ð 	õ 'ØØØ"Ô0Ø#Ô2ð	ñ ô ˆFõ   Ñ'Ô'Ð'Øð	ð 	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð 	ð 	s   ».A6Á6A:Á=A:r]   rf   rg   rh   rD   c                 ó(   — t          |||||¦  «        S r¢   )r{   )rª   r   r]   rf   rg   rh   rD   s          r,   Úchunk_tensorzDTensorExtensions.chunk_tensorr  s   € õ ˜V T¨:Ð7KÈRÑPÔPÐPr.   r   c                 ó$   — t          |||¦  «        S r¢   )r™   )rª   r   r]   r   s       r,   Úchunk_dtensorzDTensorExtensions.chunk_dtensor}  s   € õ ˜f d¨KÑ8Ô8Ð8r.   c                 ó    — t          |¦  «        S r¢   )rœ   r­   s     r,   Úpre_load_state_dict_transformz/DTensorExtensions.pre_load_state_dict_transform…  s   € õ $ FÑ+Ô+Ð+r.   r�   c                 ó"   — t          ||¦  «        S r¢   )rŸ   )rª   r   r�   s      r,   Úall_gather_dtensorz$DTensorExtensions.all_gather_dtensor‹  s   € õ
 # 6¨;Ñ7Ô7Ð7r.   )r   Nr¢   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r¤   r$   r�   Útupler   r®   r©   ÚintrO   ÚProcessGrouprD   rµ   r   r·   rC   r   r¹   r   r»   Ú__classcell__)r«   s   @r,   r   r   G  sØ  ø€ € € € € ðð ð
ð 
ð 
ð 
ð 
ð 
ð'à”ð'ð 
ˆuŒ|˜S 4™ZÐ'Ô	(ð'ð 'ð 'ð 'ðØ”lðØ58ðà	Œðð ð ð ð4 '+ð	Qð 	Qà”ð	Qð ð	Qð ð		Qð
 "ð	Qð Ôð	Qð ”˜tÑ#ð	Qð 
Œð	Qð 	Qð 	Qð 	Qð9à”ð9ð ð9ð  ð	9ð
 
Œð9ð 9ð 9ð 9ð,à”ð,ð 
ˆuŒ|˜T %œ[Ð(Ô	)ð,ð ,ð ,ð ,ð8àð8ð   $Ñ&ð8ð 
Œð	8ð 8ð 8ð 8ð 8ð 8ð 8ð 8r.   );r`   Útypingr   r   r$   Útorch.distributedÚdistributedrO   Ú&torch.distributed._shard.sharding_specÚ_shardÚsharding_specÚ
shard_specÚ"torch.distributed.distributed_c10dÚdistributed_c10dÚc10dÚ'torch.distributed._shard.sharded_tensorr   r   r   r   r	   Ú:torch.distributed._shard.sharding_spec.chunk_sharding_specr
   Ú$torch.distributed.fsdp._common_utilsr   Ú'torch.distributed.fsdp._fsdp_extensionsr   Ú#torch.distributed.fsdp._shard_utilsr   Útorch.distributed.remote_devicer   Útorch.distributed.tensorr   r   r   r"   Ú6torch.distributed.tensor.parallel._data_parallel_utilsr   r   Ú__all__rÀ   r%   r-   rÁ   r6   r:   rG   rÂ   rX   r[   ÚShardingSpecr�   re   r{   r™   rC   rœ   rŸ   r   r2   r.   r,   ú<module>rØ      s  ðà €€€Ø Ð Ð Ð Ð Ð Ð Ð à €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1ðð ð ð ð ð ð ð ð ð ð ð ð AÐ @Ð @Ð @Ð @Ð @Ø XÐ XÐ XÐ XÐ XÐ XØ DÐ DÐ DÐ DÐ DÐ DØ BÐ BÐ BÐ BÐ BÐ BØ LÐ LÐ LÐ LÐ LÐ LØ :Ð :Ð :Ð :Ð :Ð :Ø TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ Tðð ð ð ð ð ð ð ð Ð
€ð>�Wð >  u¤z°5´:Ð'=Ô!>ð >ð >ð >ð >ð$>˜ð > sð >¨u°U´ZÀÄÐ5KÔ/Lð >ð >ð >ð >ð
*˜7ð * u¨U¬Z¸¼Ð-CÔ'Dð *ð *ð *ð *ð
 ð 
¸ð 
Àð 
ð 
ð 
ð 
ð%Øð%ØÔ)ð%àð%ð %ð %ð %ðP�7ð ˜tÔ0ð ð ð ð ðØ
Ô
!ðØ+0¬<ðØ?BðàÔðð ð ð ð>I
ØŒLðI
à
ðI
ð ðI
ð ð	I
ð
 	ÔðI
ð „\ðI
ð I
ð I
ð I
ðX>
ØŒLð>
à
ð>
ð ð>
ð ð	>
ð >
ð >
ð >
ðB	7ØŒLð	7à
ˆ5Œ<˜˜eœÐ$Ô%ð	7ð 	7ð 	7ð 	7ðØðà˜dÑ"ðð „\ðð ð ð ð*I8ð I8ð I8ð I8ð I8˜ñ I8ô I8ð I8ð I8ð I8r.   