§
    N�’jY  ã            	       ó  — d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
Z
d dlmZ d dlmc mc mZ d dlmZmZmZ  e j        e¦  «        Z G d„ d¦  «        Zd	e
j        j        d
eeeeee         f         f         fd„Z dS )é    N)Údefaultdict)ÚIterator)Úcontextmanager)ÚEnum)Ú_apply_to_modulesÚ_get_module_fsdp_stateÚclean_tensor_namec                   óü   — e Zd ZU  G d„ dee¦  «        Z ee¦  «        Ze	eef         e
d<    e¦   «         Zee         e
d<   edd„¦   «         Zeededed         fd	„¦   «         ¦   «         Zed
eddfd„¦   «         ZdS )ÚSimpleProfilerc                   ó&   — e Zd ZdZdZdZdZdZdZdS )úSimpleProfiler.TypeÚallÚ
all_gatherÚall_gather_objectÚ
reshardingÚH2DÚD2HN)	Ú__name__Ú
__module__Ú__qualname__ÚALLÚ	ALLGATHERÚALLGATHER_OBJÚ
RESHARDINGr   r   © ó    úQC:\Projects\DSCons\.venv\Lib\site-packages\torch\distributed\fsdp\_debug_utils.pyÚTyper      s,   € € € € € ØˆØ ˆ	Ø+ˆØ!ˆ
ØˆØˆˆˆr   r   ÚresultsÚ	profilingÚreturnNc                 ój   — | j                              ¦   «          | j                             ¦   «          d S )N)r   Úclearr    )Úclss    r   ÚresetzSimpleProfiler.reset"   s0   € àŒ×ÒÑÔÐØŒ×ÒÑÔÐÐÐr   Úprofile_typec              #   óÂ  K  — || j         v rt          |› d�¦  «        ‚| j                              |¦  «         t          j        ¦   «         }	 d V — t          j        ¦   «         }| j        |xx         ||z
  z  cc<   | j                              |¦  «         d S # t          j        ¦   «         }| j        |xx         ||z
  z  cc<   | j                              |¦  «         w xY w)Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r    ÚAssertionErrorÚaddÚtimeÚ	monotonicr   Úremove)r$   r&   ÚbeginÚends       r   ÚprofilezSimpleProfiler.profile'   s  è è € ð ˜3œ=Ð(Ð(Ý Øð "ð "ð "ñô ð ð 	Œ×Ò˜,Ñ'Ô'Ð'Ý”Ñ Ô ˆð	/ØˆEˆEˆEå”.Ñ"Ô"ˆCØŒK˜Ð%Ð%Ô%¨¨u©Ñ4Ð%Ð%Ñ%ØŒM× Ò  Ñ.Ô.Ð.Ð.Ð.øõ ”.Ñ"Ô"ˆCØŒK˜Ð%Ð%Ô%¨¨u©Ñ4Ð%Ð%Ñ%ØŒM× Ò  Ñ.Ô.Ð.Ð.øøøs   ÁB ÂACÚmsgc                 óê   — t          j        ¦   «         dk    rGt          j        ¦   «         t           j        j        k    r!t
                               d|| j        ¦  «         |                      ¦   «          d S )Nr   z%s %s)	ÚdistÚget_rankÚget_debug_levelÚ
DebugLevelÚINFOÚloggerÚinfor   r%   )r$   r0   s     r   Údump_and_resetzSimpleProfiler.dump_and_reset:   sV   € õ Œ=‰?Œ?˜aÒÐ¥DÔ$8Ñ$:Ô$:½d¼oÔ>RÒ$RÐ$RÝ�KŠK˜  c¤kÑ2Ô2Ð2Ø�	Š	‰Œˆˆˆr   )r!   N)r   r   r   Ústrr   r   r   Úfloatr   ÚdictÚ__annotations__Úsetr    Úclassmethodr%   r   r   r/   r9   r   r   r   r   r      s	  € € € € € € ðð ð ð ð ˆs�Dñ ô ð ð !, ¨EÑ 2Ô 2€GˆT�#�u�*ÔÐ2Ð2Ñ2Ø˜#™%œ%€Iˆs�3ŒxÐÐÑàðð ð ñ „[ðð Øð/ 3ð /¨8°D¬>ð /ð /ð /ñ „^ñ „[ð/ð" ð ð ¨ð ð ð ñ „[ðð ð r   r   Úmodelr!   c                 óx   — d„ }d„ }dg}i }t          | ||d„ |                      ¦   «         D ¦   «         ||¦  «        S )a÷  
    It is used for composable fully_shard() code path, it returns
      1. sharded module tree info: each line represents a submodule name that contains the
    submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
    the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
    level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
    is like this:
        [CompositeModel] FULLY SHARDED
            l1[Linear]
            u1[UnitModule] FULLY SHARDED
                u1.l1[Linear]
                u1.seq[Sequential]
                    u1.seq.0[ReLU]
                    u1.seq.1[Linear]
                    u1.seq.2[ReLU]
                u1.l2[Linear]
            u2[UnitModule] FULLY SHARDED
                u2.l1[Linear]
                u2.seq[Sequential]
                    u2.seq.0[ReLU]
                    u2.seq.1[Linear]
                    u2.seq.2[ReLU]
                u2.l2[Linear]
            l2[Linear]
      2. a dict mapping from the concated module FQN and class name to a list of its managed
    original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
            {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
             'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
             'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
            }
    All FQNs are prefixed starting from ``model``.

    Args:
        model (torch.nn.Module): Root module (which may or may not be passed to
                                 composable `fully_shard()`).
    c                 óv  ‡— |dz  }t          ‰¦  «        dk    r‰d         dk    r
‰d d…         n‰}|dz   | j        j        z   dz   }d|z  |z   }t          | ¦  «        }	|	€|dxx         |dz   z  cc<   d S |	j                             | d ¦  «        }
|
r|dxx         |d	z   dz   z  cc<   n|dxx         |dz   z  cc<   |
r{|
j        }t          |t          j	        ¦  «        st          d
t          |¦  «        › �¦  «        ‚ˆfd„|j        D ¦   «         }||v r||                              |¦  «         d S |||<   d S d S )Né   r   éÿÿÿÿú.ú[ú]ú ú
z FULLY SHARDEDzExpected FlatParameter, got c                 ó4   •— g | ]}t          ‰|z   ¦  «        ‘ŒS r   )r	   )Ú.0ÚnameÚprefixs     €r   ú
<listcomp>zX_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fn.<locals>.<listcomp>‡   s3   ø€ ð ð ð Ø59Õ! &¨4¡-Ñ0Ô0ðð ð r   )ÚlenÚ	__class__r   r   Ú_fully_sharded_module_to_handleÚgetÚ
flat_paramÚ
isinstanceÚflat_param_fileÚFlatParameterr(   ÚtypeÚ_fqnsÚextend)ÚmodulerM   Ú
tree_levelÚsharded_tree_infoÚsharded_module_name_to_fqnsÚ
num_spacesÚtrimed_prefixÚprefixed_module_nameÚprinted_prefixed_module_nameÚstateÚhandleÚparamÚglobal_fqnss    `           r   Ú	module_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnk   sÒ  ø€ ð   !‘^ˆ
å ™KœK¨!šO˜O°°r´
¸cÒ0AÐ0AˆF�3�B�3ŒKˆKÈð 	ð  -¨sÑ2°VÔ5EÔ5NÑNÐQTÑTÐØ'*¨ZÑ'7Ð:NÑ'NÐ$å& vÑ.Ô.ˆØˆ=Ø˜aÐ Ð Ô Ð$@À4Ñ$GÑGÐ Ð Ñ ØˆFàÔ6×:Ò:¸6À4ÑHÔHˆàð 	HØ˜aÐ Ð Ô Ø,Ð/?Ñ?À$ÑFñÐ Ð Ñ Ð ð ˜aÐ Ð Ô Ð$@À4Ñ$GÑGÐ Ð Ñ àð 	PØÔ%ˆEÝ˜e¥_Ô%BÑCÔCð SÝ$Ð%QÅDÈÁKÄKÐ%QÐ%QÑRÔRÐRðð ð ð Ø=B¼[ðñ ô ˆKð $Ð'BÐBÐBØ+Ð,@ÔA×HÒHÈÑUÔUÐUÐUÐUàDOÐ+Ð,@ÑAÐAÐAð	Pð 	Pr   c                 ó   — | d         |fS )Nr   r   )r\   r]   s     r   Ú	return_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fn�   s   € Ø  Ô#Ð%@Ð@Ð@r   Ú c                 ó   — g | ]\  }}|‘ŒS r   r   )rK   ÚkeyÚ_s      r   rN   zE_get_sharded_module_tree_with_module_name_to_fqns.<locals>.<listcomp>œ   s   € Ð4Ð4Ð4‘��aˆÐ4Ð4Ð4r   )r   Únamed_parameters)r@   rf   rh   r\   r]   s        r   Ú1_get_sharded_module_tree_with_module_name_to_fqnsrn   C   s{   € ðP#Pð #Pð #PðJAð Að Að
 	ð$Ðð 9;ÐÝØØØØ4Ð4˜5×1Ò1Ñ3Ô3Ð4Ñ4Ô4ØØ#ñô ð r   )!Úloggingr*   Úcollectionsr   Úcollections.abcr   Ú
contextlibr   Úenumr   ÚtorchÚtorch.distributedÚdistributedr2   Ú"torch.distributed.fsdp._flat_paramÚfsdpÚ_flat_paramrU   Ú$torch.distributed.fsdp._common_utilsr   r   r	   Ú	getLoggerr   r7   r   ÚnnÚModuleÚtupler:   r<   Úlistrn   r   r   r   ú<module>r€      s^  ðà €€€Ø €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø $Ð $Ð $Ð $Ð $Ð $Ø %Ð %Ð %Ð %Ð %Ð %Ø Ð Ð Ð Ð Ð à €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ð <ðð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜8Ñ	$Ô	$€ð*ð *ð *ð *ð *ñ *ô *ð *ðZ\ØŒ8Œ?ð\à
ˆ3��S˜$˜sœ)�^Ô$Ð$Ô%ð\ð \ð \ð \ð \ð \r   