§
    Ÿ²Åji  ã                   ó²  — d dl Z d dlZd dlmZ d dlmZmZmZmZm	Z	m
Z
 d dlZd dlmZ d dlmZ d dlmZ d dlmZ dd	gZ ej        e¦  «        xZZ ed
¬¦  «         G d„ d¦  «        ¦   «         Zdej        j        dedefd„Z ed
¬¦  «        	 	 	 ddedej        j        deegef         de	eeef                  de	e         de	e         fd„¦   «         Z dS )é    N)ÚOrderedDict)ÚAnyÚCallableÚDictÚListÚOptionalÚSet)Úcompatibility)Úlazy_format_graph_code)ÚGraphModule)ÚNodeÚ	PartitionÚsplit_moduleT)Úis_backward_compatiblec                   ó&   — e Zd Zdefd„Zdefd„ZdS )r   Únamec                 óÚ   — || _         d|› �| _        g | _        i | _        i | _        i | _        i | _        t          j        j	         
                    ¦   «         | _	        i | _        i | _        d S )NÚsubmod_)r   Úsubmod_nameÚ
node_namesÚinputsÚoutputsÚdependenciesÚ
dependentsÚtorchÚfxÚgraphÚGraphÚenvironmentÚtargets)Úselfr   s     úMC:\Projects\Historical\venv\Lib\site-packages\torch/fx/passes/split_module.pyÚ__init__zPartition.__init__   sg   € ØˆŒ	Ø+ TÐ+Ð+ˆÔØ%'ˆŒØ')ˆŒØ(*ˆŒØ-/ˆÔØ+-ˆŒÝ+0¬8¬>×+?Ò+?Ñ+AÔ+AˆŒ
Ø-/ˆÔØ')ˆŒˆˆó    Úreturnc                 óf   — d| j         › d| j        › d| j        › d| j        › d| j        › d| j        › �S )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depended on: z,
 partition dependents: )r   r   r   r   r   r   )r!   s    r"   Ú__repr__zPartition.__repr__    sl   € ð8�T”Yð 8ð 8Ø”ð8ð 8àœð8ð 8ð œð8ð 8ð )-Ô(9ð	8ð 8ð
 '+¤oð8ð 8ð	
r$   N)Ú__name__Ú
__module__Ú__qualname__Ústrr#   r'   © r$   r"   r   r      sJ   € € € € € ð
*˜Sð 
*ð 
*ð 
*ð 
*ð
˜#ð 
ð 
ð 
ð 
ð 
ð 
r$   ÚmodÚqualnamer%   c                 ó    — | }|                      d¦  «        D ]5}t          ||¦  «        st          d|› d�¦  «        ‚t          ||¦  «        }Œ6|S )Nú.zNode target z not found!)ÚsplitÚhasattrÚAttributeErrorÚgetattr)r-   r.   Úattr_valÚatoms       r"   Ú_get_attr_from_qualnamer7   +   se   € Ø€HØ—’˜sÑ#Ô#ð +ð +ˆÝ�x Ñ&Ô&ð 	GÝ Ð!E°Ð!EÐ!EÐ!EÑFÔFÐFÝ˜8 TÑ*Ô*ˆˆØ€Or$   FÚmÚroot_mÚsplit_callbackÚqualname_mapÚkeep_original_orderÚkeep_original_node_namec                 óÞ  ‡ ‡‡‡8‡9‡:‡;‡<‡=‡>‡?‡@‡A‡B— t                                dt          d‰ d¬¦  «        ¦  «         dt          dt          t
          t          f         dt          t
          t          j        j        j	        f         fˆ9ˆˆ fd„}d	d
l
}i Š@i Š=i ŠBdt          dt          t                   fˆ@ˆBfd„ŠAˆ@ˆfd„}t          j        j        t          j        j        t          j        j        g}	t#          ¦   «         }
t#          ¦   «         }i }d
}t%          ¦   «         }‰ j        j        D �]mŠ;‰;j                             d¦  «        x}�Zt/          |t          j        t          j        f¦  «        r4t/          |j        j        x}|j        ¦  «        r|‰Bvr‰;‰B|j        j        <   ‰;j        dv rŒƒ |‰;¦  «         ‰;j        dk    �r„‰;j        |	v �rz‰;j        t          j        j        k    r\t?          ‰;j         ¦  «        dk    sJ ‚t/          ‰;j         d	         tB          ¦  «        sJ ‚‰;}t%           ‰‰;¦  «        h¦  «        |
|<   �n‰;j        t          j        j        k    rWtE          d„ ‰;j         D ¦   «         ¦  «        sJ ‚| #                    ‰;¦  «         t%           ‰‰;¦  «        h¦  «        |‰;<   d
|‰;<   n“‰;j        t          j        j        k    ryt?          ‰;j         ¦  «        dk    sJ ‚|‰;j         d	                   #                     ‰‰;¦  «        ¦  «         | $                    ‰;j         d	         ¦  «         ‰;|‰;j         d	         <   |�$|
|          #                     ‰‰;¦  «        ¦  «         |D ]&}||          #                     ‰‰;¦  «        ¦  «         Œ'�ŒotE          d„ | %                    ¦   «         D ¦   «         ¦  «        s
J d¦   «         ‚d„ | &                    ¦   «         D ¦   «         }d„ |
 &                    ¦   «         D ¦   «         }
tN           (                    tR          j*        ¦  «        r6tN                               d|¦  «         tN                               d|
¦  «         tC          |¦  «        ptC          |
¦  «        }d}‰ j        j        D ]ßŠ;‰;‰=‰;j+        <   ‰;j        dv rŒ‰;j        dk    r4t          j        j         ,                    ‰;j         d	         ˆAfd„¦  «         ŒU|r# ‰‰;¦  «        }||k    sJ d|› d|› �¦   «         ‚|}‰;j        |	vr\t          j        j         ,                    ‰;j         ˆ;ˆAfd „¦  «         t          j        j         ,                    ‰;j-        ˆ;ˆAfd!„¦  «         Œàt]          ‰@ /                    ¦   «         ¦  «        }g }‰@ &                    ¦   «         D ].\  }Š?t?          ‰?j0        ¦  «        s| 1                    |¦  «         Œ/g }|r}| 2                    ¦   «         }| 1                    |¦  «         ‰@|         j3        D ]D}‰@|         j0         2                    |¦  «         ‰@|         j0        s| 1                    |¦  «         ŒE|°}t?          |¦  «        t?          ‰@¦  «        k    rti          d"¦  «        ‚||
fD ]è}| &                    ¦   «         D ]Ñ\  Š;}t?          |¦  «        d	k    sJ ‚‰;‰@t          |d	         ¦  «                 j5        ‰;<   |dd
…         D ]‰}‰@t          |¦  «                 Š?‰?j         6                    ‰;j        ‰;j        to          d#„ ‰;j         D ¦   «         ¦  «        i ‰;j8        ¬$¦  «        }‰;j         9                    ¦   «         |_        |‰?j5        ‰;<   ŒŠŒÒŒé|D �]?}‰@|         Š?i }‰?j:        D �]"} ‰=|          }!|!j        d%k    r¬t/          |!j        t
          ¦  «        sJ ‚tw          ‰ |!j        ¦  «        }"t/          |"t          j<        j=        ¦  «        r/‰?j         >                    |!j        ¦  «        }#|"‰?j?        |!j        <   nY‰?j         @                    | ‰=|          j8        ¬&¦  «        }#d
|| <   n,‰?j         @                    | ‰=|          j8        ¬&¦  «        }#d
|| <   ‰=|          j         9                    ¦   «         |#_        |#‰?j5        ‰=|          <   �Œ$|‰?_:        �ŒA‰ j        j        D �]pŠ;tƒ          ‰;d'¦  «        �r\‰@‰;jB                 Š?‰?j5        Š:t          j        j         ,                    ‰;j         ˆ:fd(„¦  «        }$t          j        j         ,                    ‰;j-        ˆ:fd)„¦  «        }%‰;j        d*vr‰;j        }&nRtw          ‰ ‰;j        ¦  «        }'‰;j         C                    d+d,¦  «        }&|'‰?j?        |&<   |�‰?jD        › d+|&› �}(‰;j        ||(<   t/          |$tn          ¦  «        sJ ‚t/          |%tŠ          ¦  «        sJ ‚‰r‰;j+        nd
})‰?j         6                    ‰;j        |&|$|%‰;j8        |)¬-¦  «        }‰;j         9                    ¦   «         |_        |‰?j5        ‰;<   �Œr|fD ]¿}t�          |¦  «        D ]­Š;|‰;         }t?          |¦  «        d	k    sJ ‚|d
d…         D ]ƒ}‰@t          |¦  «                 Š?|‰;         }*|*€
J d.¦   «         ‚‰?j         6                    |*j        |*j        ‰?j5        ‰;         fi |*j8        ¬$¦  «        }|*j         9                    ¦   «         |_        Œ„Œ®ŒÀi Š<i Š8t          j        j         G                    ¦   «         Š9i }+|s ‰ j        j        D ]Š; |‰;‰8|+¦  «        \  Š8}+Œn‰ j        j        D ]Š;‰;‰<‰;j+        <   Œ|s|n|},t%          ¦   «         }-d/„ ‰ j        j        D ¦   «         Š>|,D �]}‰@|         Š?to          ˆ=ˆ?fd0„‰?jH        D ¦   «         ¦  «        }.t?          |.¦  «        }/|/dk    r!‰?j         I                    |.d	         ¦  «         n;|/dk    r‰?j         I                    |.¦  «         n‰?j         I                    d1¦  «         |rrˆ<ˆ>fd2„‰?j:        D ¦   «         }0‰>D ],Š;‰;|-v rŒ |‰;‰8|+¦  «        \  Š8}1|- #                    ‰;¦  «         Œ-|0D ],Š;‰;|-v rŒ |‰;‰8|+¦  «        \  Š8}+|- #                    ‰;¦  «         Œ-t          j        j         	                    ‰?j?        ‰?j        ¦  «        |+‰?jD        <   ‰9 J                    ‰?jD        to          ˆ8fd3„‰?j:        D ¦   «         ¦  «        ¦  «        }2t?          ‰?jH        ¦  «        }3|3dk    rPt          j        jK         L                    |2¦  «        }4t›          ‰?jH        ¦  «        D ]\  }5}6|4|5         j        ‰8|6<   Œ�Œë|3dk    r$|2‰8t�          tŸ          ‰?jH        ¦  «        ¦  «        <   �Œ|r!‰8s‰ j        j        D ]Š; |‰;‰8|+¦  «        \  Š8}+Œ‰ j        j        D ]SŠ;‰;j        dk    rF‰9 I                    t          j        j         ,                    ‰;j         d	         ˆ8fd4„¦  «        ¦  «         ŒTt          j        j         	                    |+‰9¦  «        }7t                                dt          d5|7d¬¦  «        ¦  «         |7S )6aô  
    Creates subgraphs out of main graph

    Args:
        m (GraphModule): Graph module to split
        root_m (torch.nn.Module): root nn module. Not currently used. Included
            because the root nn module is usually transformed via
            torch.fx._symbolic_trace.symbolic_trace (see example below)
        split_callback (Callable[[Node], int]): Callable function
            that maps a given Node instance to a numeric partition identifier.
            split_module will use this function as the policy for which operations
            appear in which partitions in the output Module.
        qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
            mapping from new target names in the module after split to old target
            names in the original module.
        keep_original_order: Optional[bool]: keep the original order of the GraphModule
            or use the Topological order of the new constructed GraphModule


    Returns:
        GraphModule: the module after split.

    Example:

        This is a sample setup:

            import torch
            from torch.fx.symbolic_trace import symbolic_trace
            from torch.fx.graph_module import GraphModule
            from torch.fx.node import Node
            from torch.fx.passes.split_module import split_module

            class MyModule(torch.nn.Module):
                def __init__(self) -> None:
                    super().__init__()
                    self.param = torch.nn.Parameter(torch.rand(3, 4))
                    self.linear = torch.nn.Linear(4, 5)

                def forward(self, x, y):
                    z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                    w = self.linear(y).clamp(min=0.0, max=1.0)
                    return z + w

            # symbolically trace model
            my_module = MyModule()
            my_module_traced = symbolic_trace(my_module)

            # random mod partitioning
            partition_counter = 0
            NPARTITIONS = 3

            def mod_partition(node: Node):
                global partition_counter
                partition = partition_counter % NPARTITIONS
                partition_counter = (partition_counter + 1) % NPARTITIONS
                return partition

            # split module in module with submodules
            module_with_submodules = split_module(
                my_module_traced, my_module, mod_partition
            )

        Output looks like this. Original graph is broken into partitions

            > print(module_with_submodules)
            GraphModule(
                (submod_0): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_1): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_2): GraphModule()
            )

            def forward(self, x, y):
                param = self.param
                submod_0 = self.submod_0(x, param, y);  x = param = y = None
                getitem = submod_0[0]
                getitem_1 = submod_0[1];  submod_0 = None
                submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
                getitem_2 = submod_1[0]
                getitem_3 = submod_1[1];  submod_1 = None
                submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
                return submod_2

        Output of split module is the same as output of input traced module.
        This is an example within a test setting:

            > orig_out = my_module_traced(x, y)
            > submodules_out = module_with_submodules(x, y)
            > self.assertEqual(orig_out, submodules_out)
            True
    z%szpre split_moduleT)ÚcoloredÚnodeÚbase_mod_envÚbase_mod_attrsc                 óä  •— | j         dk    rÐt          | j        ¦  «        dk    r| j        d         nt          j        j        }‰rD|t          j        j        u rdn|f}‰                     d| j        || j        ¬¦  «        || j        <   n*‰ 	                    | j
        | j        |¬¦  «        || j        <   | j                             ¦   «         || j                 _        n‘| j         dk    r†‰                     | j
        ¦  «        || j        <   | j                             ¦   «         || j                 _        t          | j
        t          ¦  «        sJ ‚t!          ‰| j
        ¦  «        }||| j
        <   ||fS )NÚplaceholderr   r,   )ÚargsÚ	type_expr)rF   Údefault_valueÚget_attr)ÚopÚlenrE   ÚinspectÚ	SignatureÚemptyÚcreate_noder   ÚtyperD   ÚtargetÚmetaÚcopyrH   Ú
isinstancer+   r7   )	r@   rA   rB   rG   rE   r5   Úbase_mod_graphr=   r8   s	         €€€r"   Úconstruct_graphz%split_module.<locals>.construct_graph¢   sq  ø€ ð
 Œ7�mÒ#Ð#å # D¤I¡¤°Ò 2Ð 2�”	˜!”�½Ô8IÔ8Oð ð 'ð à'­7Ô+<Ô+BÐBÐB�B�BÈÐHXð ð +9×*DÒ*DØ!Ø”IØØ"œið	 +Eñ +ô +�˜TœYÑ'Ð'ð +9×*DÒ*DØ”KØ"œiØ"/ð +Eñ +ô +�˜TœYÑ'ð
 ,0¬9¯>ª>Ñ+;Ô+;ˆL˜œÔ#Ô(Ð(ØŒW˜
Ò"Ð"Ø&4×&=Ò&=¸d¼kÑ&JÔ&JˆL˜œÑ#Ø+/¬9¯>ª>Ñ+;Ô+;ˆL˜œÔ#Ô(Ý˜dœk­3Ñ/Ô/Ð/Ð/Ð/Ý.¨q°$´+Ñ>Ô>ˆHØ*2ˆN˜4œ;Ñ'Ø˜^Ð+Ð+r$   r   NÚdef_nodeÚuse_nodec                 ót  •— ddl m} t          | dd ¦  «        }t          |dd ¦  «        }t                               d| j        ||�|j        nd|¦  «         ||k    �rW|�C‰|         }|j                             | j        ¦  «         |�|j                             |¦  «         |��‰|         }|j	                             | j        ¦  «         | j
                             d¦  «        x}�®t           ||¦  «        t          ¬¦  «        D ]Ž}‰|         }	|j	                             |	j        ¦  «         ‰|         j        dk    rTt          |	dd ¦  «        }
|
�A‰|
         }|j                             |	j        ¦  «         |j                             |¦  «         Œ�|� |j                             |¦  «         d S d S d S d S )	Nr   )Úfree_symbolsÚ_fx_partitionz*record_cross_partition_use %s (%s) %s (%s)ú-Úexample_value)ÚkeyrD   )Ú%torch.fx.experimental.symbolic_shapesrY   r4   ÚlogÚdebugr   r   Ú
setdefaultr   r   rQ   ÚgetÚsortedr+   rI   r   )rV   rW   rY   ÚdefinedÚusedÚdef_partitionÚuse_partitionÚdef_valÚsÚs_nodeÚ	s_definedÚs_def_partitionÚ
partitionsÚsymbol_to_nodes               €€r"   Úrecord_cross_partition_usez0split_module.<locals>.record_cross_partition_useÊ   sí  ø€ ØFÐFÐFÐFÐFÐFå˜( O°TÑ:Ô:ˆÝ�x °$Ñ7Ô7ˆå�	Š	Ø8ØŒMØØ%Ð1ˆHŒMˆM°sØñ	
ô 	
ð 	
ð �dŠ?‰?ØÐ"Ø *¨7Ô 3�ØÔ%×0Ò0°´Ñ?Ô?Ð?ØÐ#Ø!Ô,×7Ò7¸Ñ=Ô=Ð=àÑØ *¨4Ô 0�ØÔ$×/Ò/°´Ñ>Ô>Ð>ð  (œ}×0Ò0°ÑAÔAÐA�GÐNÝ# L L°Ñ$9Ô$9½sÐCÑCÔCð Lð L˜Ø!/°Ô!2˜Ø%Ô,×7Ò7¸¼ÑDÔDÐDØ)¨!Ô,Ô/°=Ò@Ð@õ )0°¸ÈÑ(NÔ(N˜IØ(Ð4Ø2<¸YÔ2G Ø /Ô 7× BÒ BÀ6Ä;Ñ OÔ OÐ OØ /Ô :× EÒ EÀdÑ KÔ KÐ KøØÐ&Ø!Ô.×9Ò9¸'ÑBÔBÐBÐBÐBðA ˆ?ð  Ðð0 'Ð&r$   c                 ó  •— t           ‰| ¦  «        ¦  «        }t                               d| j        |¦  «         ‰                     |¦  «        }|€t          |¦  «        x‰|<   }|j                             | j        ¦  «         || _        d S )Nz*instantiate_node_partition_mapping %s (%s))	r+   r_   r`   r   rb   r   r   ÚappendrZ   )r@   Úpartition_nameÚ	partitionrm   r:   s      €€r"   Ú"instantiate_node_partition_mappingz8split_module.<locals>.instantiate_node_partition_mappingú   s�   ø€ Ý˜^˜^¨DÑ1Ô1Ñ2Ô2ˆÝ�	Š	Ø8¸$¼)À^ñ	
ô 	
ð 	
ð
 —N’N >Ñ2Ô2ˆ	ØÐÝ5>¸~Ñ5NÔ5NÐNˆJ�~Ñ&¨àÔ×#Ò# D¤IÑ.Ô.Ð.Ø+ˆÔÐÐr$   r\   )rD   rH   ÚoutputÚcall_functioné   c              3   óB   K  — | ]}t          |t          ¦  «         V — Œd S ©N)rS   r   ©Ú.0Úargs     r"   ú	<genexpr>zsplit_module.<locals>.<genexpr>@  s/   è è € ÐJÐJ¸�z¨#­tÑ4Ô4Ð4ÐJÐJÐJÐJÐJÐJr$   c              3   ó   K  — | ]}|d uV — Œ	d S ry   r,   )r{   Úvs     r"   r}   zsplit_module.<locals>.<genexpr>P  s&   è è € Ð>Ð> ˆq˜ˆ}Ð>Ð>Ð>Ð>Ð>Ð>r$   zautocast must exitc                 ó4   — i | ]\  }}|t          |¦  «        “ŒS r,   ©rc   ©r{   Úkr   s      r"   ú
<dictcomp>z split_module.<locals>.<dictcomp>R  s$   € ÐJÐJÐJ©¨¨A˜�6 !™9œ9ÐJÐJÐJr$   c                 ó4   — i | ]\  }}|t          |¦  «        “ŒS r,   r�   r‚   s      r"   r„   z split_module.<locals>.<dictcomp>S  s$   € ÐBÐBÐB¡T Q¨�A•v˜a‘y”yÐBÐBÐBr$   zautocast_regions: %szgrad_regions: %séÿÿÿÿ)rD   rH   ru   c                 ó   •—  ‰| d ¦  «        S ry   r,   )Únro   s    €r"   ú<lambda>zsplit_module.<locals>.<lambda>f  s   ø€ Ð(BÐ(BÀ1ÀdÑ(KÔ(K€ r$   zRautocast or set_grad_enabled require monotonically increasing partitions:highest: z, this node's: c                 ó   •—  ‰| ‰¦  «        S ry   r,   ©rV   r@   ro   s    €€r"   r‰   zsplit_module.<locals>.<lambda>v  s   ø€ Ð,FÐ,FÀxÐQUÑ,VÔ,V€ r$   c                 ó   •—  ‰| ‰¦  «        S ry   r,   r‹   s    €€r"   r‰   zsplit_module.<locals>.<lambda>y  s   ø€ Ð.HÐ.HÈÐSWÑ.XÔ.X€ r$   z cycle exists between partitions!c              3   ó   K  — | ]}|V — Œd S ry   r,   rz   s     r"   r}   zsplit_module.<locals>.<genexpr>™  s"   è è € Ð8Ð8 s˜sÐ8Ð8Ð8Ð8Ð8Ð8r$   )rI   rP   rE   ÚkwargsrF   rH   )rF   rZ   c                 ó   •— ‰|          S ry   r,   ©rˆ   r   s    €r"   r‰   zsplit_module.<locals>.<lambda>É  s   ø€ ÈÐTUÌ€ r$   c                 ó   •— ‰|          S ry   r,   r�   s    €r"   r‰   zsplit_module.<locals>.<lambda>Ë  s   ø€  {°1¤~€ r$   )Úcall_modulerH   r0   Ú_)rI   rP   rE   rŽ   rF   r   zMissing exit nodec                 ó(   — g | ]}|j         d k    ¯|‘ŒS )rD   )rI   )r{   r@   s     r"   ú
<listcomp>z split_module.<locals>.<listcomp>  s$   € ÐQÐQÐQ˜t¸¼À=Ò8PÐ8P�dÐ8PÐ8PÐ8Pr$   c              3   ó>   •K  — | ]}‰j         ‰|                  V — Œd S ry   )r   )r{   r   Ú
orig_nodesrs   s     €€r"   r}   zsplit_module.<locals>.<genexpr>"  sA   øè è € ð 
ð 
Ø8<ˆIÔ! *¨TÔ"2Ô3ð
ð 
ð 
ð 
ð 
ð 
r$   r,   c                 ó(   •— g | ]}|‰v¯‰|         ‘ŒS r,   r,   )r{   r]   Úorig_mod_envÚoriginal_orders     €€r"   r•   z split_module.<locals>.<listcomp>2  s3   ø€ ð /ð /ð /àØ˜nÐ,Ð,ð ˜SÔ!à,Ð,Ð,r$   c              3   ó(   •K  — | ]}‰|         V — Œd S ry   r,   )r{   r   rA   s     €r"   r}   zsplit_module.<locals>.<genexpr>P  s(   øè è € ÐBÐB¨�,˜tÔ$ÐBÐBÐBÐBÐBÐBr$   c                 ó   •— ‰| j                  S ry   )r   )rˆ   rA   s    €r"   r‰   zsplit_module.<locals>.<lambda>j  s   ø€ ¸|ÈAÌFÔ?S€ r$   zpost split_module)Pr_   r`   r   r   r   r+   r   r   Úgraph_moduler   Úsympyr   ÚampÚ_enter_autocastÚ_exit_autocastÚ_CÚ_set_grad_enabledr   Úsetr   ÚnodesrQ   rb   rS   ÚSymIntÚSymFloatr@   ÚexprÚSymbolrI   rP   rJ   rE   ÚboolÚallÚaddÚremoveÚvaluesÚitemsÚ_LOGGERÚisEnabledForÚloggingÚDEBUGr   Úmap_argrŽ   ÚlistÚkeysr   rq   Úpopr   ÚRuntimeErrorr   rN   ÚtuplerO   rR   r   r7   ÚnnÚModulerH   r    rD   r2   rZ   Úreplacer   ÚdictÚreversedr   r   ru   r’   ÚproxyÚProxyÚ	enumerateÚnextÚiter)Cr8   r9   r:   r;   r<   r=   rU   rž   rt   ÚGLOBAL_STATE_NODESÚgrad_regionsÚautocast_regionsÚautocast_exitsÚactive_gradÚactive_autocastsÚvalÚs0ÚaÚassert_monotonically_increasingÚhighest_partitionÚpidÚoriginal_partition_orderÚroot_partitionsrr   Úsorted_partitionsÚroot_partitionÚ	dependentÚregions_mappingÚregionsÚrÚnew_nodeÚ
new_inputsÚinpÚ	orig_nodeÚ	orig_attrrD   Úgathered_argsÚgathered_kwargsrP   Útarget_attrr.   r   Ú	exit_noderB   Úconstruct_order_partitionsÚalready_constructed_attr_nodesÚoutput_valsÚnum_output_valsÚorig_mod_attr_nodesÚ_based_mod_attrsÚ
output_valÚnum_outputsÚoutput_val_proxyÚiÚoutput_nameÚretrA   rT   r   r@   r™   r—   rš   rs   rm   ro   rn   sC   ` `  `                                                  @@@@@@@@@@@r"   r   r   5   s˜  øøøøøøøøøøøøøø€ õP ‡I‚IØÝÐ1°1¸dÐCÑCÔCñô ð ð
 ,Ýð ,å�3¥˜9”oð ,õ �S¥%¤(Ô"7Ô"CÐCÔDð ,ð  ,ð  ,ð  ,ð  ,ð  ,ð  ,ð  ,ðD €L€L€Là')€JØ"$€JØ/1€Nð.C­Tð .C½XÅd¼^ð .Cð .Cð .Cð .Cð .Cð .Cð .Cð`,ð ,ð ,ð ,ð ,ð ,õ" 	Œ	Ô!ÝŒ	Ô ÝŒÔ"ðÐõ 1<±´€Lõ 5@±M´MÐØ13€Nà€KÝ‘u”uÐà””ð +:ñ +:ˆð ”I—M’M /Ñ2Ô2Ð2ˆSÐ?Ý˜3¥¤­u¬~Ð >Ñ?Ô?ð @å ¤¤Ð.˜2°´Ñ=Ô=ð @ð ˜.Ð(Ð(à,0ˆN˜3œ8œ=Ñ)àŒ7Ð;Ð;Ð;Øà*Ð*¨4Ñ0Ô0Ð0àŒ7�oÒ%Ñ%¨$¬+Ð9KÐ*KÑ*KØŒ{�eœhÔ8Ò8Ð8Ý˜4œ9‘~”~¨Ò*Ð*Ð*Ð*Ý! $¤)¨A¤,µÑ5Ô5Ð5Ð5Ð5Ø"�Ý,/°°ÀÑ1EÔ1EÐ0FÑ,GÔ,G�˜[Ñ)Ñ)Ø”¥¤	Ô 9Ò9Ð9åÐJÐJÀÄ	ÐJÑJÔJÑJÔJÐJÐJÐJØ ×$Ò$ TÑ*Ô*Ð*Ý),¨n¨n¸TÑ.BÔ.BÐ-CÑ)DÔ)DÐ  Ñ&Ø'+�˜tÑ$Ð$Ø”¥¤	Ô 8Ò8Ð8Ý˜4œ9‘~”~¨Ò*Ð*Ð*Ð*Ø  ¤¨1¤Ô.×2Ò2°>°>À$Ñ3GÔ3GÑHÔHÐHØ ×'Ò'¨¬	°!¬Ñ5Ô5Ð5Ø/3�˜tœy¨œ|Ñ,àÐ"Ø˜Ô%×)Ò)¨.¨.¸Ñ*>Ô*>Ñ?Ô?Ð?à!ð 	:ð 	:ˆAØ˜QÔ×#Ò# N N°4Ñ$8Ô$8Ñ9Ô9Ð9Ð9ñ	:õ Ð>Ð> n×&;Ò&;Ñ&=Ô&=Ð>Ñ>Ô>Ñ>Ô>ÐTÐTÐ@TÑTÔTÐ>àJÐJÐ1A×1GÒ1GÑ1IÔ1IÐJÑJÔJÐØBÐB¨\×-?Ò-?Ñ-AÔ-AÐBÑBÔB€Lå×Ò�GœMÑ*Ô*ð 8Ý�ŠÐ,Ð.>Ñ?Ô?Ð?Ý�ŠÐ(¨,Ñ7Ô7Ð7å&*Ð+;Ñ&<Ô&<Ð&RÅÀ\Ñ@RÔ@RÐ#ð ÐØ””ð ð ˆØ $ˆ
�4”9Ñð Œ7Ð1Ð1Ð1ØØŒ7�hÒÐÝŒHŒN×"Ò"Ø”	˜!”ÐKÐKÐKÐKñô ð ð à*ð 	$Ø �. Ñ&Ô&ˆCØ$¨Ò+Ð+Ð+ðDØ-ðDð DØ>AðDð Dñ ,Ô+Ð+ð !$Ðð Œ;Ð0Ð0Ð0ÝŒHŒN×"Ò"Ø”	ÐVÐVÐVÐVÐVñô ð õ ŒHŒN×"Ò"Ø”ÐXÐXÐXÐXÐXñô ð øõ  $ J§O¢OÑ$5Ô$5Ñ6Ô6Ðà!#€OØ%/×%5Ò%5Ñ%7Ô%7ð 3ð 3Ñ!ˆ˜	Ý�9Ô)Ñ*Ô*ð 	3Ø×"Ò" >Ñ2Ô2Ð2øð $&ÐØ
ð 2Ø(×,Ò,Ñ.Ô.ˆØ× Ò  Ñ0Ô0Ð0Ø# NÔ3Ô>ð 	2ð 	2ˆIØ�yÔ!Ô.×2Ò2°>ÑBÔBÐBØ˜iÔ(Ô5ð 2Ø×&Ò& yÑ1Ô1Ð1øð ð 2õ ÐÑÔ¥ Z¡¤Ò0Ð0ÝÐ=Ñ>Ô>Ð>ð -¨lÐ;ð 7ð 7ˆØ,×2Ò2Ñ4Ô4ð 	7ð 	7‰MˆD�'Ý�w‘<”< !Ò#Ð#Ð#Ð#Ø<@ˆJ•s˜7 1œ:‘”Ô'Ô3°DÑ9Ø˜Q˜R˜R”[ð 7ð 7�Ø&¥s¨1¡v¤vÔ.�	Ø$œ?×6Ò6Ø”wØœ;ÝÐ8Ð8¨d¬iÐ8Ñ8Ô8Ñ8Ô8ØØ"œið 7ñ ô �ð ”I—N’NÑ$Ô$ð ”ð /7�	Ô% dÑ+Ð+ð7ð	7ð$ ,ð &ñ &ˆØ˜~Ô.ˆ	Ø&(ˆ
ØÔ#ð 	Añ 	AˆCØ" 3œˆIð Œ|˜zÒ)Ð)Ý! )Ô"2µCÑ8Ô8Ð8Ð8Ð8å3°A°yÔ7GÑHÔH�	Ý˜i­¬¬Ñ9Ô9ð +Ø"+¤/×":Ò":¸9Ô;KÑ"LÔ"L�KØ:C�IÔ% iÔ&6Ñ7Ð7à"+¤/×"=Ò"=ØØ",¨S¤/Ô"6ð #>ñ #ô #�Kð '+�J˜s‘O�Oà'œo×9Ò9ØØ(¨œoÔ2ð :ñ ô �ð #'�
˜3‘Ø)¨#œÔ3×8Ò8Ñ:Ô:ˆKÔØ5@ˆIÔ! *¨S¤/Ñ2Ñ2Ø%ˆ	ÔÑð ””ð $3ñ $3ˆÝ�4˜Ñ)Ô)ñ #	3Ø" 4Ô#5Ô6ˆIð $Ô/ˆKÝ!œHœN×2Ò2°4´9Ð>VÐ>VÐ>VÐ>VÑWÔWˆMÝ#œhœn×4Ò4Ø”Ð5Ð5Ð5Ð5ñô ˆOð ŒwÐ9Ð9Ð9Øœ��å5°a¸¼ÑEÔE�Øœ×,Ò,¨S°#Ñ6Ô6�Ø,7�	Ô! &Ñ)àÐ+ð #,Ô"7ÐBÐB¸&ÐBÐB�HØ-1¬[�L Ñ*å˜m­UÑ3Ô3Ð3Ð3Ð3Ý˜o­tÑ4Ô4Ð4Ð4Ð4Ø 7ÐA�4”9�9¸TˆDØ ”×2Ò2Ø”7ØØ"Ø&Øœ)Øð 3ñ ô ˆHð !œIŸNšNÑ,Ô,ˆHŒMØ*2ˆIÔ! $Ñ'ùð -Ð-ð ð ˆÝ˜_Ñ-Ô-ð 	ð 	ˆDØ% dÔ+ˆGÝ�w‘<”< !Ò#Ð#Ð#Ð#Ø˜S˜b˜S”\ð ð �Ø&¥s¨1¡v¤vÔ.�	Ø*¨4Ô0�	Ø Ð,Ð,Ð.AÑ,Ô,Ð,Ø$œ?×6Ò6Ø ”|Ø$Ô+Ø#Ô/°Ô5Ð7ØØ'œnð 7ñ ô �ð ”N×'Ò'Ñ)Ô)ð ”�ðð	ð& %'€Là$&€LÝ+0¬8¬>×+?Ò+?Ñ+AÔ+A€NØCE€NØð 	+Ø”G”Mð 	ð 	ˆDØ+:¨?Ø�l Nñ,ô ,Ñ(ˆL˜.˜.ð	ð ”G”Mð 	+ð 	+ˆDØ&*ˆL˜œÑ#Ð#ð "5ÐRÐÐÐ:Rð õ &)¡U¤UÐ"ð RÐQ q¤w¤}ÐQÑQÔQ€Nà4ð <Eñ <EˆØ˜~Ô.ˆ	õ ð 
ð 
ð 
ð 
ð 
Ø@IÔ@Qð
ñ 
ô 
ñ 
ô 
ˆõ
 ˜kÑ*Ô*ˆØ˜aÒÐØŒO×"Ò" ;¨q¤>Ñ2Ô2Ð2Ð2Ø˜qÒ Ð ØŒO×"Ò" ;Ñ/Ô/Ð/Ð/ð ŒO×"Ò" 2Ñ&Ô&Ð&àð 	9ð/ð /ð /ð /ð /à$Ô+ð/ñ /ô /Ðð 'ð 9ð 9�ØÐ9Ð9Ð9ØØ1@°Ø˜,¨ñ2ô 2Ñ.�Ð.ð /×2Ò2°4Ñ8Ô8Ð8Ð8ð ,ð 9ð 9�ØÐ9Ð9Ð9ØØ/>¨Ø˜,¨ñ0ô 0Ñ,�˜nð /×2Ò2°4Ñ8Ô8Ð8Ð8å05´Ô0E×0QÒ0QØÔ˜yœñ1
ô 1
ˆ�yÔ,Ñ-ð
 $×/Ò/ØÔ!ÝÐBÐBÐBÐB°Ô1AÐBÑBÔBÑBÔBñ
ô 
ˆ
õ
 ˜)Ô+Ñ,Ô,ˆØ˜Š?ˆ?å$œxœ~×3Ò3°JÑ?Ô?ÐÝ"+¨IÔ,=Ñ">Ô">ð Eð E‘��;Ø,<¸QÔ,?Ô,D�˜[Ñ)Ð)ñEà˜AÒÐØ:DˆL��d 9Ô#4Ñ5Ô5Ñ6Ô6Ñ7ùð ð  <ð Ø”G”Mð 	ð 	ˆDØ+:¨?Ø�l Nñ,ô ,Ñ(ˆL˜.˜.ð
 ””ð ð ˆØŒ7�hÒÐØ×!Ò!Ý””×&Ò& t¤y°¤|Ð5SÐ5SÐ5SÐ5SÑTÔTñô ð øõ Œ(Ô
×
+Ò
+¨N¸NÑ
KÔ
K€CÝ‡I‚IØÝÐ2°CÀÐFÑFÔFñô ð ð €Jr$   )NFF)!rK   r²   Úcollectionsr   Útypingr   r   r   r   r   r	   r   Útorch.fx._compatibilityr
   Útorch.fx._utilsr   Útorch.fx.graph_moduler   Útorch.fx.noder   Ú__all__Ú	getLoggerr(   r_   r°   r   rº   r»   r+   r7   Úintrª   r   r,   r$   r"   ú<module>rö      sí  ðà €€€Ø €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;à €€€Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø -Ð -Ð -Ð -Ð -Ð -Ø Ð Ð Ð Ð Ð ð ˜Ð
'€Ø!�Ô! (Ñ+Ô+Ð +€€gð € dÐ+Ñ+Ô+ð
ð 
ð 
ð 
ð 
ñ 
ô 
ñ ,Ô+ð
ð0 ¤¤ð ¸Cð ÀCð ð ð ð ð € dÐ+Ñ+Ô+ð
 .2Ø*/Ø.3ð|ð |Øð|àŒHŒOð|ð ˜d˜V S˜[Ô)ð|ð ˜4  S œ>Ô*ð	|ð
 " $œð|ð & dœ^ð|ð |ð |ñ ,Ô+ð|ð |ð |r$   