§
    Ë²ÅjÌ  ã                   ó  — U d dl Z d dlmZmZ d dlZd dlmZ d dlmZ d dl	m
Z
  G d„ de¦  «        Z G d„ d	e¦  «        Z G d
„ de¦  «        Z G d„ de¦  «        Zdaeeeej                                   ed<   dej        fd„ZdS )é    N)ÚListÚOptional©Ú_get_device_index)ÚFunction)Úcommc                   ó:   — e Zd Zed„ ¦   «         Zed„ ¦   «         ZdS )Ú	Broadcastc                 ó  ‡— t          d„ |D ¦   «         ¦  «        s
J d¦   «         ‚d„ |D ¦   «         }|| _        t          |¦  «        dk    rdS t          |¦  «        | _        |d                              ¦   «         | _        t          j        || j        ¦  «        }g }t          | j	        dd …         ¦  «        D ](\  Š}|s!| 
                    ˆfd„|D ¦   «         ¦  «         Œ) | j        |Ž  t          d„ |D ¦   «         ¦  «        S )	Nc              3   ó6   K  — | ]}|j         j        d k    V — ŒdS ©ÚcpuN©ÚdeviceÚtype©Ú.0Úis     úMC:\Projects\Historical\venv\Lib\site-packages\torch/nn/parallel/_functions.pyú	<genexpr>z$Broadcast.forward.<locals>.<genexpr>   ó<   è è € ð 
ð 
Ø'(ˆAŒHŒM˜UÒ"ð
ð 
ð 
ð 
ð 
ð 
ó    z2Broadcast function not implemented for CPU tensorsc                 ó.   — g | ]}t          |d ¦  «        ‘ŒS ©Tr   ©r   Úxs     r   ú
<listcomp>z%Broadcast.forward.<locals>.<listcomp>   ó#   € ÐGÐGÐG°aÕ(¨¨DÑ1Ô1ÐGÐGÐGr   r   © é   c              3   ó(   •K  — | ]}|‰         V — Œd S ©Nr   )r   ÚoutputÚidxs     €r   r   z$Broadcast.forward.<locals>.<genexpr>   s'   øè è € Ð*MÐ*M¸6¨6°#¬;Ð*MÐ*MÐ*MÐ*MÐ*MÐ*Mr   c                 ó   — g | ]	}|D ]}|‘ŒŒ
S r   r   )r   ÚtensorsÚts      r   r   z%Broadcast.forward.<locals>.<listcomp>   s%   € Ð@Ð@Ð@˜G¸Ð@Ð@°1�aÐ@Ð@Ð@Ð@r   )ÚallÚtarget_gpusÚlenÚ
num_inputsÚ
get_deviceÚinput_devicer   Úbroadcast_coalescedÚ	enumerateÚneeds_input_gradÚextendÚmark_non_differentiableÚtuple)Úctxr)   ÚinputsÚoutputsÚnon_differentiablesÚinput_requires_gradr$   s         @r   ÚforwardzBroadcast.forward   sJ  ø€ åð 
ð 
Ø,2ð
ñ 
ô 
ñ 
ô 
ð 	@ð 	@à?ñ	@ô 	@ð 
ð HÐG¸;ÐGÑGÔGˆØ%ˆŒÝˆv‰;Œ;˜!ÒÐØ�2Ý˜V™œˆŒØ! !œ9×/Ò/Ñ1Ô1ˆÔÝÔ*¨6°3´?ÑCÔCˆØ ÐÝ(1°#Ô2FÀqÀrÀrÔ2JÑ(KÔ(Kð 	Nð 	NÑ$ˆCÐ$Ø&ð NØ#×*Ò*Ð*MÐ*MÐ*MÐ*MÀWÐ*MÑ*MÔ*MÑMÔMÐMøØ#ˆÔ#Ð%8Ð9Ð9ÝÐ@Ð@ wÐ@Ñ@Ô@ÑAÔAÐAr   c                 óB   — dt          j        | j        | j        g|¢R Ž z   S )Nr"   )ÚReduceAddCoalescedÚapplyr-   r+   ©r4   Úgrad_outputss     r   ÚbackwardzBroadcast.backward   s6   € àÕ+Ô1ØÔ˜cœnð
Ø/;ð
ð 
ð 
ñ 
ð 	
r   N©Ú__name__Ú
__module__Ú__qualname__Ústaticmethodr9   r?   r   r   r   r
   r
   
   sK   € € € € € ØðBð Bñ „\ðBð$ ð
ð 
ñ „\ð
ð 
ð 
r   r
   c                   ó:   — e Zd Zed„ ¦   «         Zed„ ¦   «         ZdS )r;   c                 óä   ‡‡— ˆfd„t          dt          ‰¦  «        ‰¦  «        D ¦   «         | _        ˆˆfd„t          dt          ‰¦  «        ‰¦  «        D ¦   «         }t          j        ||¦  «        S )Nc                 óD   •— g | ]}‰|                               ¦   «         ‘ŒS r   ©r,   )r   r   Úgradss     €r   r   z.ReduceAddCoalesced.forward.<locals>.<listcomp>(   s6   ø€ ð 
ð 
ð 
Ø&'ˆE�!ŒH×ÒÑ!Ô!ð
ð 
ð 
r   r   c                 ó*   •— g | ]}‰||‰z   …         ‘ŒS r   r   )r   r   rI   r+   s     €€r   r   z.ReduceAddCoalesced.forward.<locals>.<listcomp>,   s'   ø€ ÐVÐVÐV°�%˜˜A 
™NÐ*Ô+ÐVÐVÐVr   )Úranger*   r)   r   Úreduce_add_coalesced)r4   Údestinationr+   rI   Úgrads_s     `` r   r9   zReduceAddCoalesced.forward&   s„   øø€ ð
ð 
ð 
ð 
Ý+0°µC¸±J´JÀ
Ñ+KÔ+Kð
ñ 
ô 
ˆŒð WÐVÐVÐVÐVµU¸1½cÀ%¹j¼jÈ*Ñ5UÔ5UÐVÑVÔVˆÝÔ(¨°Ñ=Ô=Ð=r   c                 ó6   — dt          j        | j        g|¢R Ž z   S )N©NN)r
   r<   r)   r=   s     r   r?   zReduceAddCoalesced.backward/   s*   € ð
õ ŒO˜CœOÐ;¨lÐ;Ð;Ð;ñ<ð 	<r   Nr@   r   r   r   r;   r;   %   sH   € € € € € Øð>ð >ñ „\ð>ð ð<ð <ñ „\ð<ð <ð <r   r;   c                   ó:   — e Zd Zed„ ¦   «         Zed„ ¦   «         ZdS )ÚGatherc                 ó  ‡ — t          d„ |D ¦   «         ¦  «        s
J d¦   «         ‚|dk    rd‰ _        nt          |d¦  «        }|‰ _        |‰ _        t	          d„ |D ¦   «         ¦  «        ‰ _        t          d„ |D ¦   «         ¦  «        r;|dk    r5t	          d„ |D ¦   «         ¦  «        }t          j        d	¦  «         d‰ _        nd
‰ _        t	          ˆ fd„|D ¦   «         ¦  «        ‰ _	        t          j        |‰ j        ‰ j        ¦  «        S )Nc              3   ó6   K  — | ]}|j         j        d k    V — ŒdS r   r   r   s     r   r   z!Gather.forward.<locals>.<genexpr>:   r   r   z/Gather function not implemented for CPU tensorsr   Tc              3   ó>   K  — | ]}|                      ¦   «         V — Œd S r"   rH   r   s     r   r   z!Gather.forward.<locals>.<genexpr>C   s*   è è € Ð>Ð>°!˜qŸ|š|™~œ~Ð>Ð>Ð>Ð>Ð>Ð>r   c              3   óF   K  — | ]}|                      ¦   «         d k    V — ŒdS ©r   N)Údim©r   r'   s     r   r   z!Gather.forward.<locals>.<genexpr>D   s.   è è € Ð,Ð, ˆq�uŠu‰wŒw˜!Š|Ð,Ð,Ð,Ð,Ð,Ð,r   r   c              3   ó@   K  — | ]}|                      d ¦  «        V — ŒdS )r    N)ÚviewrY   s     r   r   z!Gather.forward.<locals>.<genexpr>E   s,   è è € Ð5Ð5¨˜1Ÿ6š6 !™9œ9Ð5Ð5Ð5Ð5Ð5Ð5r   zvWas asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.Fc              3   óL   •K  — | ]}|                      ‰j        ¦  «        V — Œd S r"   )ÚsizerX   )r   r   r4   s     €r   r   z!Gather.forward.<locals>.<genexpr>N   s/   øè è € Ð@Ð@°A §¢ s¤w¡¤Ð@Ð@Ð@Ð@Ð@Ð@r   )r(   Útarget_devicer   rX   r3   Ú
input_gpusÚwarningsÚwarnÚunsqueezed_scalarÚinput_sizesr   Úgather)r4   r^   rX   r5   s   `   r   r9   zGather.forward8   sJ  ø€ åð 
ð 
Ø,2ð
ñ 
ô 
ñ 
ô 
ð 	=ð 	=à<ñ	=ô 	=ð 
ð ˜EÒ!Ð!Ø %ˆCÔÐå-¨m¸TÑBÔBˆMØ -ˆCÔØˆŒÝÐ>Ð>°vÐ>Ñ>Ô>Ñ>Ô>ˆŒÝÐ,Ð, VÐ,Ñ,Ô,Ñ,Ô,ð 		*°¸²°ÝÐ5Ð5¨fÐ5Ñ5Ô5Ñ5Ô5ˆFÝŒMð'ñô ð ð
 %)ˆCÔ!Ð!à$)ˆCÔ!ÝÐ@Ð@Ð@Ð@¸Ð@Ñ@Ô@Ñ@Ô@ˆŒÝŒ{˜6 3¤7¨CÔ,=Ñ>Ô>Ð>r   c                 ó¤   — t                                | j        | j        | j        |¦  «        }| j        rt          d„ |D ¦   «         ¦  «        }d|z   S )Nc              3   ó&   K  — | ]}|d          V — ŒdS rW   r   )r   Úgs     r   r   z"Gather.backward.<locals>.<genexpr>W   s&   è è € Ð#BÐ#B¨Q A a¤DÐ#BÐ#BÐ#BÐ#BÐ#BÐ#Br   rP   )ÚScatterr<   r_   rc   rX   rb   r3   )r4   Úgrad_outputÚscattered_gradss      r   r?   zGather.backwardQ   sY   € å!Ÿ-š-ØŒN˜CœO¨S¬W°kñ
ô 
ˆð Ô ð 	CÝ#Ð#BÐ#B°/Ð#BÑ#BÔ#BÑBÔBˆOØ˜oÑ-Ð-r   Nr@   r   r   r   rR   rR   7   sH   € € € € € Øð?ð ?ñ „\ð?ð0 ð.ð .ñ „\ð.ð .ð .r   rR   c                   ó:   — e Zd Zed„ ¦   «         Zed„ ¦   «         ZdS )rh   c                 ón  — d„ |D ¦   «         }|| _         |j        j        dk    r|                     ¦   «         nd| _        d }t
          j                             ¦   «         r| j        dk    rd„ |D ¦   «         }t          j	        |||| j         |¦  «        }|� t          |¦  «        D ]�\  }}t
          j                             ||         ¦  «        5  t
          j                             ¦   «         }	|	                     ||         ¦  «         |                     |	¦  «         d d d ¦  «         n# 1 swxY w Y   Œ‘|S )Nc                 ó.   — g | ]}t          |d ¦  «        ‘ŒS r   r   r   s     r   r   z#Scatter.forward.<locals>.<listcomp>^   r   r   r   éÿÿÿÿc                 óR   — g | ]$}t          t          j        d |¦  «        ¦  «        ‘Œ%S )Úcuda)Ú_get_streamÚtorchr   )r   r   s     r   r   z#Scatter.forward.<locals>.<listcomp>d   s9   € ð ð ð Ø>D•�EœL¨°Ñ8Ô8Ñ9Ô9ðð ð r   )rX   r   r   r,   r-   rr   rp   Úis_availabler   Úscatterr/   Úcurrent_streamÚwait_streamÚrecord_stream)
r4   r)   Úchunk_sizesrX   ÚinputÚstreamsr6   r   r#   Úmain_streams
             r   r9   zScatter.forward\   s}  € àGÐG¸;ÐGÑGÔGˆØˆŒØ16´Ô1BÀeÒ1KÐ1K˜5×+Ò+Ñ-Ô-Ð-ÐQSˆÔØˆÝŒ:×"Ò"Ñ$Ô$ð 	¨Ô)9¸RÒ)?Ð)?ðð ØHSðñ ô ˆGõ ”,˜u k°;ÀÄÈÑQÔQˆàÐÝ& wÑ/Ô/ð 6ð 6‘	��6Ý”Z×&Ò& {°1¤~Ñ6Ô6ð 6ð 6Ý"'¤*×";Ò";Ñ"=Ô"=�KØ×+Ò+¨G°A¬JÑ7Ô7Ð7Ø×(Ò(¨Ñ5Ô5Ð5ð6ð 6ð 6ñ 6ô 6ð 6ð 6ð 6ð 6ð 6ð 6øøøð 6ð 6ð 6ð 6øð ˆs   ÃAD)Ä)D-	Ä0D-	c                 óD   — d d d t          j        | j        | j        g|¢R Ž fS r"   )rR   r<   r-   rX   )r4   ri   s     r   r?   zScatter.backwardq   s*   € à�T˜4¥¤¨cÔ.>ÀÄÐ!VÈ+Ð!VÐ!VÐ!VÐVÐVr   Nr@   r   r   r   rh   rh   [   sM   € € € € € Øðð ñ „\ðð( ðWð Wñ „\ðWð Wð Wr   rh   Ú_streamsr   c                 ó.  — | j         dk    rdS t          t          | j         d¦  «        }|€dS t          €dg|                     ¦   «         z  at          | j                 €'|                     | j        ¦  «        t          | j        <   t          | j                 S )zBGet a background stream for copying between CPU and target device.r   N)r   Úgetattrrr   r}   Údevice_countÚindexÚStream)r   Ú
device_mods     r   rq   rq   z   sˆ   € ð „{�eÒÐØˆtÝ� ¤¨TÑ2Ô2€JØÐØˆtÝÐØ�6˜J×3Ò3Ñ5Ô5Ñ5ˆÝ�”ÔÐ%Ø!+×!2Ò!2°6´<Ñ!@Ô!@��”ÑÝ�F”LÔ!Ð!r   )r`   Útypingr   r   rr   Útorch._utilsr   Útorch.autogradr   Útorch.nn.parallelr   r
   r;   rR   rh   r}   r‚   Ú__annotations__r   rq   r   r   r   ú<module>r‰      sb  ðØ €€€€Ø !Ð !Ð !Ð !Ð !Ð !Ð !Ð !à €€€Ø *Ð *Ð *Ð *Ð *Ð *Ø #Ð #Ð #Ð #Ð #Ð #Ø "Ð "Ð "Ð "Ð "Ð "ð
ð 
ð 
ð 
ð 
�ñ 
ô 
ð 
ð6<ð <ð <ð <ð <˜ñ <ô <ð <ð$!.ð !.ð !.ð !.ð !.ˆXñ !.ô !.ð !.ðHWð Wð Wð Wð Wˆhñ Wô Wð Wð8 48€ˆ(�4˜ ¤Ô.Ô/Ô
0Ð 7Ð 7Ñ 7ð"˜œð "ð "ð "ð "ð "ð "r   