§
    Ë²ÅjF�  ã            *       ó€  — d dl mZmZmZmZmZ d dlZd dlmZ ddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddgZ G d„ de¦  «        Zd	d
e› de› de› de	› de› de› d�z   e_        dee         dee         dee         dee         dee         dee         dee         dee         dededee ef         dee ef         dee ef         de de dededef$d „Z!dee         dee         dee         dee         dee         dee         dee         dee         dededee ef         dee ef         dee ef         de de dededef$d!„Z"dee         dee         dee         dee         dee         dee         dee         dee         dedede de dee ef         de de dededed"df&d#„Z# ee!¬$¦  «        	 	 	 	 	 	 	 d)dee         dee         dee         dee         dee         dee         d&ee         deded'ee         dee         dee         dedede de dee ef         de de def(d(„¦   «         Z$dS )*é    )ÚcastÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_device_dtype_check_for_fusedÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ
_fused_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_params_docÚ_stack_if_compilingÚ_use_grad_for_differentiableÚ_view_as_realÚ
DeviceDictÚDeviceDtypeDictÚ	OptimizerÚParamsTÚAdamÚadamc                   óæ   ‡ — e Zd Z	 	 	 	 	 dddddddœded	eeef         d
eeeef         eeef         f         dededede	e         dededede	e         fˆ fd„Z
ˆ fd„Zd„ Zedd„¦   «         Zˆ xZS )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   FN)ÚforeachÚmaximizeÚ
capturableÚdifferentiableÚfusedÚparamsÚlrÚbetasÚepsÚweight_decayÚamsgradr#   r$   r%   r&   r'   c                ó  •— t          |t          ¦  «        r:|r|	st          d¦  «        ‚|                     ¦   «         dk    rt          d¦  «        ‚d|k    st          d|› �¦  «        ‚d|k    st          d|› �¦  «        ‚d|d         cxk    rdk     sn t          d	|d         › �¦  «        ‚d|d         cxk    rdk     sn t          d
|d         › �¦  «        ‚d|k    st          d|› �¦  «        ‚t          |d         t          ¦  «        rt          |d         t          ¦  «        sEt          |d         t          ¦  «        rt          |d         t          ¦  «        st          d¦  «        ‚t          |d         t          ¦  «        r@|	s|rt          d¦  «        ‚|d                              ¦   «         dk    rt          d¦  «        ‚t          |d         t          ¦  «        r@|	s|rt          d¦  «        ‚|d                              ¦   «         dk    rt          d¦  «        ‚t          ||||||||	|
|¬¦
  «
        }t          ¦   «                              ||¦  «         |r)|
rt          d¦  «        ‚d| _	        |rt          d¦  «        ‚d S d S )NúElr as a Tensor is not supported for capturable=False and foreach=Truer	   zTensor lr must be 1-elementç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: z0betas must be either both floats or both TensorszKbetas[0] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[0] must be 1-elementzKbetas[1] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[1] must be 1-element)
r)   r*   r+   r,   r-   r$   r#   r%   r&   r'   z)`fused` does not support `differentiable`Tz0`fused` and `foreach` cannot be `True` together.)
Ú
isinstancer   Ú
ValueErrorÚnumelÚfloatÚdictÚsuperÚ__init__ÚRuntimeErrorÚ_step_supports_amp_scaling)Úselfr(   r)   r*   r+   r,   r-   r#   r$   r%   r&   r'   ÚdefaultsÚ	__class__s                €úAC:\Projects\Historical\venv\Lib\site-packages\torch/optim/adam.pyr8   zAdam.__init__#   s  ø€ õ �b�&Ñ!Ô!ð 	@Øð ˜zð Ý Ø[ñô ð ð �xŠx‰zŒz˜QŠˆÝ Ð!>Ñ?Ô?Ð?Ø�bŠyˆyÝÐ;°rÐ;Ð;Ñ<Ô<Ð<Ø�cŠzˆzÝÐ<°sÐ<Ð<Ñ=Ô=Ð=Ø�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�lÒ"Ð"ÝÐJ¸LÐJÐJÑKÔKÐKå˜˜aœ¥%Ñ(Ô(ð	QÝ-7¸¸a¼Å%Ñ-HÔ-Hð	Qå˜5 œ8¥VÑ,Ô,ð	Qå1;¸EÀ!¼HÅfÑ1MÔ1Mð	Qõ ÐOÑPÔPÐPÝ�e˜A”h¥Ñ'Ô'ð 	FØð  'ð Ý Øañô ð ð �QŒx�~Š~ÑÔ 1Ò$Ð$Ý Ð!DÑEÔEÐEÝ�e˜A”h¥Ñ'Ô'ð 	FØð  'ð Ý Øañô ð ð �QŒx�~Š~ÑÔ 1Ò$Ð$Ý Ð!DÑEÔEÐEåØØØØ%ØØØØ!Ø)Øð
ñ 
ô 
ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*àð 		WØð PÝ"Ð#NÑOÔOÐOØ.2ˆDÔ+ð
 ð WÝ"Ð#UÑVÔVÐVð		Wð 		WðWð Wó    c                 óø  •— t          ¦   «                              |¦  «         | j        D �]N}|                     dd¦  «         |                     dd¦  «         |                     dd ¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd ¦  «        }|d         D ]¾}| j                             |g ¦  «        }t          |¦  «        d	k    rŽt          j        |d
         ¦  «        stt          |d
         ¦  «        }|d         s|d         r*t          j
        |t          |¬¦  «        |j        ¬¦  «        n!t          j
        |t          ¦   «         ¬¦  «        |d
<   Œ¿�ŒPd S )Nr-   Fr$   r#   r%   r&   r'   r(   r   Ústep©Úis_fused©ÚdtypeÚdevice©rE   )r7   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorr5   Útensorr   rF   )r;   rK   Úgroupr'   ÚpÚp_stateÚstep_valr=   s          €r>   rH   zAdam.__setstate__p   s�  ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	ñ 	ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×Ò˜Z¨Ñ/Ô/Ð/Ø×Ò˜Y¨Ñ-Ô-Ð-Ø×Ò˜\¨5Ñ1Ô1Ð1Ø×ÒÐ-¨uÑ5Ô5Ð5Ø×$Ò$ W¨dÑ3Ô3ˆEØ˜8”_ð ð �Øœ*Ÿ.š.¨¨BÑ/Ô/�Ý�w‘<”< 1Ò$Ð$­U¬_¸WÀV¼_Ñ-MÔ-MÐ$Ý$ W¨V¤_Ñ5Ô5�Hð ! Ô.ðOð
 38¸´.ðO�œØ$Ý"3¸UÐ"CÑ"CÔ"CØ#$¤8ðñ ô ð õ #œ\¨(Õ:KÑ:MÔ:MÐNÑNÔNð ˜F‘Oøñ	ð	ð 	r?   c                 ó²  — d}|d         D �]J}	|	j         ��?|t          j        |	¦  «        z  }|                     |	¦  «         |	j         j        rt          d¦  «        ‚|                     |	j         ¦  «         | j        |	         }
t          |
¦  «        dk    rí|d         rt          |	¦  «         |d         s|d         r0t          j	        dt          |d         ¬¦  «        |	j        ¬	¦  «        n!t          j        d
t          ¦   «         ¬¦  «        |
d<   t          j        |	t          j        ¬¦  «        |
d<   t          j        |	t          j        ¬¦  «        |
d<   |d         r#t          j        |	t          j        ¬¦  «        |
d<   |                     |
d         ¦  «         |                     |
d         ¦  «         |d         r|                     |
d         ¦  «         |d         r|
d         j        rt          d¦  «        ‚|d         r1t          j        |d         ¦  «        r|d         st          d¦  «        ‚|                     |
d         ¦  «         �ŒL|S )NFr(   zJAdam does not support sparse gradients, please consider SparseAdam insteadr   r'   r%   © rB   rD   r0   rG   rA   )Úmemory_formatÚexp_avgÚ
exp_avg_sqr-   Úmax_exp_avg_sqr&   zB`requires_grad` is not supported for `step` in differentiable moder#   r)   r/   )ÚgradrN   Ú
is_complexÚappendÚ	is_sparser9   rK   rM   r   Úzerosr   rF   rP   Ú
zeros_likeÚpreserve_formatÚrequires_gradrO   )r;   rQ   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsÚhas_complexrR   rK   s              r>   Ú_init_groupzAdam._init_group‡   s¥  € ð ˆØ�x”ð =	2ñ =	2ˆAØŒvÑ!Ø�uÔ/°Ñ2Ô2Ñ2�Ø ×'Ò'¨Ñ*Ô*Ð*Ø”6Ô#ð Ý&Ødñô ð ð —’˜QœVÑ$Ô$Ð$àœ
 1œ�å�u‘:”: ’?�?Ø˜W”~ð 9Ý5°aÑ8Ô8Ð8ð ! Ô.ðJð
 38¸´.ðJ�œØÝ"3¸UÀ7¼^Ð"LÑ"LÔ"LØ#$¤8ðñ ô ð õ #œ\¨#Õ5FÑ5HÔ5HÐIÑIÔIð ˜&‘Mõ (-Ô'7Ø­Ô)>ð(ñ (ô (�E˜)Ñ$õ +0Ô*:Ø­Ô)>ð+ñ +ô +�E˜,Ñ'ð ˜YÔ'ð å27Ô2BØ­UÔ-Bð3ñ 3ô 3˜Ð.Ñ/ð —’  iÔ 0Ñ1Ô1Ð1Ø×"Ò" 5¨Ô#6Ñ7Ô7Ð7à˜Ô#ð DØ#×*Ò*¨5Ð1AÔ+BÑCÔCÐCØÐ)Ô*ð ¨u°V¬}Ô/Jð Ý&Ø\ñô ð ð ˜)Ô$ðåœ¨¨d¬Ñ4Ô4ðð " ,Ô/ðõ
 'Ø_ñô ð ð ×"Ò" 5¨¤=Ñ1Ô1Ð1ùØÐr?   c                 óþ  — |                       ¦   «          d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D ]§}g }g }g }g }g }g }	|d         \  }
}|                      |||||||	¦  «        }t          ||||||	f|d         ||
||d         |d         |d         |d         |d         |d	         |d
         |d         t          | dd¦  «        t          | dd¦  «        dœŽ Œ¨|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr*   r-   r)   r,   r+   r$   r#   r%   r&   r'   Ú
grad_scaleÚ	found_inf)r-   ri   Úbeta1Úbeta2r)   r,   r+   r$   r#   r%   r&   r'   rl   rm   )Ú _cuda_graph_capture_health_checkrN   Úenable_gradrI   rj   r   Úgetattr)r;   ÚclosureÚlossrQ   rc   rd   re   rf   rg   rh   rn   ro   ri   s                r>   rA   z	Adam.stepÒ   s¥  € ð 	×-Ò-Ñ/Ô/Ð/àˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð (	ð (	ˆEØ-/ÐØ"$ˆEØ%'ˆHØ(*ˆKØ,.ˆOØ(*ˆKØ  œ>‰LˆE�5à×*Ò*ØØ ØØØØØñô ˆKõ Ø ØØØØØðð ˜iÔ(Ø'ØØØ˜”;Ø" >Ô2Ø˜%”LØ˜zÔ*Ø˜iÔ(Ø  Ô.Ø$Ð%5Ô6Ø˜G”nÝ" 4¨°tÑ<Ô<Ý! $¨°TÑ:Ô:ð)ð ð ð ð ð. ˆs   ¬AÁAÁ
A)r    r!   r"   r   F©N)Ú__name__Ú
__module__Ú__qualname__r   r   r5   r   r   Úboolr   r8   rH   rj   r   rA   Ú__classcell__)r=   s   @r>   r   r   "   s}  ø€ € € € € ð $(ØCOØØØðKWð #'ØØ Ø$Ø $ðKWð KWð KWàðKWð �%˜�-Ô ðKWð �U˜5 &˜=Ô)¨5°¸°Ô+?Ð?Ô@ð	KWð
 ðKWð ðKWð ðKWð ˜$”ðKWð ðKWð ðKWð ðKWð ˜Œ~ðKWð KWð KWð KWð KWð KWðZð ð ð ð ð.Ið Ið IðV "ð8ð 8ð 8ñ "Ô!ð8ð 8ð 8ð 8ð 8r?   a  Implements Adam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \beta_1, \beta_2
                \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)}          \\
            &\hspace{13mm}      \lambda \text{ (weight decay)},  \: \textit{amsgrad},
                \:\textit{maximize},  \: \epsilon \text{ (epsilon)}                              \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0\leftarrow 0 \text{ (second moment)},\: \widehat{v_0}^{max}\leftarrow 0\\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\

            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm}\textbf{if} \: \lambda \neq 0                                           \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
            &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_{t-1}}^{max},
                \widehat{v_t})                                                                   \\
            &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big)                                 \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_.
    z
    Args:
        a  
        lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR
            is not yet supported for all our implementations. Please use a float
            LR if you are not also specifying fused=True or capturable=True.
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        amsgrad (bool, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        z	
        a=  
    .. Note::
        A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`.
    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ

    r(   rd   re   rf   rg   rh   rl   rm   r-   ri   rn   ro   r)   r,   r+   r$   r%   r&   c       
         ó	  — |€|�J ‚t           j                             ¦   «         rEt          |t          ¦  «        sJ ‚t          |
t          ¦  «        sJ ‚t          |t          ¦  «        sJ ‚t          |
t
          ¦  «        r|
j        |
j        f|
i}nd }t          | ¦  «        D �]ã\  }}|s||         n||          }||         }||         }||         }t           j	         
                    ¦   «         sF|rDt          ¦   «         }|j        j        |j        j        k    r|j        j        |v sJ d|› d�¦   «         ‚|dz  }|dk    r|                     ||¬¦  «        }t          j        |¦  «        rot          j        |¦  «        }t          j        |¦  «        }t          j        |¦  «        }|rt          j        ||         ¦  «        ||<   t          j        |¦  «        }|j        }|�3|j        }||f}||vr|
                     ||d¬¦  «        ||<   ||         }n|
}|                     |d|z
  ¦  «         |                     |¦  «                             ||                     ¦   «         d|z
  ¬¦  «         |s|�r|}d|
|z  z
  }d||z  z
  } ||z  }!|!                     ¦   «         }"|                      ¦   «         }#|rŠ|r||                              ¦   «         }$n||         }$||                              t          j        |$|¦  «        ¦  «         ||                              ¦   «         |#|"z  z                       ||"z  ¦  «        }%n0|                     ¦   «         |#|"z  z                       ||"z  ¦  «        }%|                     ||%¦  «         nÂt9          |¦  «        }d|
|z  z
  }d||z  z
  } ||z  }!| d	z  }#|rTt          j        ||         |||         ¬
¦  «         ||                              ¦   «         |#z                       |¦  «        }%n*|                     ¦   «         |#z                       |¦  «        }%|                     ||%|! ¬¦  «         |r7t          j        | |         ¦  «        rt          j        ||         ¦  «        ||<   �Œåd S )NúIIf capturable=True, params and state_steps must be on supported devices: ú.r	   r   ©ÚalphaT)rF   rE   Únon_blocking)Úvalueç      à?)Úout)rN   ÚjitÚis_scriptingr2   r5   r   rF   rE   Ú	enumerateÚcompilerÚis_compilingr   ÚtypeÚaddr\   Úview_as_realÚtoÚlerp_Úmul_Úaddcmul_ÚconjÚnegÚsqrtÚcloneÚcopy_ÚmaximumÚadd_Úaddcdiv_r   Úview_as_complex)&r(   rd   re   rf   rg   rh   rl   rm   r-   ri   rn   ro   r)   r,   r+   r$   r%   r&   Ú
beta1_dictÚiÚparamr[   rX   rY   Ústep_tÚcapturable_supported_devicesrF   rE   ÚkeyÚdevice_beta1rA   Úbias_correction1Úbias_correction2Ú	step_sizeÚstep_size_negÚbias_correction2_sqrtrZ   Údenoms&                                         r>   Ú_single_tensor_adamr¦   T  sÿ  € ð* Ð )Ð"3Ð"3Ð3å„y×ÒÑÔð (õ ˜"�eÑ$Ô$Ð$Ð$Ð$Ý˜%¥Ñ'Ô'Ð'Ð'Ð'Ý˜%¥Ñ'Ô'Ð'Ð'Ð'õ �%�Ñ Ô ð Ø27´,ÀÄÐ1LÈeÐ0Tˆ
ˆ
àˆ
å˜fÑ%Ô%ð fKñ fK‰ˆˆ5Ø'Ð6ˆu�QŒxˆx¨e°A¬h¨YˆØ˜1”+ˆØ  ”^ˆ
Ø˜Q”ˆõ Œ~×*Ò*Ñ,Ô,ð 	{°ð 	{Ý+LÑ+NÔ+NÐ(à”Ô! V¤]Ô%7Ò7Ð7Ø”LÔ%Ð)EÐEÐEÐEØzÐ[wÐzÐzÐzñ FÔEðFð
 	�!‰ˆà˜1ÒÐØ—8’8˜E¨�8Ñ6Ô6ˆDåÔ˜EÑ"Ô"ð 	.ÝÔ% dÑ+Ô+ˆDÝÔ(¨Ñ1Ô1ˆGÝÔ+¨JÑ7Ô7ˆJØð LÝ%*Ô%7¸ÈÔ8JÑ%KÔ%K� Ñ"ÝÔ& uÑ-Ô-ˆEà”ˆàÐ!Ø”KˆEð ˜5�/ˆCØ˜*Ð$Ð$Ø"'§(¢(°&ÀÐTX (Ñ"YÔ"Y�
˜3‘à1;¸C´ˆLˆLà ˆLð 	�Š�d˜A Ñ,Ñ-Ô-Ð-à�Š˜ÑÔ×'Ò'¨¨d¯iªi©k¬kÀÀUÁÐ'ÑKÔKÐKàð 3	=˜ñ 3	=ØˆDà  5¨$¡;™ÐØ  5¨$¡;™ÐàÐ-Ñ-ˆIØ%ŸMšM™OœOˆMà$4×$9Ò$9Ñ$;Ô$;Ð!àð ,à!ð 8Ø%4°QÔ%7×%=Ò%=Ñ%?Ô%?�N�Nà%4°QÔ%7�Nà Ô"×(Ò(­¬°~ÀzÑ)RÔ)RÑSÔSÐSð $ AÔ&×+Ò+Ñ-Ô-Ð1FÈÑ1VÑWß’$�s˜]Ñ*Ñ+Ô+ð �ð
 —O’OÑ%Ô%Ð)>ÀÑ)NÑOß’$�s˜]Ñ*Ñ+Ô+ð ð �NŠN˜7 EÑ*Ô*Ð*Ð*å˜fÑ%Ô%ˆDà  5¨$¡;™ÐØ  5¨$¡;™ÐàÐ-Ñ-ˆIà$4°cÑ$9Ð!àð Nå”˜o¨aÔ0°*À/ÐRSÔBTÐUÑUÔUÐUð )¨Ô+×0Ò0Ñ2Ô2Ð5JÑJ×PÒPÐQTÑUÔU��à#ŸšÑ*Ô*Ð-BÑB×HÒHÈÑMÔM�à�NŠN˜7 E°)°ˆNÑ<Ô<Ð<ð ð 	K•uÔ'¨¨q¬	Ñ2Ô2ð 	KÝ!&Ô!6°ÀqÔ7IÑ!JÔ!JˆO˜AÑùðMfKð fKr?   c       
         óà  ‡
‡‡‡*— t          | ¦  «        dk    rd S t          ‰t          ¦  «        r|st          d¦  «        ‚t          ‰
t          ¦  «        r8|st	          d¦  «        ‚‰
                     ¦   «         dk    rt	          d¦  «        ‚t          ‰t          ¦  «        r8|st	          d¦  «        ‚‰                     ¦   «         dk    rt	          d¦  «        ‚t          j                             ¦   «         sI|rGt          d¬	¦  «        Š*t          ˆ*fd
„t          | |¦  «        D ¦   «         ¦  «        sJ d‰*› d�¦   «         ‚|€|�J ‚|r
J d¦   «         ‚t          j        | |||||g¦  «        }t          ‰
t          ¦  «        r!t          ‰
j        ¦  «        dk    r	‰
j        ‰
ind }|                     ¦   «         D �]³\  \  }}}}}}}t#          t$          t                   |¦  «        }t#          t$          t                   |¦  «        }t#          t$          t                   |¦  «        }t#          t$          t                   |¦  «        }t#          t$          t                   |¦  «        }|d         j        } |�| |vr‰
                     | d¬¦  «        || <   |r||          n‰
}!|	rH|r4t#          t$          t                   |¦  «        }"t)          |||||"¦  «         nt)          ||||¦  «         |rt          j        |¦  «        }t          j                             ¦   «         s9|d         j        r,t          j        |t          j        dd¬¦  «        d¬¦  «         nt          j        |d¦  «         |dk    r1|rt          j        |||¬¦  «         nt          j        |||¬¦  «        }t          j        ||d|!z
  ¦  «         t          j        |‰¦  «         t          ‰t          j        ¦  «        rt          j        |d‰z
  ¦  «        }#d}$n|}#d‰z
  }$t          j        ||#||$¦  «         ~~#|�r`t          j        ‰
|¦  «        }%t          j        ‰|¦  «        }&t          j        |%d¦  «         t          j        |&d¦  «         t          j         |&¦  «         t          j!        |%‰¦  «         t          j"        |%¦  «         t          j#        |&¦  «         |%}'|&}(|rJt#          t$          t                   |¦  «        }"t          j$        |"|¦  «         t          j%        |"¦  «        })nt          j%        |¦  «        })t          j!        |)|(¦  «         t          j        |)|¦  «         t          j!        |)|'¦  «         t          j&        |||)¦  «         �ŒÏˆ
fd„|D ¦   «         }%ˆfd„|D ¦   «         }&tO          ˆfd„|%D ¦   «         ¦  «        }'d„ |&D ¦   «         }(|rJt#          t$          t                   |¦  «        }"t          j$        |"|¦  «         t          j%        |"¦  «        })nt          j%        |¦  «        })t          j!        |)|(¦  «         t          j        |)|¦  «         t          j&        |||)|'¦  «         �Œµd S )Nr   r/   zHbeta1 as a Tensor is not supported for capturable=False and foreach=Truer	   zTensor beta1 must be 1-elementzHbeta2 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta2 must be 1-elementF)Úsupports_xlac              3   ón   •K  — | ]/\  }}|j         j        |j         j        k    o|j         j        ‰v V — Œ0d S ru   )rF   r‰   )Ú.0rR   rA   r�   s      €r>   ú	<genexpr>z%_multi_tensor_adam.<locals>.<genexpr>  s]   øè è € ð 
ð 
ñ ��4ð ŒHŒM˜Tœ[Ô-Ò-ð >Ø””Ð!=Ð=ð
ð 
ð 
ð 
ð 
ð 
r?   r|   r}   z#_foreach ops don't support autogradÚcpuT©rF   r€   r1   )rF   r~   c                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS ©r	   ©r   )rª   rA   rn   s     €r>   ú
<listcomp>z&_multi_tensor_adam.<locals>.<listcomp>¬  ó8   ø€ ð  ð  ð  Ø26��E�Z¨Ñ-Ô-Ñ-Ñ-ð ð  ð  r?   c                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS r¯   r°   )rª   rA   ro   s     €r>   r±   z&_multi_tensor_adam.<locals>.<listcomp>¯  r²   r?   c                 ó    •— g | ]
}‰|z  d z  ‘ŒS )éÿÿÿÿrV   )rª   Úbcr)   s     €r>   r±   z&_multi_tensor_adam.<locals>.<listcomp>³  s!   ø€ Ð,WÐ,WÐ,WÀ¨b°2©g¸©^Ð,WÐ,WÐ,Wr?   c                 ó   — g | ]}|d z  ‘ŒS )r‚   rV   )rª   r¶   s     r>   r±   z&_multi_tensor_adam.<locals>.<listcomp>µ  s   € Ð$HÐ$HÐ$H° R¨¡WÐ$HÐ$HÐ$Hr?   )(rM   r2   r   r9   r3   r4   rN   r‡   rˆ   r   ÚallÚzipr   Ú"_group_tensors_by_device_and_dtypeÚstrrF   Úvaluesr   r   rŒ   r   Ú_foreach_negÚis_cpuÚ_foreach_add_rP   Ú_foreach_addÚ_foreach_lerp_Ú_foreach_mul_Ú_foreach_mulÚ_foreach_addcmul_Ú_foreach_powÚ_foreach_sub_Ú_foreach_neg_Ú_foreach_div_Ú_foreach_reciprocal_Ú_foreach_sqrt_Ú_foreach_maximum_Ú_foreach_sqrtÚ_foreach_addcdiv_r   )+r(   rd   re   rf   rg   rh   rl   rm   r-   ri   rn   ro   r)   r,   r+   r$   r%   r&   Úgrouped_tensorsr™   Údevice_params_Údevice_grads_Údevice_exp_avgs_Údevice_exp_avg_sqs_Údevice_max_exp_avg_sqs_Údevice_state_steps_Ú_Údevice_paramsÚdevice_gradsÚdevice_exp_avgsÚdevice_exp_avg_sqsÚdevice_state_stepsrF   rŸ   Údevice_max_exp_avg_sqsÚscaled_device_gradsr�   r    r¡   r¢   r¤   Úexp_avg_sq_sqrtr�   s+             ```                             @r>   Ú_multi_tensor_adamrÞ   å  sŒ  øøøø€ õ* ˆ6�{„{�aÒÐØˆå�"•fÑÔð 
 jð 
ÝØSñ
ô 
ð 	
õ �%�Ñ Ô ð ?Øð 	ÝØZñô ð ð �;Š;‰=Œ=˜AÒÐÝÐ=Ñ>Ô>Ð>å�%�Ñ Ô ð ?Øð 	ÝØZñô ð ð �;Š;‰=Œ=˜AÒÐÝÐ=Ñ>Ô>Ð>õ Œ>×&Ò&Ñ(Ô(ð w¨Zð wÝ'HØð(
ñ (
ô (
Ð$õ ð 
ð 
ð 
ð 
õ ˜v {Ñ3Ô3ð
ñ 
ô 
ñ 
ô 
ð 	wð 	wð wÐWsÐvÐvÐvñ		wô 	wð 
ð Ð )Ð"3Ð"3Ð3àÐDÐDÐDÑDÔDÐåÔBØ	�˜ +¨ÀÐLñô €Oõ �e�VÑ$Ô$ð	Ý),¨U¬\Ñ):Ô):¸eÒ)CÐ)Cð 
Œ�uÐÐàð ð ×"Ò"Ñ$Ô$ðXñ Xñ 		ñ 	ØØØØØØØÝ�T¥&œ\¨>Ñ:Ô:ˆÝ�D¥œL¨-Ñ8Ô8ˆÝ�t¥Fœ|Ð-=Ñ>Ô>ˆÝ!¥$¥v¤,Ð0CÑDÔDÐÝ!¥$¥v¤,Ð0CÑDÔDÐà˜qÔ!Ô(ˆØÐ! f°JÐ&>Ð&>Ø!&§¢°Àd Ñ!KÔ!KˆJ�vÑà-7ÐB�z &Ô)Ð)¸Uˆð ð 	Øð Ý)-­dµ6¬lÐ<SÑ)TÔ)TÐ&ÝØ!Ø Ø#Ø&Ø*ñô ð ð õ Ø! <°ÐBTñô ð ð ð 	<Ý Ô-¨lÑ;Ô;ˆLõ Œ~×*Ò*Ñ,Ô,ð 	7Ð1CÀAÔ1FÔ1Mð 	7ÝÔØ"¥E¤L°¸UÐ$CÑ$CÔ$CÈ3ðñ ô ð ð õ ÔÐ 2°AÑ6Ô6Ð6à˜1ÒÐàð ÝÔ# L°-À|ÐTÑTÔTÐTÐTå$Ô1Ø  -°|ð ñ  ô  �õ 	Ô˜_¨l¸AÀÑ<LÑMÔMÐMåÔÐ.°Ñ6Ô6Ð6õ �e�Uœ\Ñ*Ô*ð 	Ý"'Ô"4°\À1ÀuÁ9Ñ"MÔ"MÐØˆEˆEà".ÐØ˜‘IˆEåÔØÐ 3°\À5ñ	
ô 	
ð 	
ð
 Øð ñ ?	Ý$Ô1°%Ð9KÑLÔLÐÝ$Ô1°%Ð9KÑLÔLÐåÔÐ 0°!Ñ4Ô4Ð4ÝÔÐ 0°!Ñ4Ô4Ð4åÔÐ 0Ñ1Ô1Ð1õ ÔÐ 0°"Ñ5Ô5Ð5ÝÔ&Ð'7Ñ8Ô8Ð8åÔ Ð!1Ñ2Ô2Ð2ð
 )ˆIØ$4Ð!àð JÝ)-­dµ6¬lÐ<SÑ)TÔ)TÐ&åÔ'Ð(>Ð@RÑSÔSÐSõ #(Ô"5Ð6LÑ"MÔ"M��å"'Ô"5Ð6HÑ"IÔ"I�åÔ Ð1FÑGÔGÐGÝÔ °Ñ5Ô5Ð5ÝÔ °Ñ;Ô;Ð;õ Ô# M°?ÀOÑTÔTÐTÑTð ð  ð  ð  Ø:Lð ñ  ô  Ðð ð  ð  ð  Ø:Lð ñ  ô  Ðõ ,Ð,WÐ,WÐ,WÐ,WÐFVÐ,WÑ,WÔ,WÑXÔXˆIà$HÐ$HÐ7GÐ$HÑ$HÔ$HÐ!àð JÝ)-­dµ6¬lÐ<SÑ)TÔ)TÐ&åÔ'Ð(>Ð@RÑSÔSÐSõ #(Ô"5Ð6LÑ"MÔ"M��å"'Ô"5Ð6HÑ"IÔ"I�åÔ Ð1FÑGÔGÐGÝÔ °Ñ5Ô5Ð5ÝÔ#Ø˜°Àñô ð ñ ðmXð Xr?   Úreturnc       
         óŠ  — | sd S |rt          d¦  «        ‚|�	|j        |ini }|�	|j        |ini }t          |t          ¦  «        r!t	          |j        ¦  «        dk    r	|j        |ind }t          j        | |||||g¦  «        }|                     ¦   «         D �]«\  \  }}\  \  }}}}}}}t          t          t                   |¦  «        }t          t          t                   |¦  «        }t          t          t                   |¦  «        } t          t          t                   |¦  «        }!t          t          t                   |¦  «        }"|j
        dk    r|€|�J ‚d\  }#}$|�+|                     ||                     |d¬¦  «        ¦  «        }#|�+|                     ||                     |d¬¦  «        ¦  «        }$|�&||vr"|                     |d¬¦  «        ||<   ||         }t          j        |"d¦  «         t          j        ||| |!||"|||
|||||#|$¬	¦  «         |$�&t          j        |"|$gt#          |"¦  «        z  ¦  «         �Œ­d S )
Nz9Adam with fused=True does not support differentiable=Truer¬   Úmps)NNT)r€   r­   r	   )	r-   r)   rn   ro   r,   r+   r$   rl   rm   )r9   rF   r2   r   r»   r   rº   Úitemsr   r   r‰   rJ   rŒ   rN   r¿   Ú_fused_adam_rÆ   rM   )%r(   rd   re   rf   rg   rh   rl   rm   r-   ri   rn   ro   r)   r,   r+   r$   r%   r&   Úgrad_scale_dictÚfound_inf_dictÚlr_dictrÎ   rF   rÕ   rÏ   rÐ   rÑ   rÒ   rÛ   rÔ   rÖ   r×   rØ   rÙ   rÚ   Údevice_grad_scaleÚdevice_found_infs%                                        r>   Ú_fused_adamré   È  sÁ  € ð* ð ØˆØð XÝÐVÑWÔWÐWð ,6Ð+AˆÔ	˜JÐ'Ð'Àrð ð *3Ð)>ˆÔ	˜9Ð%Ð%ÀBð õ & b­&Ñ1Ô1ÐWµc¸"¼)±n´nÈÒ6MÐ6MˆŒ�BˆˆÐSWð õ  ÔBØ	�˜ +¨ÀÐLñô €Oð 
×	Ò	Ñ	 Ô	 ð5ñ 5ñ 
	‰ˆ�ñ 
ñ	
ØØØØØ"Øà	å�T¥&œ\¨>Ñ:Ô:ˆÝ�D¥œL¨-Ñ8Ô8ˆÝ�t¥Fœ|Ð-=Ñ>Ô>ˆÝ!¥$¥v¤,Ð0CÑDÔDÐÝ!¥$¥v¤,Ð0CÑDÔDÐàŒ;˜%ÒÐØÐ$¨Ð);Ð);Ð;à.8Ñ+ÐÐ+ØÐ!Ø /× :Ò :Ø˜
Ÿš f¸4˜Ñ@Ô@ñ!ô !Ðð Ð Ø-×8Ò8Ø˜	Ÿš V¸$˜Ñ?Ô?ñ ô  Ðð Ð 6°Ð#8Ð#8Ø Ÿeše¨6À˜eÑEÔEˆG�F‰OØ˜”ˆBÝÔÐ.°Ñ2Ô2Ð2ÝÔØØØØØ"ØØØØØØ%ØØØ(Ø&ð	
ñ 	
ô 	
ð 	
ð" Ð'ÝÔØ"Ð%5Ð$6½Ð=OÑ9PÔ9PÑ$Pñô ð ùðg5ð 5r?   )Úsingle_tensor_fnFr#   r'   c                ó®  — |	€2|€0t          | |d¬¦  «        \  }}|rt          |t          ¦  «        r|sd}|	€d}	|€d}t          j                             ¦   «         s(t          d„ |D ¦   «         ¦  «        st          d¦  «        ‚|r-t          j         	                    ¦   «         rt          d¦  «        ‚|	r-t          j         	                    ¦   «         rt          d¦  «        ‚|	r&t          j         	                    ¦   «         st          }n/|r&t          j         	                    ¦   «         st          }nt          } || ||||||||||||||||
|¬¦  «         dS )	znFunctional API that performs Adam algorithm computation.

    See :class:`~torch.optim.Adam` for details.
    NF)Ú	use_fusedc              3   óJ   K  — | ]}t          |t          j        ¦  «        V — Œd S ru   )r2   rN   r   )rª   Úts     r>   r«   zadam.<locals>.<genexpr>Y  s?   è è € ð 5ð 5Ø()�
�1•e”lÑ#Ô#ð5ð 5ð 5ð 5ð 5ð 5r?   zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers)r-   ri   rn   ro   r)   r,   r+   r$   r%   r&   rl   rm   )r   r2   r   rN   r‡   rˆ   r¸   r9   r„   r…   ré   rÞ   r¦   )r(   rd   re   rf   rg   rh   r#   r%   r&   r'   rl   rm   ri   r-   rn   ro   r)   r,   r+   r$   rÕ   Úfuncs                         r>   r   r   )  sÈ  € ðD €}˜˜Ý1Ø�N¨eð
ñ 
ô 
‰
ˆˆ7ð ð 	•z "¥fÑ-Ô-ð 	°jð 	ØˆGØ€}ØˆØ€Øˆõ Œ>×&Ò&Ñ(Ô(ð 
µð 5ð 5Ø-8ð5ñ 5ô 5ñ 2ô 2ð 
õ Ø^ñ
ô 
ð 	
ð ð U•5”9×)Ò)Ñ+Ô+ð UÝÐSÑTÔTÐTØð S•”×'Ò'Ñ)Ô)ð SÝÐQÑRÔRÐRàð #•U”Y×+Ò+Ñ-Ô-ð #ÝˆˆØ	ð #�œ×/Ò/Ñ1Ô1ð #Ý!ˆˆå"ˆà€DØØØØØØØØØØØØ!ØØØØ%ØØð%ñ ô ð ð ð r?   )NFFNNNF)%Útypingr   r   r   r   r   rN   r   Ú	optimizerr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__ry   r5   r¦   rÞ   ré   r   rV   r?   r>   ú<module>rô      s”  ðð 6Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5à €€€Ø Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð. �6Ð
€ðið ið ið ið iˆ9ñ iô ið iðZ&ðNà	ðð ð 
ðð ð 
ðð ð  
ð!ð ð" 
ð#ð ð$ 
ð%ð ð ñOAð „ðLNKØ�ŒLðNKà�Œ<ðNKð �6ŒlðNKð �f”ð	NKð
 ˜&”\ðNKð �f”ðNKð ˜Ô ðNKð ˜ÔðNKð ðNKð ðNKð �˜�ÔðNKð �˜�ÔðNKð 	ˆe�VˆmÔðNKð ðNKð  
ð!NKð" ð#NKð$ ð%NKð& ð'NKð NKð NKð NKðb`Ø�ŒLð`à�Œ<ð`ð �6Œlð`ð �f”ð	`ð
 ˜&”\ð`ð �f”ð`ð ˜Ô ð`ð ˜Ôð`ð ð`ð ð`ð �˜�Ôð`ð �˜�Ôð`ð 	ˆe�VˆmÔð`ð ð`ð  
ð!`ð" ð#`ð$ ð%`ð& ð'`ð `ð `ð `ðF^Ø�ŒLð^à�Œ<ð^ð �6Œlð^ð �f”ð	^ð
 ˜&”\ð^ð �f”ð^ð ˜Ô ð^ð ˜Ôð^ð ð^ð ð^ð ð^ð ð^ð 	ˆe�VˆmÔð^ð ð^ð  
ð!^ð" ð#^ð$ ð%^ð& ð'^ð( 
ð)^ð ^ð ^ð ^ðB  ÐÐ1DÐEÑEÔEð #ØØ Ø Ø#'Ø"&ØðUð UØ�ŒLðUà�Œ<ðUð �6ŒlðUð �f”ð	Uð
 ˜&”\ðUð �f”ðUð �dŒ^ðUð ðUð ðUð �DŒ>ðUð ˜Ô ðUð ˜ÔðUð ðUð" ð#Uð$ ð%Uð& ð'Uð( 	ˆe�VˆmÔð)Uð* ð+Uð, 
ð-Uð. ð/Uð Uð Uñ FÔEðUð Uð Ur?   