§
    Ë²Åj…Œ  ã            *       ó€  — d dl mZmZmZmZmZ d dlZd dlmZ ddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddgZ G d„ de¦  «        Zd	d
e› de› de› de	› de› de› d�z   e_        dee         dee         dee         dee         dee         dee         dee         dee         dedeee f         deee f         deee f         de de dedededef$d „Z!dee         dee         dee         dee         dee         dee         dee         dee         dedeee f         deee f         deee f         de de dedededef$d!„Z"dee         dee         dee         dee         dee         dee         dee         dee         dede de deee f         de de dedededed"df&d#„Z# ee!¬$¦  «        	 	 	 	 	 	 	 d)dee         dee         dee         dee         dee         dee         d&ee         deded'ee         dee         dee         dedede de dee ef         de de def(d(„¦   «         Z$dS )*é    )ÚcastÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_device_dtype_check_for_fusedÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ
_fused_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_params_docÚ_stack_if_compilingÚ_use_grad_for_differentiableÚ_view_as_realÚ
DeviceDictÚDeviceDtypeDictÚ	OptimizerÚParamsTÚAdamWÚadamwc                   óæ   ‡ — e Zd Z	 	 	 	 	 dddddddœded	eeef         d
eeeef         eeef         f         dedededede	e         dedede	e         fˆ fd„Z
ˆ fd„Zd„ Zedd„¦   «         Zˆ xZS )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>ç{®Gáz„?FN)ÚmaximizeÚforeachÚ
capturableÚdifferentiableÚfusedÚparamsÚlrÚbetasÚepsÚweight_decayÚamsgradr$   r%   r&   r'   r(   c                ó  •— t          |t          ¦  «        r:|r|	st          d¦  «        ‚|                     ¦   «         dk    rt          d¦  «        ‚d|k    st          d|› �¦  «        ‚d|k    st          d|› �¦  «        ‚d|d         cxk    rdk     sn t          d	|d         › �¦  «        ‚d|d         cxk    rdk     sn t          d
|d         › �¦  «        ‚d|k    st          d|› �¦  «        ‚t          |d         t          ¦  «        rt          |d         t          ¦  «        sEt          |d         t          ¦  «        rt          |d         t          ¦  «        st          d¦  «        ‚t          |d         t          ¦  «        r@|	s|rt          d¦  «        ‚|d                              ¦   «         dk    rt          d¦  «        ‚t          |d         t          ¦  «        r@|	s|rt          d¦  «        ‚|d                              ¦   «         dk    rt          d¦  «        ‚t          ||||||||	|
|¬¦
  «
        }t          ¦   «                              ||¦  «         |r)|
rt          d¦  «        ‚d| _	        |rt          d¦  «        ‚d S d S )NúElr as a Tensor is not supported for capturable=False and foreach=Truer	   zTensor lr must be 1-elementç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: z0betas must be either both floats or both TensorszKbetas[0] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[0] must be 1-elementzKbetas[1] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[1] must be 1-element)
r*   r+   r,   r-   r.   r%   r$   r&   r'   r(   z)`fused` does not support `differentiable`Tz0`fused` and `foreach` cannot be `True` together.)
Ú
isinstancer   Ú
ValueErrorÚnumelÚfloatÚdictÚsuperÚ__init__ÚRuntimeErrorÚ_step_supports_amp_scaling)Úselfr)   r*   r+   r,   r-   r.   r$   r%   r&   r'   r(   ÚdefaultsÚ	__class__s                €úBC:\Projects\Historical\venv\Lib\site-packages\torch/optim/adamw.pyr9   zAdamW.__init__#   s  ø€ õ �b�&Ñ!Ô!ð 	@Øð ˜zð Ý Ø[ñô ð ð �xŠx‰zŒz˜QŠˆÝ Ð!>Ñ?Ô?Ð?à�bŠyˆyÝÐ;°rÐ;Ð;Ñ<Ô<Ð<Ø�cŠzˆzÝÐ<°sÐ<Ð<Ñ=Ô=Ð=Ø�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�lÒ"Ð"ÝÐJ¸LÐJÐJÑKÔKÐKå˜˜aœ¥%Ñ(Ô(ð	QÝ-7¸¸a¼Å%Ñ-HÔ-Hð	Qå˜5 œ8¥VÑ,Ô,ð	Qå1;¸EÀ!¼HÅfÑ1MÔ1Mð	Qõ ÐOÑPÔPÐPÝ�e˜A”h¥Ñ'Ô'ð 	FØð  'ð Ý Øañô ð ð �QŒx�~Š~ÑÔ 1Ò$Ð$Ý Ð!DÑEÔEÐEÝ�e˜A”h¥Ñ'Ô'ð 	FØð  'ð Ý Øañô ð ð �QŒx�~Š~ÑÔ 1Ò$Ð$Ý Ð!DÑEÔEÐEåØØØØ%ØØØØ!Ø)Øð
ñ 
ô 
ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*àð 	WØð PÝ"Ð#NÑOÔOÐOØ.2ˆDÔ+Øð WÝ"Ð#UÑVÔVÐVð	Wð 	WðWð Wó    c                 óø  •— t          ¦   «                              |¦  «         | j        D �]N}|                     dd¦  «         |                     dd¦  «         |                     dd ¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd ¦  «        }|d         D ]¾}| j                             |g ¦  «        }t          |¦  «        d	k    rŽt          j        |d
         ¦  «        stt          |d
         ¦  «        }|d         s|d         r*t          j
        |t          |¬¦  «        |j        ¬¦  «        n!t          j
        |t          ¦   «         ¬¦  «        |d
<   Œ¿�ŒPd S )Nr.   Fr$   r%   r&   r'   r(   r)   r   Ústep©Úis_fused©ÚdtypeÚdevice©rF   )r8   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorr6   Útensorr   rG   )r<   rL   Úgroupr(   ÚpÚp_stateÚstep_valr>   s          €r?   rI   zAdamW.__setstate__m   s�  ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	ñ 	ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×Ò˜Z¨Ñ/Ô/Ð/Ø×Ò˜Y¨Ñ-Ô-Ð-Ø×Ò˜\¨5Ñ1Ô1Ð1Ø×ÒÐ-¨uÑ5Ô5Ð5Ø×$Ò$ W¨dÑ3Ô3ˆEØ˜8”_ð ð �Øœ*Ÿ.š.¨¨BÑ/Ô/�Ý�w‘<”< 1Ò$Ð$­U¬_¸WÀV¼_Ñ-MÔ-MÐ$Ý$ W¨V¤_Ñ5Ô5�Hð ! Ô.ðOð
 38¸´.ðO�œØ$Ý"3¸UÐ"CÑ"CÔ"CØ#$¤8ðñ ô ð õ #œ\¨(Õ:KÑ:MÔ:MÐNÑNÔNð ˜F‘Oøñ	ð	ð 	r@   c	                 ó¨  — d}	|d         D �]E}
|
j         €Œ|	t          j        |
¦  «        z  }	|                     |
¦  «         |
j         j        rt          d¦  «        ‚|                     |
j         ¦  «         | j        |
         }t          |¦  «        dk    rç|d         rt          |
¦  «         |d         s|d         r0t          j	        dt          |d         ¬¦  «        |
j        ¬	¦  «        n!t          j        d
t          ¦   «         ¬¦  «        |d<   t          j        |
t          j        ¬¦  «        |d<   t          j        |
t          j        ¬¦  «        |d<   |r#t          j        |
t          j        ¬¦  «        |d<   |                     |d         ¦  «         |                     |d         ¦  «         |d         r|                     |d         ¦  «         |d         r|d         j        rt          d¦  «        ‚|d         r2t!          |d         t"          ¦  «        r|d         st          d¦  «        ‚|                     |d         ¦  «         �ŒG|	S )NFr)   z'AdamW does not support sparse gradientsr   r(   r&   © rC   rE   r1   rH   rB   )Úmemory_formatÚexp_avgÚ
exp_avg_sqÚmax_exp_avg_sqr.   r'   zB`requires_grad` is not supported for `step` in differentiable moder%   r*   r0   )ÚgradrO   Ú
is_complexÚappendÚ	is_sparser:   rL   rN   r   Úzerosr   rG   rQ   Ú
zeros_likeÚpreserve_formatÚrequires_gradr3   r   )r<   rR   Úparams_with_gradÚgradsr.   Úexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsÚhas_complexrS   rL   s               r?   Ú_init_groupzAdamW._init_group„   sŸ  € ð ˆØ�x”ð <	.ñ <	.ˆAØŒvˆ~ØØ�5Ô+¨AÑ.Ô.Ñ.ˆKØ×#Ò# AÑ&Ô&Ð&ØŒvÔð NÝ"Ð#LÑMÔMÐMØ�LŠL˜œÑ Ô Ð à”J˜q”MˆEõ �5‰zŒz˜QŠˆØ˜”>ð 5Ý1°!Ñ4Ô4Ð4ð ˜\Ô*ðFð
 /4°G¬nðF•E”KØÝ/¸¸w¼ÐHÑHÔHØ œxðñ ô ð õ œ cÕ1BÑ1DÔ1DÐEÑEÔEð �f‘õ $)Ô#3Ø¥UÔ%:ð$ñ $ô $��iÑ õ ',Ô&6Ø¥UÔ%:ð'ñ 'ô '��lÑ#ð ð å.3Ô.>Ø­Ô)>ð/ñ /ô /�EÐ*Ñ+ð �OŠO˜E )Ô,Ñ-Ô-Ð-Ø×Ò˜u \Ô2Ñ3Ô3Ð3à�YÔð @Ø×&Ò& uÐ-=Ô'>Ñ?Ô?Ð?ØÐ%Ô&ð ¨5°¬=Ô+Fð Ý"ØXñô ð ð �iÔ ðå˜u Tœ{­FÑ3Ô3ðð ˜lÔ+ðõ
 #Ø[ñô ð ð ×Ò˜u Vœ}Ñ-Ô-Ð-Ñ-ØÐr@   c                 óN  — |                       ¦   «          d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D ]Ï}g }g }g }g }g }g }	|d         }
t	          t
          t          t          f         |d         ¦  «        \  }}|                      ||||
||||	¦  «        }t          ||||||	f|
|||d         |d         |d         |d         |d         |d	         |d
         |d         t          | dd¦  «        t          | dd¦  «        |dœŽ ŒÐ|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr.   r+   r*   r-   r,   r$   r%   r&   r'   r(   Ú
grad_scaleÚ	found_inf)r.   Úbeta1Úbeta2r*   r-   r,   r$   r%   r&   r'   r(   rm   rn   rj   )
Ú _cuda_graph_capture_health_checkrO   Úenable_gradrJ   r   r   r6   rk   r   Úgetattr)r<   ÚclosureÚlossrR   rd   re   rf   rg   rh   ri   r.   ro   rp   rj   s                 r?   rB   z
AdamW.stepÏ   sÀ  € ð 	×-Ò-Ñ/Ô/Ð/àˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð *	ð *	ˆEØ-/ÐØ"$ˆEØ%'ˆHØ(*ˆKØ,.ˆOØ(*ˆKØ! )Ô,ˆGÝ¥¥e­U lÔ 3°U¸7´^ÑDÔD‰LˆE�5à×*Ò*ØØ ØØØØØØñ	ô 	ˆKõ Ø ØØØØØðð  ØØØ˜”;Ø" >Ô2Ø˜%”LØ˜zÔ*Ø˜iÔ(Ø  Ô.Ø$Ð%5Ô6Ø˜G”nÝ" 4¨°tÑ<Ô<Ý! $¨°TÑ:Ô:Ø'ð)ð ð ð ð ð. ˆs   ¬AÁAÁ
A)r    r!   r"   r#   F©N)Ú__name__Ú
__module__Ú__qualname__r   r   r6   r   r   Úboolr   r9   rI   rk   r   rB   Ú__classcell__)r>   s   @r?   r   r   "   s}  ø€ € € € € ð $(ØCOØØ"ØðHWð Ø"&Ø Ø$Ø $ðHWð HWð HWàðHWð �%˜�-Ô ðHWð �U˜5 &˜=Ô)¨5°¸°Ô+?Ð?Ô@ð	HWð
 ðHWð ðHWð ðHWð ðHWð ˜$”ðHWð ðHWð ðHWð ˜Œ~ðHWð HWð HWð HWð HWð HWðTð ð ð ð ð.Ið Ið IðV "ð:ð :ð :ñ "Ô!ð:ð :ð :ð :ð :r@   a  Implements AdamW algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{(lr)}, \: \beta_1, \beta_2
                \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)},
                \: \epsilon \text{ (epsilon)}                                                    \\
            &\hspace{13mm}      \lambda \text{(weight decay)},  \: \textit{amsgrad},
                \: \textit{maximize}                                                             \\
            &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0
                \text{ ( second moment)}, \: \widehat{v_0}^{max}\leftarrow 0              \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\

            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1}         \\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
            &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_{t-1}}^{max},
                \widehat{v_t})                                                                   \\
            &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big)                                 \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Decoupled Weight Decay Regularization`_.
    z
    Args:
        a  
        lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR
            is not yet supported for all our implementations. Please use a float
            LR if you are not also specifying fused=True or capturable=True.
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay coefficient (default: 1e-2)
        amsgrad (bool, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        z	
        a8  
    .. Note::
        A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`.
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ

    r)   re   rf   rg   rh   ri   rm   rn   r.   ro   rp   r*   r-   r,   r$   r&   r'   rj   c       
         ó	  — |€|�J ‚t           j                             ¦   «         rEt          |t          ¦  «        sJ ‚t          |	t          ¦  «        sJ ‚t          |
t          ¦  «        sJ ‚t          |	t
          ¦  «        r|	j        |	j        f|	i}nd }t          | ¦  «        D �]Ý\  }}|s||         n||          }||         }||         }||         }t           j	         
                    ¦   «         sF|rDt          ¦   «         }|j        j        |j        j        k    r|j        j        |v sJ d|› d�¦   «         ‚t          j        |¦  «        rot          j        |¦  «        }t          j        |¦  «        }t          j        |¦  «        }|rt          j        ||         ¦  «        ||<   t          j        |¦  «        }|dz  }|                     d||z  z
  ¦  «         |j        }|j        }|j        }|�3|j        }||f}||vr|	                     ||d¬¦  «        ||<   ||         }n|	}|                     |d|z
  ¦  «         |                     |
¦  «                             ||d|
z
  ¬¦  «         |s|�r|}d|	|z  z
  }d|
|z  z
  } ||z  }!|!                     ¦   «         }"|                      ¦   «         }#|rŠ|r||                              ¦   «         }$n||         }$||                              t          j        |$|¦  «        ¦  «         ||                              ¦   «         |#|"z  z                       ||"z  ¦  «        }%n0|                     ¦   «         |#|"z  z                       ||"z  ¦  «        }%|                     ||%¦  «         nÂt5          |¦  «        }d|	|z  z
  }d|
|z  z
  } ||z  }!| dz  }#|rTt          j        ||         |||         ¬¦  «         ||                              ¦   «         |#z                       |¦  «        }%n*|                     ¦   «         |#z                       |¦  «        }%|                     ||%|! ¬¦  «         |r7t          j        | |         ¦  «        rt          j        ||         ¦  «        ||<   �Œßd S )	NúIIf capturable=True, params and state_steps must be on supported devices: ú.r	   T)rG   rF   Únon_blocking)Úvalueç      à?)Úout)rO   ÚjitÚis_scriptingr3   r6   r   rG   rF   Ú	enumerateÚcompilerÚis_compilingr   Útyper]   Úview_as_realÚmul_ÚtoÚlerp_Úaddcmul_ÚnegÚsqrtÚcloneÚcopy_ÚmaximumÚadd_Úaddcdiv_r   Úview_as_complex)&r)   re   rf   rg   rh   ri   rm   rn   r.   ro   rp   r*   r-   r,   r$   r&   r'   rj   Ú
beta1_dictÚiÚparamr\   rY   rZ   Ústep_tÚcapturable_supported_devicesrG   rF   ÚkeyÚdevice_beta1rB   Úbias_correction1Úbias_correction2Ú	step_sizeÚstep_size_negÚbias_correction2_sqrtr[   Údenoms&                                         r?   Ú_single_tensor_adamwr£   S  s  € ð* Ð )Ð"3Ð"3Ð3å„y×ÒÑÔð (õ ˜"�eÑ$Ô$Ð$Ð$Ð$Ý˜%¥Ñ'Ô'Ð'Ð'Ð'Ý˜%¥Ñ'Ô'Ð'Ð'Ð'õ �%�Ñ Ô ð Ø27´,ÀÄÐ1LÈeÐ0Tˆ
ˆ
àˆ
å˜fÑ%Ô%ð hKñ hK‰ˆˆ5Ø'Ð6ˆu�QŒxˆx¨e°A¬h¨YˆØ˜1”+ˆØ  ”^ˆ
Ø˜Q”ˆõ Œ~×*Ò*Ñ,Ô,ð 	{°ð 	{Ý+LÑ+NÔ+NÐ(à”Ô! V¤]Ô%7Ò7Ð7Ø”LÔ%Ð)EÐEÐEÐEØzÐ[wÐzÐzÐzñ FÔEðFõ Ô˜EÑ"Ô"ð 	.ÝÔ% dÑ+Ô+ˆDÝÔ(¨Ñ1Ô1ˆGÝÔ+¨JÑ7Ô7ˆJØð LÝ%*Ô%7¸ÈÔ8JÑ%KÔ%K� Ñ"ÝÔ& uÑ-Ô-ˆEð 	�!‰ˆð 	�
Š
�1�r˜LÑ(Ñ(Ñ)Ô)Ð)à”ˆà”ˆØ”ˆàÐ!Ø”KˆEð ˜5�/ˆCØ˜*Ð$Ð$Ø"'§(¢(°&ÀÐTX (Ñ"YÔ"Y�
˜3‘à1;¸C´ˆLˆLà ˆLð 	�Š�d˜A Ñ,Ñ-Ô-Ð-Ø�Š˜ÑÔ×'Ò'¨¨d¸!¸e¹)Ð'ÑDÔDÐDàð 3	=˜ñ 3	=ØˆDà  5¨$¡;™ÐØ  5¨$¡;™ÐàÐ-Ñ-ˆIØ%ŸMšM™OœOˆMà$4×$9Ò$9Ñ$;Ô$;Ð!àð ,à!ð 8Ø%4°QÔ%7×%=Ò%=Ñ%?Ô%?�N�Nà%4°QÔ%7�Nà Ô"×(Ò(­¬°~ÀzÑ)RÔ)RÑSÔSÐSð $ AÔ&×+Ò+Ñ-Ô-Ð1FÈÑ1VÑWß’$�s˜]Ñ*Ñ+Ô+ð �ð
 —O’OÑ%Ô%Ð)>ÀÑ)NÑOß’$�s˜]Ñ*Ñ+Ô+ð ð �NŠN˜7 EÑ*Ô*Ð*Ð*å˜fÑ%Ô%ˆDà  5¨$¡;™ÐØ  5¨$¡;™ÐàÐ-Ñ-ˆIà$4°cÑ$9Ð!àð Nå”˜o¨aÔ0°*À/ÐRSÔBTÐUÑUÔUÐUð )¨Ô+×0Ò0Ñ2Ô2Ð5JÑJ×PÒPÐQTÑUÔU��à#ŸšÑ*Ô*Ð-BÑB×HÒHÈÑMÔM�à�NŠN˜7 E°)°ˆNÑ<Ô<Ð<ð ð 	K•uÔ'¨¨q¬	Ñ2Ô2ð 	KÝ!&Ô!6°ÀqÔ7IÑ!JÔ!JˆO˜AÑùðQhKð hKr@   c       
         ó´  ‡	‡
‡‡*— t          | ¦  «        dk    rd S t          ‰t          ¦  «        r|st          d¦  «        ‚t          ‰	t          ¦  «        r8|st	          d¦  «        ‚‰	                     ¦   «         dk    rt	          d¦  «        ‚t          ‰
t          ¦  «        r8|st	          d¦  «        ‚‰
                     ¦   «         dk    rt	          d¦  «        ‚t          j                             ¦   «         sI|rGt          d¬	¦  «        Š*t          ˆ*fd
„t          | |¦  «        D ¦   «         ¦  «        sJ d‰*› d�¦   «         ‚|r
J d¦   «         ‚|€|�J ‚t          j        | |||||g¦  «        }t          ‰	t          ¦  «        r!t          ‰	j        ¦  «        dk    r	‰	j        ‰	ind }|                     ¦   «         D �]�\  \  }}}}}}}t#          t$          t                   |¦  «        }t#          t$          t                   |¦  «        }t#          t$          t                   |¦  «        }t#          t$          t                   |¦  «        }t#          t$          t                   |¦  «        }|d         j        } |�| |vr‰	                     | d¬¦  «        || <   |r||          n‰	}!|rH|r4t#          t$          t                   |¦  «        }"t)          |||||"¦  «         nt)          ||||¦  «         |rt          j        |¦  «        }t          j                             ¦   «         s9|d         j        r,t          j        |t          j        dd¬¦  «        d¬¦  «         nt          j        |d¦  «         |dk    rt          j        |d‰|z  z
  ¦  «         t          j        ||d|!z
  ¦  «         t          j        |‰
¦  «         t          ‰
t          j        ¦  «        rt          j        |d‰
z
  ¦  «        }#d}$n|}#d‰
z
  }$t          j        ||#||$¦  «         ~~#|�r`t          j        ‰	|¦  «        }%t          j        ‰
|¦  «        }&t          j        |%d¦  «         t          j        |&d¦  «         t          j        |&¦  «         t          j         |%‰¦  «         t          j!        |%¦  «         t          j"        |&¦  «         |%}'|&}(|rJt#          t$          t                   |¦  «        }"t          j#        |"|¦  «         t          j$        |"¦  «        })nt          j$        |¦  «        })t          j         |)|(¦  «         t          j        |)|¦  «         t          j         |)|'¦  «         t          j%        |||)¦  «         �Œ¹ˆ	fd„|D ¦   «         }%ˆ
fd„|D ¦   «         }&tM          ˆfd„|%D ¦   «         ¦  «        }'d„ |&D ¦   «         }(|rJt#          t$          t                   |¦  «        }"t          j#        |"|¦  «         t          j$        |"¦  «        })nt          j$        |¦  «        })t          j         |)|(¦  «         t          j        |)|¦  «         t          j%        |||)|'¦  «         �ŒŸd S )Nr   r0   zHbeta1 as a Tensor is not supported for capturable=False and foreach=Truer	   zTensor beta1 must be 1-elementzHbeta2 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta2 must be 1-elementF)Úsupports_xlac              3   ón   •K  — | ]/\  }}|j         j        |j         j        k    o|j         j        ‰v V — Œ0d S rv   )rG   rˆ   )Ú.0rS   rB   rš   s      €r?   ú	<genexpr>z&_multi_tensor_adamw.<locals>.<genexpr>  s]   øè è € ð 
ð 
ñ ��4ð ŒHŒM˜Tœ[Ô-Ò-ð >Ø””Ð!=Ð=ð
ð 
ð 
ð 
ð 
ð 
r@   r}   r~   z#_foreach ops don't support autogradÚcpuT©rG   r   r2   )rG   )Úalphac                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS ©r	   ©r   )r§   rB   ro   s     €r?   ú
<listcomp>z'_multi_tensor_adamw.<locals>.<listcomp>¥  ó8   ø€ ð  ð  ð  Ø26��E�Z¨Ñ-Ô-Ñ-Ñ-ð ð  ð  r@   c                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS r­   r®   )r§   rB   rp   s     €r?   r¯   z'_multi_tensor_adamw.<locals>.<listcomp>¨  r°   r@   c                 ó    •— g | ]
}‰|z  d z  ‘ŒS )éÿÿÿÿrW   )r§   Úbcr*   s     €r?   r¯   z'_multi_tensor_adamw.<locals>.<listcomp>¬  s!   ø€ Ð,WÐ,WÐ,WÀ¨b°2©g¸©^Ð,WÐ,WÐ,Wr@   c                 ó   — g | ]}|d z  ‘ŒS )r�   rW   )r§   r´   s     r?   r¯   z'_multi_tensor_adamw.<locals>.<listcomp>®  s)   € ð %ð %ð %Ø��C‘ð%ð %ð %r@   )'rN   r3   r   r:   r4   r5   rO   r†   r‡   r   ÚallÚzipr   Ú"_group_tensors_by_device_and_dtypeÚstrrG   Úvaluesr   r   r‹   r   Ú_foreach_negÚis_cpuÚ_foreach_add_rQ   Ú_foreach_mul_Ú_foreach_lerp_Ú_foreach_mulÚ_foreach_addcmul_Ú_foreach_powÚ_foreach_sub_Ú_foreach_neg_Ú_foreach_div_Ú_foreach_reciprocal_Ú_foreach_sqrt_Ú_foreach_maximum_Ú_foreach_sqrtÚ_foreach_addcdiv_r   )+r)   re   rf   rg   rh   ri   rm   rn   r.   ro   rp   r*   r-   r,   r$   r&   r'   rj   Úgrouped_tensorsr–   Údevice_params_Údevice_grads_Údevice_exp_avgs_Údevice_exp_avg_sqs_Údevice_max_exp_avg_sqs_Údevice_state_steps_Ú_Údevice_paramsÚdevice_gradsÚdevice_exp_avgsÚdevice_exp_avg_sqsÚdevice_state_stepsrG   rœ   Údevice_max_exp_avg_sqsÚscaled_device_gradsr€   r�   rž   rŸ   r¡   Úexp_avg_sq_sqrtrš   s+            ```                              @r?   Ú_multi_tensor_adamwrÛ   æ  s|  øøøø€ õ* ˆ6�{„{�aÒÐØˆå�"•fÑÔð 
 jð 
ÝØSñ
ô 
ð 	
õ �%�Ñ Ô ð ?Øð 	ÝØZñô ð ð �;Š;‰=Œ=˜AÒÐÝÐ=Ñ>Ô>Ð>å�%�Ñ Ô ð ?Øð 	ÝØZñô ð ð �;Š;‰=Œ=˜AÒÐÝÐ=Ñ>Ô>Ð>õ Œ>×&Ò&Ñ(Ô(ð w¨Zð wÝ'HØð(
ñ (
ô (
Ð$õ ð 
ð 
ð 
ð 
õ ˜v {Ñ3Ô3ð
ñ 
ô 
ñ 
ô 
ð 	wð 	wð wÐWsÐvÐvÐvñ		wô 	wð 
ð ÐDÐDÐDÑDÔDÐàÐ )Ð"3Ð"3Ð3åÔBØ	�˜ +¨ÀÐLñô €Oõ �e�VÑ$Ô$ð	Ý),¨U¬\Ñ):Ô):¸eÒ)CÐ)Cð 
Œ�uÐÐàð ð ×"Ò"Ñ$Ô$ðVñ Vñ 		ñ 	ØØØØØØØÝ�T¥&œ\¨>Ñ:Ô:ˆÝ�D¥œL¨-Ñ8Ô8ˆÝ�t¥Fœ|Ð-=Ñ>Ô>ˆÝ!¥$¥v¤,Ð0CÑDÔDÐÝ!¥$¥v¤,Ð0CÑDÔDÐà˜qÔ!Ô(ˆØÐ! f°JÐ&>Ð&>Ø!&§¢°Àd Ñ!KÔ!KˆJ�vÑà-7ÐB�z &Ô)Ð)¸Uˆàð 	Øð Ý)-­dµ6¬lÐ<SÑ)TÔ)TÐ&ÝØ!Ø Ø#Ø&Ø*ñô ð ð õ Ø! <°ÐBTñô ð ð ð 	<Ý Ô-¨lÑ;Ô;ˆLõ Œ~×*Ò*Ñ,Ô,ð 	7Ð1CÀAÔ1FÔ1Mð 	7ÝÔØ"¥E¤L°¸UÐ$CÑ$CÔ$CÈ3ðñ ô ð ð õ ÔÐ 2°AÑ6Ô6Ð6ð ˜1ÒÐÝÔ ¨q°2¸Ñ3DÑ/DÑEÔEÐEõ 	Ô˜_¨l¸AÀÑ<LÑMÔMÐMåÔÐ.°Ñ6Ô6Ð6õ
 �e�Uœ\Ñ*Ô*ð 	Ý"'Ô"4°\À1ÀuÁ9Ñ"MÔ"MÐØˆEˆEà".ÐØ˜‘IˆEåÔØÐ 3°\À5ñ	
ô 	
ð 	
ð
 Øð ñ F	Ý$Ô1°%Ð9KÑLÔLÐÝ$Ô1°%Ð9KÑLÔLÐåÔÐ 0°!Ñ4Ô4Ð4ÝÔÐ 0°!Ñ4Ô4Ð4åÔÐ 0Ñ1Ô1Ð1õ ÔÐ 0°"Ñ5Ô5Ð5ÝÔ&Ð'7Ñ8Ô8Ð8åÔ Ð!1Ñ2Ô2Ð2ð
 )ˆIØ$4Ð!àð 	JÝ)-­dµ6¬lÐ<SÑ)TÔ)TÐ&õ Ô'Ð(>Ð@RÑSÔSÐSõ #(Ô"5Ð6LÑ"MÔ"M��å"'Ô"5Ð6HÑ"IÔ"I�åÔ Ð1FÑGÔGÐGÝÔ °Ñ5Ô5Ð5ÝÔ °Ñ;Ô;Ð;õ Ô# M°?ÀOÑTÔTÐTÑTð ð  ð  ð  Ø:Lð ñ  ô  Ðð ð  ð  ð  Ø:Lð ñ  ô  Ðõ ,Ð,WÐ,WÐ,WÐ,WÐFVÐ,WÑ,WÔ,WÑXÔXˆIð%ð %Ø"2ð%ñ %ô %Ð!ð ð 	JÝ)-­dµ6¬lÐ<SÑ)TÔ)TÐ&õ Ô'Ð(>Ð@RÑSÔSÐSõ #(Ô"5Ð6LÑ"MÔ"M��å"'Ô"5Ð6HÑ"IÔ"I�åÔ Ð1FÑGÔGÐGÝÔ °Ñ5Ô5Ð5ÝÔ#ØØØØñ	ô ð ñ ðcVð Vr@   Úreturnc       
         óœ  — | sd S |rt          d¦  «        ‚|�	|j        |ini }|�	|j        |ini }t          |t          ¦  «        r!t	          |j        ¦  «        dk    r	|j        |ind }t          j        | |||||g¦  «        }|                     ¦   «         D �]´\  \  }}\  \  }}}}}}}t          t          t                   |¦  «        }t          t          t                   |¦  «        }t          t          t                   |¦  «        } t          t          t                   |¦  «        }!t          t          t                   |¦  «        }"|j
        dk    r|€|�J ‚d\  }#}$|�+|                     ||                     |d¬¦  «        ¦  «        }#|�+|                     ||                     |d¬¦  «        ¦  «        }$|�/||vr+|                     ||                     |d¬¦  «        ¦  «        }t          j        |"d¦  «         t          j        ||| |!||"|||	|
||||#|$¬	¦  «         |$�&t          j        |"|$gt#          |"¦  «        z  ¦  «         �Œ¶d S )
Nz9Adam with fused=True does not support differentiable=Truer©   Úmps)NNT)r   rª   r	   )	r.   r*   ro   rp   r-   r,   r$   rm   rn   )r:   rG   r3   r   r¹   r   r¸   Úitemsr   r   rˆ   rK   r‹   rO   r½   Ú_fused_adamw_rÃ   rN   )%r)   re   rf   rg   rh   ri   rm   rn   r.   ro   rp   r*   r-   r,   r$   r&   r'   rj   Úgrad_scale_dictÚfound_inf_dictÚlr_dictrË   rG   rÒ   rÌ   rÍ   rÎ   rÏ   rØ   rÑ   rÓ   rÔ   rÕ   rÖ   r×   Údevice_grad_scaleÚdevice_found_infs%                                        r?   Ú_fused_adamwræ   Ç  sË  € ð* ð ØˆØð XÝÐVÑWÔWÐWð ,6Ð+AˆÔ	˜JÐ'Ð'Àrð ð *3Ð)>ˆÔ	˜9Ð%Ð%ÀBð õ & b­&Ñ1Ô1ÐWµc¸"¼)±n´nÈÒ6MÐ6MˆŒ�BˆˆÐSWð õ  ÔBØ	�˜ +¨ÀÐLñô €Oð 
×	Ò	Ñ	 Ô	 ð6ñ 6ñ 
	‰ˆ�ñ 
ñ	
ØØØØØ"Øà	å�T¥&œ\¨>Ñ:Ô:ˆÝ�D¥œL¨-Ñ8Ô8ˆÝ�t¥Fœ|Ð-=Ñ>Ô>ˆÝ!¥$¥v¤,Ð0CÑDÔDÐÝ!¥$¥v¤,Ð0CÑDÔDÐàŒ;˜%ÒÐØÐ$¨Ð);Ð);Ð;à.8Ñ+ÐÐ+ØÐ!Ø /× :Ò :Ø˜
Ÿš f¸4˜Ñ@Ô@ñ!ô !Ðð Ð Ø-×8Ò8Ø˜	Ÿš V¸$˜Ñ?Ô?ñ ô  Ðð Ð 6°Ð#8Ð#8Ø×#Ò#Ø˜Ÿš V¸$˜Ñ?Ô?ñô ˆBõ 	ÔÐ.°Ñ2Ô2Ð2ÝÔØØØØØ"ØØØØØØ%ØØØ(Ø&ð	
ñ 	
ô 	
ð 	
ð" Ð'ÝÔØ"Ð%5Ð$6½Ð=OÑ9PÔ9PÑ$Pñô ð ùði6ð 6r@   )Úsingle_tensor_fnFr%   r(   c                ó®  — t           j                             ¦   «         s(t          d„ |D ¦   «         ¦  «        st	          d¦  «        ‚|	€2|€0t          | |d¬¦  «        \  }}|rt          |t          ¦  «        r|sd}|	€d}	|€d}|r-t           j         	                    ¦   «         rt	          d¦  «        ‚|	r-t           j         	                    ¦   «         rt	          d¦  «        ‚|	r&t           j         	                    ¦   «         st          }n/|r&t           j         	                    ¦   «         st          }nt          } || |||||||||||||||
||¬¦  «         dS )	zpFunctional API that performs AdamW algorithm computation.

    See :class:`~torch.optim.AdamW` for details.
    c              3   óJ   K  — | ]}t          |t          j        ¦  «        V — Œd S rv   )r3   rO   r   )r§   Úts     r?   r¨   zadamw.<locals>.<genexpr>H  s?   è è € ð 5ð 5Ø()�
�1•e”lÑ#Ô#ð5ð 5ð 5ð 5ð 5ð 5r@   zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers)r.   ro   rp   r*   r-   r,   r$   r&   r'   rm   rn   rj   )rO   r†   r‡   r¶   r:   r   r3   r   rƒ   r„   ræ   rÛ   r£   )r)   re   rf   rg   rh   ri   r%   r&   r'   r(   rm   rn   rj   r.   ro   rp   r*   r-   r,   r$   rÒ   Úfuncs                         r?   r   r   *  sÅ  € õ< Œ>×&Ò&Ñ(Ô(ð 
µð 5ð 5Ø-8ð5ñ 5ô 5ñ 2ô 2ð 
õ Ø^ñ
ô 
ð 	
ð €}˜˜Ý1Ø�N¨eð
ñ 
ô 
‰
ˆˆ7ð ð 	•z "¥fÑ-Ô-ð 	°jð 	ØˆGØ€}ØˆØ€Øˆàð U•5”9×)Ò)Ñ+Ô+ð UÝÐSÑTÔTÐTØð S•”×'Ò'Ñ)Ô)ð SÝÐQÑRÔRÐRàð $•U”Y×+Ò+Ñ-Ô-ð $ÝˆˆØ	ð $�œ×/Ò/Ñ1Ô1ð $Ý"ˆˆå#ˆà€DØØØØØØØØØØØ!ØØØØ%ØØØð%ñ ô ð ð ð r@   )NFFNNNF)%Útypingr   r   r   r   r   rO   r   Ú	optimizerr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__rz   r6   r£   rÛ   ræ   r   rW   r@   r?   ú<module>rñ      s”  ðð 6Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5à €€€Ø Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð. �GÐ
€ðhð hð hð hð hˆIñ hô hð hðX&ðNà	ðð ð 
ðð ð 
ðð ð  
ð!ð ð" 
ð#ð ð$ 
ð%ð ð ñOAð „ðLPKØ�ŒLðPKà�Œ<ðPKð �6ŒlðPKð �f”ð	PKð
 ˜&”\ðPKð �f”ðPKð ˜Ô ðPKð ˜ÔðPKð ðPKð �˜�ÔðPKð �˜�ÔðPKð 	ˆf�eˆmÔðPKð ðPKð 
ðPKð  ð!PKð" ð#PKð$ ð%PKð& ð'PKð PKð PKð PKðf^Ø�ŒLð^à�Œ<ð^ð �6Œlð^ð �f”ð	^ð
 ˜&”\ð^ð �f”ð^ð ˜Ô ð^ð ˜Ôð^ð ð^ð �˜�Ôð^ð �˜�Ôð^ð 	ˆf�eˆmÔð^ð ð^ð 
ð^ð  ð!^ð" ð#^ð$ ð%^ð& ð'^ð ^ð ^ð ^ðB`Ø�ŒLð`à�Œ<ð`ð �6Œlð`ð �f”ð	`ð
 ˜&”\ð`ð �f”ð`ð ˜Ô ð`ð ˜Ôð`ð ð`ð ð`ð ð`ð 	ˆf�eˆmÔð`ð ð`ð 
ð`ð  ð!`ð" ð#`ð$ ð%`ð& ð'`ð( 
ð)`ð `ð `ð `ðF  ÐÐ1EÐFÑFÔFð #ØØ Ø Ø#'Ø"&ØðSð SØ�ŒLðSà�Œ<ðSð �6ŒlðSð �f”ð	Sð
 ˜&”\ðSð �f”ðSð �dŒ^ðSð ðSð ðSð �DŒ>ðSð ˜Ô ðSð ˜ÔðSð ðSð" ð#Sð$ ð%Sð& ð'Sð( 	ˆe�VˆmÔð)Sð* ð+Sð, 
ð-Sð. ð/Sð Sð Sñ GÔFðSð Sð Sr@   