§
    [�’j‡/  ã                  óø   — d dl mZ d dlZd dlZd dlZd dlmZ ddlmZm	Z	m
Z
mZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZmZmZmZmZmZmZ  ej        e¦  «        Z erd dl!m"Z"  G d„ d¦  «        Z#dS )é    )ÚannotationsN)ÚTYPE_CHECKINGé   )ÚALL_PARALLEL_STYLESÚapply_tensor_parallelismÚgather_state_dict_for_saveÚinitialize_tensor_parallelism)Úis_torch_greater_or_equalÚlogging)Úcreate_and_tag_model_cardé   )ÚDistributedConfig)Ú$apply_fully_sharded_data_parallelismÚis_fsdp_managed_module)Ú_distributed_barrierÚ_ensure_torch_distributedÚ_get_torch_distributed_rankÚ!_get_torch_distributed_world_sizeÚ!_is_torch_distributed_initializedÚgather_full_state_dictÚ)initialize_fully_sharded_data_parallelismÚ!save_model_checkpoint_distributedc                  óX  — e Zd ZU dZdZdZded<   dZded<   dZdZ	ded<   dZ
ded<   d5d„Zed6d„¦   «         Zed6d„¦   «         Zed7d„¦   «         Zej        d8d„¦   «         Zej        d9d„¦   «         Zedddœd:d„¦   «         Zed;d„¦   «         Zd<d„Zd d!ddddd d"œd=d/„Zd!d0œd>d3„Zd?d4„ZdS )@ÚDistributedMixinzèDistributed orchestration and save/load hooks for [`PreTrainedModel`].

    Stateless heavy lifting stays in `transformers.distributed.*` and
    `integrations.tensor_parallel`. This mixin owns orchestration and instance state.
    Núdict[str, str] | NoneÚ_tp_planÚ_ep_planú!dict[str, tuple[str, str]] | NoneÚ_pp_planÚ
_fsdp_planÚreturnÚNonec                ó°  ‡— t          | ¦  «        }t          t          |dd¦  «        pi ¦  «        | _        t          t          |dd¦  «        pi ¦  «        | _        t          t          |dd¦  «        pi ¦  «        | _        t          t          |dd¦  «        pi ¦  «        | _        | j        | u r˜| j                             | j	        j
        pi ¦  «         | j                             | j	        j        pi ¦  «         | j                             | j	        j        pi ¦  «         | j                             | j	        j        pi ¦  «         |                      ¦   «         D �]z\  Š}t          |dd¦  «        x}rJ| j                             ˆfd„|                     ¦   «                              ¦   «         D ¦   «         ¦  «         t          |dd¦  «        x}rJ| j                             ˆfd„|                     ¦   «                              ¦   «         D ¦   «         ¦  «         t          |dd¦  «        x}rJ| j                             ˆfd„|                     ¦   «                              ¦   «         D ¦   «         ¦  «         t          |dd¦  «        x}rJ| j                             ˆfd	„|                     ¦   «                              ¦   «         D ¦   «         ¦  «         �Œ|dS )
zQCopy class-level plans onto the instance and merge config/children contributions.r   Nr   r   r    c                ó&   •— i | ]\  }}‰› d |› �|“ŒS ©ú.© ©Ú.0ÚkÚvÚnames      €úLC:\Projects\DSCons\.venv\Lib\site-packages\transformers\distributed\mixin.pyú
<dictcomp>z8DistributedMixin.init_parallel_plans.<locals>.<dictcomp>P   ó)   ø€ Ð%WÐ%WÐ%W¹4¸1¸a¨ m m° m m°QÐ%WÐ%WÐ%Wó    c                ó&   •— i | ]\  }}‰› d |› �|“ŒS r%   r'   r(   s      €r-   r.   z8DistributedMixin.init_parallel_plans.<locals>.<dictcomp>R   r/   r0   c                ó&   •— i | ]\  }}‰› d |› �|“ŒS r%   r'   r(   s      €r-   r.   z8DistributedMixin.init_parallel_plans.<locals>.<dictcomp>T   r/   r0   c                ó&   •— i | ]\  }}‰› d |› �|“ŒS r%   r'   r(   s      €r-   r.   z8DistributedMixin.init_parallel_plans.<locals>.<dictcomp>V   s)   ø€ Ð'YÐ'YÐ'Y¹T¸QÀ¨4¨¨°!¨¨°qÐ'YÐ'YÐ'Yr0   )ÚtypeÚdictÚgetattrr   r   r   r    Ú
base_modelÚupdateÚconfigÚbase_model_pp_planÚbase_model_tp_planÚbase_model_ep_planÚbase_model_fsdp_planÚnamed_childrenÚcopyÚitems)ÚselfÚ	model_clsÚmoduleÚplanr,   s       @r-   Úinit_parallel_plansz$DistributedMixin.init_parallel_plans@   s­  ø€ å˜‘J”Jˆ	Ý�W Y°
¸DÑAÔAÐGÀRÑHÔHˆŒÝ�W Y°
¸DÑAÔAÐGÀRÑHÔHˆŒÝ�W Y°
¸DÑAÔAÐGÀRÑHÔHˆŒÝ�w y°,ÀÑEÔEÐKÈÑLÔLˆŒàŒ?˜dÐ"Ð"ØŒM× Ò  ¤Ô!?Ð!EÀ2ÑFÔFÐFØŒM× Ò  ¤Ô!?Ð!EÀ2ÑFÔFÐFØŒM× Ò  ¤Ô!?Ð!EÀ2ÑFÔFÐFØŒO×"Ò" 4¤;Ô#CÐ#IÀrÑJÔJÐJà ×/Ò/Ñ1Ô1ð 	[ñ 	[‰LˆD�&Ý˜v z°4Ñ8Ô8Ð8ˆtð YØ”×$Ò$Ð%WÐ%WÐ%WÐ%WÀ4Ç9Â9Á;Ä;×CTÒCTÑCVÔCVÐ%WÑ%WÔ%WÑXÔXÐXÝ˜v z°4Ñ8Ô8Ð8ˆtð YØ”×$Ò$Ð%WÐ%WÐ%WÐ%WÀ4Ç9Â9Á;Ä;×CTÒCTÑCVÔCVÐ%WÑ%WÔ%WÑXÔXÐXÝ˜v z°4Ñ8Ô8Ð8ˆtð YØ”×$Ò$Ð%WÐ%WÐ%WÐ%WÀ4Ç9Â9Á;Ä;×CTÒCTÑCVÔCVÐ%WÑ%WÔ%WÑXÔXÐXÝ˜v |°TÑ:Ô:Ð:ˆtð [Ø”×&Ò&Ð'YÐ'YÐ'YÐ'YÀTÇYÂYÁ[Ä[×EVÒEVÑEXÔEXÐ'YÑ'YÔ'YÑZÔZÐZùð	[ð 	[r0   údict[str, str]c                ó²   — t          | j        d¦  «        r<| j        j        j        r+| j        st          d| j        j        › d�¦  «        ‚| j        S | j        S )z)The full tp plan for the model's modules.Údistributed_configzGExpert parallelism was requested (`enable_expert_parallel=True`), but `zs` does not define an expert-parallel plan. Add a `base_model_ep_plan` to its config, or disable expert parallelism.)	Úhasattrr9   rH   Úenable_expert_parallelr   Ú
ValueErrorÚ	__class__Ú__name__r   ©rA   s    r-   Útp_planzDistributedMixin.tp_planX   sv   € õ �4”;Ð 4Ñ5Ô5ð 	!¸$¼+Ô:XÔ:oð 	!Ø”=ð Ý ðZØœÔ/ðZð Zð Zñô ð ð
 ”=Ð ØŒ}Ðr0   c                ó   — | j         S ©N)r    rN   s    r-   Ú	fsdp_planzDistributedMixin.fsdp_plane   s
   € àŒÐr0   údict[str, tuple[str, str]]c                ó   — | j         S rQ   )r   rN   s    r-   Úpp_planzDistributedMixin.pp_plani   s
   € àŒ}Ðr0   rD   c                ó.  — |€	i | _         d S t          |t          ¦  «        st          d¦  «        ‚|                     ¦   «         D ]D\  }}|t
          vr6t          d|› d|› dt          t          j        ¦   «         ¦  «        › �¦  «        ‚ŒEd„ |                      ¦   «         D ¦   «         }|                     ¦   «         D ]R}| 	                    dd¦  «        }d}|D ]}t          j        ||¦  «        rd	} nŒ|st          j        d
|› d�¦  «         ŒS|| _         d S )Nz&Can only set a dictionary as `tp_plan`z#Unsupported tensor parallel style 'z' for layer 'z'. Supported styles are c                ó   — g | ]\  }}|‘ŒS r'   r'   )r)   r,   Ú_s      r-   ú
<listcomp>z,DistributedMixin.tp_plan.<locals>.<listcomp>|   s   € ÐIÐIÐI¡g d¨A˜TÐIÐIÐIr0   Ú*z\d+FTzLayer pattern 'z�' does not match any parameters in the model. This rule may not be applied during tensor parallelization, or may lead to dimension mismatches)r   Ú
isinstancer5   rK   r@   r   ÚlistÚkeysÚnamed_parametersÚreplaceÚreÚmatchÚwarningsÚwarn)rA   rD   Úlayer_patternÚparallel_styleÚmodel_param_namesÚregex_patternÚpattern_matchedÚ
param_names           r-   rO   zDistributedMixin.tp_planm   s„  € àˆ<ØˆDŒMØˆFÝ˜$¥Ñ%Ô%ð 	GÝÐEÑFÔFÐFà-1¯ZªZ©\¬\ð 	ð 	Ñ)ˆM˜>ØÕ%8Ð8Ð8Ý ðO¸.ð Oð OÐWdð Oð OÝ,0Õ1DÔ1IÑ1KÔ1KÑ,LÔ,LðOð Oñô ð ð 9ð JÐI°×1FÒ1FÑ1HÔ1HÐIÑIÔIÐØ!ŸYšY™[œ[ð 	ð 	ˆMØ)×1Ò1°#°vÑ>Ô>ˆMØ#ˆOØ/ð ð �
Ý”8˜M¨:Ñ6Ô6ð Ø&*�OØ�Eðð #ð Ý”ðd mð dð dð dñô ð øð
 ˆŒˆˆr0   c                ór   — |€	i | _         d S t          |t          ¦  «        st          d¦  «        ‚|| _         d S )Nz&Can only set a dictionary as `pp_plan`)r   r[   r5   rK   )rA   rD   s     r-   rU   zDistributedMixin.pp_planŒ   s@   € àˆ<ØˆDŒMØˆFÝ˜$¥Ñ%Ô%ð 	GÝÐEÑFÔFÐFàˆŒˆˆr0   )Údevice_meshÚ
device_maprH   úDistributedConfig | dict | Noneú/tuple[DistributedConfig | None, object, object]c          	     óê  — |€d ||fS t          |t          ¦  «        rt          j        |¦  «        }|j        dk    s|j        dk    rRt          ¦   «          t          ¦   «         }|j        |j        z  |k    r#t          d|j        › d|j        › d|› d�¦  «        ‚|j        dk    r/|j	        €d|_	        t          |j	        |j        ||¬¦  «        \  }}n|j        dk    rt          |¦  «        \  }}|||fS )Nr   z	tp_size (z) * fsdp_size (z) is not equal to world_size (ú)Úauto)Útp_sizerk   rl   )r[   r5   r   Ú	from_dictrr   Ú	fsdp_sizer   r   ÚRuntimeErrorrO   r	   r   )ÚclsrH   rk   rl   Ú
world_sizes        r-   Úprepare_distribute_modelz)DistributedMixin.prepare_distribute_model–   sW  € ð Ð%Ø˜ [Ð0Ð0åÐ(­$Ñ/Ô/ð 	QÝ!2Ô!<Ð=OÑ!PÔ!PÐàÔ%¨Ò)Ð)Ð-?Ô-IÈAÒ-MÐ-MÝ%Ñ'Ô'Ð'Ý:Ñ<Ô<ˆJØ!Ô)Ð,>Ô,HÑHÈJÒVÐVÝ"ðAÐ 2Ô :ð Að AÐK]ÔKgð Að AØ3=ðAð Að Añô ð ð
 Ô%¨Ò)Ð)Ø!Ô)Ð1Ø-3Ð"Ô*Ý&CØ"Ô*Ø*Ô2Ø'Ø%ð	'ñ 'ô 'Ñ#ˆJ˜˜ð  Ô)¨AÒ-Ð-Ý&OÐPbÑ&cÔ&cÑ#ˆJ˜à! :¨{Ð:Ð:r0   Úmodelú	nn.ModuleúDistributedConfig | Nonec                óÖ   — |�f||j         _        ||_        |j        dk    rt	          ||j        ||¦  «        }n0|j        dk    r%|j        dk    r|d         n|}t          ||¦  «        }|S )z:Apply TP or FSDP2 after model init, before weight loading.Nr   Úfsdp)	r9   rH   Ú_device_meshrr   r   rO   rt   Úndimr   )rv   ry   rH   rk   Ú	fsdp_meshs        r-   Úmaybe_distribute_modelz'DistributedMixin.maybe_distribute_model»   sŠ   € ð Ð"Ø.@ˆEŒLÔ+Ø!,ˆEÔà!Ô)¨AÒ-Ð-Ý0ØØ&Ô.Ø&Øñ	ô ��ð $Ô-°Ò1Ð1Ø3>Ô3CÀaÒ3GÐ3G˜K¨Ô/Ð/È[�	Ý<¸UÀIÑNÔN�Øˆr0   Úis_main_processÚboolc                óN   — |}t          ¦   «         r|ot          ¦   «         dk    }|S )z7Return whether this rank should write checkpoint files.r   )r   r   )rA   r‚   Úsave_on_this_ranks      r-   Úshould_save_on_this_rankz)DistributedMixin.should_save_on_this_rankÓ   s5   € à+ÐÝ,Ñ.Ô.ð 	YØ 1Ð XÕ6QÑ6SÔ6SÐWXÒ6XÐØ Ð r0   FT)Úpush_to_hubr…   Úrepo_idÚfiles_timestampsÚcommit_messageÚtokenÚ	create_prÚsave_directoryústr | os.PathLiker‡   r…   rˆ   ú
str | Noner‰   údict | NonerŠ   r‹   ústr | bool | NonerŒ   c               ó¸  — t          d¦  «        st          d¦  «        ‚t          |¦  «        st          d¦  «        ‚t	          |dd¦  «        €t          d¦  «        ‚t          ||¦  «         |ri|rit          || j        |¬¦  «        }
|
                     t          j
                             |d¦  «        ¦  «         |                      ||||||	¬	¦  «         dS dS dS )
zBSave an FSDP-wrapped model via DCP and optionally push to the Hub.z2.7zFsave_pretrained(..., distributed_checkpoint=True) requires torch>=2.7.z\save_pretrained(..., distributed_checkpoint=True) is only supported for FSDP-wrapped models.r~   Nz�save_pretrained(..., distributed_checkpoint=True) requires the model to have been initialized with a distributed_config (_device_mesh is None).)r‹   z	README.md)rŠ   r‹   rŒ   )r
   ÚOSErrorr   rK   r6   r   r   Ú
model_tagsÚsaveÚosÚpathÚjoinÚ_upload_modified_files)rA   Úmodel_to_saver�   r‡   r…   rˆ   r‰   rŠ   r‹   rŒ   Ú
model_cards              r-   Úsave_distributed_checkpointz,DistributedMixin.save_distributed_checkpointÚ   s  € õ )¨Ñ/Ô/ð 	dÝÐbÑcÔcÐcÝ% mÑ4Ô4ð 	ÝØnñô ð õ �= .°$Ñ7Ô7Ð?ÝðPñô ð õ 	*¨-¸ÑHÔHÐHàð 
	Ð,ð 
	Ý2°7¸D¼OÐSXÐYÑYÔYˆJØ�OŠO�BœGŸLšL¨¸ÑEÔEÑFÔFÐFØ×'Ò'ØØØ Ø-ØØ#ð (ñ ô ð ð ð ð
	ð 
	ð 
	ð 
	r0   )r…   Ú
state_dictr5   c               óà   — |€|S |j         dk    r't          || j        | j        | j        ¦  «        }|si }|S |j        dk    r,t          ¦   «         st          d¦  «        ‚t          |¦  «        S |S )zNGather TP- or FSDP-sharded weights to full CPU tensors for checkpoint writing.Nr   z‰Saving an FSDP-wrapped model requires torch.distributed to be initialized. Call save_pretrained from every rank after init_process_group.)	rr   r   r   r~   Ú_tp_sizert   r   rK   r   )rA   rš   r�   rH   r…   s        r-   Ú"gather_sharded_state_dict_for_savez3DistributedMixin.gather_sharded_state_dict_for_save  s—   € ð Ð%ØÐàÔ%¨Ò)Ð)Ý3°JÀÄÈtÔO`ÐbfÔboÑpÔpˆJØ$ð  Ø�
ØÐàÔ'¨!Ò+Ð+Ý4Ñ6Ô6ð Ý ðUñô ð õ *¨-Ñ8Ô8Ð8àÐr0   c                óZ   — |€dS |j         dk    s|j        dk    rt          ¦   «          dS dS )zQBarrier so non-writer ranks wait for rank 0 to finish gathered checkpoint writes.Nr   )rr   rt   r   )rA   rH   s     r-   Ú&barrier_after_gathered_checkpoint_savez7DistributedMixin.barrier_after_gathered_checkpoint_save  sD   € àÐ%ØˆFØÔ%¨Ò)Ð)Ð-?Ô-IÈAÒ-MÐ-MÝ Ñ"Ô"Ð"Ð"Ð"ð .NÐ-Mr0   )r!   r"   )r!   rF   )r!   rS   )rD   r   )rD   r   )rH   rm   r!   rn   )ry   rz   rH   r{   )r‚   rƒ   r!   rƒ   )r�   rŽ   r‡   rƒ   r…   rƒ   rˆ   r�   r‰   r�   rŠ   r�   r‹   r‘   rŒ   rƒ   r!   r"   )r�   r5   rH   r{   r…   rƒ   r!   r5   )rH   r{   r!   r"   )rM   Ú
__module__Ú__qualname__Ú__doc__r~   r   Ú__annotations__r   rŸ   r   r    rE   ÚpropertyrO   rR   rU   ÚsetterÚclassmethodrx   r�   r†   rœ   r    r¢   r'   r0   r-   r   r   2   s   € € € € € € ðð ð €LØ&*€HÐ*Ð*Ð*Ñ*Ø&*€HÐ*Ð*Ð*Ñ*Ø€HØ26€HÐ6Ð6Ð6Ñ6Ø(,€JÐ,Ð,Ð,Ñ,ð[ð [ð [ð [ð0 ð
ð 
ð 
ñ „Xð
ð ðð ð ñ „Xðð ðð ð ñ „Xðð „^ðð ð ñ „^ðð< „^ðð ð ñ „^ðð ð
 Øð";ð ";ð ";ð ";ð ";ñ „[ð";ðH ðð ð ñ „[ðð.!ð !ð !ð !ð "Ø"&Ø"Ø(,Ø%)Ø#'Øð%ð %ð %ð %ð %ð %ðZ #'ðð ð ð ð ð ð8#ð #ð #ð #ð #ð #r0   r   )$Ú
__future__r   r–   r`   rb   Útypingr   Úintegrations.tensor_parallelr   r   r   r	   Úutilsr
   r   Ú	utils.hubr   Úconfiguration_utilsr   r}   r   r   r   r   r   r   r   r   r   r   Ú
get_loggerrM   ÚloggerÚtorch.nnÚnnr   r'   r0   r-   ú<module>r´      s¨  ðð #Ð "Ð "Ð "Ð "Ð "à 	€	€	€	Ø 	€	€	€	Ø €€€Ø  Ð  Ð  Ð  Ð  Ð  ðð ð ð ð ð ð ð ð ð ð ð ð 7Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø NÐ NÐ NÐ NÐ NÐ NÐ NÐ Nð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 
ˆÔ	˜HÑ	%Ô	%€ð ð ØÐÐÐÐÐðp#ð p#ð p#ð p#ð p#ñ p#ô p#ð p#ð p#ð p#r0   