§
    [�’jLÌ  ã                  óš  — U d dl mZ d dlZd dlZd dlmZ d dlmZ d dlZd dl	m
Z
 d dl	mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'  ej(        e)¦  «        Z*ej+        Z, ej-        e,¦  «        j.        Z/ ej-        e,¦  «        j0        Z1ej2        dZd„¦   «         Z3d„ Z4 ed¬¦  «         G d„ d¦  «        ¦   «         Z5da6de7d<   ej8        j9        d[d„¦   «         Z:d\d„Z;d]d"„Z<	 	 	 	 d^d_d.„Z= ed/d0¬1¦  «        	 	 	 	 d`dad<„¦   «         Z> ed/d0¬1¦  «        	 	 	 	 	 dbdcd?„¦   «         Z? G d@„ dAe
j@        ¦  «        ZA G dB„ dCeA¦  «        ZBdddI„ZCdddJ„ZD G dK„ dLe
jE        ¦  «        ZF G dM„ dNe$¦  «        ZG eG¦   «         ZHdedQ„ZI	 dfdgdU„ZJ G dV„ dWe¦  «        ZK G dX„ dYe¦  «        ZLdS )hé    )ÚannotationsN)ÚCallable)Ú	dataclass)Ú
functionalé   )ÚACT2FN)ÚConversionOps)Úget_module_from_nameÚshould_convert_module)Úlogging)Údeprecate_kwarg)Úis_kernels_availableé   )Ú deepgemm_fp8_fp4_experts_forwardÚdeepgemm_fp8_fp4_linearÚ(deepgemm_fp8_fp4_megamoe_experts_forwardÚis_deepgemm_loadableÚis_sm100)Ú_MISSING_KERNELS_MESSAGEÚlazy_load_kernel)ÚExpertsInterfaceÚuse_experts_implementation)Úto_localÚreturnútorch.dtypec                 ó~   — t          t          d¦  «        st          dt          j        › d�¦  «        ‚t          j        S )uS  Return ``torch.float8_e8m0fnu`` or raise a clear error on torch without FP8 support.

    UE8M0 scales are always stored/consumed as this single dtype â€” the kernels (Triton
    finegrained + DeepGEMM) read it natively, and supporting the same scales in mixed
    container dtypes would be a mess â€” so fail loudly rather than fall back.Úfloat8_e8m0fnuzbscale_fmt='ue8m0' requires torch.float8_e8m0fnu, which is only available in PyTorch >= 2.7 (found z.). Upgrade torch to use UE8M0 FP8 checkpoints.)ÚhasattrÚtorchÚRuntimeErrorÚ__version__r   © ó    úWC:\Projects\DSCons\.venv\Lib\site-packages\transformers\integrations\finegrained_fp8.pyÚ_get_ue8m0_dtyper%   3   sR   € õ •5Ð*Ñ+Ô+ð 
ÝðgÝ%*Ô%6ðgð gð gñ
ô 
ð 	
õ ÔÐr#   c                óœ   — |D ]$}t          | |¦  «        rt          | |¦  «        c S Œ%t          t          | ¦  «        j        › d|› �¦  «        ‚)Nz has none of: )r   ÚgetattrÚAttributeErrorÚtypeÚ__name__)ÚobjÚnamesÚnames      r$   Ú_first_attrr.   B   sc   € Øð &ð &ˆÝ�3˜ÑÔð 	&Ý˜3 Ñ%Ô%Ð%Ð%Ð%ð	&å
�D ™IœIÔ.ÐEÐE¸eÐEÐEÑ
FÔ
FÐFr#   T)Úfrozenc                  ó2   — e Zd ZU dZded<   ded<   ded<   dS )ÚFineGrainedFP8zNEntry points exposed by the `kernels-community/finegrained-fp8` Triton kernel.r   ÚmatmulÚbatched_matmulÚgrouped_matmulN)r*   Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__r"   r#   r$   r1   r1   I   s<   € € € € € € àXÐXàÐÐÑØÐÐÑØÐÐÑÐÐr#   r1   zFineGrainedFP8 | NoneÚ_FINEGRAINED_FP8ÚNonec                 ó´  — t           �dS t          ¦   «         st          dt          › �¦  «        ‚t	          d¦  «        } | €t          d¦  «        ‚t          | dd¦  «        }t          | dd¦  «        }t          | dd¦  «        }d„ d|fd|fd|ffD ¦   «         }|r-t          d	d
                     |¦  «        › dt          › �¦  «        ‚t          |||¬¦  «        a dS )aç  
    Load the finegrained-fp8 Triton kernel once into the `_FINEGRAINED_FP8` module global.

    `@allow_in_graph` makes `torch.compile` treat the untraceable hub download + dynamic import as a
    single opaque node instead of tracing into it; it returns `None` (proxyable) and populates the
    global, which `load_finegrained_fp8_kernel` then returns.

    Under NO circumstances may this function return a value: an `@allow_in_graph` fx node's
    return must be proxyable, and returning the bundle (e.g. from the warm-cache
    short-circuit) breaks torch.compile with `Unsupported: torch.* op returned non-Tensor`.

    Raises `ImportError` if the `kernels` package is missing, or the kernel or required
    symbols cannot be found.
    Nz$finegrained-fp8 kernel unavailable: zfinegrained-fp8u‰   Failed to load the finegrained-fp8 kernel â€” check that `kernels-community/finegrained-fp8` has a build matching the current torch/CUDA.Ú	matmul_2dÚmatmul_batchedÚmatmul_groupedc                ó   — g | ]	\  }}|­|‘Œ
S ©Nr"   )Ú.0r-   Úattrs      r$   ú
<listcomp>z0_load_finegrained_fp8_kernel.<locals>.<listcomp>z   s-   € ð ð ð áˆD�$ð
 ˆ<ð 	ð ˆ<ˆ<r#   z4finegrained-fp8 kernel is missing required symbols: ú, z. )r2   r3   r4   )r9   r   ÚImportErrorr   r   r'   Újoinr1   )Úkernelr2   r3   r4   Úmissings        r$   Ú_load_finegrained_fp8_kernelrI   X   s3  € õ" Ð#ØˆåÑ!Ô!ð ]ÝÐ[ÕAYÐ[Ð[Ñ\Ô\Ð\åÐ/Ñ0Ô0€FØ€~Ýð;ñ
ô 
ð 	
õ
 �V˜[¨$Ñ/Ô/€FÝ˜VÐ%5°tÑ<Ô<€NÝ˜VÐ%5°tÑ<Ô<€Nðð ð ˜&Ð!Ø˜~Ð.Ø˜~Ð.ð
ðñ ô €Gð ð 
ÝØsÀ4Ç9Â9ÈWÑCUÔCUÐsÐsÕYqÐsÐsñ
ô 
ð 	
õ &ØØ%Ø%ðñ ô ÐÐÐr#   c                 ó,   — t          ¦   «          t          S r@   )rI   r9   r"   r#   r$   Úload_finegrained_fp8_kernelrK   �   s   € Ý Ñ"Ô"Ð"ÝÐr#   ÚaÚintÚbc                ó   — | |z   dz
  |z  S )zCeiling division.r   r"   )rL   rN   s     r$   Ú_cdivrP   ”   s   € à�‰E�A‰I˜!ÑÐr#   Únum_expertsÚproj_outÚproj_inÚweight_dtypeÚsf_dtypeÚweight_k_divÚ	sf_gran_nú
int | NoneÚ	sf_gran_kÚ
min_sf_outú!tuple[nn.Parameter, nn.Parameter]c	                ó|  — t          j        | |||z  |¬¦  «        }	t          j        |	|	                     ¦   «         ¬¦  «        }
t          |�t          ||¦  «        nd|¦  «        }|�t          ||¦  «        nd}t          j        | |||¬¦  «        }t          j        ||                     ¦   «         ¬¦  «        }|
|fS )u×  Allocate `(weight, weight_scale_inv)` parameters for one expert projection.

    `weight_k_div` halves the K dim for FP4-packed storage (2 e2m1 values per byte).
    `sf_gran_n` / `sf_gran_k` set per-block (None â†’ per-row/per-tensor) SF granularity.
    `min_sf_out` floors the SF tensor's output dim â€” used by the fused gate_up
    projection to keep room for both halves (pass `2`) even when `proj_out < sf_gran_n`
    would otherwise collapse the SF dim to 1.
    ©Údtype©Úrequires_gradNr   )r   ÚemptyÚnnÚ	ParameterÚis_floating_pointÚmaxrP   )rQ   rR   rS   rT   rU   rV   rW   rY   rZ   Úweight_tÚweightÚsf_outÚsf_inÚsf_tÚsfs                  r$   Ú_alloc_expert_projrl   ™   sÀ   € õ& Œ{˜;¨°'¸\Ñ2IÐQ]Ð^Ñ^Ô^€HÝŒ\˜(°(×2LÒ2LÑ2NÔ2NÐOÑOÔO€FÝ¨yÐ/D•�x Ñ+Ô+Ð+È!ÈZÑXÔX€FØ)2Ð)>�E�'˜9Ñ%Ô%Ð%ÀA€EÝŒ;�{ F¨E¸ÐBÑBÔB€DÝ	Œ�d¨$×*@Ò*@Ñ*BÔ*BÐ	CÑ	CÔ	C€BØ�2ˆ:Ðr#   Úoutput_dtypezv5.16)ÚversionÚinputútorch.Tensorrg   Úweight_scale_invÚ
block_sizeúlist[int] | NoneÚbiasútorch.Tensor | NoneÚactivation_scaleútorch.dtype | Nonec                ó�   — t          ¦   «         }|                     | |||| j        |¬¦  «        }|�|                     |¦  «         |S )uu  Triton FP8/FP4 linear: fused act-quant + matmul, then optional bias add.

    ``activation_scale=None`` â†’ dynamic per-K-block scales (inline); set it for
    static per-tensor quant. ``weight_scale_inv`` accepts fp32 or UE8M0; the
    dispatcher routes FP4 (``int8``-packed) weights automatically. ``output_dtype``
    defaults to ``input.dtype`` when left ``None``.
    ©rv   )rK   r2   r^   Úadd_)	ro   rg   rq   rr   rt   rv   rm   Úfinegrained_fp8Úoutputs	            r$   Úfinegrained_fp8_linearr}   µ   s\   € õ" 2Ñ3Ô3€OØ×#Ò#ØØØØØŒØ)ð $ñ ô €Fð ÐØ�Š�DÑÔÐØ€Mr#   Úallow_deepgemmÚboolc                ó   — |oŽt          ¦   «         o€|du o|t          ¦   «         o|j        t          j        k     oX|j        t          j        k    p|duo|d         |d         cxk    odk    nc o#t          j                             dd¦  «        dk    }|rX	 t          | |||||¬¦  «        S # t          t          t          f$ r(}	t                               d	|	› d
�¦  «         Y d}	~	nd}	~	ww xY wt          | |||||¦  «        S )u  End-to-end FP8/FP4 linear used by `FP8Linear` and the eager `FP8Experts` loop.

    Dispatch order â€” both backends handle FP8 and FP4 weights with fp32 or UE8M0 scales:
      1. DeepGEMM (`deepgemm_fp8_fp4_linear`) â€” 3-6Ã— faster on the shapes it supports.
         Preferred for FP4, UE8M0 SFs, and 128Ã—128 block FP8.
      2. Triton finegrained-fp8 fallback â€” used when DeepGEMM is unavailable, when the
         caller passes ``activation_scale`` (DeepGEMM is dynamic-only), or for any
         shape DeepGEMM declined.

    Args:
        input: (..., K) bf16/fp16 activations.
        weight: (N, K) `float8_e4m3fn` or (N, K // 2) `int8` (FP4-packed).
        weight_scale_inv: per-block weight scales â€” `float32` (V3-style) or `float8_e8m0fnu`
            (V4-style; reinterpreted as int32 at the DeepGEMM kernel boundary).
        block_size: [block_n, block_k] for FP8 block-wise quant, or None/[N, K] for per-tensor.
            Ignored for FP4 weights (the kernel infers SF granularity from the dtype).
        bias: optional bias added to the matmul output.
        activation_scale: pass a per-tensor scalar to use static activation quant; leave `None`
            for dynamic (per-token) quant.
        allow_deepgemm: set ``False`` to force the Triton fallback for this call. Used when the
            model spans multiple CUDA devices in one process â€” DeepGEMM's cached kernels are bound
            to a single CUDA context and produce garbage across devices (see the multi-device guard
            in ``quantizer_finegrained_fp8.py``).
    Nr   r   é€   Ú$TRANSFORMERS_DISABLE_DEEPGEMM_LINEARÚ0Ú1)rr   rv   rt   z=DeepGEMM declined this call, falling back to Triton. Reason: zW Set `TRANSFORMERS_DISABLE_DEEPGEMM_LINEAR=1` to skip DeepGEMM for FP8 linear entirely.)r   r   r^   r   Úfloat32Úint8ÚosÚenvironÚgetr   rE   ÚNotImplementedErrorÚ
ValueErrorÚloggerÚwarning_oncer}   )
ro   rg   rq   rr   rt   rv   rm   r~   Údeepgemm_preferredÚes
             r$   Ú
fp8_linearr�   Ô   s„  € ðZ 	ð 	OÝ Ñ"Ô"ð	Oà Ð$ð	Oõ ‘”ÐGÐ 0Ô 6½%¼-Ò GÐHð	Oð Œ\�UœZÒ'Ðm¨J¸dÐ,BÐ,lÀzÐRSÄ}ÐXbÐcdÔXeÐGlÐGlÒGlÐGlÐilÒGlÐGlÐGlÐGlð		Oõ
 ŒJ�NŠNÐAÀ3ÑGÔGÈ3ÒNð ð ð ð	Ý*ØØØ Ø%Ø!1Øðñ ô ð øõ Õ0µ*Ð=ð 	ð 	ð 	õ ×ÒðiÐPQð ið ið iñô ð ð ð ð ð ð øøøøð	øøøõ " %¨Ð1AÀ:ÈtÐUeÑfÔfÐfs   ÂB* Â*C)ÃC$Ã$C)c                  ó6   ‡ — e Zd ZdZ	 	 	 	 ddˆ fd„Zdd„Zˆ xZS )Ú	FP8LinearFNÚdynamicÚfloatÚin_featuresrM   Úout_featuresrr   útuple[int, int] | NoneÚactivation_schemeÚstrÚ	scale_fmtÚhas_biasr   c                óî  •— t          ¦   «                              ||¦  «         || _        || _        || _        t
          j                             t          j        ||t          ¬¦  «        ¦  «        | _
        | j        €8t          j        t          j        dt
          j        ¬¦  «        ¦  «        | _        n—|dk    rt          ¦   «         nt
          j        }|| j        d         z   dz
  | j        d         z  }|| j        d         z   dz
  | j        d         z  }	t          j        t          j        ||	|¬¦  «        |j        ¬¦  «        | _        | j        dk    r8t          j        t          j        dt
          j        ¬¦  «        ¦  «        | _        n|                      dd ¦  «         | j        r2t          j        t          j        | j        ¦  «        ¦  «        | _        d S |                      d	d ¦  «         d S )
Nr]   ç      ð?Úue8m0r   r   r_   Ústaticrv   rt   )ÚsuperÚ__init__r›   rr   r˜   r   rb   rc   ra   Ú
_FP8_DTYPErg   Útensorr…   rq   r%   rd   rv   Úregister_parameterr–   rt   )Úselfr•   r–   rr   r˜   rš   r›   rU   Úscale_out_featuresÚscale_in_featuresÚ	__class__s             €r$   r¡   zFP8Linear.__init__)  s»  ø€ õ 	‰Œ×Ò˜ lÑ3Ô3Ð3à ˆŒØ$ˆŒØ!2ˆÔÝ”h×(Ò(­¬°\À;ÕV`Ð)aÑ)aÔ)aÑbÔbˆŒàŒ?Ð"å$&¤Lµ´¸cÍÌÐ1WÑ1WÔ1WÑ$XÔ$XˆDÔ!Ð!à-6¸'Ò-AÐ-AÕ'Ñ)Ô)Ð)ÅuÄ}ˆHØ".°´ÀÔ1CÑ"CÀaÑ"GÈDÌOÐ\]ÔL^Ñ!^ÐØ!,¨t¬¸qÔ/AÑ!AÀAÑ!EÈ$Ì/ÐZ[ÔJ\Ñ \ÐÝ$&¤LÝ”Ð.Ð0AÈÐRÑRÔRØ&Ô8ð%ñ %ô %ˆDÔ!ð
 Ô! XÒ-Ð-Ý$&¤Lµ´¸cÍÌÐ1WÑ1WÔ1WÑ$XÔ$XˆDÔ!Ð!à×#Ò#Ð$6¸Ñ=Ô=Ð=àŒ=ð 	2Ýœ¥U¤[°Ô1BÑ%CÔ%CÑDÔDˆDŒIˆIˆIà×#Ò# F¨DÑ1Ô1Ð1Ð1Ð1r#   ro   rp   r   c           	     ó"  — | j                              ¦   «         dk    r t          j        || j         | j        ¦  «        S t          | j         ¦  «        }t          | j        ¦  «        }t          |||| j        | j	        | j        | j
         ¬¦  «        S )Nr   )rr   rv   rt   r~   )rg   Úelement_sizeÚFÚlinearrt   r   rq   r�   rr   rv   Ú_deepgemm_disabled)r¥   ro   rg   Ú	scale_invs       r$   ÚforwardzFP8Linear.forwardO  s‹   € ØŒ;×#Ò#Ñ%Ô%¨Ò)Ð)Ý”8˜E 4¤;°´	Ñ:Ô:Ð:å˜$œ+Ñ&Ô&ˆÝ˜TÔ2Ñ3Ô3ˆ	åØØØØ”Ø!Ô2Ø”Ø#Ô6Ð6ð
ñ 
ô 
ð 	
r#   ©Nr“   r”   F)r•   rM   r–   rM   rr   r—   r˜   r™   rš   r™   r›   r   )ro   rp   r   rp   )r*   r5   r6   r­   r¡   r¯   Ú__classcell__©r¨   s   @r$   r’   r’   #  sm   ø€ € € € € ð Ðð .2Ø!*Ø Øð$2ð $2ð $2ð $2ð $2ð $2ð $2ðL
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r#   r’   c                  ó6   ‡ — e Zd ZdZ	 	 	 	 ddˆ fd„Zdd„Zˆ xZS )ÚFP8GroupedLinearuï  FP8 drop-in for block-diagonal grouped linears.

    The underlying nn.Linear stores a single `(n_groups * out_per_group, in_per_group)`
    weight; logically that's `n_groups` independent `(out_per_group, in_per_group)`
    sub-matrices, each consuming a disjoint slice of the input's last-but-one dim.
    Forward expects input of shape `(..., n_groups, in_per_group)` and returns
    `(..., n_groups, out_per_group)` â€” same contract as the vanilla bf16 grouped
    linear it replaces.

    Nr“   r”   FÚin_features_per_grouprM   r–   Ún_groupsrr   r—   r˜   r™   rš   r›   r   c                ód   •— t          ¦   «                              ||||||¬¦  «         || _        d S )N©r•   r–   rr   r˜   rš   r›   )r    r¡   r¶   )	r¥   rµ   r–   r¶   rr   r˜   rš   r›   r¨   s	           €r$   r¡   zFP8GroupedLinear.__init__m  sE   ø€ õ 	‰Œ×ÒØ-Ø%Ø!Ø/ØØð 	ñ 	
ô 	
ð 	
ð !ˆŒˆˆr#   Úxrp   r   c                óÂ  — |j         d d…         }|j         d         }| j                             ¦   «         dk    rà| j                             | j        d|¦  «                             dd¦  «        }|                     d| j        |¦  «                             dd¦  «        }t          j        ||¦  «                             dd¦  «        } |j        g |¢| j        ‘d‘R Ž }| j	        r3| 
                    | j                             | j        d¦  «        ¦  «         |S t          | j        ¦  «        }t          | j        ¦  «        }|                     | j        d|¦  «        }|                     dd¦  «                             d|¦  «        }|                     | j        |                     d¦  «        | j        z  |                     d¦  «        ¦  «        }|                     d¦  «        | j        z  }t          j        | j        f||j        t          j        ¬¦  «        }t          j        d| j        dz   |j        t          j        ¬¦  «        |z  }	t)          ¦   «         }
|
                     ||||	|| j        ¬¦  «        } |j        | j        g|¢d‘R Ž                      dd¦  «        }| j	        r3| 
                    | j                             | j        d¦  «        ¦  «         |S )Néþÿÿÿéÿÿÿÿr   r   r   )Údevicer^   ©ÚoffsetsÚtokens_per_expertrr   )Úshaperg   rª   Úviewr¶   Ú	transposeÚreshaper   Úbmmr›   rz   rt   r   rq   ÚmovedimÚsizeÚfullr½   Úint32ÚarangerK   r4   rr   )r¥   r¹   Úinput_shapeÚ
hidden_dimÚwÚyr®   Útokens_per_grouprÀ   r¿   r{   s              r$   r¯   zFP8GroupedLinear.forward�  s”  € Ø”g˜c˜r˜c”lˆØ”W˜R”[ˆ
àŒ;×#Ò#Ñ%Ô%¨Ò)Ð)Ø”× Ò  ¤°°JÑ?Ô?×IÒIÈ!ÈQÑOÔOˆAØ—	’	˜"˜dœm¨ZÑ8Ô8×BÒBÀ1ÀaÑHÔHˆAÝ”	˜!˜Q‘”×)Ò)¨!¨QÑ/Ô/ˆAØ�”	Ð:˜;Ð:¨¬Ð:°rÐ:Ð:Ð:ˆAØŒ}ð :Ø—’�t”y—~’~ d¤m°RÑ8Ô8Ñ9Ô9Ð9ØˆHå�T”[Ñ!Ô!ˆÝ˜TÔ2Ñ3Ô3ˆ	à�FŠF�4”= " jÑ1Ô1ˆØ�IŠI�b˜!ÑÔ×$Ò$ R¨Ñ4Ô4ˆØ—N’N 4¤=°)·.².ÀÑ2CÔ2CÀtÄ}Ñ2TÐV_×VdÒVdÐefÑVgÔVgÑhÔhˆ	àŸ6š6 !™9œ9¨¬Ñ5ÐÝ!œJ¨¬Ð'7Ð9IÐRSÔRZÕbgÔbmÐnÑnÔnÐÝ”,˜q $¤-°!Ñ"3¸A¼HÍEÌKÐXÑXÔXÐ[kÑkˆå5Ñ7Ô7ˆØ×*Ò*ØØØØØ/Ø”ð +ñ 
ô 
ˆð ˆAŒI�d”mÐ6 kÐ6°2Ð6Ð6Ð6×>Ò>¸qÀ"ÑEÔEˆØŒ=ð 	6Ø�FŠF�4”9—>’> $¤-°Ñ4Ô4Ñ5Ô5Ð5Øˆr#   r°   )rµ   rM   r–   rM   r¶   rM   rr   r—   r˜   r™   rš   r™   r›   r   )r¹   rp   r   rp   )r*   r5   r6   r7   r¡   r¯   r±   r²   s   @r$   r´   r´   a  sn   ø€ € € € € ð	ð 	ð  .2Ø!*Ø Øð!ð !ð !ð !ð !ð !ð !ð($ð $ð $ð $ð $ð $ð $ð $r#   r´   r¥   útorch.nn.ModuleÚhidden_statesÚtop_k_indexÚtop_k_weightsc                ó\  — | j         dk    rt          d¦  «        ‚t          ¦   «         }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }|                     |d¬¦  «        }|                     d¦  «        }	|                     d¦  «        }
|
| j        k                         d¦  «        }t          | j	        r| j
        n| j        ¦  «        }t          | j	        r| j        n| j        ¦  «        }t          | j        ¦  «        }t          | j        ¦  «        }|                     |||| j        |
¬¦  «        }| j	        r|                      |¦  «        }n|                      |¦  «        }|                     |||| j        |
¬¦  «        }||	                     |j        ¦  «                             d¦  «        z  }|                     |d¦  «         |                     |||¦  «                             d¬¦  «        }|                     |j        ¦  «        S )	NrŸ   z‘batched_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.r¼   r   ©Údim)rr   Ú
expert_idsç        r   )r˜   rŠ   rK   rÇ   Úrepeat_interleaverÄ   rQ   Ú	unsqueezer   Úhas_gateÚgate_up_projÚup_projÚgate_up_proj_scale_invÚup_proj_scale_invÚ	down_projÚdown_proj_scale_invr3   rr   Ú_apply_gateÚact_fnÚtor^   Úmasked_fill_rÂ   Úsum)r¥   rÑ   rÒ   rÓ   r{   Ú	num_top_kÚ
num_tokensrÌ   Úselected_hidden_statesÚsample_weightsr×   Úsentinel_maskÚ	weight_upÚweight_scale_upÚweight_downÚweight_scale_downrR   Úweighted_outÚfinal_hidden_statess                      r$   Úfp8_batched_mm_experts_forwardrò   ¨  s?  € ð Ô Ò)Ð)Ý!ðWñ
ô 
ð 	
õ
 2Ñ3Ô3€Oà× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jð +×<Ò<¸YÈAÐ<ÑNÔNÐØ"×*Ò*¨2Ñ.Ô.€NØ×$Ò$ RÑ(Ô(€Jð
   4Ô#3Ò3×>Ò>¸rÑBÔB€Må¨d¬mÐM˜Ô*Ð*ÀÄÑNÔN€IÝ¸d¼mÐg˜tÔ:Ð:ÐQUÔQgÑhÔh€OÝ˜4œ>Ñ*Ô*€KÝ  Ô!9Ñ:Ô:Ðð ×-Ò-ØØØØ”?Øð .ñ ô €Hð „}ð )à×#Ò# HÑ-Ô-ˆˆð —;’;˜xÑ(Ô(ˆð ×-Ò-ØØØØ”?Øð .ñ ô €Hð ˜n×/Ò/°´Ñ?Ô?×IÒIÈ"ÑMÔMÑM€Lð ×Ò˜m¨SÑ1Ô1Ð1ð '×+Ò+¨J¸	À:ÑNÔN×RÒRÐWXÐRÑYÔYÐà×!Ò! -Ô"5Ñ6Ô6Ð6r#   c                ó  — | j         dk    rt          d¦  «        ‚t          ¦   «         }|j        }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }	|                     d¦  «        }
t          j        |
¦  «        \  }}|||z           }|	|         }|j        dk    r| 	                    ¦   «         n| 
                    ¦   «         }t          j        || j        d| j        dz
  ¬¦  «        }t          j        |dt          j        ¬¦  «        }|| j        k                         d¦  «        }t!          | j        r| j        n| j        ¦  «        }t!          | j        r| j        n| j        ¦  «        }t!          | j        ¦  «        }t!          | j        ¦  «        }|                     |||||| j        ¬	¦  «        }| j        r|                      |¦  «        }n|                      |¦  «        }|                     |||||| j        ¬	¦  «        }||                     |j        ¦  «                             d¦  «        z  }|                     |d
¦  «         t          j        |¦  «        }t          j         |                     d¦  «        |¬¦  «        ||<   ||         }| !                    |||¦  «         "                    d¬¦  «        }|                     |j        ¦  «        S )NrŸ   z‘grouped_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.r¼   r   Úcpur   )ÚbinsÚminre   )rÖ   r^   r¾   rØ   )r½   rÕ   )#r˜   rŠ   rK   r½   rÇ   rÄ   r   Úsortr)   r”   rM   ÚhistcrQ   ÚcumsumrÉ   rÚ   r   rÛ   rÜ   rÝ   rÞ   rß   rà   rá   r4   rr   râ   rã   rä   r^   rå   Ú
empty_likerÊ   rÂ   ræ   )r¥   rÑ   rÒ   rÓ   r{   r½   rç   rè   rÌ   rê   r×   Úexpert_ids_gÚpermÚselected_hidden_states_gÚsample_weights_gÚhistc_inputrÀ   r¿   rë   rì   rí   rî   rï   rR   rð   Úinv_permrñ   s                              r$   Úfp8_grouped_mm_experts_forwardr  ò  s  € ð Ô Ò)Ð)Ý!ðWñ
ô 
ð 	
õ
 2Ñ3Ô3€OàÔ!€FØ× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jð #×*Ò*¨2Ñ.Ô.€NØ×$Ò$ RÑ(Ô(€Jõ œ JÑ/Ô/Ñ€L�$Ø,¨T°YÑ->Ô?ÐØ% dÔ+Ðð
 +1¬+¸Ò*>Ð*>�,×$Ò$Ñ&Ô&Ð&ÀL×DTÒDTÑDVÔDV€KÝœ K°dÔ6FÈAÐSWÔScÐfgÑSgÐhÑhÔhÐÝŒlÐ,°!½5¼;ÐGÑGÔG€Gð " TÔ%5Ò5×@Ò@ÀÑDÔD€Må¨d¬mÐM˜Ô*Ð*ÀÄÑNÔN€IÝ¸d¼mÐg˜tÔ:Ð:ÐQUÔQgÑhÔh€OÝ˜4œ>Ñ*Ô*€KÝ  Ô!9Ñ:Ô:Ðð ×-Ò-Ø ØØØØ+Ø”?ð .ñ ô €Hð „}ð )à×#Ò# HÑ-Ô-ˆˆð —;’;˜xÑ(Ô(ˆð ×-Ò-ØØØØØ+Ø”?ð .ñ ô €Hð Ð.×1Ò1°(´.ÑAÔA×KÒKÈBÑOÔOÑO€Lð ×Ò˜m¨SÑ1Ô1Ð1õ Ô Ñ%Ô%€HÝ”\ $§)¢)¨A¡,¤,°vÐ>Ñ>Ô>€HˆT�NØ Ô)€Lð '×+Ò+¨J¸	À:ÑNÔN×RÒRÐWXÐRÑYÔYÐà×!Ò! -Ô"5Ñ6Ô6Ð6r#   c                  óf   ‡ — e Zd ZU dZddddœiZded<   	 	 	 	 	 d#d$ˆ fd„Zd%d„Zd&d„Z	 d'd(d"„Z	ˆ xZ
S ))Ú
FP8ExpertsFÚdeepgemm_megamoeÚmegamoe_expertsÚmegamoe_router)Úmoe_tp_expertsÚ	ep_routerzdict[str, dict[str, str]]Ú_impl_tp_layer_overridesNr“   r”   Trr   r—   r˜   r™   rš   r›   r   rÛ   c                ó^  •— t          ¦   «                              ¦   «          |du s
J d¦   «         ‚|| _        || _        || _        || _        |j        | _        || _        t          |dd¦  «        | _
        t          |dd¦  «        | _        t          |dd ¦  «        | _        t          |dd ¦  «        | _        t          t          |d	d
¦  «                 | _        t          |dd ¦  «        | _        t          |dd¦  «        dk    }|dk    rt%          ¦   «         nt&          j        }|rt&          j        |ddddœ}	n t,          ||�|d         nd |�|d         nd dœ}	| j        rGt/          | j
        d| j        z  | j        fddi|	¤Ž\  | _        | _        |                      dd ¦  «         nAt/          | j
        | j        | j        fi |	¤Ž\  | _        | _        |                      dd ¦  «         t/          | j
        | j        | j        fi |	¤Ž\  | _        | _        |                      dd ¦  «         | j        dk    rzt?          j         t'          j!        | j
        t&          j        ¬¦  «        ¦  «        | _"        t?          j         t'          j!        | j
        t&          j        ¬¦  «        ¦  «        | _#        d S d S )NFzWFP8Experts does not support bias for now, please open an issue if you want this featureÚnum_local_expertsrQ   Úmoe_intermediate_sizeÚintermediate_sizeÚswiglu_alphaÚswiglu_limitÚhidden_activationÚ
hidden_actÚexpert_dtypeÚfp8Úfp4rž   r   r   é    )rT   rU   rV   rW   rY   r   )rT   rU   rW   rY   rZ   Úgate_up_proj_biasÚup_proj_biasÚdown_proj_biasrŸ   r]   )$r    r¡   Úconfigr›   rÛ   rr   Úhidden_sizerÌ   r˜   r.   rQ   Úintermediate_dimr'   r  r  r   rã   Úlimitr%   r   r…   r†   r¢   rl   rÜ   rÞ   r¤   rÝ   rß   rà   rá   rb   rc   ÚonesÚgate_up_proj_activation_scaleÚdown_proj_activation_scale)r¥   r  rr   r˜   rš   r›   rÛ   Úis_fp4rU   Úalloc_kwargsr¨   s             €r$   r¡   zFP8Experts.__init__d  sè  ø€ õ 	‰Œ×ÒÑÔÐà˜5Ð Ð Ð Øeñ !Ô Ð ð ˆŒØ ˆŒØ ˆŒØ$ˆŒØ Ô,ˆŒØ!2ˆÔÝ& vÐ/BÀMÑRÔRˆÔÝ +¨FÐ4KÐM`Ñ aÔ aˆÔÝ# F¨N¸DÑAÔAˆÔÝ# F¨N¸DÑAÔAˆÔÝ�[¨Ð1DÀlÑSÔSÔTˆŒÝ˜V ^°TÑ:Ô:ˆŒ
õ ˜ °Ñ7Ô7¸5Ò@ˆØ)2°gÒ)=Ð)=Õ#Ñ%Ô%Ð%Å5Ä=ˆØð 	å %¤
Ø$Ø !ØØðð ˆLˆLõ !+Ø$Ø.8Ð.D˜Z¨œ]˜]È$Ø.8Ð.D˜Z¨œ]˜]È$ð	ð ˆLð Œ=ð 		:Ý=OØÔ  ! dÔ&;Ñ";¸T¼_ð>ð >ØYZð>Ø^jð>ð >Ñ:ˆDÔ˜tÔ:ð ×#Ò#Ð$7¸Ñ>Ô>Ð>Ð>å3EØÔ  $Ô"7¸¼ð4ð 4ØLXð4ð 4Ñ0ˆDŒL˜$Ô0ð ×#Ò# N°DÑ9Ô9Ð9å3EØÔ˜dœo¨tÔ/Dð4
ð 4
ØHTð4
ð 4
Ñ0ˆŒ˜Ô0ð 	×ÒÐ 0°$Ñ7Ô7Ð7àÔ! XÒ-Ð-Ý13´½e¼jÈÔIYÕafÔanÐ>oÑ>oÔ>oÑ1pÔ1pˆDÔ.Ý.0¬l½5¼:ÀdÔFVÕ^cÔ^kÐ;lÑ;lÔ;lÑ.mÔ.mˆDÔ+Ð+Ð+ð .Ð-r#   Úgate_uprp   r   c                óÄ  — |                      dd¬¦  «        \  }}| j        �d|                     | j        ¬¦  «        }|                     | j         | j        ¬¦  «        }|t	          j        || j        z  ¦  «        z  }|dz   |z  S | j        �=|                     | j        ¬¦  «        }|                     | j         | j        ¬¦  «        }|                      |¦  «        |z  S )Nr   r¼   rÕ   )re   ©rö   re   r�   )Úchunkr  Úclampr  r   Úsigmoidr  rã   )r¥   r"  ÚgateÚupÚglus        r$   râ   zFP8Experts._apply_gateª  sÒ   € Ø—=’= ¨�=Ñ+Ô+‰ˆˆbØÔÐ(à—:’: $Ô"3�:Ñ4Ô4ˆDØ—’˜tÔ0Ð0°dÔ6G�ÑHÔHˆBØ�œ t¨dÔ.?Ñ'?Ñ@Ô@Ñ@ˆCØ˜‘H Ñ#Ð#ØŒZÐ#Ø—:’: $¤*�:Ñ-Ô-ˆDØ—’˜tœz˜k¨t¬z�Ñ:Ô:ˆBØ�{Š{˜4Ñ Ô  2Ñ%Ð%r#   rÑ   rÒ   rÓ   c                óê  — t          j        |t           j        ¬¦  «        }t          j        ¦   «         5  t           j        j                             || j        dz   ¬¦  «        }|                     ddd¦  «        }t          j	        | 
                    d¬¦  «        d¦  «                             d¬	¦  «                             d
¦  «        }d d d ¦  «         n# 1 swxY w Y   |D �]v}|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
| j        dk    r| j        |         nd }|                      |
| j        r| j        |         n| j        |         | j        r| j        |         n| j        |         |¬¦  «        }| j        r|                      |¦  «        n|                      |¦  «        }| j        dk    r| j        |         nd }|                      || j        |         | j        |         |¬¦  «        }||	|d f         }||                     |j        ¦  «        z  }|                     d|	|                     |j        ¦  «        ¦  «         �Œx|                     |j        ¦  «        S )Nr]   r   )Únum_classesr   r   )r¼   r»   rÕ   F)Úas_tupler¼   rŸ   ry   )r   Ú
zeros_liker…   Úno_gradrb   r   Úone_hotrQ   ÚpermuteÚgreaterræ   ÚnonzerorÂ   Úwherer˜   r  r¬   rÛ   rÜ   rÝ   rÞ   rß   râ   rã   r  rà   rá   rä   r^   Ú
index_add_)r¥   rÑ   rÒ   rÓ   rñ   Úexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgate_up_act_scalerR   Údown_act_scaleÚrouting_weightsrð   s                   r$   r¯   zFP8Experts.forward·  sË  € õ
 $Ô.¨}ÅEÄMÐRÑRÔRÐåŒ]‰_Œ_ð 	jð 	jÝœ(Ô-×5Ò5°kÈtÔO_ÐbcÑOcÐ5ÑdÔdˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÐZ_ÐPÑ`Ô`×eÒeÐfhÑiÔiˆJð	jð 	jð 	jñ 	jô 	jð 	jð 	jð 	jð 	jð 	jð 	jøøøð 	jð 	jð 	jð 	jð
 %ð 	eñ 	eˆJØ˜TÔ-Ò-Ð-Øå#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMàBFÔBXÐ\dÒBdÐBd�Ô2°:Ô>Ð>Ðjnð ð —{’{ØØ15´Ð\�Ô! *Ô-Ð-ÀDÄLÐQ[ÔD\Ø;?¼=Ðp�Ô+¨JÔ7Ð7ÈdÔNdÐeoÔNpØ!2ð	 #ñ ô ˆHð 6:´]Ð]�t×'Ò'¨Ñ1Ô1Ð1ÈÏÊÐT\ÑH]ÔH]ˆHà?CÔ?UÐYaÒ?aÐ?a�Ô/°
Ô;Ð;Ðgkð ð —{’{ØØ”˜zÔ*ØÔ(¨Ô4Ø!/ð	 #ñ ô ˆHð ,¨I°yÀ$Ð,FÔGˆOØ# o×&8Ò&8¸¼Ñ&HÔ&HÑHˆLØ×*Ò*¨1¨i¸¿ºÐI\ÔIbÑ9cÔ9cÑdÔdÐdÑdØ"×%Ò% mÔ&9Ñ:Ô:Ð:s   ´BCÃCÃCro   rg   rq   rv   ru   c                óž   — |                      ¦   «         dk    rt          j        ||d ¦  «        S t          |||| j        || j         ¬¦  «        S )Nr   )rv   r~   )rª   r«   r¬   r�   rr   r­   )r¥   ro   rg   rq   rv   s        r$   r¬   zFP8Experts.lineará  s`   € ð ×ÒÑ Ô  1Ò$Ð$Ý”8˜E 6¨4Ñ0Ô0Ð0åØØØØŒOØ-Ø#Ô6Ð6ð
ñ 
ô 
ð 	
r#   )Nr“   r”   FT)
rr   r—   r˜   r™   rš   r™   r›   r   rÛ   r   )r"  rp   r   rp   )rÑ   rp   rÒ   rp   rÓ   rp   r   rp   r@   )
ro   rp   rg   rp   rq   rp   rv   ru   r   rp   )r*   r5   r6   r­   r	  r8   r¡   râ   r¯   r¬   r±   r²   s   @r$   r  r  P  sä   ø€ € € € € € ð Ðð 	Ø/Ø)ð
ð 
ð;Ðð ð ð ñ ð .2Ø!*Ø ØØðDnð Dnð Dnð Dnð Dnð Dnð DnðL&ð &ð &ð &ð(;ð (;ð (;ð (;ð^ 15ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r#   r  c                  ó    — e Zd ZdZeeeedœZdS )ÚFP8ExpertsInterfacez?Interface for registering custom FP8 experts forward functions.)Ú
batched_mmÚ
grouped_mmÚdeepgemmr  N)	r*   r5   r6   r7   rò   r  r   r   Ú_global_mappingr"   r#   r$   rA  rA  õ  s.   € € € € € ØIÐIð 5Ø4Ø4ØDð	ð €O€O€Or#   rA  Úmodelú	nn.Modulec                ó„  — d„ |                       ¦   «         D ¦   «         }t          ¦   «         }|D ]U}t          |                     ¦   «         d¦  «        }|�/|j        j        dk    r|                     |j        j        ¦  «         ŒVt          |¦  «        dk    rdS |D ]	}d|_	        Œ
t                               d¦  «         dS )a4  Internal, temporary helper (not public API): flag every FP8 module to skip DeepGEMM when the
    model spans >1 CUDA device in one process.

    DeepGEMM loads each kernel via `cuKernelGetFunction`, which binds the `CUfunction` handle to the
    CUDA context live at load time; driving that cached handle from another device launches it against
    the wrong context and produces garbage. (Build-time fix: compile DeepGEMM with
    `DG_JIT_USE_RUNTIME_API=1` for a context-free `cudaKernel_t` loader; until our wheel picks that up
    we avoid single-process multi-device.) Setting `_deepgemm_disabled` routes both the linear and
    experts paths through Triton/grouped_mm. A model that fits on one device keeps DeepGEMM even with
    other GPUs visible; TP/EP put one device per process, so this is a no-op there.
    c                óJ   — g | ] }t          |t          t          f¦  «        ¯|‘Œ!S r"   )Ú
isinstancer’   r  )rA   Úms     r$   rC   z5_disable_deepgemm_on_multi_device.<locals>.<listcomp>  s,   € ÐXÐXÐX˜µ¸AÅ	Í:Ð?VÑ1WÔ1WÐX�1ÐXÐXÐXr#   NÚcudar   TaB  This FP8 model spans multiple CUDA devices in one process; routing its FP8 linear and experts layers through Triton/grouped_mm instead of DeepGEMM (DeepGEMM's cached kernels are bound to a single CUDA context and corrupt across devices). Run tensor/expert parallel (one device per process) to use the faster DeepGEMM path.)ÚmodulesÚsetÚnextÚ
parametersr½   r)   ÚaddÚindexÚlenr­   rŒ   r�   )rF  Úfp8_modulesÚcuda_devicesrK  Úparams        r$   Ú!_disable_deepgemm_on_multi_devicerW    s×   € ð YÐX˜eŸmšm™oœoÐXÑXÔX€KÝ‘5”5€LØð 1ð 1ˆÝ�Q—\’\‘^”^ TÑ*Ô*ˆØÐ ¤Ô!2°fÒ!<Ð!<Ø×Ò˜Uœ\Ô/Ñ0Ô0Ð0øÝ
ˆ<ÑÔ˜AÒÐØˆØð $ð $ˆØ#ˆÔÐÝ
×Òð	4ñô ð ð ð r#   FÚmodules_to_not_convertúlist[str] | Nonec                óð  — |j         r| S d}|                      ¦   «         D �]¸\  }}t          ||¦  «        sŒd}t          j        d¦  «        5  |                     d¦  «        rˆt          |dd¦  «        }t          |dd¦  «        }	t          |d| j                             ¦   «         ¦  «        }
t          t          t          |	|¬	¦  «        } ||
|j        |j        |j        |	|¬
¦  «        }n¼t          |¦  «        t           j        u r6t%          |j        |j        |j        |j        |j        |j        du¬¦  «        }nkt-          |t           j        ¦  «        rQdt          |¦  «        j        v r;t1          |j        |j        |j        |j        |j        |j        |j        du¬¦  «        }|�|                      ||¦  «         d}ddd¦  «         n# 1 swxY w Y   �Œº|st6                               d¦  «         | S )a®  
    A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.

    Parameters:
        model (`torch.nn.Module`):
            Input model or `torch.nn.Module` as the function is run recursively.
        modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
            Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons.
        quantization_config (`FineGrainedFP8Config`):
            The quantization config object that contains the quantization parameters.
        pre_quantized (`bool`, defaults to `False`):
            Whether the model is pre-quantized or not
    FNÚmetaz.expertsrÛ   Tr›   r  )Úexperts_classÚexperts_interfacer›   rÛ   )r  rr   r˜   rš   r›   rÛ   r¸   ÚGroupedLinear)rµ   r–   r¶   rr   r˜   rš   r›   z�You are loading your model using fp8 but no linear modules were found in your model. Please double check your model architecture.)Ú
dequantizeÚnamed_modulesr   r   r½   Úendswithr'   r  Úget_text_configr   r  ÚALL_FP8_EXPERTS_FUNCTIONSÚweight_block_sizer˜   rš   r)   rb   ÚLinearr’   r•   r–   rt   rJ  r*   r´   r¶   Úset_submodulerŒ   Úwarning)rF  rX  Úquantization_configÚpre_quantizedÚhas_been_replacedÚmodule_nameÚmoduleÚ
new_modulerÛ   r›   r  Ú	new_classs               r$   Úreplace_with_fp8_linearro  !  sx  € ð" Ô%ð ØˆàÐØ$×2Ò2Ñ4Ô4ð 4)ñ 4)Ñˆ�VÝ$ [Ð2HÑIÔIð 	Øàˆ
ÝŒ\˜&Ñ!Ô!ð /	)ð /	)Ø×#Ò# JÑ/Ô/ð +Ý" 6¨:°tÑ<Ô<�Ý" 6¨:°uÑ=Ô=�Ý  ¨°5´<×3OÒ3OÑ3QÔ3QÑRÔR�Ý6Ý",Ý&?Ø%Ø%ð	ñ ô �	ð '˜YØ!Ø2ÔDØ&9Ô&KØ1Ô;Ø%Ø%ðñ ô �
�
õ �f‘”¥¤Ð*Ð*å&Ø &Ô 2Ø!'Ô!4Ø2ÔDØ&9Ô&KØ1Ô;Ø#œ[°Ð4ðñ ô �
�
õ ˜F¥B¤IÑ.Ô.ð °?ÅdÈ6ÁlÄlÔF[Ð3[Ð3[õ .Ø*0Ô*<Ø!'Ô!4Ø#œ_Ø2ÔDØ&9Ô&KØ1Ô;Ø#œ[°Ð4ðñ ô �
ð Ð%Ø×#Ò# K°Ñ<Ô<Ð<Ø$(Ð!ð_/	)ð /	)ð /	)ñ /	)ô /	)ð /	)ð /	)ð /	)ð /	)ð /	)ð /	)øøøð /	)ð /	)ð /	)ð /	)ùðb ð 
Ý�Šð<ñ	
ô 	
ð 	
ð €Ls   ÁE4GÇG	ÇG	c                  óH   — e Zd ZdZd„ Zdd„Zdd„Zdd„Zedd„¦   «         Z	dS )ÚFp8Quantizez^
    A quantization operation that creates two tensors, weight and scale out of a weight.
    c                ó   — || _         d S r@   ©Úhf_quantizer©r¥   rt  s     r$   r¡   zFp8Quantize.__init__y  ó   € Ø(ˆÔÐÐr#   Úvaluerp   r   útuple[int, int]c                ó(  — d }| j         j        �Zt          | j         j        t          ¦  «        r | j         j                             d¦  «        }nt          | j         j        dd ¦  «        }|€|j        d         |j        d         f}t          |¦  «        S )Nrd  r»   r¼   )rt  rh  rJ  Údictr‰   r'   rÁ   Útuple)r¥   rw  rr   s      r$   Ú_resolve_block_sizezFp8Quantize._resolve_block_size|  s‹   € Øˆ
ØÔÔ0Ð<Ý˜$Ô+Ô?ÅÑFÔFð gØ!Ô.ÔB×FÒFÐGZÑ[Ô[�
�
å$ TÔ%6Ô%JÐL_ÐaeÑfÔf�
ØÐØœ+ bœ/¨5¬;°r¬?Ð;ˆJÝ�ZÑ Ô Ð r#   Úkeyr™   údict[str, torch.Tensor]c                ó°  — |j         dk     r||iS |                      |¦  «        \  }}|j        d         |j        d         }}||z  dk    s	||z  dk    r||iS |j        d d…         }||z  }||z  }	|j        }
|                     t          j        ¦  «        } |j        g |¢|‘|‘|	‘|‘R Ž }|                     ¦   «                              d¬¦  «        }t	          j	        |dk    |t	          j
        |¦  «        ¦  «        }t          |z  }t	          j	        |dk    |t	          j
        |¦  «        ¦  «        }d|z                       t          j        ¦  «        }| j        j        j        dk    r±t	          j        d	t	          j        t	          j        |                     t	          j        t          j        ¦  «        j        ¬
¦  «        ¦  «        ¦  «        ¦  «        }|                     t+          ¦   «         ¦  «        }d|                     t          j        ¦  «        z  }|                     d¦  «                             d¦  «        }||z  }t	          j        |t.          t          ¬¦  «                             t0          ¦  «        }|                     |
¦  «        }|                     d¦  «        r|                     dd¦  «        d         dz   n|dz   }||||iS )Nr   r»   r¼   r   )éýÿÿÿr¼   rÕ   r�   rž   ç       @)rö   r€  r$  ú.weightú.r   ú.weight_scale_invÚ
_scale_inv)Úndimr|  rÁ   rä   r   r…   rÄ   ÚabsÚamaxr4  Ú	ones_likeÚ_FP8_MAXrt  rh  rš   ÚpowÚceilÚlog2r&  ÚfinfoÚtinyr%   rÚ   Ú_FP8_MINr¢   ra  Úrsplit)r¥   r}  rw  Úblock_mÚblock_nÚrowsÚcolsÚleading_shapeÚ
rows_tilesÚ
cols_tilesÚoriginal_shapeÚ
value_fp32ÚreshapedÚmax_absÚsafe_max_absÚscalesÚ
inv_scalesÚscales_broadcastÚscaledÚ	quantizedÚ	scale_keys                        r$   Ú_quantize_onezFp8Quantize._quantize_one‡  s™  € ð Œ:˜Š>ˆ>Ø˜�<ÐØ×3Ò3°EÑ:Ô:Ñˆ�Ø”[ ”_ e¤k°"¤oˆdˆØ�'‰>˜QÒÐ $¨¡.°AÒ"5Ð"5Ø˜�<Ðð œ C R CÔ(ˆØ˜W‘_ˆ
Ø˜W‘_ˆ
ØœˆØ—X’X�eœmÑ,Ô,ˆ
à%�:Ô%Ð_ }Ð_°jÐ_À'Ð_È:Ð_ÐW^Ð_Ð_Ð_ˆà—,’,‘.”.×%Ò%¨(Ð%Ñ3Ô3ˆÝ”{ 7¨Q¢;°½¼ÈÑ9QÔ9QÑRÔRˆå˜LÑ(ˆÝ”˜W qš[¨&µ%´/À&Ñ2IÔ2IÑJÔJˆØ˜F‘l×&Ò&¥u¤}Ñ5Ô5ˆ
ð ÔÔ0Ô:¸gÒEÐEÝœ 3­¬
µ5´:¸j×>NÒ>NÕSXÔS^Õ_dÔ_lÑSmÔSmÔSrÐ>NÑ>sÔ>sÑ3tÔ3tÑ(uÔ(uÑvÔvˆJØ#ŸšÕ'7Ñ'9Ô'9Ñ:Ô:ˆJØ˜:Ÿ=š=­¬Ñ7Ô7Ñ7ˆFà!×+Ò+¨BÑ/Ô/×9Ò9¸"Ñ=Ô=ÐØÐ,Ñ,ˆÝ”K ­H½(ÐCÑCÔC×FÒFÅzÑRÔRˆ	Ø×%Ò% nÑ5Ô5ˆ	ØCFÇ<Â<ÐPYÑCZÔCZÐr�C—J’J˜s AÑ&Ô& qÔ)Ð,?Ñ?Ð?Ð`cÐfrÑ`rˆ	Ø�Y 	¨:Ð6Ð6r#   Ú
input_dictc                óÎ   — i }|                      ¦   «         D ]M\  }}t          |t          ¦  «        r|d         n|}|                     |                      ||¦  «        ¦  «         ŒN|S )Nr   )ÚitemsrJ  ÚlistÚupdater¤  )r¥   r¥  ÚkwargsÚresultr}  rw  r£   s          r$   ÚconvertzFp8Quantize.convert¯  sn   € ð +-ˆØ$×*Ò*Ñ,Ô,ð 	;ð 	;‰JˆC�Ý!+¨Eµ4Ñ!8Ô!8ÐC�U˜1”X�X¸eˆFØ�MŠM˜$×,Ò,¨S°&Ñ9Ô9Ñ:Ô:Ð:Ð:Øˆr#   r	   c                ó*   — t          | j        ¦  «        S r@   )ÚFp8Dequantizert  ©r¥   s    r$   Ú
reverse_opzFp8Quantize.reverse_op¹  s   € å˜TÔ.Ñ/Ô/Ð/r#   N)rw  rp   r   rx  )r}  r™   rw  rp   r   r~  )r¥  rp   r   r~  ©r   r	   )
r*   r5   r6   r7   r¡   r|  r¤  r¬  Úpropertyr°  r"   r#   r$   rq  rq  t  sŠ   € € € € € ðð ð)ð )ð )ð	!ð 	!ð 	!ð 	!ð&7ð &7ð &7ð &7ðPð ð ð ð ð0ð 0ð 0ñ „Xð0ð 0ð 0r#   rq  c                  óf   — e Zd ZdZd„ Zdd„ZdZdd
„Z	 ddd„Zdd„Z		 	 d d!d„Z
ed"d„¦   «         ZdS )#r®  u¬  Dequantize FP8 weights using their per-block ``weight_scale_inv``.

    Designed to run as the *first* op in any :class:`WeightConverter` chain when
    loading with ``dequantize=True`` â€” :meth:`update_weight_conversions` on the
    FP8 quantizer attaches it to each existing model-specific converter so that
    per-expert (weight, scale) pairs are folded into full-precision tensors before
    the chain's merge / concat ops collapse the per-expert structure.

    Pattern semantics
        Input ``input_dict`` carries one entry per source pattern; each value is a
        list of tensors (one per ``*`` match). For every weight pattern that has a
        sibling ``*.weight_scale_inv`` pattern in the dict, this op pairs them up by
        index, dequantizes per-pair, and emits the dequantized list under the
        original *weight* key. Scale entries are dropped from the output so the
        remaining ops only see weights.
    c                ó   — || _         d S r@   rs  ru  s     r$   r¡   zFp8Dequantize.__init__Ð  rv  r#   Úweight_patternr™   r   c                óØ   — |                      d¦  «        }|r
|d d…         n|}|                      d¦  «        r|d t          d¦  «         …         dz   }n|dk    rd}n|dz   }|r|dz   n|S )Nú$r¼   r‚  r„  rg   rq   r…  )ra  rS  )r¥   rµ  ÚanchoredÚbaseÚscales        r$   Ú_scale_pattern_forz Fp8Dequantize._scale_pattern_forÓ  s�   € à!×*Ò*¨3Ñ/Ô/ˆØ&.ÐBˆ~˜c˜r˜cÔ"Ð"°NˆØ�=Š=˜Ñ#Ô#ð 	(ØÐ*�C 	™NœN˜?Ð*Ô+Ð.AÑAˆEˆEØ�XÒÐØ&ˆEˆEà˜<Ñ'ˆEØ&Ð1ˆu�s‰{ˆ{¨EÐ1r#   )rØ   g      à?r�   g      ø?r�  g      @g      @g      @g       €g      à¿g      ð¿g      ø¿g       Àg      Àg      Àg      ÀÚpackedrp   c                ó¸  — t          j        | j        t           j        |j        ¬¦  «        }|                     ¦   «                              t           j        ¦  «        }|dz                       ¦   «         }|dz	  dz                       ¦   «         }t          j	        ||         ||         gd¬¦  «        } |j
        g |j        dd…         ¢d|j        d         z  ‘R Ž S )uR   Two ``e2m1`` FP4 values per byte â†’ float32 tensor twice as wide on the last dim.)r^   r½   é   é   r¼   rÕ   Nr   )r   r£   Ú_FP4_E2M1_LUTr…   r½   Ú
contiguousrÂ   Úuint8ÚlongÚstackrÄ   rÁ   )r¥   r¼  ÚlutÚu8ÚlowÚhighÚunpackeds          r$   Ú_unpack_fp4zFp8Dequantize._unpack_fp4ä  sÂ   € åŒl˜4Ô-µU´]È6Ì=ÐYÑYÔYˆØ×ÒÑ Ô ×%Ò%¥e¤kÑ2Ô2ˆØ�C‰x�oŠoÑÔˆØ�q‘˜C‘×%Ò%Ñ'Ô'ˆÝ”;  C¤¨#¨d¬)Ð4¸"Ð=Ñ=Ô=ˆØˆxÔÐI ¤¨c¨r¨cÔ!2ÐI°A¸¼ÀRÔ8HÑ4HÐIÐIÐIÐIr#   Nr¢  rž  rm   rw   c                ó  — t          t          dd ¦  «        }|j        t          j        k    s|�!|j        |k    r|                      |¦  «        }n|                     t          j        ¦  «        }|j        dd …         \  }}	 |j        dd …         \  }}	n# t          $ r d\  }}	Y nw xY w||z  s||	z  rt          d|› d|› d|› d|	› d�	¦  «        ‚||z  }
||	z  }|€7|j        j
        r|                     ¦   «         dk    r|j        nt          j        }|j        t          j        k    r5|                     t          j        ¦  «        d	z
                       ¦   «         }n|                     t          j        ¦  «        }|j        }|                     d
||
|	|¦  «        }|                     d
||	¦  «                             d
¦  «                             d¦  «        }||z                       |¦  «                             |¦  «        S )NÚfloat4_e2m1fn_x2r»   )r   r   zWeight shape (rD   z) not divisible by scale grid (z).r   g     À_@r¼   )r'   r   r^   r†   rÊ  rä   r…   rÁ   Ú	Exceptionr‹   rd   rª   Úbfloat16rÂ  Úexp2rÄ   rÚ   )r¥   r¢  rž  rm   Ú	fp4_dtypeÚquantized_fp32r”  r•  Ú
scale_rowsÚ
scale_colsr’  r“  Ús_fp32r™  ÚqÚss                   r$   Ú_dequantize_onezFp8Dequantize._dequantize_oneí  s   € õ
 �EÐ#5°tÑ<Ô<ˆ	ØŒ?�eœjÒ(Ð(¨YÐ-BÀyÄÐZcÒGcÐGcØ!×-Ò-¨iÑ8Ô8ˆNˆNà&Ÿ\š\­%¬-Ñ8Ô8ˆNØ#Ô)¨"¨#¨#Ô.‰
ˆˆdð	*Ø%+¤\°"°#°#Ô%6Ñ"ˆJ˜
˜
øÝð 	*ð 	*ð 	*à%)Ñ"ˆJ˜
˜
˜
ð	*øøøð �*Ñð 	  zÑ 1ð 	ÝØj ÐjÐj¨ÐjÐjÈjÐjÐjÐ\fÐjÐjÐjñô ð ð ˜*Ñ$ˆØ˜*Ñ$ˆð
 Ðà &¤Ô >ÐqÀ6×CVÒCVÑCXÔCXÐ\]ÒC]ÐC]�”�ÕchÔcqð ð Œ<�5œ;Ò&Ð&Ø—i’i¥¤Ñ.Ô.°Ñ6×<Ò<Ñ>Ô>ˆFˆFà—Y’Y�uœ}Ñ-Ô-ˆFØ'Ô-ˆØ×"Ò" 2 z°7¸JÈÑPÔPˆØ�NŠN˜2˜z¨:Ñ6Ô6×@Ò@ÀÑDÔD×NÒNÈqÑQÔQˆØ�A‘�zŠz˜,Ñ'Ô'×/Ò/°Ñ?Ô?Ð?s   ÂB ÂB&Â%B&rF  útorch.nn.Module | NoneÚfull_layer_nameú
str | Nonec                óx   — |�|€d S t          ||¦  «        \  }}t          ||d ¦  «        }t          |dd ¦  «        S )Nr^   )r
   r'   )r¥   rF  rÙ  rl  Útensor_namerV  s         r$   Ú_get_target_dtypezFp8Dequantize._get_target_dtype  sH   € Øˆ=˜OÐ3Ø�4Ý2°5¸/ÑJÔJÑˆ�Ý˜ ¨TÑ2Ô2ˆÝ�u˜g tÑ,Ô,Ð,r#   r¥  ú,dict[str, list[torch.Tensor] | torch.Tensor]c                ó  ‡ ‡— ‰                       ||¦  «        Šd|v rv|�|nd}|d         }t          |t          ¦  «        r|d         n|}d|v rA|d         }t          |t          ¦  «        r|d         n|}|‰                      ||‰¬¦  «        iS ||iS i }|                     ¦   «         D ]Ü\  }	}
d|	v sd|	v rŒ‰                      |	¦  «        }||vr|
||	<   Œ-t          |
t          ¦  «        r|
n|
g}||         }t          |t          ¦  «        r|n|g}t          |¦  «        t          |¦  «        k    r3t          d|	› dt          |¦  «        › d	t          |¦  «        › d
�¦  «        ‚ˆˆ fd„t          ||¦  «        D ¦   «         ||	<   ŒÝ|S )Nzweight$rg   r   rq   ©rm   rv   z/Fp8Dequantize: weight/scale count mismatch for z (z weights vs z	 scales).c                óF   •— g | ]\  }}‰                      ||‰¬ ¦  «        ‘ŒS )rà  )r×  )rA   rÍ   rÖ  rm   r¥   s      €€r$   rC   z)Fp8Dequantize.convert.<locals>.<listcomp>L  s5   ø€ ÐrÐrÐrÑUYÐUVÐXY˜4×/Ò/°°1À<Ð/ÑPÔPÐrÐrÐrr#   )	rÝ  rJ  r¨  r×  r§  r»  rS  r‹   Úzip)r¥   r¥  rÙ  rF  rª  Ú
target_keyr¢  rž  r«  r}  rw  r£  Úweightsrm   s   `            @r$   r¬  zFp8Dequantize.convert!  s  øø€ ð ×-Ò-¨e°_ÑEÔEˆð
 ˜
Ð"Ð"ð -<Ð,G˜˜ÈXˆJØ" 9Ô-ˆIÝ(2°9½dÑ(CÔ(CÐR˜	 !œ˜ÈˆIØ! ZÐ/Ð/Ø#Ð$6Ô7�Ý&0°½Ñ&>Ô&>ÐJ˜ œ˜ÀF�Ø" D×$8Ò$8¸ÀFÐYeÐ$8Ñ$fÔ$fÐgÐgØ 	Ð*Ð*ð @BˆØ$×*Ò*Ñ,Ô,ð 	sð 	s‰JˆC�Ø! SÐ(Ð(Ð,>À#Ð,EÐ,EØØ×/Ò/°Ñ4Ô4ˆIØ 
Ð*Ð*à#��s‘ØÝ)¨%µÑ6Ô6ÐC�e�e¸U¸GˆGØ 	Ô*ˆFÝ)¨&µ$Ñ7Ô7ÐE�V�V¸f¸XˆFÝ�7‰|Œ|�s 6™{œ{Ò*Ð*Ý ðIÀcð Ið IÝ˜G™œðIð IÝ25°f±+´+ðIð Ið Iñô ð ð sÐrÐrÐrÐrÕ]`ÐahÐjpÑ]qÔ]qÐrÑrÔrˆF�3‰KˆKØˆr#   r	   c                ó*   — t          | j        ¦  «        S r@   )rq  rt  r¯  s    r$   r°  zFp8Dequantize.reverse_opO  s   € õ
 ˜4Ô,Ñ-Ô-Ð-r#   )rµ  r™   r   r™   )r¼  rp   r   rp   r@   )r¢  rp   rž  rp   rm   rw   r   rp   )rF  rØ  rÙ  rÚ  r   rw   )NN)r¥  rÞ  rÙ  rÚ  rF  rØ  r   rÞ  r±  )r*   r5   r6   r7   r¡   r»  rÀ  rÊ  r×  rÝ  r¬  r²  r°  r"   r#   r$   r®  r®  ¾  sÝ   € € € € € ðð ð")ð )ð )ð
2ð 
2ð 
2ð 
2ð m€MðJð Jð Jð Jð aeð+@ð +@ð +@ð +@ð +@ðZ-ð -ð -ð -ð '+Ø(,ð	,ð ,ð ,ð ,ð ,ð\ ð.ð .ð .ñ „Xð.ð .ð .r#   r®  )r   r   )r   r:   )r   r1   )rL   rM   rN   rM   r   rM   )r   NNr   )rQ   rM   rR   rM   rS   rM   rT   r   rU   r   rV   rM   rW   rX   rY   rX   rZ   rM   r   r[   )NNNN)ro   rp   rg   rp   rq   rp   rr   rs   rt   ru   rv   ru   rm   rw   r   rp   )NNNNT)ro   rp   rg   rp   rq   rp   rr   rs   rt   ru   rv   ru   rm   rw   r~   r   r   rp   )
r¥   rÐ   rÑ   rp   rÒ   rp   rÓ   rp   r   rp   )rF  rG  r   r:   )NNF)rX  rY  )MÚ
__future__r   Ú	functoolsr‡   Úcollections.abcr   Údataclassesr   r   Útorch.nnrb   r   r«   Úactivationsr   Úcore_model_loadingr	   Úquantizers.quantizers_utilsr
   r   Úutilsr   Úutils.deprecationr   Úutils.import_utilsr   rD  r   r   r   r   r   Úhub_kernelsr   r   Úmoer   r   Útensor_parallelr   Ú
get_loggerr*   rŒ   Úfloat8_e4m3fnr¢   rŽ  rö   r�  re   rŠ  Úcacher%   r.   r1   r9   r8   Ú_dynamoÚallow_in_graphrI   rK   rP   rl   r}   r�   re  r’   r´   rò   r  ÚModuler  rA  rc  rW  ro  rq  r®  r"   r#   r$   ú<module>rú     s  ðð #Ð "Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø 	€	€	€	Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $à  Ð  Ð  Ð  Ð  Ð  Ø .Ð .Ð .Ð .Ð .Ð .Ø UÐ UÐ UÐ UÐ UÐ UÐ UÐ UØ Ð Ð Ð Ð Ð Ø /Ð /Ð /Ð /Ð /Ð /Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5ðð ð ð ð ð ð ð ð ð ð ð ð ð ð DÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ =Ð =Ð =Ð =Ð =Ð =Ð =Ð =Ø %Ð %Ð %Ð %Ð %Ð %ð 
ˆÔ	˜HÑ	%Ô	%€ð Ô €
Øˆ5Œ;�zÑ"Ô"Ô&€Øˆ5Œ;�zÑ"Ô"Ô&€ð „ð ð  ð  ñ „ð ðGð Gð Gð €�$ÐÑÔðð ð ð ð ñ ô ñ Ôðð +/Ð Ð .Ð .Ð .Ñ .ð „Ôð3ð 3ð 3ñ Ôð3ðlð ð ð ð
ð ð ð ð Ø Ø Øðð ð ð ð ð8 €�¨Ð1Ñ1Ô1ð
 $(Ø $Ø,0Ø'+ðð ð ð ñ 2Ô1ðð< €�¨Ð1Ñ1Ô1ð
 $(Ø $Ø,0Ø'+ØðKgð Kgð Kgð Kgñ 2Ô1ðKgð\;
ð ;
ð ;
ð ;
ð ;
�”	ñ ;
ô ;
ð ;
ð|Dð Dð Dð Dð D�yñ Dô Dð DðNG7ð G7ð G7ð G7ðT[7ð [7ð [7ð [7ð|b
ð b
ð b
ð b
ð b
�”ñ b
ô b
ð b
ðJð ð ð ð Ð*ñ ô ð ð 0Ð/Ñ1Ô1Ð ðð ð ð ð> ejðPð Pð Pð Pð PðfG0ð G0ð G0ð G0ð G0�-ñ G0ô G0ð G0ðTV.ð V.ð V.ð V.ð V.�Mñ V.ô V.ð V.ð V.ð V.r#   