§
    [�’jè®  ã                  óŒ  — U d Z ddlmZ ddlZddlZddlZddlZddlZddlm	Z	 ddl
mZ ddlZddlmZ ddlmZ dd	lmZmZmZ d
dlmZmZ d
dlmZ  ej        e¦  «        Z ed¬¦  «         G d„ d¦  «        ¦   «         Zej        d^d„¦   «         Zej        d_d„¦   «         Z ej!        j"        d`d„¦   «         Z#ej!        j"        dadbd„¦   «         Z$da%de&d<   ej!        j'        dcd„¦   «         Z(ddd„Z)ded#„Z*dfd%„Z+dgdhd)„Z,did.„Z-djd5„Z.dkd9„Z/dld;„Z0dmdA„Z1dndK„Z2 edLdM¬N¦  «        	 	 	 	 dodpdT„¦   «         Z3dqdW„Z4dqdX„Z5drdZ„Z6	 dgdsd]„Z7dS )tuS  DeepGEMM integration: fused grouped GEMM kernels from `kernels-community/deep-gemm`.

Provides:
- `deepgemm_bf16_experts_forward`: BF16 M-grouped experts forward.
- `deepgemm_fp8_fp4_linear`: end-to-end FP8/FP4 linear (output dtype follows the input).
- `deepgemm_fp8_fp4_experts_forward`: FP8 (or FP4 on SM100+) M-grouped experts forward.
- `deepgemm_fp8_fp4_megamoe_experts_forward`: FP8xFP4 Mega MoE forward (SM100+).

Requirements: CUDA, Hopper (SM90+), CUDA runtime â‰¥ 12.3, kernels-community/deep-gemm
â‰¥ 2.5 (Mega MoE symbols required). Mega MoE additionally needs SM100+ at call time.
é    )ÚannotationsN)ÚCallable)Ú	dataclassé   )Úlogging)Údeprecate_kwarg)Úis_kernels_availableÚmaybe_import_errorÚresolve_internal_importé   )Ú_MISSING_KERNELS_MESSAGEÚlazy_load_kernel)Úto_localT)Úfrozenc                  ó‚   — e Zd ZU dZded<   ded<   ded<   ded<   ded<   ded<   ded	<   ded
<   ded<   ded<   ded<   dS )ÚDeepGEMMz>Curated entry points exposed by `kernels-community/deep-gemm`.r   Úfp8_fp4_matmulÚgrouped_fp8_fp4_matmul_ntÚgrouped_fp8_fp4_matmul_nnÚgrouped_bf16_matmul_ntÚgrouped_bf16_matmul_nnÚper_token_cast_to_fp8Ú!transform_sf_into_required_layoutÚtransform_weights_for_mega_moeÚget_symm_buffer_for_mega_moeÚfp8_fp4_mega_moeÚintÚm_alignmentN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__© ó    úPC:\Projects\DSCons\.venv\Lib\site-packages\transformers\integrations\deepgemm.pyr   r   7   sž   € € € € € € àHÐHàÐÐÑØ'Ð'Ð'Ñ'Ø'Ð'Ð'Ñ'Ø$Ð$Ð$Ñ$Ø$Ð$Ð$Ñ$Ø#Ð#Ð#Ñ#Ø/Ð/Ð/Ñ/Ø,Ð,Ð,Ñ,Ø*Ð*Ð*Ñ*ØÐÐÑð ÐÐÑÐÐr%   r   Úreturnú
str | Nonec                 óp  — t           j                             d¦  «        pt           j                             d¦  «        } | r| S t          j        d¦  «        }|r<t           j                             t           j                             |¦  «        ¦  «        S t           j                             d¦  «        rdS dS )u‚  Resolve the CUDA toolkit root the way DeepGEMM's JIT does:
    ``CUDA_HOME`` â†’ ``CUDA_PATH`` â†’ dir of ``which nvcc`` â†’ ``/usr/local/cuda`` (``None`` if none found).

    Mirrors DeepGEMM's own ``_find_cuda_home`` so we agree on the path it will actually use, rather than
    reusing ``torch.utils.cpp_extension.CUDA_HOME`` whose resolution inits a CUDA context (fork-unsafe).
    Ú	CUDA_HOMEÚ	CUDA_PATHÚnvccz/usr/local/cudaN)ÚosÚenvironÚgetÚshutilÚwhichÚpathÚdirnameÚisdir)Ú	cuda_homer,   s     r&   Ú_get_cuda_homer6   N   s“   € õ ”
—’˜{Ñ+Ô+ÐJ­r¬z¯~ª~¸kÑ/JÔ/J€IØð ØÐÝŒ<˜ÑÔ€DØð 6ÝŒw�Š�rœwŸš¨tÑ4Ô4Ñ5Ô5Ð5Ý	„w‡}‚}Ð&Ñ'Ô'ð !Ø Ð Øˆ4r%   útuple[int, int] | Nonec                 óÚ  — t          ¦   «         } | €dS t          j                             | d¦  «        }t          j                             |¦  «        rÕ	 t          |¦  «        5 }t          j        |¦  «        }ddd¦  «         n# 1 swxY w Y   |                     d|                     di ¦  «        ¦  «                             dd¦  «        }| 	                    d¦  «        dd…         \  }}t          |¦  «        t          |¦  «        fS # t          t          t          f$ r Y nw xY wt          j                             | d	¦  «        }t          j                             |¦  «        r­	 t          |¦  «        5 }t          j        d
|                     ¦   «         ¦  «        }ddd¦  «         n# 1 swxY w Y   |rDt          |                     d¦  «        ¦  «        t          |                     d¦  «        ¦  «        fS n# t          t          f$ r Y nw xY wt          j                             | dd¦  «        }	t          j                             |	¦  «        r˜	 t          |	¦  «        5 }t          j        d|                     ¦   «         ¦  «        }ddd¦  «         n# 1 swxY w Y   |r/t          |                     d¦  «        ¦  «        }
|
dz  |
dz  dz  fS n# t          t          f$ r Y nw xY wdS )a‹  Version of the CUDA toolkit nvcc will use, as ``(major, minor)``, read off disk without a
    subprocess from (in order) ``{CUDA_HOME}/version.json``, ``version.txt``, or the ``CUDA_VERSION``
    define in ``include/cuda.h``. ``None`` if unreadable. This is the compiler that builds the kernels,
    unlike ``torch.version.cuda`` (torch's bundled runtime, which never drives a JIT compile).
    Nzversion.jsonÚ	cuda_nvccÚcudaÚversionÚ ú.r   zversion.txtzCUDA Version (\d+)\.(\d+)r   Úincludezcuda.hz#define CUDA_VERSION (\d+)iè  é
   )r6   r-   r2   ÚjoinÚisfileÚopenÚjsonÚloadr/   Úsplitr   ÚOSErrorÚ
ValueErrorÚAttributeErrorÚreÚsearchÚreadÚgroup)r5   Úversion_jsonÚfÚ
componentsr;   ÚmajorÚminorÚversion_txtÚmatchÚcuda_hÚcuda_versions              r&   Ú_get_nvcc_versionrV   a   s�  € õ Ñ Ô €IØÐØˆtå”7—<’< 	¨>Ñ:Ô:€LÝ	„w‡~‚~�lÑ#Ô#ð ð	Ý�lÑ#Ô#ð * qÝ!œY q™\œ\�
ð*ð *ð *ñ *ô *ð *ð *ð *ð *ð *ð *øøøð *ð *ð *ð *à —n’n [°*·.².ÀÈÑ2LÔ2LÑMÔM×QÒQÐR[Ð]_Ñ`Ô`ˆGØ"Ÿ=š=¨Ñ-Ô-¨b¨q¨bÔ1‰LˆE�5Ý�u‘:”:�s 5™zœzÐ)Ð)øÝ�¥^Ð4ð 	ð 	ð 	ØˆDð	øøøõ ”'—,’,˜y¨-Ñ8Ô8€KÝ	„w‡~‚~�kÑ"Ô"ð ð	Ý�kÑ"Ô"ð J aÝœ	Ð">ÀÇÂÁÄÑIÔI�ðJð Jð Jñ Jô Jð Jð Jð Jð Jð Jð Jøøøð Jð Jð Jð Jàð @Ý˜5Ÿ;š; q™>œ>Ñ*Ô*­C°·²¸A±´Ñ,?Ô,?Ð?Ð?ð@øå�Ð$ð 	ð 	ð 	ØˆDð	øøøõ ŒW�\Š\˜) Y°Ñ9Ô9€FÝ	„w‡~‚~�fÑÔð ð	Ý�f‘”ð K Ýœ	Ð"?ÀÇÂÁÄÑJÔJ�ðKð Kð Kñ Kô Kð Kð Kð Kð Kð Kð Køøøð Kð Kð Kð Kàð IÝ" 5§;¢;¨q¡>¤>Ñ2Ô2�Ø# tÑ+¨l¸TÑ.AÀbÑ-HÐHÐHðIøõ �Ð$ð 	ð 	ð 	ØˆDð	øøøð ˆ4s¤   ÁD
 Á"BÁ7D
 ÂBÂD
 Â
BÂA>D
 Ä
D$Ä#D$Å'G< Å6(F*ÆG< Æ*F.Æ.G< Æ1F.Æ2AG< Ç<HÈHÉK É#(JÊK ÊJÊK ÊJÊ3K ËK(Ë'K(Úboolc                 óR   — t           j                             ¦   «         d         dk    S )u®  Whether the current CUDA device is Blackwell (SM100+). `@assume_constant_result` makes dynamo
    evaluate it once at trace time and inline the bool â€” keeping the untraceable
    `torch.cuda.get_device_capability()` pybind out of the graph â€” so the branches on it (SF layout, cast
    kwargs, psum layout, and the arch guards) stay compile-safe. Re-queries each eager call, so a faked
    capability in tests is honoured.
    r   r?   )Útorchr:   Úget_device_capabilityr$   r%   r&   Úis_sm100r[   �   s"   € õ Œ:×+Ò+Ñ-Ô-¨aÔ0°BÒ6Ð6r%   FÚraise_errorc                óŠ  — t          ¦   «         st          dt          › �| ¬¦  «        S t          j                             ¦   «         st          d| ¬¦  «        S t          j                             ¦   «         \  }}|dvrt          d|› |› d�| ¬¦  «        S |dk    rdnd	}t          ¦   «         }|€$t          d|d         › d|d         › d�| ¬¦  «        S t          j	         
                    t          j	                             |dd¦  «        ¦  «        s't          d|› d|d         › d|d         › d�| ¬¦  «        S t          ¦   «         }|€'t          d|› d|d         › d|d         › d�| ¬¦  «        S ||k     rPt          d|› |› d|d         › d|d         › d|d         › d|d         › d|› d|d         › d|d         › d�| ¬¦  «        S dS )uì  Whether the DeepGEMM kernel can be loaded in this environment: `kernels` installed, a CUDA GPU on a
    supported arch (Hopper SM90 or Blackwell SM100), and a CUDA toolkit/nvcc new enough to JIT-compile it.
    A one-glance gate for callers â€” including external stacks â€” deciding whether to dispatch to DeepGEMM.
    With `raise_error=True` (used by the loader) it re-raises the specific `ImportError` explaining what's
    missing instead of returning `False`.

    `@assume_constant_result` makes dynamo evaluate this once at trace time and inline the bool (its probe
    is untraceable â€” an `lru_cache`'d import check + the `get_device_capability` pybind + nvcc lookup), so
    the check can sit in a compiled dispatch gate (e.g. `fp8_linear`) without breaking the graph.

    FP4 / Mega MoE additionally need Blackwell (SM100); the relevant forwards enforce that via
    `_assert_sm100_requirements` / `is_sm100()`, since this env check can't see which dtype path the
    caller will take.
    zDeepGEMM kernel unavailable: ©r\   z9DeepGEMM kernel requires CUDA, but CUDA is not available.)é	   r?   zJDeepGEMM requires Hopper (SM90) or Blackwell (SM100); current device is SMr=   r?   )é   r_   )r`   é   Nu(   DeepGEMM's JIT needs a CUDA toolkit â‰¥ r   r   z8, but none was found. Set `CUDA_HOME` to a CUDA toolkit.Úbinr,   z:DeepGEMM's JIT compiles with nvcc, but none was found in `u,   /bin`. Point `CUDA_HOME` at a full CUDA â‰¥ z& toolkit (not a runtime-only install).zDeepGEMM found nvcc in `u–   /bin` but could not read its CUDA version (no parseable `version.json`, `version.txt`, or `include/cuda.h`). Point `CUDA_HOME` at a complete CUDA â‰¥ z	 toolkit.zDeepGEMM on SMu    needs a CUDA â‰¥ z toolkit, but nvcc z in `u.   ` is too old. Point `CUDA_HOME` at a CUDA â‰¥ T)r	   r
   r   rY   r:   Úis_availablerZ   r6   r-   r2   rA   r@   rV   )r\   rP   rQ   Úmin_cudar5   Únvcc_versions         r&   Úis_deepgemm_loadablerf   ›   sÊ  € õ   Ñ!Ô!ð wÝ!Ð"\ÕBZÐ"\Ð"\ÐjuÐvÑvÔvÐvÝŒ:×"Ò"Ñ$Ô$ð xÝ!Ð"]ÐkvÐwÑwÔwÐwå”:×3Ò3Ñ5Ô5�L€Eˆ5ð �GÐÐÝ!ØhÐY^ÐhÐ`eÐhÐhÐhØ#ð
ñ 
ô 
ð 	
ð   2š+˜+ˆwˆw¨7€HÝÑ Ô €IØÐÝ!ð1°xÀ´{ð 1ð 1ÀXÈaÄ[ð 1ð 1ð 1à#ð
ñ 
ô 
ð 	
õ Œ7�>Š>�"œ'Ÿ,š, y°%¸Ñ@Ô@ÑAÔAð 
Ý!ðpÈð pð pØ.6°q¬kðpð pØ<DÀQ¼Kðpð pð pà#ð
ñ 
ô 
ð 	
õ %Ñ&Ô&€LØÐÝ!ðF yð Fð Fà!)¨!¤ðFð Fà/7¸¬{ðFð Fð Fð $ð	
ñ 
ô 
ð 	
ð �hÒÐÝ!ð=˜Uð = Eð =ð =¸XÀa¼[ð =ð =È8ÐTUÌ;ð =ð =Ø˜AŒð=ð =Ø!-¨a¤ð=ð =Ø7@ð=ð =à  œð=ð =à&.¨q¤kð=ð =ð =ð $ð	
ñ 
ô 
ð 	
ð ˆ4r%   zDeepGEMM | NoneÚ	_DEEPGEMMÚNonec                 óê  — t           �dS t          d¬¦  «         t          d¦  «        } | €t          d¦  «        ‚t	          | dd¦  «        }t	          | dd¦  «        }t	          | dd¦  «        }t	          | d	d¦  «        }t	          | d
d¦  «        }t          | d¬¦  «        }t	          | dd¦  «        }t	          | dd¦  «        }t	          | dd¦  «        }	t	          | dd¦  «        }
t	          | dd¦  «        }d„ d|fd|fd|fd	|fd
|fd|fd|fd|fd|	fd|
fd|ffD ¦   «         }|r-t          dd                     |¦  «        › dt          › �¦  «        ‚t          |||||||||	| |
¦   «         ¬¦  «        a dS )uý  Load DeepGEMM once into the `_DEEPGEMM` module global, raising `ImportError` if the env or any
    required symbol is missing. Under NO circumstances may this function return a value: it rides
    through `@allow_in_graph` as an opaque fx node, whose return must be proxyable â€” returning the
    bundle (e.g. from the warm-cache short-circuit) breaks torch.compile with
    `Unsupported: torch.* op returned non-Tensor`. Callers read `_DEEPGEMM` back from the global.

    `@allow_in_graph` makes `torch.compile` treat the untraceable cold path (hub download + dynamic
    import via `lazy_load_kernel`) as a single opaque node instead of tracing into it; it returns `None`
    (proxyable) and populates the global, which `load_deepgemm_kernel` then returns.
    NTr^   z	deep-gemmuc   Failed to load `kernels-community/deep-gemm` â€” check that a build matches the current torch/CUDA.Úfp8_fp4_gemm_ntÚ$m_grouped_fp8_fp4_gemm_nt_contiguousÚ$m_grouped_fp8_fp4_gemm_nn_contiguousÚ!m_grouped_bf16_gemm_nt_contiguousÚ!m_grouped_bf16_gemm_nn_contiguouszutils.per_token_cast_to_fp8)Úchained_pathr   r   r   Ú&get_mk_alignment_for_contiguous_layoutr   c                ó   — g | ]	\  }}|­|‘Œ
S ©Nr$   )Ú.0ÚnameÚattrs      r&   ú
<listcomp>z)_load_deepgemm_kernel.<locals>.<listcomp>
  s-   € ð ð ð áˆD�$ð ˆ<ð 	ð ˆ<ˆ<r%   z-DeepGEMM kernel is missing required symbols: z, z. )r   r   r   r   r   r   r   r   r   r   r   )	rg   rf   r   ÚImportErrorÚgetattrr   r@   r   r   )Úkernelr   r   r   r   r   r   r   r   r   Úget_mk_alignmentr   Úmissings                r&   Ú_load_deepgemm_kernelr|   ç   s&  € õ ÐØˆå TÐ*Ñ*Ô*Ð*Ý˜kÑ*Ô*€FØ€~ÝØqñ
ô 
ð 	
õ ˜VÐ%6¸Ñ=Ô=€NÝ '¨Ð0VÐX\Ñ ]Ô ]ÐÝ '¨Ð0VÐX\Ñ ]Ô ]ÐÝ$ VÐ-PÐRVÑWÔWÐÝ$ VÐ-PÐRVÑWÔWÐÝ3°FÐIfÐgÑgÔgÐÝ(/°Ð8[Ð]aÑ(bÔ(bÐ%Ý%,¨VÐ5UÐW[Ñ%\Ô%\Ð"Ý#*¨6Ð3QÐSWÑ#XÔ#XÐ Ý˜vÐ'OÐQUÑVÔVÐÝ˜vÐ'9¸4Ñ@Ô@Ððð ð  Ð/Ø3Ð5NÐOØ3Ð5NÐOØ0Ð2HÐIØ0Ð2HÐIØ*Ð,AÐBØ0Ð2SÐTØ-Ð/MÐNØ+Ð-IÐJØ5Ð7GÐHØÐ!1Ð2ð
ðñ ô €Gð" ð 
ÝØl¸D¿IºIÀgÑ<NÔ<NÐlÐlÕRjÐlÐlñ
ô 
ð 	
õ Ø%Ø";Ø";Ø5Ø5Ø3Ø*KØ'EØ%AØ)Ø$Ð$Ñ&Ô&ðñ ô €I€I€Ir%   c                 ó,   — t          ¦   «          t          S rr   )r|   rg   r$   r%   r&   Úload_deepgemm_kernelr~   /  s   € ÝÑÔÐÝÐr%   Úweightútorch.TensorÚscalec                ó¶   — t          ¦   «         s&| j        t          j        k    rt	          d¦  «        ‚dS |j        t          j        k    rt	          d¦  «        ‚dS )um  Before-load guard for DeepGEMM's FP8/FP4 arch constraints on the given weight/scale dtypes:

      - FP4 (``int8``-packed) weights have no Hopper (SM90) kernel â€” they need Blackwell (SM100+).
      - Blackwell has no float32 scale-factor path: ``_coerce_sf_for_kernel`` would silently round a
        ``float32`` scale to UE8M0 and corrupt the output. UE8M0 scales load as ``float8_e8m0fnu`` (the
        loader normalizes even float32-container checkpoints like dsv4-flash-base), so a ``float32`` scale
        on SM100 means a genuine non-UE8M0 checkpoint.

    Uses `is_sm100()` (compile-safe via `assume_constant_result`), so the whole guard folds to a constant under
    ``torch.compile``: the valid case compiles away to nothing, while an unsupported combo fails loud
    rather than letting the hot path silently corrupt (unlike an ``is_compiling`` skip, which would miss a
    model compiled from cold with no eager warmup). Both raise ``NotImplementedError``, which
    ``fp8_linear`` treats as "DeepGEMM declined" and falls back to Triton (SM90 consuming float32 SFs
    directly is fine, so those cases are no-ops).
    z DeepGEMM's FP4 (int8-packed) path requires a Blackwell (SM100+) GPU; FP4 weights have no Hopper (SM90) kernel. Use an FP8 checkpoint, or run on a Blackwell GPU.Nu§  DeepGEMM has no float32 scale-factor path on Blackwell (SM100): these scales are plain float32 (quantization_config.scale_fmt='float'), and rounding them to UE8M0 would silently corrupt the output. Use a checkpoint quantized with scale_fmt='ue8m0', or a path that consumes float32 block scales directly â€” the FP8 linear falls back to Triton automatically; for experts use `model.set_experts_implementation('grouped_mm')`.)r[   ÚdtyperY   Úint8ÚNotImplementedErrorÚfloat32)r   r�   s     r&   Ú_assert_sm100_requirementsr‡   7  so   € õ  ‰:Œ:ð àŒ<�5œ:Ò%Ð%Ý%ðZñô ð ð 	ˆð „{•e”mÒ#Ð#Ý!ð@ñ
ô 
ð 	
ð $Ð#r%   Úsfc                óª   — |                       t          j        ¦  «        }|dz                        d¦  «                              t          j        ¦  «        S )uÅ  Round each fp32 SF up to the nearest power of 2 (zero mantissa).

    Mirrors `deep_gemm.utils.math.ceil_to_ue8m0`. On SM100 the kernel's
    `pack_fp32_into_ue8m0` cleanly extracts the biased exponent only when the
    mantissa is already zero â€” its inner shifts (`>> 15`, `>> 7`, `<< 1`)
    otherwise leak mantissa bits into adjacent UE8M0 byte slots and silently
    corrupt the SF. SM90 consumes raw fp32 SFs without going through this path.
    iÿÿ i  €ÿ)ÚviewrY   Úint32Úbitwise_and_Úfloat)rˆ   Úint_views     r&   Ú_ceil_to_ue8m0r�   [  sA   € ð �wŠw•u”{Ñ#Ô#€HØ˜Ñ&×4Ò4Ð5EÑFÔF×KÒKÍEÌKÑXÔXÐXr%   r[   Úexpected_mnú
int | Nonec                óè  — | j         t          j        k    r“|�H|                      d¦  «        |k     r/||                      d¦  «        z  }|                      |d¬¦  «        } |r2|                      ¦   «                              t          j        ¦  «        } n;|                      ¦   «         } n&| j         t          j	        k    r|rt          | ¦  «        } |                      ¦   «         dvr%t          d|                      ¦   «         › d�¦  «        ‚|s|                      ¦   «         S |                      d¦  «        }|                      d¦  «        }d|                      ¦   «         z  }| |z   |z  }|                      ¦   «         d	k    rd
|fn||z  d
|f}t          |                      ¦   «         ¦  «        |k    r| S t          j        | j        || j         | j        ¬¦  «        }	|	                     | ¦  «         |	S )u¢  Lay out `sf` as DeepGEMM's dispatch expects, per arch.

    On SM100 the int-SF path only *checks* the SF (`tma_stride_check`) and never
    transforms it, so we hand it a TMA-aligned MN-major layout (`stride(-2) == 1`,
    `stride(-1) == align(mn, 16/esize)`). On SM90 DeepGEMM transforms SFA itself
    (`get_mn_major_tma_aligned_tensor`) and only *checks* SFB against
    `sm90_sfb_check`, which rejects TMA padding (`stride(-1)` must equal `size(-2)`,
    not `align(mn, â€¦)`); a padded weight SF trips `layout.hpp` whenever `mn` isn't a
    multiple of `16/esize` (e.g. N=576 â†’ mn=5). So on SM90 we return the raw
    row-major SF and let DeepGEMM lay it out.

    Inputs come in three flavors:
      - `float8_e8m0fnu` on SM100: raw UE8M0 bytes â€” pack 4 K-bytes â†’ int32
        (last dim /4) for the kernel's `(INT, 1, gran_k)` path.
      - `float8_e8m0fnu` on SM90: SM90 dispatch only accepts FP32 SFs, so cast
        UE8M0 â†’ FP32 (exact upcast â€” UE8M0 is the biased-exponent half of a
        pow-of-2 FP32, so `.float()` rebuilds the original FP32 scale exactly).
      - `float32`: per-token / per-block SFs from `per_token_cast_to_fp8` or
        on-disk weights â€” round to UE8M0 on SM100 (see `_ceil_to_ue8m0`).
      - `int32`: already-packed UE8M0 â€” pass through.

    When `expected_mn` is set and the SF's M-dim is smaller (block-quantized
    UE8M0, e.g. DSv4-Flash compressor weights with `(N/128, K/128)` SFs), we
    repeat the SF on the M-axis to per-row before packing â€” the `(INT, 1, gran_k)`
    DeepGEMM kernel branch is the only UE8M0 path on SM100; for `gran_mn > 1`
    the kernel only handles FP32 SFs and would otherwise reject our INT SF here.
    Néþÿÿÿ©Údim)r   ra   z"DeepGEMM SF must be 2D or 3D, got ÚDéÿÿÿÿé   r   r   ©rƒ   Údevice)rƒ   rY   Úfloat8_e8m0fnuÚsizeÚrepeat_interleaveÚ
contiguousrŠ   r‹   r�   r†   r�   r•   rG   Úelement_sizeÚtupleÚstrideÚempty_stridedÚshaperš   Úcopy_)
rˆ   r[   r�   Úgran_mnÚmnÚkfÚalign_toÚ
aligned_mnÚtarget_stridesÚouts
             r&   Ú_coerce_sf_for_kernelr¬   h  s¿  € ð8 
„x•5Ô'Ò'Ð'ØÐ" r§w¢w¨r¡{¤{°[Ò'@Ð'@Ø! R§W¢W¨R¡[¤[Ñ0ˆGØ×%Ò% g°2Ð%Ñ6Ô6ˆBØð 	Ø—’‘”×%Ò%¥e¤kÑ2Ô2ˆBˆBà—’‘”ˆBˆBØ	Œ•U”]Ò	"Ð	" xÐ	"Ý˜BÑÔˆà	‡v‚v�x„x�vÐÐÝÐI¸b¿fºf¹h¼hÐIÐIÐIÑJÔJÐJð ð Ø�}Š}‰ŒÐà	�Š�‰Œ€BØ	�Š�‰Œ€BØ�R—_’_Ñ&Ô&Ñ&€HØ�3˜(‘?Ð# hÑ.€JØ(*¯ª©¬°Aª¨�a˜�_�_¸BÀ¹OÈQÐPZÐ;[€NåˆR�YŠY‰[Œ[ÑÔ˜^Ò+Ð+Øˆ	Ý
Ô
˜bœh¨¸b¼hÈrÌyÐ
YÑ
YÔ
Y€CØ‡I‚Iˆb�M„M€MØ€Jr%   Úweight_scale_invÚ
block_sizeútuple | NoneÚdictc                óê   — | j         t          j        k    rddddœS |€t          d¦  «        ‚t	          |¦  «        }|dvrt          d|› d�¦  «        ‚|j         t          j        k    r|rdd	ddœS d
d	dœS )uC  Pick the `per_token_cast_to_fp8` kwargs from weight dtype + SF dtype + arch.

    Cases mirror the kernel's recipes:
      - FP4 weights (`int8`): gran_k=32 packed-UE8M0 SF. SM100+ only.
      - FP8 weights + UE8M0 SF on SM100: gran_k=128 packed-UE8M0 SF (DSv4).
      - FP8 weights + UE8M0 SF on SM90: gran_k=128 FP32 SF â€” the SM90 dispatch in
        `layout.hpp` only matches FP32 SFs, so we keep act SFs as FP32 (and float
        the weight SF in `_coerce_sf_for_kernel`; UE8M0 â†’ FP32 is an exact upcast).
      - FP8 weights + float SF: gran_k=128 float SF (DSv3).
    Té    ©Ú	use_ue8m0Úgran_kÚuse_packed_ue8m0Nz]DeepGEMM requires block-wise quantized FP8 weights, but the experts have no `block_size` set.))é€   r·   )r   r·   u?   DeepGEMM requires `block_size` âˆˆ {(128, 128), (1, 128)}, got r=   r·   F)r´   rµ   )rƒ   rY   r„   rG   r    r›   )r   r­   r®   r[   s       r&   Ú_select_fp8_cast_kwargsr¸   ¤  s¨   € ð „|•u”zÒ!Ð!Ø!¨RÀTÐJÐJÐJàÐÝØkñ
ô 
ð 	
õ �zÑ"Ô"€JØÐ/Ð/Ð/ÝÐjÐ]gÐjÐjÐjÑkÔkÐkØÔ¥Ô!5Ò5Ð5¸(Ð5Ø!¨SÀdÐKÐKÐKØ¨#Ð.Ð.Ð.r%   Úexpert_ids_sortedÚnum_expertsr   Ú	alignmentÚuse_psum_layoutú&tuple[torch.Tensor, torch.Tensor, int]c                ó  — | j         }|                      d¦  «        }|j        dk    r|                      ¦   «         n|                      ¦   «         }t          j        ||d|dz
  ¬¦  «                             ¦   «         }||z   dz
  |z  |z  }|t          ||¦  «        |dz
  z  z   }	||z
  }
t
          j	        j
                             |
                     d¦  «        d¦  «        }t          j        ||¬¦  «        ||          z   }|r(|                     d¦  «                             ¦   «         }nRt          j        |	fd|t
          j        ¬¦  «        }t          j        | |k     |                      ¦   «         d¦  «        ||<   |||	fS )	aŽ  Build the TMA-aligned grouped layout DeepGEMM expects.

    Returns `(sorted_to_padded, grouped_layout, total_padded_rows)`:
      - `grouped_layout` is per-row expert id (Hopper, with `-1` for padding /
        sentinels) or a cumsum of aligned per-expert counts (Blackwell).
      - EP sentinels (values == `num_experts`) are routed past the last expert
        block so DeepGEMM skips them.
    r   Úcpur   )ÚbinsÚminÚmax)r   r   ©rš   r—   ©rš   rƒ   )rš   rœ   Útyper�   r   rY   ÚhistcÚlongrÁ   ÚnnÚ
functionalÚpadÚcumsumÚarangeÚfullr‹   Úwhere)r¹   rº   r»   r¼   rš   Ú
num_tokensÚhistc_inputÚtokens_per_expertÚaligned_tokens_per_expertÚtotal_padded_rowsÚpadding_per_expertÚcumulative_paddingÚsorted_to_paddedÚgrouped_layouts                 r&   Ú!_build_deepgemm_contiguous_layoutrØ   Ã  s“  € ð Ô%€FØ"×'Ò'¨Ñ*Ô*€Jð 06¬{¸eÒ/CÐ/CÐ#×)Ò)Ñ+Ô+Ð+ÐIZ×I^ÒI^ÑI`ÔI`€KÝœ K°kÀqÈkÐ\]ÉoÐ^Ñ^Ô^×cÒcÑeÔeÐØ"3°iÑ"?À!Ñ"CÈ	Ñ!QÐU^Ñ ^Ðà"¥S¨°[Ñ%AÔ%AÀYÐQRÁ]Ñ%SÑSÐð 3Ð5FÑFÐÝœÔ,×0Ò0Ð1C×1JÒ1JÈ1Ñ1MÔ1MÈvÑVÔVÐÝ”| J°vÐ>Ñ>Ô>ÐASÐTeÔAfÑfÐàð uØ2×9Ò9¸!Ñ<Ô<×@Ò@ÑBÔBˆˆåœÐ%6Ð$8¸"ÀVÕSXÔS^Ð_Ñ_Ô_ˆÝ+0¬;Ð7HÈ;Ò7VÐXi×XmÒXmÑXoÔXoÐqsÑ+tÔ+tˆÐ'Ñ(à˜^Ð->Ð>Ð>r%   ÚxrÖ   rÓ   c                ój   — t          j        |g| j        dd…         ¢R | j        | j        dœŽ}| ||<   |S )z;Pad a sorted tensor into the TMA-aligned contiguous layout.r   NrÄ   )rY   Úemptyr£   rš   rƒ   )rÙ   rÖ   rÓ   Úpaddeds       r&   Ú_pad_for_deepgemmrÝ   ç  sD   € åŒ[Ð*ÐY¨Q¬W°Q°R°R¬[ÐYÐYÀÄÐQRÔQXÐYÐYÐY€FØ €FÐÑØ€Mr%   Úx_paddedc                ó   — | |         S rr   r$   )rÞ   rÖ   s     r&   Ú&_unpad_from_deepgemm_contiguous_layoutrà   î  s   € ØÐ$Ô%Ð%r%   Úhidden_statesÚtop_k_indexÚtop_k_weightsr   r    c                óx  — |                      d¦  «        }|                     d¦  «        }|                     d¦  «        }t          j        |¦  «        \  }	}
| |
|z           }||
         }t	          |	|||¦  «        \  }}}|	|k                         d¦  «        }|	                     |dz
  ¬¦  «         |||	||
|||fS )zóSort tokens by expert id and build the M-grouped padded layout.

    Returns `(sorted_hidden_states_g, sample_weights_g, expert_ids_g,
              sentinel_mask, perm, sorted_to_padded, grouped_layout,
              total_padded_rows)`.
    r—   r   )rÂ   )rœ   ÚreshaperY   ÚsortrØ   Ú	unsqueezeÚclamp_)rá   râ   rã   rº   r   r¼   Ú	num_top_kÚ
expert_idsÚsample_weightsÚexpert_ids_gÚpermÚsorted_hidden_states_gÚsample_weights_grÖ   r×   rÓ   Úsentinel_masks                    r&   Ú_dispatch_routed_inputrñ   õ  sé   € ð × Ò  Ñ$Ô$€IØ×$Ò$ RÑ(Ô(€JØ"×*Ò*¨2Ñ.Ô.€Nõ œ JÑ/Ô/Ñ€L�$Ø*¨4°9Ñ+<Ô=ÐØ% dÔ+Ðõ
 ;\Ø�k ;°ñ;ô ;Ñ7Ð�nÐ&7ð " [Ò0×;Ò;¸BÑ?Ô?€MØ×Ò˜K¨!™OÐÑ,Ô,Ð,àØØØØØØØð	ð 	r%   Ú
out_paddedÚsorted_weightsrð   rí   rÏ   ré   Ú
hidden_dimÚ	out_dtypeútorch.dtypec	                óÀ  — t          | |¦  «        }	|	|                     |	j        ¦  «                             d¦  «        z  }
|
                     |d¦  «         t          j        |¦  «        }t          j        |                     d¦  «        |	j	        ¬¦  «        ||<   |
|          
                    |||¦  «                             d¬¦  «                             |¦  «        S )uR   Unpad â†’ weighted multiply â†’ mask sentinels â†’ restore order â†’ top-k reduce.r—   g        r   rÃ   r   r”   )rà   Útorƒ   rç   Úmasked_fill_rY   Ú
empty_likerÌ   rœ   rš   rŠ   Úsum)rò   ró   rð   rí   rÖ   rÏ   ré   rô   rõ   r«   ÚweightedÚinv_perms               r&   Ú_combine_routed_outputrþ   '  sÅ   € õ 1°Ð=MÑ
NÔ
N€CØ�^×&Ò& s¤yÑ1Ô1×;Ò;¸BÑ?Ô?Ñ?€Hð ×Ò˜-¨Ñ-Ô-Ð-ÝÔ Ñ%Ô%€HÝ”\ $§)¢)¨A¡,¤,°s´zÐBÑBÔB€HˆT�Nà�HÔ×"Ò" :¨y¸*ÑEÔE×IÒIÈaÐIÑPÔP×SÒSÐT]Ñ^Ô^Ð^r%   Úoutput_dtypezv5.16)r;   ÚinputÚbiasútorch.Tensor | Noneútorch.dtype | NoneÚactivation_scalec                óª  — |�t          d¦  «        ‚| j        t          j        t          j        fvrt          d| j        › �¦  «        ‚t          ||¦  «         t          ¦   «         }t          |||t          ¦   «         ¦  «        }|  
                    d| j        d         ¦  «        }	 |j        |	fi |¤Ž\  }
}t          j        |
j        d         |j        d         | j        | j        ¬¦  «        }|                     d¦  «        rdd|d	         fnd}|                     |
t#          |t          ¦   «         |
                     d¦  «        ¬
¦  «        f|t#          |t          ¦   «         |                     d¦  «        ¬
¦  «        f||¬¦  «         | 
                    | j        dd…         |j        d         fz   ¦  «        }|�|                     |¦  «         |S )uó   End-to-end DeepGEMM linear: per-token activation quant + FP8/FP4 matmul.

    Static (per-tensor) activation quantization is rejected â€” DeepGEMM needs
    per-row SFs. Callers should route static activations through the Triton fallback.
    Nz@DeepGEMM linear does not support static activation quantization.z7DeepGEMM linear requires FP16 or BF16 activations, got r—   r   rÄ   r¶   r   rµ   ©r�   )Úrecipe)r…   rƒ   rY   Úbfloat16Úfloat16rG   r‡   r~   r¸   r[   rŠ   r£   r   rÛ   rš   r/   r   r¬   rœ   Úadd_)r   r   r­   r  r®   rÿ   r  ÚdeepgemmÚcast_kwargsÚinput_2dÚ	qinput_2dÚscale_2dÚoutputÚ	sf_recipes                 r&   Údeepgemm_fp8_fp4_linearr  A  sÓ  € ð Ð#Ý!Ð"dÑeÔeÐeØ„{�5œ>­5¬=Ð9Ð9Ð9ÝÐ`ÐSXÔS^Ð`Ð`ÑaÔaÐaõ ˜vÐ'7Ñ8Ô8Ð8å#Ñ%Ô%€HÝ)¨&Ð2BÀJÕPXÑPZÔPZÑ[Ô[€Kà�zŠz˜"˜eœk¨"œoÑ.Ô.€HØ8˜(Ô8¸ÐQÐQÀ[ÐQÐQÑ€IˆxÝŒ[˜œ¨Ô+¨V¬\¸!¬_ÀUÄ\ÐY^ÔYdÐeÑeÔe€Fð 2=·²ÐASÑ1TÔ1TÐ^��A�{ 8Ô,Ð-Ð-ÐZ^€IØ×ÒØ	Õ)¨(µH±J´JÈIÏNÊNÐ[\ÑL]ÔL]Ð^Ñ^Ô^Ð_Ø	Õ&Ð'7½¹¼ÐQW×Q\ÒQ\Ð]^ÑQ_ÔQ_Ð`Ñ`Ô`ÐaØØð	 ñ ô ð ð �[Š[˜œ S b SÔ)¨V¬\¸!¬_Ð,>Ñ>Ñ?Ô?€FØÐØ�Š�DÑÔÐØ€Mr%   Úselfútorch.nn.Modulec                ó   — |j         t          j        k    rt          d|j         › �¦  «        ‚t	          ¦   «         }| j        r|j        n|j        }|j        }| 	                    d¦  «        }| 	                    d¦  «        }| 	                    d¦  «        }	t          |||| j        |j        t          ¦   «         ¦  «        \  }
}}}}}}}t          | j        r| j        n| j        ¦  «        }t          | j        ¦  «        }| j        r"t          | j        r| j        n| j        ¦  «        nd }| j        rt          | j        ¦  «        nd }| j        r|j        d         n|j        d         }t1          |
||¦  «        }t          j        ||||j         ¬¦  «        } |||||t          ¦   «         ¬¦  «         | j        r|                     d|||         ¦  «         | j        r|                      |¦  «        n|                      |¦  «        }t          j        ||	||j         ¬¦  «        } |||||t          ¦   «         ¬¦  «         | j        r|                     d|||         ¦  «         t;          ||||||||	|j         ¦	  «	        S )Nú;DeepGEMM experts path requires bfloat16 hidden states, got r—   r   r   rÄ   )r¼   )rƒ   rY   r  rG   r~   Úis_transposedr   r   rš   rœ   rñ   rº   r   r[   r   Úhas_gateÚgate_up_projÚup_projÚ	down_projÚhas_biasÚgate_up_proj_biasÚup_proj_biasÚdown_proj_biasr£   rÝ   rÛ   Ú
index_add_Ú_apply_gateÚact_fnrþ   )r  rá   râ   rã   r  Úgrouped_bf16_matmulrš   ré   rÏ   rô   Úsorted_hiddenró   rì   rð   rí   rÖ   r×   rÓ   Ú	weight_upÚweight_downÚup_biasÚ	down_biasÚ
up_out_dimÚactÚproj_outr«   s                             r&   Údeepgemm_bf16_experts_forwardr,  n  s´  € ð Ô�eœnÒ,Ð,ÝÐlÐWdÔWjÐlÐlÑmÔmÐmå#Ñ%Ô%€Hà=AÔ=OÐt˜(Ô9Ð9ÐU]ÔUtÐàÔ!€FØ× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jõ 	Ø�{ M°4Ô3CÀXÔEYÕ[cÑ[eÔ[eñ	ô 	ñ	ØØØØØØØØõ
 ¨d¬mÐM˜Ô*Ð*ÀÄÑNÔN€IÝ˜4œ>Ñ*Ô*€KØZ^ÔZgÐq�h°´ÐU�tÔ-Ð-ÀDÔDUÑVÔVÐVÐmq€GØ15´ÐH•˜Ô,Ñ-Ô-Ð-ÀD€Ið )-Ô(:ÐR�” Ô$Ð$À	ÄÐPQÔ@R€JÝ
˜MÐ+;Ð=NÑ
OÔ
O€CÝŒ{Ð,¨jÀÈ}ÔObÐcÑcÔc€HØÐ˜˜Y¨°.ÕRZÑR\ÔR\Ð]Ñ]Ô]Ð]Ø„}ð HØ×Ò˜AÐ/°¸Ô1FÑGÔGÐGà-1¬]ÐUˆt×Ò Ñ)Ô)Ð)ÀÇÂÈHÑ@UÔ@U€Hõ Œ+Ð'¨¸FÈ-ÔJ]Ð
^Ñ
^Ô
^€CØÐ˜ +¨s°NÕT\ÑT^ÔT^Ð_Ñ_Ô_Ð_Ø„}ð EØ�Š�qÐ*¨I°lÔ,CÑDÔDÐDå!ØØØØØØØØØÔñ
ô 
ð 
r%   c                óø  — | j         rt          d¦  «        ‚t          | j        | j        ¦  «         t          ¦   «         }| j        dk    rt          d¦  «        ‚|j        t          j
        k    rt          d|j        › �¦  «        ‚| j        r|j        n|j        }|j        }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }	t#          | j        r| j        n| j        ¦  «        }
t#          | j        r| j        n| j        ¦  «        }t#          | j        ¦  «        }t#          | j        ¦  «        }t/          |
|| j        t3          ¦   «         ¦  «        }t5          |||| j        |j        t3          ¦   «         ¦  «        \  }}}}}}}}|                     d¦  «        rdd|d	         fnd } |j        |fi |¤Ž\  }}t?          |||¦  «        }t?          |||¦  «        }t          j         ||
j!        d         |t          j
        ¬
¦  «        } ||tE          |t3          ¦   «         |¬¦  «        f|
tE          |t3          ¦   «         |
                     d¦  «        ¬¦  «        f|||t3          ¦   «         ¬¦  «         | j        r|  #                    |¦  «        n|  $                    |¦  «        } |j        |fi |¤Ž\  }}t          j         ||	|t          j
        ¬
¦  «        } ||tE          |t3          ¦   «         |¬¦  «        f|tE          |t3          ¦   «         |                     d¦  «        ¬¦  «        f|||t3          ¦   «         ¬¦  «         tK          ||||||||	|j        ¦	  «	        S )NzðDeepGEMM experts selected on a model spanning multiple CUDA devices in one process; its kernels are bound to a single CUDA context and corrupt across devices. Use `experts_implementation='grouped_mm'`, or run one device per process (TP/EP).ÚstaticzJDeepGEMM experts dispatch does not support static activation quantization.r  r—   r   r¶   r   rµ   rÄ   r  r“   )r  r¼   )&Ú_deepgemm_disabledÚRuntimeErrorr‡   r  Údown_proj_scale_invr~   Úactivation_schemer…   rƒ   rY   r  rG   r  r   r   rš   rœ   r   r  r  r  Úgate_up_proj_scale_invÚup_proj_scale_invr¸   r®   r[   rñ   rº   r   r/   r   rÝ   rÛ   r£   r¬   r!  r"  rþ   )r  rá   râ   rã   r  Úgrouped_fp8_fp4_matmulrš   ré   rÏ   rô   r%  Úweight_scale_upr&  Úweight_scale_downr  r$  ró   Ú_expert_ids_grð   rí   rÖ   r×   rÓ   r  Úact_fp8Ú
act_scalesr+  Úproj_fp8Úproj_scalesr«   s                                 r&   Ú deepgemm_fp8_fp4_experts_forwardr=  ¯  sÉ  € ð Ôð 
õ ð\ñ
ô 
ð 	
õ ˜tœ~¨tÔ/GÑHÔHÐHå#Ñ%Ô%€HàÔ Ò)Ð)Ý!Ð"nÑoÔoÐoØÔ�eœnÒ,Ð,ÝÐlÐWdÔWjÐlÐlÑmÔmÐmà.2Ô.@ÐhˆÔ*Ð*ÀhÔFhð ð Ô!€FØ× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jå¨d¬mÐM˜Ô*Ð*ÀÄÑNÔN€IÝ¸d¼mÐg˜tÔ:Ð:ÐQUÔQgÑhÔh€OÝ˜4œ>Ñ*Ô*€KÝ  Ô!9Ñ:Ô:Ðå)¨)°_ÀdÄoÕW_ÑWaÔWaÑbÔb€Kõ 	Ø�{ M°4Ô3CÀXÔEYÕ[cÑ[eÔ[eñ	ô 	ñ	ØØØØØØØØð 2=·²ÐASÑ1TÔ1TÐ^��A�{ 8Ô,Ð-Ð-ÐZ^€Ið 9˜(Ô8¸ÐVÐVÈ+ÐVÐVÑ€GˆZÝ Ð)9Ð;LÑMÔM€GÝ" :Ð/?ÐARÑSÔS€JÝŒ{Ð,¨i¬o¸aÔ.@ÈÕW\ÔWeÐfÑfÔf€HØÐØ	Õ'¨
µH±J´JÐL]Ð^Ñ^Ô^Ð_Ø	Õ)¨/½8¹:¼:ÐS\×SaÒSaÐbdÑSeÔSeÐfÑfÔfÐgØØØÝ ™
œ
ðñ ô ð ð .2¬]ÐUˆt×Ò Ñ)Ô)Ð)ÀÇÂÈHÑ@UÔ@U€Hð ;˜HÔ:¸8ÐSÐSÀ{ÐSÐSÑ€HˆkÝ
Œ+Ð'¨¸FÍ%Ì.Ð
YÑ
YÔ
Y€CØÐØ	Õ(¨µh±j´jÐN_Ð`Ñ`Ô`ÐaØ	Õ+Ð,=½x¹z¼zÐWb×WgÒWgÐhjÑWkÔWkÐlÑlÔlÐmØØØÝ ™
œ
ðñ ô ð õ "ØØØØØØØØØÔñ
ô 
ð 
r%   Úmodulec                óL  — t          ¦   «         }t          | j        j        ¦  «        }t          | j        j        ¦  «        }t          | j        j        ¦  «                             t          j        ¦  «         	                    ¦   «         }t          | j
        j        ¦  «                             t          j        ¦  «         	                    ¦   «         }| j        }| j        }| j        }|dz  dk    s	|dz  dk    rt          d|› d|› d�¦  «        ‚|                     |                     ¦   «         d|z  |d|¬¦  «        }	|                     |                     ¦   «         ||d|¬¦  «        }
|                     ||	f||
f¦  «        \  \  }}	\  }}
t          j                             |d	¬
¦  «        | _        t          j                             |	d	¬
¦  «        | _        t          j                             |d	¬
¦  «        | _
        t          j                             |
d	¬
¦  «        | _        dS )u!  One-shot pack + permute of an FP8Experts module's L1/L2 weights into the
    Mega MoE UTCCP layout. Called lazily on the first megamoe forward; idempotent
    via the caller's ``_megamoe_transformed`` flag.

    Steps:
      1. Cast UE8M0 SF â†’ FP32 and call ``transform_sf_into_required_layout`` â†’
         packed int32 in MN-major TMA-aligned layout.
      2. Run ``transform_weights_for_mega_moe``: interleaves gate/up on L1 and
         transposes both SFs for UTCCP.
      3. Overwrite the loader-side parameters in place; the interleave preserves
         the ``[E_local, 2*I, *]`` leading dims so downstream ``.size(...)`` reads
         stay valid.

    Unwraps any ``DTensor`` wrappers FSDP2/EP may have placed around the loader-
    side Parameters â€” the kernel takes raw pointers.
    r²   r   zwDeepGEMM Mega MoE requires `hidden_dim` and `intermediate_hidden` divisible by 32 (FP8 SF granularity); got hidden_dim=z, intermediate_hidden=r=   r   )r   r²   )r  Ú
num_groupsF)Úrequires_gradN)r~   r   r3  Údatar1  r  rŠ   rY   r„   rž   r  Úintermediate_dimrº   rô   rG   r   r�   r   rÈ   Ú	Parameter)r>  r  Úgate_up_sf_rawÚdown_sf_rawÚ	gate_up_wÚdown_wÚintermediate_hiddenÚnum_local_expertsrô   Ú
gate_up_sfÚdown_sfÚgate_upÚdowns                r&   Úsetup_megamoe_weightsrO    s.  € õ" $Ñ%Ô%€HÝ˜fÔ;Ô@ÑAÔA€NÝ˜6Ô5Ô:Ñ;Ô;€Kå˜Ô,Ô1Ñ2Ô2×7Ò7½¼
ÑCÔC×NÒNÑPÔP€IÝ�fÔ&Ô+Ñ,Ô,×1Ò1µ%´*Ñ=Ô=×HÒHÑJÔJ€Fà Ô1ÐØÔ*ÐØÔ"€Jà�B�˜!ÒÐÐ2°RÑ7¸1Ò<Ð<ÝðmØ4>ðmð mØViðmð mð mñ
ô 
ð 	
ð
 ×;Ò;Ø×ÒÑÔØ	ÐÑØØØ$ð <ñ ô €Jð ×8Ò8Ø×ÒÑÔØØØØ$ð 9ñ ô €Gð .6×-TÒ-TØ	�JÐØ	�Ðñ.ô .Ñ*Ñ€Wˆj™?˜D 'õ  œ(×,Ò,¨WÀEÐ,ÑJÔJ€FÔÝ$)¤H×$6Ò$6°zÐQVÐ$6Ñ$WÔ$W€FÔ!Ý”x×)Ò)¨$¸eÐ)ÑDÔD€FÔÝ!&¤×!3Ò!3°GÈ5Ð!3Ñ!QÔ!Q€FÔÐÐr%   Úprocess_groupú%torch.distributed.ProcessGroup | Nonec                ój  — t          | j        | j        ¦  «         | j        j        t          j        k    rt          d| j        j        › d�¦  «        ‚|€t          d¦  «        ‚t          ¦   «         }t          | dd¦  «        st          | ¦  «         d| _        |                     d¦  «        }|                     d	¦  «        }|                     d¦  «        }| j                             d	¦  «        }	| j                             d
¦  «        dz  }
|	|                     ¦   «         z  }t          | dd¦  «        �| j        j        |k     r |                     ||||||
¬¦  «        | _        |                     |ddd¬¦  «        \  }}| j        j        d|…                              |¦  «         | j        j        d|…                              |¦  «         | j        j        d|…                              |¦  «         | j        j        d|…                              |¦  «         t	          j        ||ft          j        |j        ¬¦  «        }|                     || j        | j        f| j        | j        f| j        t          t          | dd¦  «        dd¦  «        ¬¦  «         |                     |j        ¦  «        S )uï  FP8 acts Ã— FP4 weights Mega MoE forward (SM100+).

    Fuses EP dispatch + L1 + SwiGLU + L2 + EP combine into one kernel,
    overlapping NVLink with tensor-core compute. The kernel handles the full
    `(num_tokens, hidden) â†’ (num_tokens, hidden)` MoE forward including the
    weighted top-k reduction; the caller must NOT all-reduce the output.

    `process_group` is supplied automatically by `MoeTensorParalellExperts._prepare_input_fn`
    when the module is wrapped for TP â€” it's required for the symm-buffer rendezvous
    on first forward. `top_k_index` is GLOBAL expert ids (`-1` marks skipped slots).

    Caller-managed `self` attributes:
      - `gate_up_proj`, `gate_up_proj_scale_inv`: L1 weight + UE8M0 SF.
      - `down_proj`, `down_proj_scale_inv`: L2 weight + UE8M0 SF.
      Both pairs must be transformed together via
      `transform_weights_for_mega_moe((gate_up, gate_up_sf), (down, down_sf))`.
      - `config.swiglu_limit` (optional): SwiGLU clamp; absent â†’ unclamped.
    zJDeepGEMM Mega MoE requires FP4-packed expert weights (dtype=`int8`), got `z/`. Use the 'deepgemm' dispatch for FP8 experts.Nz©DeepGEMM Mega MoE requires a `process_group` for the EP group. The TP wrapping (MoeTensorParalellMegaMoeExperts) supplies it automatically; pass it explicitly otherwise.Ú_megamoe_transformedFTr—   r   r   r   Úsymm_buffer)ÚhiddenÚnum_topkrº   Únum_max_tokens_per_rankrI  r²   r³   r™   ÚconfigÚswiglu_limit)Úactivation_clamp)r‡   r  r1  rƒ   rY   r„   r…   rG   r~   rx   rO  rS  rœ   rT  rW  r   r   rÙ   r¤   Úx_sfÚtopk_idxÚtopk_weightsrÛ   r  rš   r   r3  r  rø   )r  rá   râ   rã   rP  r  ré   rÏ   rô   rJ  rI  Únum_global_expertsÚx_fp8r[  Úys                  r&   Ú(deepgemm_fp8_fp4_megamoe_experts_forwardra  F  sØ  € õ8 ˜tÔ0°$Ô2JÑKÔKÐKàÔÔ¥%¤*Ò,Ð,Ý!ðYØÔ!Ô'ðYð Yð Yñ
ô 
ð 	
ð
 ÐÝðiñ
ô 
ð 	
õ
 $Ñ%Ô%€Hõ �4Ð/°Ñ7Ô7ð )Ý˜dÑ#Ô#Ð#Ø$(ˆÔ!à× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€JØÔ)×.Ò.¨qÑ1Ô1ÐØÔ+×0Ò0°Ñ3Ô3°qÑ8ÐØ*¨]×-?Ò-?Ñ-AÔ-AÑAÐõ ˆt�] DÑ)Ô)Ð1°TÔ5EÔ5]Ð`jÒ5jÐ5jØ#×@Ò@ØØØØ*Ø$.Ø 3ð Añ 
ô 
ˆÔð ×0Ò0°È$ÐWYÐlpÐ0ÑqÔq�K€Eˆ4ØÔÔ�{˜
�{Ô#×)Ò)¨%Ñ0Ô0Ð0ØÔÔ˜+˜:˜+Ô&×,Ò,¨TÑ2Ô2Ð2ØÔÔ˜k˜z˜kÔ*×0Ò0°Ñ=Ô=Ð=ØÔÔ! + : +Ô.×4Ò4°]ÑCÔCÐCõ 	Œ�Z Ð,µE´NÈ=ÔK_Ð`Ñ`Ô`€AØ×ÒØ	Ø	Ô	˜DÔ7Ð8Ø	Œ˜Ô1Ð2ØÔÝ ¥¨¨x¸Ñ!>Ô!>ÀÐPTÑUÔUð ñ ô ð ð �4Š4�Ô#Ñ$Ô$Ð$r%   )r'   r(   )r'   r7   )r'   rW   )F)r\   rW   r'   rW   )r'   rh   )r'   r   )r   r€   r�   r€   r'   rh   )rˆ   r€   r'   r€   rr   )rˆ   r€   r[   rW   r�   r‘   r'   r€   )
r   r€   r­   r€   r®   r¯   r[   rW   r'   r°   )
r¹   r€   rº   r   r»   r   r¼   rW   r'   r½   )rÙ   r€   rÖ   r€   rÓ   r   r'   r€   )rÞ   r€   rÖ   r€   r'   r€   )rá   r€   râ   r€   rã   r€   rº   r   r   r   r¼   rW   r'   r    )rò   r€   ró   r€   rð   r€   rí   r€   rÖ   r€   rÏ   r   ré   r   rô   r   rõ   rö   r'   r€   )NNNN)r   r€   r   r€   r­   r€   r  r  r®   r7   rÿ   r  r  r  r'   r€   )
r  r  rá   r€   râ   r€   rã   r€   r'   r€   )r>  r  r'   rh   )r  r  rá   r€   râ   r€   rã   r€   rP  rQ  r'   r€   )8r"   Ú
__future__r   Ú	functoolsrC   r-   rI   r0   Úcollections.abcr   Údataclassesr   rY   Úutilsr   Úutils.deprecationr   Úutils.import_utilsr	   r
   r   Úhub_kernelsr   r   Útensor_parallelr   Ú
get_loggerr   Úloggerr   Úcacher6   rV   Ú_dynamoÚassume_constant_resultr[   rf   rg   r#   Úallow_in_graphr|   r~   r‡   r�   r¬   r¸   rØ   rÝ   rà   rñ   rþ   r  r,  r=  rO  ra  r$   r%   r&   ú<module>rq     sÌ  ðð
ð 
ð 
ð #Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø €€€Ø 	€	€	€	Ø 	€	€	€	Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€à Ð Ð Ð Ð Ð Ø /Ð /Ð /Ð /Ð /Ð /ðð ð ð ð ð ð ð ð ð ð
 DÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ %Ð %Ð %Ð %Ð %Ð %ð 
ˆÔ	˜HÑ	%Ô	%€ð
 €�$ÐÑÔðð ð ð ð ñ ô ñ Ôðð, „ðð ð ñ „ðð$ „ð+ð +ð +ñ „ð+ð\ „Ô%ð7ð 7ð 7ñ &Ô%ð7ð „Ô%ðBð Bð Bð Bñ &Ô%ðBðP "€	Ð !Ð !Ð !Ñ !ð „ÔðDð Dð Dñ ÔðDðNð ð ð ð!
ð !
ð !
ð !
ðH
Yð 
Yð 
Yð 
Yð9ð 9ð 9ð 9ð 9ðx/ð /ð /ð /ð>!?ð !?ð !?ð !?ðHð ð ð ð&ð &ð &ð &ð/ð /ð /ð /ðd_ð _ð _ð _ð4 €�¨Ð1Ñ1Ô1ð
 !%Ø)-Ø'+Ø,0ð)ð )ð )ð )ñ 2Ô1ð)ðX>ð >ð >ð >ðBZð Zð Zð Zðz7Rð 7Rð 7Rð 7Rð~ <@ðV%ð V%ð V%ð V%ð V%ð V%ð V%r%   