§
    2�ÅjYO  ã                  ó  — d dl mZ d dlZd dlZd dlmZ d dlmZ dZ	dZ
dZdZd'd(d„Zd„ Zd„ Zd„ Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 d)d*d„Z G d„ de¦  «        Z G d„ d e¦  «        Z G d!„ d"e¦  «        Z G d#„ d$e¦  «        Z G d%„ d&e¦  «        ZdS )+é    )ÚannotationsN)ÚOpRun)Únumpy_matmulé   é   é   éÿÿÿÿÚxú
np.ndarrayÚaxisÚintÚreturnc                óŠ  — t          j        | |d¬¦  «        }t          j        |¦  «        }t          j        d| j        ¬¦  «        }t          j        d| j        ¬¦  «        }t          j        |||¦  «        }t          j        | |z
  ¦  «        }t          j        ||d¬¦  «        }t          j        ||k    ||¦  «        }||z  S )NT©r   Úkeepdims© ©Údtype)	ÚnpÚmaxÚisneginfÚzerosr   ÚonesÚwhereÚexpÚsum)	r
   r   Úx_maxÚrow_all_maskedÚzeroÚoneÚsafe_maxÚtmpÚss	            úPC:\Projects\Historical\venv\Lib\site-packages\onnx/reference/ops/op_attention.pyÚ_softmaxr%      s±   € ÝŒF�1˜4¨$Ð/Ñ/Ô/€Eõ ”[ Ñ'Ô'€NÝŒ8�B˜aœgÐ&Ñ&Ô&€DÝ
Œ'�"˜AœGÐ
$Ñ
$Ô
$€CÝŒx˜¨¨eÑ4Ô4€HÝ
Œ&��X‘Ñ
Ô
€CÝ
Œˆs˜¨Ð-Ñ-Ô-€AÝ
Œ��d’˜C Ñ#Ô#€AØ�‰7€Nó    c                óN   — |dk    r| |z  }t          j        |¦  «        }||z  S | S )Nr   )r   Útanh)ÚXÚsoftcapÚYs      r$   Ú_softcapr,   $   s0   € Ø�‚{€{Ø�‰KˆÝŒG�A‰JŒJˆØ�7‰{ÐØ€Hr&   c                ó`  — | j         dd…         \  }}t          j        |¦  «                             |d¦  «        }t          j        |¦  «                             d|¦  «        }t          j        |¦  «        dk    }|rt          j        |d¦  «        }|||z   k    }n||t          |¦  «        z   k    }t          j        || j                             d¦  «        | j                             t          j	         ¦  «        ¦  «        }	|r&| |	                     |	j         d         d||¦  «        z   S | |	z   S )aU  Adds an offset-aligned (bottom-right) causal bias to `base`.

    A query at in-block index ``i`` attends key ``j`` iff ``j <= i + offset``;
    attended positions contribute ``0`` and masked positions ``-inf`` (a softmax
    then maps ``-inf`` to 0 and 0 to 1). This is the select-not-multiply form that
    mirrors the function body's ``Where(allowed, 0, -inf)`` in ``utils.cc``, so the
    reference and the ``_expanded`` graph agree bit-for-bit.

    ``offset`` is either:

    * a scalar -- the same causal frontier for every batch. Used for the internal
      cache (``offset = past_key.shape[2]``) and the no-cache case (``offset = 0``,
      i.e. ordinary top-left causal). The result keeps the 2-D ``(q, kv)`` shape of
      ``base``.
    * a 1-D per-batch array of shape ``(batch,)`` -- the external/static cache,
      where ``offset[b] = nonpad_kv_seqlen[b] - q_len``. The result is promoted to
      ``(batch, 1, q, kv)`` so the downstream padding-mask block is a no-op reshape.

    Note: ``offset`` is intentionally not clamped to ``>= 0``. A negative offset
    (the over-long-query / out-of-contract regime) fully masks the affected rows,
    which the caller's fully-masked-row guard then zeroes -- matching the spec's
    fully-masked ``Y = 0`` / mode-3 ``= 0`` behavior. Clamping would change that
    result, so it is deliberately omitted.
    éþÿÿÿNé   r   ©r	   r/   r/   )
Úshaper   ÚarangeÚreshapeÚndimr   r   r   ÚtypeÚinf)
ÚbaseÚoffsetÚq_sequence_lengthÚkv_sequence_lengthÚi_idxÚj_idxÚ	per_batchÚoffsetsÚallowedÚcausals
             r$   Ú_apply_causalrA   ,   s#  € ð2 -1¬J°r°s°s¬OÑ)ÐÐ)ÝŒIÐ'Ñ(Ô(×0Ò0Ð1BÀAÑFÔF€EÝŒIÐ(Ñ)Ô)×1Ò1°!Ð5GÑHÔH€EÝ”˜‘” !Ò#€IØð 1Ý”*˜V ZÑ0Ô0ˆØ˜E G™OÒ,ˆˆà˜E¥C¨¡K¤KÑ/Ò0ˆÝŒX�g˜tœzŸš¨qÑ1Ô1°4´:·?²?ÅBÄFÀ7Ñ3KÔ3KÑLÔL€FØð 
ð �f—n’nØŒL˜ŒO˜QÐ 1Ð3Eñ
ô 
ñ 
ð 	
ð �&‰=Ðr&   c                óÚ  — | j         dd…         \  }}t          j        |¦  «                             |d¦  «        }t          j        |¦  «                             d|¦  «        }t          j        |¦  «        dk    }|rt          j        |d¦  «        }	||	z   |z
  }
n|t          |¦  «        z   |z
  }
t          j        |
t          j        ¬¦  «        }|dk    r	||
|k    z  }|dk    r
||
 |k    z  }t          j        || j	         
                    d¦  «        | j	         
                    t          j         ¦  «        ¦  «        }|r&| |                     |j         d         d||¦  «        z   S | |z   S )aŠ  Adds an offset-aligned sliding-window bias to ``base``.

    A query at absolute position ``p = offset + i`` attends key ``j`` iff
    ``p - left_window_size <= j <= p + right_window_size`` for each nonnegative
    bound. A value of ``-1`` leaves that side unbounded.

    ``offset`` semantics match ``_apply_causal``: scalar for internal/no cache,
    1-D ``(batch,)`` for external cache.
    r.   Nr/   r   r0   r   )r1   r   r2   r3   r4   r   Ú	ones_likeÚbool_r   r   r5   r6   )r7   Úleft_window_sizeÚright_window_sizer8   r9   r:   r;   r<   r=   r>   Údiffr?   Úwindows                r$   Ú_apply_sliding_windowrI   Y   sq  € ð -1¬J°r°s°s¬OÑ)ÐÐ)ÝŒIÐ'Ñ(Ô(×0Ò0Ð1BÀAÑFÔF€EÝŒIÐ(Ñ)Ô)×1Ò1°!Ð5GÑHÔH€EÝ”˜‘” !Ò#€IØð -Ý”*˜V ZÑ0Ô0ˆØ˜‘ 5Ñ(ˆˆà�˜F™œÑ# uÑ,ˆÝŒl˜4¥r¤xÐ0Ñ0Ô0€GØ˜1ÒÐØ�4Ð+Ò+Ñ+ˆØ˜AÒÐØ�D�5Ð-Ò-Ñ-ˆÝŒX�g˜tœzŸš¨qÑ1Ô1°4´:·?²?ÅBÄFÀ7Ñ3KÔ3KÑLÔL€FØð 
ð �f—n’nØŒL˜ŒO˜QÐ 1Ð3Eñ
ô 
ñ 
ð 	
ð �&‰=Ðr&   FTÚQÚKÚVÚ	attn_maskúnp.ndarray | NoneÚpast_keyÚ
past_valueÚnonpad_kv_seqlenc                óV  — |€dn|}|€dn|}|dk     rt          d|› �¦  «        ‚|dk     rt          d|› �¦  «        ‚|d u |d u k    rt          d¦  «        ‚|�|�t          d¦  «        ‚t          | j        ¦  «        t          |j        ¦  «        cxk    rt          |j        ¦  «        k    sn J ‚|r#| j        t          k    r|	€|
�t          d¦  «        ‚t          | j        ¦  «        }| j        d         }t          | j        ¦  «        t
          k    �r| j        d         }|j        d         }|j        d         }|	�|
€J ‚t          ||	z  ¦  «        }|| j        d	         |	|g}t          j        | |¦  «        } t          j	        | d
¦  «        } t          ||
z  ¦  «        }||j        d	         |
|g}t          j        ||¦  «        }t          j	        |d
¦  «        }t          ||
z  ¦  «        }||j        d	         |
|g}t          j        ||¦  «        }t          j	        |d
¦  «        }t          | j        ¦  «        t          k    r:t          |j        ¦  «        t          k    rt          |j        ¦  «        t          k    sJ ‚|€$| j        d         }d	t          j
        |¦  «        z  }t          j
        |¦  «        }| j                             |¦  «        }|�t          j        ||fd¬¦  «        }n|}|�t          j        ||fd¬¦  «        }n|}|}|}| j        d         }|j        d         } t          j        || f| j        ¬¦  «        }!|�f| |j        d         z
  }"|"dk    rPdg|j        d	z
  z  d|"fgz   }#|j        t          j        k    rdnt          j         }$t          j        ||#d|$¬¦  «        }|�r|�f|j        t          j        k    rQt          j        || j                             d¦  «        | j                             t          j         ¦  «        ¦  «        }|€t          j        || f| j        ¬¦  «        n|                     ¦   «         }%|dk    s|dk    r t          j        || f| j        ¬¦  «        |%z   }%|€+|�)|                     d¦  «        |z
  }&t)          |%|&¦  «        }!no|�|j        d         nd}&t)          |%|&¦  «        }!nM|�K|j        t          j        k    r1d	|z
                       | j        ¦  «        }t          j         ||d	k    <   |!|z   }!|dk    s|dk    ra|€|�|                     d¦  «        |z
  }'n|�|j        d         }'nd}'t          j        || f| j        ¬¦  «        |!z   }!t-          |!|||'¦  «        }!|�‚t          j        | ¦  «        |d d …t          j        f         k     }(|(                     |d	d	| ¦  «        }(t          j        |(dt          j         ¦  «                             |!j        ¦  «        }(|!|(z   }!|	€| j        d	         }	|
€|j        d	         })|j        d	         }*n|
})|
}*|	|)k    rB|	|)z  dk    r9|)|*k    r3|	|)z  }+t          j        ||+d	¬¦  «        }t          j        ||+d	¬¦  «        }t          j	        |d¦  «        },t5          | |z  |,|z  ¦  «        }-|�t7          |-|¦  «        }-|-|!z   }.|d	k    r|�n|t8          k    r|.                     ¦   «         }-|�2|.                     t:          j                             |¦  «        ¦  «        }.tA          |.¦  «        }/t          j!        t          j"        |!dd¬¦  «        ¦  «        }0t          j        |0t          j        d|/j        ¬¦  «        |/¦  «        }/|tF          k    r|/}-|-                     | j        ¦  «        }-|/                     | j        ¦  «        }/t5          |/|¦  «        }1|t
          k    rCt          j	        |1d
¦  «        }1t          j        |1|1j        d         |1j        d	         df¦  «        }1|1|||-fS )Nr	   z0left_window_size must be -1 or nonnegative, got z1right_window_size must be -1 or nonnegative, got z1past_key and past_value must be provided togetherz;nonpad_kv_seqlen cannot be combined with past cache tensorsz@q_num_heads and kv_num_heads must not be specified for 4D inputsr   r   r/   )r   r   r/   r   r   )r   r   )r   r   FÚconstant)ÚmodeÚconstant_values)Úrepeatsr   )r   r/   r   r   Tr   r   )$Ú
ValueErrorÚlenr1   r4   Ú_INPUT_RANK_4DÚ_INPUT_RANK_3Dr   r   r3   Ú	transposeÚsqrtr   r5   Úconcatenater   rD   r6   Úpadr   ÚcopyrA   ÚastyperI   r2   ÚnewaxisÚrepeatr   r,   Ú_QK_MATMUL_OUTPUT_WITH_BIASÚonnxÚhelperÚtensor_dtype_to_np_dtyper%   r   r   Ú_QK_MATMUL_OUTPUT_AFTER_SOFTMAX)2rJ   rK   rL   rM   rO   rP   rQ   ÚscaleÚ	is_causalÚq_num_headsÚkv_num_headsÚsoftmax_precisionr*   Úqk_matmul_output_moderE   rF   Ú_validate_attention25Úinput_shape_lenÚ
batch_sizeÚhidden_size_qÚhidden_size_kÚhidden_size_vÚhead_size_qÚintermediate_shape_qÚhead_size_kÚintermediate_shape_kÚhead_size_vÚintermediate_shape_vÚq_head_sizeÚpresent_keyÚpresent_valuer9   r:   Ú	attn_biasÚ	pad_widthÚ	pad_shapeÚ	pad_valuer7   r8   Ú
win_offsetÚpadding_maskÚk_num_headsÚv_num_headsÚseq_repsÚk_transposeÚqk_matmul_outputÚqk_with_biasÚ
qk_softmaxr   Úoutputs2                                                     r$   Ú_compute_attentionr‹   {   sË  € ð& .Ð5�r�rÐ;KÐØ/Ð7˜˜Ð=NÐØ˜"ÒÐÝØQÐ?OÐQÐQñ
ô 
ð 	
ð ˜2ÒÐÝØSÐ@QÐSÐSñ
ô 
ð 	
ð 	�DÐ˜j¨DÐ0Ò1Ð1ÝÐLÑMÔMÐMØÐ#¨Ð(<ÝÐVÑWÔWÐWÝˆqŒw‰<Œ<�3˜qœw™<œ<Ð7Ð7Ò7Ð7­3¨q¬w©<¬<Ò7Ð7Ð7Ð7Ð7Ð7àð
àŒF•nÒ$Ð$ØÐ$¨Ð(@åØNñ
ô 
ð 	
õ ˜!œ'‘l”l€OØ”˜”€JÝ
ˆ1Œ7�|„|•~Ò%Ñ%Øœ œ
ˆØœ œ
ˆØœ œ
ˆØÐ&¨<Ð+CÐ+CÐCå˜-¨+Ñ5Ñ6Ô6ˆà *¨A¬G°A¬J¸À[ÐQÐÝŒJ�qÐ.Ñ/Ô/ˆåŒL˜˜LÑ)Ô)ˆå˜-¨,Ñ6Ñ7Ô7ˆà *¨A¬G°A¬J¸ÀkÐRÐÝŒJ�qÐ.Ñ/Ô/ˆåŒL˜˜LÑ)Ô)ˆå˜-¨,Ñ6Ñ7Ô7ˆà *¨A¬G°A¬J¸ÀkÐRÐÝŒJ�qÐ.Ñ/Ô/ˆåŒL˜˜LÑ)Ô)ˆåˆAŒG‰Œ�Ò&Ð&Ý�”‰LŒL�NÒ*Ð*Ý�”‰LŒL�NÒ*Ð*Ð*ð	+ð €}Ø”g˜a”jˆØ•B”G˜KÑ(Ô(Ñ(ˆÝŒG�E‰NŒN€Eð ŒG�LŠL˜ÑÔ€Eð ÐÝ”n h° ]¸Ð;Ñ;Ô;ˆˆàˆØÐÝœ¨
°A ¸QÐ?Ñ?Ô?ˆˆàˆØ€AØ€Að œ œ
ÐØœ œÐÝ”Ð+Ð-?Ð@ÈÌÐPÑPÔP€Ið ÐØ&¨¬¸Ô)<Ñ<ˆ	Ø�qŠ=ˆ=Ø˜ I¤N°QÑ$6Ñ7¸A¸y¸>Ð:JÑJˆIØ!*¤µB´HÒ!<Ð!<˜˜Å2Ä6À'ˆIÝœØ˜9¨:Àyðñ ô ˆIð ñ !*ØÐ  Y¤_½¼Ò%@Ð%@õ œ ¨A¬G¯LªL¸©O¬O¸Q¼W¿\º\Í2Ì6È'Ñ=RÔ=RÑSÔSˆIð Ð õ ŒHÐ'Ð);Ð<ÀAÄGÐLÑLÔLÐLà—’Ñ!Ô!ð 	ð
 ˜qÒ Ð Ð$5¸Ò$:Ð$:õ ”Ð+Ð-?Ð@ÈÌÐPÑPÔPÐSWÑWð ð ÐÐ 0Ð <ð &×-Ò-¨bÑ1Ô1Ð4EÑEˆFÝ% d¨FÑ3Ô3ˆIˆIð +3Ð*>�X”^ AÔ&Ð&ÀAˆFÝ% d¨FÑ3Ô3ˆIˆIØ	Ð	ØŒ?�bœhÒ&Ð&Ø˜Y™×.Ò.¨q¬wÑ7Ô7ˆIÝ)+¬¨ˆI�i 1’nÑ%Ø 	Ñ)ˆ	ð ˜1ÒÐÐ 1°QÒ 6Ð 6ØÐÐ 0Ð <Ø)×1Ò1°"Ñ5Ô5Ð8IÑIˆJˆJØÐ!Ø!œ¨Ô*ˆJˆJàˆJåŒHÐ'Ð);Ð<ÀAÄGÐLÑLÔLÈyÑXð 	õ *ØÐ'Ð):¸Jñ
ô 
ˆ	ð Ð#Ý”yÐ!3Ñ4Ô4Ð7GÈÈÈÍ2Ì:ÈÔ7VÒVˆØ#×+Ò+¨J¸¸1Ð>PÑQÔQˆÝ”x ¨aµ"´&°Ñ9Ô9×@Ò@ÀÄÑQÔQˆØ Ñ,ˆ	ð ÐØ”g˜a”jˆØÐØ”g˜a”jˆØ”g˜a”jˆˆà"ˆØ"ˆà	˜Ò	#Ð	#Ø˜;Ñ&¨!Ò+Ð+Ø˜KÒ'Ð'à +Ñ-ˆåŒI�a °Ð2Ñ2Ô2ˆÝŒI�a °Ð2Ñ2Ô2ˆõ( ”,˜q ,Ñ/Ô/€KÝ# A¨¡I¨{¸UÑ/BÑCÔCÐð ÐÝ#Ð$4°gÑ>Ô>Ðà# iÑ/€LØ Ò!Ð! gÐ&9ØØ	Õ"=Ò	=Ð	=Ø'×,Ò,Ñ.Ô.ÐàÐ$Ø#×*Ò*ÝŒK×0Ò0Ð1BÑCÔCñ
ô 
ˆõ ˜,Ñ'Ô'€Jõ ”[Ý
Œˆy˜r¨DÐ1Ñ1Ô1ñô €Nõ ”Ø�œ ¨:Ô+;Ð<Ñ<Ô<¸jñô €Jð Õ ?Ò?Ð?ð
 &ÐØ'×.Ò.¨q¬wÑ7Ô7Ðð ×"Ò" 1¤7Ñ+Ô+€Jå˜* aÑ(Ô(€FØ�.Ò(Ð(Ý”˜f lÑ3Ô3ˆÝ”˜F V¤\°!¤_°f´lÀ1´oÀrÐ$JÑKÔKˆØ�; Ð/?Ð?Ð?r&   c                  ó6   — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 ddd„ZdS )Ú	AttentionTNFrJ   r   rK   rL   rM   rN   rO   rP   rQ   r   c                óD   — t          |||f||||||	|
||||||| j        dœŽS )N)rM   rO   rP   rQ   rh   ri   rj   rk   rl   r*   rm   rE   rF   rn   )r‹   rn   )ÚselfrJ   rK   rL   rM   rO   rP   rQ   rh   ri   rj   rk   rl   r*   rm   rE   rF   s                    r$   Ú_runzAttention._run•  sX   € õ& "ØØØð
ð  ØØ!Ø-ØØØ#Ø%Ø/ØØ"7Ø-Ø/Ø"&Ô"<ð#
ð 
ð 
ð 	
r&   )NNNNNFNNNNNNN©rJ   r   rK   r   rL   r   rM   rN   rO   rN   rP   rN   rQ   rN   r   r   )Ú__name__Ú
__module__Ú__qualname__rn   r�   r   r&   r$   r�   r�   ’  s\   € € € € € Ø Ðð (,Ø&*Ø(,Ø.2ØØØØØØØ"ØØð#%
ð %
ð %
ð %
ð %
ð %
ð %
r&   r�   c                  ó   — e Zd ZdZdS )ÚAttention_1FN©r’   r“   r”   rn   r   r&   r$   r–   r–   ½  ó   € € € € € ð "ÐÐÐr&   r–   c                  ó   — e Zd ZdZdS )ÚAttention_23FNr—   r   r&   r$   rš   rš   Ã  r˜   r&   rš   c                  ó   — e Zd ZdS )ÚAttention_24N)r’   r“   r”   r   r&   r$   rœ   rœ   É  s   € € € € € Ø€Dr&   rœ   c                  ó   — e Zd ZdZdS )ÚAttention_25TNr—   r   r&   r$   rž   rž   Í  s   € € € € € Ø ÐÐÐr&   rž   )r	   )r
   r   r   r   r   r   )NNNNNFNNNNNNNTr‘   )Ú
__future__r   Únumpyr   rd   Úonnx.reference.op_runr   Úonnx.reference.ops.op_matmulr   rZ   rY   rc   rg   r%   r,   rA   rI   r‹   r�   r–   rš   rœ   rž   r   r&   r$   ú<module>r£      sÝ  ðð #Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð à €€€Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5à€Ø€ØÐ Ø"#Ð ðð ð ð ð ð$ð ð ð*ð *ð *ðZð ð ðL $(Ø"&Ø$(Ø*.Ø
ØØØØØØØØØð#T@ð T@ð T@ð T@ð T@ðn(
ð (
ð (
ð (
ð (
�ñ (
ô (
ð (
ðV"ð "ð "ð "ð "�)ñ "ô "ð "ð"ð "ð "ð "ð "�9ñ "ô "ð "ð	ð 	ð 	ð 	ð 	�<ñ 	ô 	ð 	ð!ð !ð !ð !ð !�9ñ !ô !ð !ð !ð !r&   