§
    ¹¯Åj5O  ã                   óz   — d dl Z d dlZd dlmZmZ d dlmZ d dlm	Z	  e j
        e¦  «        Z G d„ de¦  «        ZdS )é    N)ÚAttentionMaskÚFusionAttention)Úhelper)Ú	OnnxModelc                   ó:   ‡ — e Zd ZdZdedededefˆ fd„Zd„ Zˆ xZ	S )ÚFusionBartAttentionz?
    Fuse Bart Attention subgraph into one Attention node.
    ÚmodelÚhidden_sizeÚ	num_headsÚattention_maskc                 óP   •— t          ¦   «                              ||||¦  «         d S )N)ÚsuperÚ__init__)Úselfr	   r
   r   r   Ú	__class__s        €ú_C:\Projects\Historical\venv\Lib\site-packages\onnxruntime/transformers/fusion_bart_attention.pyr   zFusionBartAttention.__init__   s)   ø€ õ 	‰Œ×Ò˜ ¨Y¸ÑGÔGÐGÐGÐGó    c                 ó  — | j                              |g d¢g d¢¦  «        }|�	|\  }}}}}	nt                               d¦  «         d S g }
|j        D ]4}||vrŒ||d         j        d         k    rŒ|
                     |¦  «         Œ5t          |
¦  «        dk    rd S |
d         }	 ||         }|j        dv r | j          	                    |¦  «        d         }|j        D ]6}|sŒ||         }d„ |D ¦   «         }| 
                    d¦  «        dk    r|} nŒ7d	„ | j                              ¦   «         j        D ¦   «         }d
„ | j                              ¦   «         j        D ¦   «         }| j                              |	g d¢g d¢¦  «        }| j                              |	g d¢g d¢¦  «        }| j                              |	g d¢g d¢¦  «        }d\  }}g d d }}}|�º|}|\  }}}}||j        d                  }|D ]W}|j        dk    rJ| j                              |ddgddg¦  «        }|�|d         j        d         }||j        d                  } nŒX|D ]@}||j        d                  } | D ] }!|!j        d         |v r|!j        d         } nŒ!|dk    r nŒAnx|�%|}|\  }"}}}}|"j        d         }|"j        d         }nQ|	j        d         |v r|	j        d         }n4|�|}|d         j        d         }nt                               d¦  «         d S ||v r|nd}||v r|nd}| j                              |	ddgddg¦  «        }#| j                              |	g d¢g d¢¦  «        }$g d }&}%|#�|#\  }'}(|#}%n'|$�	|$\  }'}&}(|$}%nt                               d¦  «         d S | j                              |(g d¢g d¢¦  «        })| j                              |(g d¢g d¢¦  «        }*g }+|)�|)}+|+\  },}-}.}/}0n)|*�|*}+|+\  }.},}-}/}0nt                               d¦  «         d S | j                              |(g d ¢g d!¢¦  «        }1| j                              |(g d"¢g d#¢¦  «        }2| j                              |(g d$¢g d¢¦  «        }3| j                              |(g d%¢g d&¢¦  «        }4d\  }5}6g d d }9}8}7|1�@|1}7|7\  }:};}9||;j        d                  }<|<D ] }=|=j        d         |v r|=j        d         }6 nŒ!�n`|2�&|2}7|7\  }'}>}:};}9|>j        d         }5|>j        d         }6�n8||(j        d                  j        d         |v r(||(j        d                  g}7|7d         j        d         }5nð|3�º|3}7|7\  }'}:};}9||9j        d                  }|D ]W}|j        dk    rJ| j                              |ddgddg¦  «        }?|?�|?d         j        d         }5||j        d                  } nŒX|D ]@}||j        d                  } | D ] }!|!j        d         |v r|!j        d         }6 nŒ!|6dk    r nŒAn4|4�|4}7|7d         j        d         }5nt                               d'¦  «         d S |5|v r|5nd}5|6|v r|6nd}6|9�á|8€ß| j                              |j        d         ¦  «        }@|@j        d         }A|@j        }Bd(}C| j                              |C¦  «        }D|D€D|                      |C|B|Agt!          j        d)g|Az  t%          j        |B¦  «        ¬*¦  «        ¬+¦  «         | j                              d,¦  «        }Et%          j        d,|C|9j        d         g|;j        g|E¦  «        }8t/          |5¦  «        ot/          |¦  «        o|9d u o|d u }F|F o2|0j        d         |k    o!|9j        d         |k    o|j        d         |k    }G|F oH|0j        d         |k    o7|9j        d         |j        d         k    o|9j        d         |0j        d         k    }H|Go|%|#k    }I|Go|%|$k    }J|Ho|%|#k    }K|Jot/          |5¦  «        ot/          |¦  «        }L|Fo|%|#k    }M|%|$k    }Ng }O|NrÃ| j                              |&d-gdg¦  «        }P| j                              |&g d.¢g d/¢¦  «        }Q| j                              |&g d0¢g d1¢¦  «        }R| j                              |&d2d2gddg¦  «        }S|Q�|Q}On+|R�|R}On&|S�|S}On!|P�|P}Ont                               d3¦  «         d S t          |O¦  «        dk    sJ ‚|Is	|Js|Ks|Ls|M�r(|}T|                      |-¦  «        \  }U}V|Udk    s|Vdk    s	|V|Uz  dk    rt                               d4¦  «         d S d }W|Ls|Ks|MrX| j        rN|                      |0|Ks|Lr|9n|5|Ks|Lr|n||/|Ks|Lr|8nd |Ks|Lr|nd |U|V|Tj        d         |N|Lr|5nd|Lr|nd|6|¬5¦  «        nd }WnE| j        }Xd6| _        |                      d |0|9||/|8||U|V||Tj        d         |N|5||6|¬7¦  «        }W|X| _        |W€t                               d8¦  «         d S | j                             |W¦  «         | j        | j        |Wj        <   | j                              |T||	g¦  «         | j                              |%¦  «         |Ls|Ks|M�rWt          |+¦  «        dk    r%|+d         j        dk    r|+ !                    ¦   «          t          |7¦  «        dk    r%|7d         j        dk    r|7 !                    ¦   «          t          |¦  «        dk    r%|d         j        dk    r| !                    ¦   «          | j"        r¨t          |+¦  «        dk    r%|+d         j        d,k    r|+ !                    ¦   «          t          |7¦  «        dk    r%|7d         j        d,k    r|7 !                    ¦   «          t          |¦  «        dk    r%|d         j        d,k    r| !                    ¦   «          | j                              |+¦  «         | j                              |7¦  «         | j                              |¦  «         d9| _#        d S d S ):N)ÚAddÚMatMulÚReshapeÚ	Transposer   )é   r   r   r   r   z(fuse_attention: failed to match qkv pathr   r   >   r   ÚClipc                 ó   — g | ]	}|j         ‘Œ
S © )Úop_type)Ú.0Úchilds     r   ú
<listcomp>z,FusionBartAttention.fuse.<locals>.<listcomp>O   s   € ÐBÐBÐB°˜eœmÐBÐBÐBr   r   c                 ó   — h | ]	}|j         ’Œ
S r   ©Úname©r   Únodes     r   ú	<setcomp>z+FusionBartAttention.fuse.<locals>.<setcomp>T   s   € ÐLÐLÐL¨4˜TœYÐLÐLÐLr   c                 ó   — h | ]	}|j         ’Œ
S r   r"   r$   s     r   r&   z+FusionBartAttention.fuse.<locals>.<setcomp>U   s   € ÐNÐNÐN¨D˜dœiÐNÐNÐNr   )r   r   r   r   )r   r   r   N)ÚConcatr   r   r   r   )r   r   r   r   N)r   r   r   r   )r   r   r   r   )Ú r)   r(   r   r   éÿÿÿÿr)   z&fuse_attention: failed to match v pathÚSoftmax)r+   r   r   )r   r   r   z'fuse_attention: failed to match qk path)r   r   ÚMulr   r   )r   r   r   r   r   )r,   r   r   r   r   z&fuse_attention: failed to match q path)r   r   r   )r   r   r   )r   r(   r   r   r   )r   r   r   r   r   )r,   r   r   r   )r,   r   r   r   r   )r   r   r   r   r   z&fuse_attention: failed to match k pathÚ
empty_biasg        )Údtype)ÚdimsÚvalsr   ÚWhere)ÚSliceÚExpandr1   )r   r   r   )r2   Ú	UnsqueezeÚGatherÚShaper   )r   é   r   r   r   r2   z*fuse_attention: failed to match mask nodesz9fuse_attention: failed to detect num_heads or hidden_size)Úq_matmulÚk_matmulÚv_matmulÚq_addÚk_addÚv_addr   r
   ÚoutputÚunidirectionalÚpast_kÚpast_vÚ	present_kÚ	present_vF)Ú
mask_indexr8   r9   r:   r;   r<   r=   r   r
   Úfirst_inputr>   Úcausalr@   rA   rB   rC   z+fuse_attention: failed to create fused nodeT)$r	   Úmatch_parent_pathÚloggerÚdebugÚinputr>   ÚappendÚlenr   Úget_childrenÚcountÚgraphÚget_initializerr/   Ú	data_typeÚadd_initializerÚnpÚarrayr   Útensor_dtype_to_np_dtypeÚcreate_node_nameÚ	make_noder#   ÚboolÚget_num_heads_and_hidden_sizeÚuse_multi_head_attentionÚcreate_multihead_attention_nodeÚcreate_attention_nodeÚnodes_to_addÚthis_graph_nameÚnode_name_to_graph_nameÚnodes_to_removeÚextendÚpopÚ!disable_multi_head_attention_biasÚprune_graph)Yr   Únormalize_nodeÚinput_name_to_nodesÚoutput_name_to_nodeÚ	qkv_nodesÚadd_outÚ
matmul_outÚreshape_qkvÚtranspose_qkvÚ
matmul_qkvÚother_inputsÚinput_Ú
root_inputÚskip_layernormr>   ÚchildrenÚchildren_typesÚgraph_input_namesÚgraph_output_namesÚv_nodes_past_or_presentÚv_nodes_with_pastÚv_nodes_past_only_oairA   rC   Úv_nodesÚadd_vÚmatmul_vÚtranspose_vÚ	reshape_vÚstart_child_nodesÚstart_child_nodeÚconcat_v_nodesÚstart_grandchild_nodesÚstart_grandchild_nodeÚconcat_vÚqk_nodes_no_maskÚqk_nodes_with_maskÚqk_nodesÚadd_qkÚ_Ú	matmul_qkÚ
q_nodes_hfÚq_nodes_oaiÚq_nodesÚtranspose_qÚ	reshape_qÚmul_qÚadd_qÚmatmul_qÚk_nodes_no_past_hfÚk_nodes_with_past_hfÚk_nodes_past_or_present_oaiÚk_nodes_past_only_oair@   rB   Úk_nodesÚadd_kÚmatmul_kÚtranspose_kÚ	reshape_kÚtranspose_k_nodesÚtranspose_k_nodeÚconcat_kÚconcat_k_nodesÚadd_v_tensorÚbias_dimr.   Úempty_bias_nameÚempty_tensorÚadd_nameÚthree_root_inputsÚone_root_inputÚtwo_root_inputsÚencoder_attentionÚdecoder_self_attentionÚdecoder_cross_attentionÚ decoder_self_attention_with_pastÚ!decoder_cross_attention_with_pastÚcausal_maskÚ
mask_nodesÚmask_nodes_bartÚmask_nodes_whisper_hfÚmask_nodes_whisper_oaiÚ mask_nodes_whisper_oai_unit_testÚattention_last_noder   r
   Únew_nodeÚ%use_multi_head_attention_ground_truthsY                                                                                            r   ÚfusezFusionBartAttention.fuse   s  € à”J×0Ò0ØØ?Ð?Ð?ØˆOˆOñ
ô 
ˆ	ð
 Ð ð ñØØØØØ�
õ �LŠLÐCÑDÔDÐDØˆFàˆØ$Ô*ð 	(ð 	(ˆFØÐ0Ð0Ð0ØØ˜ 1œÔ,¨QÔ/Ò/Ð/ØØ×Ò Ñ'Ô'Ð'Ð'Ýˆ|ÑÔ Ò!Ð!ØˆFØ! !”_ˆ
ð	ð -¨ZÔ8ˆð Ô! _Ð4Ð4Ø!œZ×4Ò4°^ÑDÔDÀQÔGˆNØ$Ô+ð 	ð 	ˆFØð ØØ*¨6Ô2ˆHØBÐB¸ÐBÑBÔBˆNØ×#Ò# HÑ-Ô-°Ò2Ð2Ø#�
Ø�ð 3ð MÐL°4´:×3CÒ3CÑ3EÔ3EÔ3KÐLÑLÔLÐØNÐN°D´J×4DÒ4DÑ4FÔ4FÔ4MÐNÑNÔNÐà"&¤*×">Ò">ØØ5Ð5Ð5ØˆOˆOñ#
ô #
Ðð
 !œJ×8Ò8ØØ?Ð?Ð?ØÐÐñ
ô 
Ðð
 !%¤
× <Ò <ØØ<Ð<Ð<ØˆLˆLñ!
ô !
Ðð
 #Ñˆ�	Ø#% t¨T˜�ˆØ"Ð.Ø-ˆGØ8?Ñ5ˆ[˜) U¨Hð !4°E´LÀ´OÔ DÐØ$5ð 
ð 
Ð Ø#Ô+¨xÒ7Ð7Ø%)¤Z×%AÒ%AØ(Ø" KÐ0Ø˜A˜ñ&ô &�Nð
 &Ð1Ø!/°Ô!3Ô!9¸!Ô!<˜Ø(;Ð<LÔ<SÐTUÔ<VÔ(WÐ%Ø�Eð 8ð %6ð ð Ð Ø)<Ð=MÔ=TÐUVÔ=WÔ)XÐ&Ø-Cð ð Ð)Ø,Ô3°AÔ6Ð:LÐLÐLØ$9Ô$@ÀÔ$C˜	Ø˜ð Mð  ’?�?Ø�Eð #øàÐ*Ø'ˆGØBIÑ?ˆX�{ I¨u°hØ”^ AÔ&ˆFØ œ¨Ô*ˆIˆIØÔ˜aÔ Ð$5Ð5Ð5àÔ% aÔ(ˆFˆFØ"Ð.à+ˆGØ˜R”[Ô& qÔ)ˆFˆFå�LŠLÐAÑBÔBÐBØˆFØ!Ð%6Ð6Ð6��¸BˆØ!*Ð.@Ð!@Ð!@�I�IÀbˆ	àœ:×7Ò7¸
ÀYÐPXÐDYÐ\]Ð_`Ð[aÑbÔbÐØ!œZ×9Ò9¸*ÐFbÐFbÐFbÐdmÐdmÐdmÑnÔnÐØ˜t�&ˆØÐ'Ø+‰LˆAˆyØ'ˆHˆHØÐ+Ø#5Ñ ˆAˆv�yØ)ˆHˆHå�LŠLÐBÑCÔCÐCØˆFà”Z×1Ò1ØØ<Ð<Ð<ØˆOˆOñ
ô 
ˆ
ð
 ”j×2Ò2ØØ<Ð<Ð<ØˆOˆOñ
ô 
ˆð
 ˆØÐ!Ø ˆGØ?FÑ<ˆ[˜) U¨E°8°8ØÐ$Ø!ˆGØ?FÑ<ˆU�K ¨E°8°8å�LŠLÐAÑBÔBÐBØˆFà!œZ×9Ò9ØØ.Ð.Ð.ØˆIˆIñ
ô 
Ðð
  $œz×;Ò;ØØEÐEÐEØˆOˆOñ 
ô  
Ðð
 '+¤j×&BÒ&BØØ5Ð5Ð5ØˆLˆLñ'
ô '
Ð#ð
 !%¤
× <Ò <ØØCÐCÐCØˆOˆOñ!
ô !
Ðð
 #Ñˆ�	Ø#% t¨T˜�ˆØÐ)Ø(ˆGØ18Ñ.ˆ[˜) Xð !4°IÔ4DÀQÔ4GÔ HÐØ$5ð ð Ð Ø#Ô*¨1Ô-Ð1CÐCÐCØ 0Ô 7¸Ô :�IØ�Eð Dùð "Ð-Ø*ˆGØ>EÑ;ˆQ�˜+ y°(Ø”^ AÔ&ˆFØ œ¨Ô*ˆI‰IØ  ¤°Ô!3Ô4Ô:¸1Ô=ÐARÐRÐRà*¨9¬?¸1Ô+=Ô>Ð?ˆGØ˜Q”ZÔ% aÔ(ˆFˆFØ(Ð4Ø1ˆGØ4;Ñ1ˆQ�˜Y¨ð !4°H´OÀAÔ4FÔ GÐØ$5ð 
ð 
Ð Ø#Ô+¨xÒ7Ð7Ø%)¤Z×%AÒ%AØ(Ø" KÐ0Ø˜A˜ñ&ô &�Nð
 &Ð1Ø!/°Ô!3Ô!9¸!Ô!<˜Ø(;Ð<LÔ<SÐTUÔ<VÔ(WÐ%Ø�Eð 8ð %6ð ð Ð Ø)<Ð=MÔ=TÐUVÔ=WÔ)XÐ&Ø-Cð ð Ð)Ø,Ô3°AÔ6Ð:LÐLÐLØ$9Ô$@ÀÔ$C˜	Ø˜ð Mð  ’?�?Ø�Eð #øà"Ð.à+ˆGØ˜R”[Ô& qÔ)ˆFˆFå�LŠLÐAÑBÔBÐBØˆFØ!Ð%6Ð6Ð6��¸BˆØ!*Ð.@Ð!@Ð!@�I�IÀbˆ	àÐ E Màœ:×5Ò5°e´kÀ!´nÑEÔEˆLØ#Ô(¨Ô+ˆHØ Ô*ˆEØ*ˆOØœ:×5Ò5°oÑFÔFˆLØÐ#Ø×$Ò$Ø#ØØ"˜Ýœ 3 %¨(Ñ"2½&Ô:YÐZ_Ñ:`Ô:`ÐaÑaÔað	 %ñ ô ð ð ”z×2Ò2°5Ñ9Ô9ˆHÝÔ$ U¨_¸h¼oÈaÔ>PÐ,QÐT]ÔTbÐScÐemÑnÔnˆEå  ™LœLÐc­T°&©\¬\Ðc¸hÈ$Ð>NÐcÐS[Ð_cÐScÐà!Ð!ð 0Ø”˜qÔ! ZÒ/ð0à”˜qÔ! ZÒ/ð0ð ”˜qÔ! ZÒ/ð	 	ð "Ð!ð 7Ø”˜qÔ! ZÒ/ð7à”˜qÔ! X¤^°AÔ%6Ò6ð7ð ”˜qÔ! X¤^°AÔ%6Ò6ð	 	ð +ÐK¨xÐ;KÒ/KÐØ!/Ð!R°HÐ@RÒ4RÐØ"1Ð"R°hÐBRÒ6RÐØ+AÐ+cÅdÈ6ÁlÄlÐ+cÕW[Ð\bÑWcÔWcÐ(Ø,=Ð,^À(ÐN^ÒB^Ð)ð Ð"4Ò4ˆØˆ
Øð  	'Ø"œj×:Ò:ØØ�	Ø�ñô ˆOð
 %)¤J×$@Ò$@ØØ,Ð,Ð,Ø�	�	ñ%ô %Ð!ð
 &*¤Z×%AÒ%AØØ@Ð@Ð@Ø��ñ&ô &Ð"ð
 04¬z×/KÒ/KØØ˜'Ð"Ø�A�ñ0ô 0Ð,ð
 %Ð0Ø2�
�
Ø'Ð3Ø3�
�
Ø1Ð=Ø=�
�
Ø Ð,Ø,�
�
å—’ÐIÑJÔJÐJØ�Ý�z‘?”? QÒ&Ð&Ð&Ð&ð ð]	$à%ð]	$ð 'ð]	$ð 0ð	]	$ð
 1ñ]	$ð #.ÐØ%)×%GÒ%GÈ	Ñ%RÔ%RÑ"ˆI�{à˜AŠ~ˆ~ °Ò!1Ð!1°kÀIÑ6MÐRSÒ5SÐ5SÝ—’ÐXÑYÔYÐYØ�àˆHØ/ð .VÐ3Jð .VÐNoð .Vð* Ô4ð!�D×8Ò8Ø!)Ø-DÐ!tÐHhÐ!t  ÐntØ-DÐ!tÐHhÐ!t  ÐntØ#Ø'>ÐlÐBbÐl˜e˜eÐhlØ'>ÐlÐBbÐl˜e˜eÐhlØ"+Ø$/Ø2Ô9¸!Ô<Ø'2Ø)IÐQ˜v˜vÈrØ)IÐQ˜v˜vÈrØ"+Ø"+ð 9ñ ô ð ð" ð% �ð, 9=Ô8UÐ5Ø05�Ô-Ø×5Ò5Ø#Ø%Ø%Ø%ØØØØ'Ø +Ø *Ø.Ô5°aÔ8Ø&Ø!Ø!Ø'Ø'ð! 6ñ ô �ð$ 1V�Ô-ØÐÝ—’ÐJÑKÔKÐKØ�àÔ×$Ò$ XÑ.Ô.Ð.Ø:>Ô:NˆDÔ(¨¬Ñ7àÔ ×'Ò'Ð)<¸mÈZÐ(XÑYÔYÐYØÔ ×'Ò'¨Ñ1Ô1Ð1ð 0ð &Ð3Jð &ÐNoñ &Ý�w‘<”< !Ò#Ð#¨°¬Ô(;¸xÒ(GÐ(GØ—K’K‘M”M�MÝ�w‘<”< !Ò#Ð#¨°¬Ô(;¸xÒ(GÐ(GØ—K’K‘M”M�MÝ�w‘<”< !Ò#Ð#¨°¬Ô(;¸xÒ(GÐ(GØ—K’K‘M”M�MØÔ9ð &Ý˜7‘|”| aÒ'Ð'¨G°B¬KÔ,?À5Ò,HÐ,HØŸš™œ˜Ý˜7‘|”| aÒ'Ð'¨G°B¬KÔ,?À5Ò,HÐ,HØŸš™œ˜Ý˜7‘|”| aÒ'Ð'¨G°B¬KÔ,?À5Ò,HÐ,HØŸš™œ˜àÔ ×'Ò'¨Ñ0Ô0Ð0ØÔ ×'Ò'¨Ñ0Ô0Ð0ØÔ ×'Ò'¨Ñ0Ô0Ð0ð  $ˆDÔÐÐð{]	$ð ]	$r   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   r   rµ   Ú__classcell__)r   s   @r   r   r      s�   ø€ € € € € ðð ðHàðHð ðHð ð	Hð
 &ðHð Hð Hð Hð Hð HðV$ð V$ð V$ð V$ð V$ð V$ð V$r   r   )ÚloggingÚnumpyrS   Úfusion_attentionr   r   Úonnxr   Ú
onnx_modelr   Ú	getLoggerr¶   rH   r   r   r   r   ú<module>rÂ      s¨   ðð
 €€€à Ð Ð Ð Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ø Ð Ð Ð Ð Ð Ø  Ð  Ð  Ð  Ð  Ð  à	ˆÔ	˜8Ñ	$Ô	$€ðd$ð d$ð d$ð d$ð d$˜/ñ d$ô d$ð d$ð d$ð d$r   