§
    ¹¯ÅjðK  ã                   ó0  — d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZd dl	Z	d dl
mZ d dlmZmZ eeeee         z  f         Z ej        e¦  «        Z G d„ d¦  «        Z G d	„ d
¦  «        Z G d„ d¦  «        Z G d„ de¦  «        Z G d„ d¦  «        ZdS )é    N)ÚOrderedDict)ÚMapping)ÚAny)ÚTensorProto)ÚInferenceSessionÚ
RunOptionsc                   ó–  — e Zd Zedededefd„¦   «         Zededefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Z	edede
eef         fd	„¦   «         Zedefd
„¦   «         Zedej        fd„¦   «         Zedej        fd„¦   «         Zedede
eej        f         fd„¦   «         Zedede
eej        f         fd„¦   «         ZdS )Ú
TypeHelperÚort_sessionÚnameÚreturnc                 óž   — t          |                      ¦   «         ¦  «        D ]\  }}|j        |k    r	|j        c S Œt	          d|› d�¦  «        ‚)Nzinput name ú
 not found)Ú	enumerateÚ
get_inputsr   ÚtypeÚ
ValueError)r   r   Ú_iÚinputs       ú[C:\Projects\Historical\venv\Lib\site-packages\onnxruntime/transformers/io_binding_helper.pyÚget_input_typezTypeHelper.get_input_type   sc   € å" ;×#9Ò#9Ñ#;Ô#;Ñ<Ô<ð 	"ð 	"‰IˆB�ØŒz˜TÒ!Ð!Ø”zÐ!Ð!Ð!ð "åÐ7 tÐ7Ð7Ð7Ñ8Ô8Ð8ó    c                 óž   — t          |                      ¦   «         ¦  «        D ]\  }}|j        |k    r	|j        c S Œt	          d|› d�¦  «        ‚)Nzoutput name r   )r   Úget_outputsr   r   r   )r   r   r   Úoutputs       r   Úget_output_typezTypeHelper.get_output_type   se   € å# K×$;Ò$;Ñ$=Ô$=Ñ>Ô>ð 	#ð 	#‰JˆB�ØŒ{˜dÒ"Ð"Ø”{Ð"Ð"Ð"ð #õ Ð8¨Ð8Ð8Ð8Ñ9Ô9Ð9r   Úort_typec                 óÔ   — t           j        t           j        t           j        t           j        t
          t           j        t           j        dœ}| |vrt          | › d�¦  «        ‚||          S )N)útensor(int64)útensor(int32)útensor(float)útensor(float16)útensor(bool)útensor(uint8)útensor(int8)ú not found in map)	ÚnumpyÚlonglongÚintcÚfloat32Úfloat16ÚboolÚuint8Úint8r   )r   Úort_type_to_numpy_type_maps     r   Úort_type_to_numpy_typez!TypeHelper.ort_type_to_numpy_type#   sb   € õ #œ^Ý"œZÝ"œ]Ý$œ}Ý Ý"œ[Ý!œJð&
ð &
Ð"ð Ð5Ð5Ð5Ý Ð;Ð;Ð;Ñ<Ô<Ð<à)¨(Ô3Ð3r   c           	      óô   — t           j        t           j        t           j        t           j        t           j        t           j        t           j        t           j        dœ}| |vrt          | › d�¦  «        ‚||          S ©N)r   r    r!   r"   ztensor(bfloat16)r#   r$   r%   r&   )
ÚtorchÚint64Úint32r*   r+   Úbfloat16r,   r-   r.   r   )r   Úort_type_to_torch_type_maps     r   Úort_type_to_torch_typez!TypeHelper.ort_type_to_torch_type3   si   € õ #œ[Ý"œ[Ý"œ]Ý$œ}Ý %¤Ý!œJÝ"œ[Ý!œJð	&
ð 	&
Ð"ð Ð5Ð5Ð5Ý Ð;Ð;Ð;Ñ<Ô<Ð<à)¨(Ô3Ð3r   c                 ó  — i }|                       ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|                      ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|S )z9Create a mapping from input/output name to onnx data type)r   r
   Úort_type_to_onnx_typer   r   r   )r   Úname_to_onnx_typer   r   s       r   Úget_io_onnx_type_mapzTypeHelper.get_io_onnx_type_mapD   sŠ   € ð ÐØ ×+Ò+Ñ-Ô-ð 	Yð 	YˆEÝ,6×,LÒ,LÈUÌZÑ,XÔ,XÐ˜eœjÑ)Ð)à!×-Ò-Ñ/Ô/ð 	[ð 	[ˆFÝ-7×-MÒ-MÈfÌkÑ-ZÔ-ZÐ˜fœkÑ*Ð*Ø Ð r   c           	      óô   — t           j        t           j        t           j        t           j        t           j        t           j        t           j        t           j        dœ}| |vrt          | › d�¦  «        ‚||          S r2   )
r   ÚINT64ÚINT32ÚFLOATÚFLOAT16ÚBFLOAT16ÚBOOLÚUINT8ÚINT8r   )r   Úort_type_to_onnx_type_maps     r   r:   z TypeHelper.ort_type_to_onnx_typeO   sq   € õ )Ô.Ý(Ô.Ý(Ô.Ý*Ô2Ý +Ô 4Ý'Ô,Ý(Ô.Ý'Ô,ð	%
ð 	%
Ð!ð Ð4Ð4Ð4Ý Ð;Ð;Ð;Ñ<Ô<Ð<à(¨Ô2Ð2r   Ú
numpy_typec                 ó˜  — t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t          t          j        t           j	        t          j	        t           j
        t          j
        i}| |vrt          | › d�¦  «        ‚||          S ©Nr&   )r'   r(   r3   r4   r)   r5   r*   r+   r,   r-   r.   r   )rG   Únumpy_type_to_torch_type_maps     r   Únumpy_type_to_torch_typez#TypeHelper.numpy_type_to_torch_type`   s‚   € õ ŒN�EœKÝŒJ�œÝŒK�œÝŒM�5œ=ÝŒM�5œ=Ý•%”*ÝŒK�œÝŒJ�œ
ð	(
Ð$ð Ð9Ð9Ð9Ý 
Ð=Ð=Ð=Ñ>Ô>Ð>à+¨JÔ7Ð7r   Ú
torch_typec                 ó@  — t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          t           j	        t          j	        i}| |vrt          | › d�¦  «        ‚||          S rI   )r3   r4   r'   r(   r5   r)   r*   r+   r,   r-   r   )rL   Útorch_type_to_numpy_type_maps     r   Útorch_type_to_numpy_typez#TypeHelper.torch_type_to_numpy_typeq   sp   € õ ŒK�œÝŒK�œÝŒM�5œ=ÝŒM�5œ=ÝŒJ�ÝŒK�œð(
Ð$ð Ð9Ð9Ð9Ý 
Ð=Ð=Ð=Ñ>Ô>Ð>à+¨JÔ7Ð7r   c                 ó  — i }|                       ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|                      ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|S )z:Create a mapping from input/output name to numpy data type)r   r
   r0   r   r   r   )r   Úname_to_numpy_typer   r   s       r   Úget_io_numpy_type_mapz TypeHelper.get_io_numpy_type_map€   óŒ   € ð  ÐØ ×+Ò+Ñ-Ô-ð 	[ð 	[ˆEÝ-7×-NÒ-NÈuÌzÑ-ZÔ-ZÐ˜uœzÑ*Ð*à!×-Ò-Ñ/Ô/ð 	]ð 	]ˆFÝ.8×.OÒ.OÐPVÔP[Ñ.\Ô.\Ð˜vœ{Ñ+Ð+Ø!Ð!r   c                 ó  — i }|                       ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|                      ¦   «         D ])}t                               |j        ¦  «        ||j        <   Œ*|S )z:Create a mapping from input/output name to torch data type)r   r
   r8   r   r   r   )r   Úname_to_torch_typer   r   s       r   Úget_io_torch_type_mapz TypeHelper.get_io_torch_type_map‹   rS   r   N)Ú__name__Ú
__module__Ú__qualname__Ústaticmethodr   Ústrr   r   r0   r8   ÚdictÚintr<   r:   r'   ÚdtyperK   r3   rO   rR   rV   © r   r   r
   r
      sê  € € € € € Øð9Ð$4ð 9¸Cð 9ÀCð 9ð 9ð 9ñ „\ð9ð ð:¨3ð :°3ð :ð :ð :ñ „\ð:ð ð4¨ð 4ð 4ð 4ñ „\ð4ð ð4¨ð 4ð 4ð 4ñ „\ð4ð  ð!Ð*:ð !¸tÀCÈÀH¼~ð !ð !ð !ñ „\ð!ð ð3¨ð 3ð 3ð 3ñ „\ð3ð  ð8¨U¬[ð 8ð 8ð 8ñ „\ð8ð  ð8¨U¬[ð 8ð 8ð 8ñ „\ð8ð ð"Ð+;ð "ÀÀSÈ%Ì+ÐEUÔ@Vð "ð "ð "ñ „\ð"ð ð"Ð+;ð "ÀÀSÈ%Ì+ÐEUÔ@Vð "ð "ð "ñ „\ð"ð "ð "r   r
   c            
       óž   — e Zd Zedefd„¦   «         Zedej        dej        dej        deej                 fd„¦   «         Z	edd	„¦   «         Z
d
S )ÚIOBindingHelperr   c                 ó   — i }|                      ¦   «         D ]f\  }}t                               | |¦  «        }t                               |¦  «        }t	          j        t          j        |¦  «        ||¬¦  «        ||<   Œg|S )zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.)r^   Údevice)Úitemsr
   r   r8   r3   Úemptyr'   Úprod)r   Úoutput_shapesrc   Úoutput_buffersr   Úshaper   rL   s           r   Úget_output_buffersz"IOBindingHelper.get_output_buffers˜   s�   € ð ˆØ(×.Ò.Ñ0Ô0ð 	cð 	c‰KˆD�%Ý!×1Ò1°+¸tÑDÔDˆHÝ#×:Ò:¸8ÑDÔDˆJÝ#(¤;­u¬z¸%Ñ/@Ô/@È
Ð[aÐ#bÑ#bÔ#bˆN˜4Ñ Ð ØÐr   Ú	input_idsÚposition_idsÚattention_maskÚpastc                 óÊ  — t                                | ¦  «        }|                      ¦   «         }|                     ¦   «         sJ ‚|                     d|j        j        d|d         t          |                     ¦   «         ¦  «        | 	                    ¦   «         ¦  «         |�¨t          |¦  «        D ]˜\  }	}
|
                     ¦   «         sJ ‚|
 	                    ¦   «         }|dk    r| 	                    ¦   «         }|                     d|	› �|
j        j        d|d|	› �         t          |
                     ¦   «         ¦  «        |¦  «         Œ™|�q|                     ¦   «         sJ ‚|                     d|j        j        d|d         t          |                     ¦   «         ¦  «        | 	                    ¦   «         ¦  «         |�q|                     ¦   «         sJ ‚|                     d|j        j        d|d         t          |                     ¦   «         ¦  «        | 	                    ¦   «         ¦  «         |                      ¦   «         D ]ž}|j        }||         }t                               |› d|j        j        › dt          |                     ¦   «         ¦  «        › �¦  «         |                     ||j        j        d||         ||         | 	                    ¦   «         ¦  «         ŒŸ|S )	zdIO binding for a session: bind inputs (input_ids, position_ids, attention_mask, past_*) and outputs.rk   r   NÚpast_rm   rl   z device type=z shape=)r
   r<   Ú
io_bindingÚis_contiguousÚ
bind_inputrc   r   ÚlistÚsizeÚdata_ptrr   r   r   ÚloggerÚdebugÚbind_output)r   rk   rl   rm   rn   rh   rg   r;   rq   ÚiÚpast_irv   r   Úoutput_nameÚoutput_buffers                  r   Úprepare_io_bindingz"IOBindingHelper.prepare_io_binding¢   s  € õ '×;Ò;¸KÑHÔHÐð !×+Ò+Ñ-Ô-ˆ
ð ×&Ò&Ñ(Ô(Ð(Ð(Ð(Ø×ÒØØÔÔ!ØØ˜kÔ*Ý�—’Ñ!Ô!Ñ"Ô"Ø×ÒÑ Ô ñ	
ô 	
ð 	
ð ÐÝ& t™_œ_ð ð ‘	��6Ø×+Ò+Ñ-Ô-Ð-Ð-Ð-à!Ÿ?š?Ñ,Ô,�Ø˜q’=�=ð  )×1Ò1Ñ3Ô3�Hà×%Ò%Ø˜A�K�KØ”MÔ&ØØ% k¨a k kÔ2Ý˜Ÿš™œÑ'Ô'Øñô ð ð ð Ð%Ø!×/Ò/Ñ1Ô1Ð1Ð1Ð1Ø×!Ò!Ø ØÔ%Ô*ØØ!Ð"2Ô3Ý�^×(Ò(Ñ*Ô*Ñ+Ô+Ø×'Ò'Ñ)Ô)ñô ð ð Ð#Ø×-Ò-Ñ/Ô/Ð/Ð/Ð/Ø×!Ò!ØØÔ#Ô(ØØ! .Ô1Ý�\×&Ò&Ñ(Ô(Ñ)Ô)Ø×%Ò%Ñ'Ô'ñô ð ð "×-Ò-Ñ/Ô/ð 	ð 	ˆFØ œ+ˆKØ*¨;Ô7ˆMÝ�LŠL˜KÐtÐt°mÔ6JÔ6OÐtÐtÕX\Ð]j×]oÒ]oÑ]qÔ]qÑXrÔXrÐtÐtÑuÔuÐuØ×"Ò"ØØÔ$Ô)ØØ! +Ô.Ø˜kÔ*Ø×&Ò&Ñ(Ô(ñô ð ð ð Ðr   Tc                 ó®  — g }|                       ¦   «         D ]½}|j        }||         }||         }|dt          j        |¦  «        …                              |¦  «                             ¦   «                              ¦   «         }	|r:|                     |	                     ¦   «                              ¦   «         ¦  «         Œ¨|                     |	¦  «         Œ¾|S )z3Copy results to cpu. Returns a list of numpy array.r   )	r   r   r'   rf   ÚreshapeÚcloneÚdetachÚappendÚcpu)
r   rh   rg   Úreturn_numpyÚort_outputsr   r|   Úbufferri   Úcopy_tensors
             r   Ú"get_outputs_from_io_binding_bufferz2IOBindingHelper.get_outputs_from_io_binding_buffer÷   sÐ   € ð ˆØ!×-Ò-Ñ/Ô/ð 	0ð 	0ˆFØ œ+ˆKØ# KÔ0ˆFØ! +Ô.ˆEØ  ¥U¤Z°Ñ%6Ô%6Ð!6Ô7×?Ò?ÀÑFÔF×LÒLÑNÔN×UÒUÑWÔWˆKØð 0Ø×"Ò" ;§?¢?Ñ#4Ô#4×#:Ò#:Ñ#<Ô#<Ñ=Ô=Ð=Ð=à×"Ò" ;Ñ/Ô/Ð/Ð/ØÐr   N)T)rW   rX   rY   rZ   r   rj   r3   ÚTensorrt   r~   r‰   r_   r   r   ra   ra   —   s¶   € € € € € ØðÐ(8ð ð ð ñ „\ðð ðRà”<ðRð ”lðRð œð	Rð
 �5”<Ô ðRð Rð Rñ „\ðRðh ðð ð ñ „\ðð ð r   ra   c                   óÔ   — e Zd ZdZddedej        fd„Zdedefd„Z	d	„ Z
d
edej        fd„Zdefd„Zddeeej        f         dedefd„Zeddedededeeef         fd„¦   «         ZdS )ÚCudaSessionzLInference Session with IO Binding for ONNX Runtime CUDA or TensorRT providerFr   rc   c                 óö  — || _         d„ | j                              ¦   «         D ¦   «         | _        d„ | j                              ¦   «         D ¦   «         | _        t
                               | j         ¦  «        | _        t
                               | j         ¦  «        | _	        | j          
                    ¦   «         | _
        || _        t          ¦   «         | _        t          ¦   «         | _        || _        i | _        d S )Nc                 ó   — g | ]	}|j         ‘Œ
S r_   ©r   )Ú.0r   s     r   ú
<listcomp>z(CudaSession.__init__.<locals>.<listcomp>  s   € ÐRÐRÐR¨5˜EœJÐRÐRÐRr   c                 ó   — g | ]	}|j         ‘Œ
S r_   r�   )r�   r   s     r   r‘   z(CudaSession.__init__.<locals>.<listcomp>  s   € ÐVÐVÐV¨V˜Vœ[ÐVÐVÐVr   )r   r   Úinput_namesr   Úoutput_namesr
   r<   Úio_name_to_onnx_typerV   Úio_name_to_torch_typerq   Úenable_cuda_graphr   Úinput_tensorsÚoutput_tensorsrc   Úbuffer_sharing)Úselfr   rc   r—   s       r   Ú__init__zCudaSession.__init__
  sÓ   € Ø&ˆÔØRÐR°DÔ4D×4OÒ4OÑ4QÔ4QÐRÑRÔRˆÔØVÐV°tÔ7G×7SÒ7SÑ7UÔ7UÐVÑVÔVˆÔÝ$.×$CÒ$CÀDÔDTÑ$UÔ$UˆÔ!Ý%/×%EÒ%EÀdÔFVÑ%WÔ%WˆÔ"ØÔ*×5Ò5Ñ7Ô7ˆŒØ!2ˆÔå(™]œ]ˆÔÝ)™mœmˆÔØˆŒð /1ˆÔÐÐr   Ú
input_namer|   c                 óZ   — || j         v sJ ‚|| j        v sJ ‚|| j        |<   || j        |<   d S ©N)r“   r”   rš   )r›   r�   r|   s      r   Úset_buffer_sharingzCudaSession.set_buffer_sharing  sJ   € Ø˜TÔ-Ð-Ð-Ð-Ð-Ø˜dÔ/Ð/Ð/Ð/Ð/Ø*5ˆÔ˜JÑ'Ø+5ˆÔ˜KÑ(Ð(Ð(r   c                 ó   — | ` | `| `d S rŸ   )r˜   r™   rq   )r›   s    r   Ú__del__zCudaSession.__del__   s   € ØÐØÐØˆOˆOˆOr   r   Útensorc           	      ó  — |j         j        �|j         j        nd}t          |j        ¦  «        dk    rdgnt	          |j        ¦  «        }| j                             ||j         j        || j        |         || 	                    ¦   «         ¦  «         || j
        v rh| j                             | j
        |         |j         j        || j        |         || 	                    ¦   «         ¦  «         || j        | j
        |         <   d S d S )Nr   é   )rc   ÚindexÚlenri   rt   rq   rs   r   r•   rv   rš   ry   r™   )r›   r   r£   Ú	device_idÚtensor_shapes        r   Úbind_input_and_buffer_sharingz)CudaSession.bind_input_and_buffer_sharing%  s	  € Ø+1¬=Ô+>Ð+J�F”MÔ'Ð'ÐPQˆ	Ý! &¤,Ñ/Ô/°1Ò4Ð4˜�s�s½$¸v¼|Ñ:LÔ:LˆàŒ×"Ò"ØØŒMÔØØÔ% dÔ+ØØ�OŠOÑÔñ	
ô 	
ð 	
ð �4Ô&Ð&Ð&ØŒO×'Ò'ØÔ# DÔ)Ø”Ô"ØØÔ)¨$Ô/ØØ—’Ñ!Ô!ñô ð ð >DˆDÔ Ô 3°DÔ 9Ñ:Ð:Ð:ð 'Ð&r   Ú
shape_dictc                 ó2  — | j         rÕ|                     ¦   «         D ]À\  }}|| j        v r²|| j        v r@t	          | j        |         j        ¦  «        t	          |¦  «        k    rŒHt          d¦  «        ‚| j        |         }t          j	        t	          |¦  «        |¬¦  «         
                    | j        ¬¦  «        }|| j        |<   |                      ||¦  «         ŒÁ|                     ¦   «         D �]$\  }}|| j        v �r|| j        v r1t	          | j        |         j        ¦  «        t	          |¦  «        k    rŒJ|| j        v rŒT| j        |         }t          j	        t	          |¦  «        |¬¦  «         
                    | j        ¬¦  «        }|| j        |<   | j                             ||j        j        |j        j        �|j        j        nd| j        |         t+          |                     ¦   «         ¦  «        |                     ¦   «         ¦  «         �Œ&dS )z Allocate tensors for I/O Bindingz(Expect static input shape for cuda graph)r^   )rc   Nr   )r—   rd   r“   r˜   Útupleri   ÚRuntimeErrorr–   r3   re   Útorc   rª   r”   r™   rš   rq   ry   r   r¦   r•   rt   ru   rv   )r›   r«   r   ri   Útorch_dtyper£   s         r   Úallocate_bufferszCudaSession.allocate_buffers=  s  € àÔ!ð 	EØ)×/Ò/Ñ1Ô1ð Eð E‘��eØ˜4Ô+Ð+Ð+à˜tÔ1Ð1Ð1Ý  Ô!3°DÔ!9Ô!?Ñ@Ô@ÅEÈ%ÁLÄLÒPÐPØ$Ý*Ð+UÑVÔVÐVà"&Ô"<¸TÔ"B�KÝ"œ[­¨u©¬¸[ÐIÑIÔI×LÒLÐTXÔT_ÐLÑ`Ô`�FØ/5�DÔ& tÑ,Ø×6Ò6°t¸VÑDÔDÐDøà%×+Ò+Ñ-Ô-ð 	ñ 	‰KˆD�%Ø�tÔ(Ð(Ñ(à˜4Ô.Ð.Ð.µ5¸Ô9LÈTÔ9RÔ9XÑ3YÔ3YÕ]bÐchÑ]iÔ]iÒ3iÐ3iØà˜4Ô.Ð.Ð.Øà"Ô8¸Ô>�Ýœ¥U¨5¡\¤\¸ÐEÑEÔE×HÒHÐPTÔP[ÐHÑ\Ô\�Ø,2�Ô# DÑ)à”×+Ò+ØØ”MÔ&Ø+1¬=Ô+>Ð+J�F”MÔ'Ð'ÐPQØÔ-¨dÔ3Ý˜Ÿš™œÑ'Ô'Ø—O’OÑ%Ô%ñô ð ùð	ð 	r   NTÚ	feed_dictÚrun_optionsÚsynchronizec                 óè  — |                      ¦   «         D ]â\  }}t          |t          j        ¦  «        r|                     ¦   «         sJ ‚|| j        v r¤| j        r‡| j        |                              ¦   «         |                     ¦   «         k    sJ ‚| j        |         j	        |j	        k    sJ ‚|j
        j        dk    sJ ‚| j        |                              |¦  «         ŒÌ|                      ||¦  «         Œã|rS| j                             ¦   «          | j                             | j        |¦  «         | j                             ¦   «          n | j                             | j        |¦  «         | j        S )z$Bind input tensors and run inferenceÚcuda)rd   Ú
isinstancer3   rŠ   rr   r“   r—   r˜   Únelementr^   rc   r   Úcopy_rª   rq   Úsynchronize_inputsr   Úrun_with_iobindingÚsynchronize_outputsr™   )r›   r²   r³   r´   r   r£   s         r   ÚinferzCudaSession.inferc  sp  € à%ŸOšOÑ-Ô-ð 		Eð 		E‰LˆD�&Ý˜f¥e¤lÑ3Ô3ÐN¸×8LÒ8LÑ8NÔ8NÐNÐNÐNØ�tÔ'Ð'Ð'ØÔ)ð EØÔ-¨dÔ3×<Ò<Ñ>Ô>À&Ç/Â/ÑBSÔBSÒSÐSÐSÐSØÔ-¨dÔ3Ô9¸V¼\ÒIÐIÐIÐIØ!œ=Ô-°Ò7Ð7Ð7Ð7ØÔ& tÔ,×2Ò2°6Ñ:Ô:Ð:Ð:à×6Ò6°t¸VÑDÔDÐDøàð 	NØŒO×.Ò.Ñ0Ô0Ð0ØÔ×/Ò/°´ÀÑMÔMÐMØŒO×/Ò/Ñ1Ô1Ð1Ð1àÔ×/Ò/°´ÀÑMÔMÐMàÔ"Ð"r   r   r¨   r—   Ústreamr   c                 óB   — | d|dœ}|dk    rt          |¦  «        |d<   |S )NÚkSameAsRequested)r¨   Úarena_extend_strategyr—   r   Úuser_compute_stream)r[   )r¨   r—   r¾   Úoptionss       r   Úget_cuda_provider_optionsz%CudaSession.get_cuda_provider_optionsy  s:   € ð #Ø%7Ø!2ð
ð 
ˆð �QŠ;ˆ;Ý-0°©[¬[ˆGÐ)Ñ*àˆr   ©F)NT)r   )rW   rX   rY   Ú__doc__r   r3   rc   rœ   r[   r    r¢   rŠ   rª   Ú	ShapeDictr±   r\   r   r,   r½   rZ   r]   r   rÄ   r_   r   r   rŒ   rŒ     sE  € € € € € ØVÐVð1ð 1Ð$4ð 1¸e¼lð 1ð 1ð 1ð 1ð 6¨Sð 6¸sð 6ð 6ð 6ð 6ðð ð ð
D°#ð D¸u¼|ð Dð Dð Dð Dð0$¨9ð $ð $ð $ð $ðL#ð #˜t C¨¬Ð$5Ô6ð #ÀZð #Ðeið #ð #ð #ð #ð, ðð ¨Sð ÀTð ÐSVð Ð_cÐdgÐilÐdlÔ_mð ð ð ñ „\ðð ð r   rŒ   c                   ó¨   ‡ — e Zd Z	 	 	 	 ddedej        deded	ed
ede	e
e
f         dz  fˆ fd„Zddedefd„Zdde	e
ej        f         defˆ fd„Zˆ xZS )Ú
GpuBindingFéÿÿÿÿr   Nr   rc   r«   Úenable_gpu_graphÚgpu_graph_idr¾   rš   c                 ó@  •— t          ¦   «                              |||¦  «         |r0|                     ¦   «         D ]\  }}	|                      ||	¦  «         Œ|                      |¦  «         || _        |rt          j        |¦  «        nd | _        || _	        d | _
        d S rŸ   )Úsuperrœ   rd   r    r±   rÌ   ÚcopyÚdeepcopyr«   r¾   Úlast_run_gpu_graph_id)r›   r   rc   r«   rË   rÌ   r¾   rš   r�   r|   Ú	__class__s             €r   rœ   zGpuBinding.__init__‰  s²   ø€ õ 	‰Œ×Ò˜ fÐ.>Ñ?Ô?Ð?Øð 	AØ+9×+?Ò+?Ñ+AÔ+Að Að AÑ'�
˜KØ×'Ò'¨
°KÑ@Ô@Ð@Ð@à×Ò˜jÑ)Ô)Ð)Ø(ˆÔà7GÐQ�$œ-¨
Ñ3Ô3Ð3ÈTˆŒØˆŒà%)ˆÔ"Ð"Ð"r   Údisable_cuda_graph_in_runr   c                 óŒ   — t          ¦   «         }|rdn| j        }|                     dt          |¦  «        ¦  «         || _        |S )NrÊ   rÌ   )r   rÌ   Úadd_run_config_entryr[   rÑ   )r›   rÓ   rÃ   rÌ   s       r   Úget_run_optionszGpuBinding.get_run_options   sG   € Ý‘,”,ˆà6ÐM�r�r¸DÔ<Mˆà×$Ò$ ^µS¸Ñ5FÔ5FÑGÔGÐGà%1ˆÔ"àˆr   r²   c                 ó¬   •— |                       |¦  «        }| j        r|                     dd¦  «         t          ¦   «                              ||¦  «        S )NÚ'disable_synchronize_execution_providersÚ1)rÖ   r¾   rÕ   rÎ   r½   )r›   r²   rÓ   r³   rÒ   s       €r   r½   zGpuBinding.infer«  sR   ø€ Ø×*Ò*Ð+DÑEÔEˆàŒ;ð 	]Ø×,Ò,Ð-VÐX[Ñ\Ô\Ð\å‰wŒw�}Š}˜Y¨Ñ4Ô4Ð4r   )FrÊ   r   NrÅ   )rW   rX   rY   r   r3   rc   rÇ   r,   r]   r\   r[   rœ   r   rÖ   rŠ   r½   Ú__classcell__)rÒ   s   @r   rÉ   rÉ   ˆ  s  ø€ € € € € ð "'ØØØ04ð*ð *à%ð*ð ”ð*ð ð	*ð
 ð*ð ð*ð ð*ð ˜S #˜Xœ¨Ñ-ð*ð *ð *ð *ð *ð *ð.	ð 	¸ð 	È*ð 	ð 	ð 	ð 	ð5ð 5˜t C¨¬Ð$5Ô6ð 5ÐSWð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5r   rÉ   c            	       ój   — e Zd ZdZddedej        dedefd„Z	 	 dde	de
deeef         d
z  defd„Zd
S )ÚGpuBindingManagerz­A manager for I/O bindings that support multiple CUDA Graphs.
    One cuda graph is reused for same input shape. Automatically add a new cuda graph for new input shape.
    r   r¥   r   rc   r¾   Úmax_cuda_graphsc                 óZ   — || _         || _        g | _        d | _        || _        || _        d S rŸ   )r   rc   Úgraph_bindingsÚno_graph_bindingr¾   rÝ   )r›   r   rc   r¾   rÝ   s        r   rœ   zGpuBindingManager.__init__¹  s:   € Ø&ˆÔØˆŒð !ˆÔð !%ˆÔàˆŒà.ˆÔÐÐr   FNr«   Úuse_cuda_graphrš   r   c           	      óº  — | j         D ]}|j        |k    r|c S Œt          | j         ¦  «        | j        k    s|sQ| j        €)t          | j        | j        || j        |¬¦  «        | _        n| j         	                    |¦  «         | j        S t          | j        | j        |dt          | j         ¦  «        | j        |¬¦  «        }| j          
                    |¦  «         |S )N)r¾   rš   T)rË   rÌ   r¾   rš   )rß   r«   r§   rÝ   rà   rÉ   r   rc   r¾   r±   rƒ   )r›   r«   rá   rš   Úgpu_graph_bindings        r   Úget_bindingzGpuBindingManager.get_bindingÇ  s  € ð "&Ô!4ð 	)ð 	)Ðà Ô+¨zÒ9Ð9Ø(Ð(Ð(Ð(ð :õ ˆtÔ"Ñ#Ô# tÔ';Ò;Ð;ÀNÐ;ØÔ$Ð,Ý(2ØÔ$ d¤k°:ÀdÄkÐbpð)ñ )ô )�Ô%Ð%ð Ô%×6Ò6°zÑBÔBÐBØÔ(Ð(õ 'ØÔØŒKØØ!Ý˜TÔ0Ñ1Ô1Ø”;Ø)ð
ñ 
ô 
Ðð 	Ô×"Ò"Ð#4Ñ5Ô5Ð5Ø Ð r   )r   r¥   )FN)rW   rX   rY   rÆ   r   r3   rc   r]   rœ   rÇ   r,   r\   r[   rÉ   rä   r_   r   r   rÜ   rÜ   ´  s¬   € € € € € ðð ð/ð /Ð$4ð /¸e¼lð /ÐTWð /Ðnqð /ð /ð /ð /ð"  %Ø04ð	 !ð  !àð !ð ð !ð ˜S #˜Xœ¨Ñ-ð	 !ð
 
ð !ð  !ð  !ð  !ð  !ð  !r   rÜ   )rÏ   ÚloggingÚcollectionsr   Úcollections.abcr   Útypingr   r'   r3   Úonnxr   Úonnxruntimer   r   r[   r­   rt   r]   rÇ   Ú	getLoggerrW   rw   r
   ra   rŒ   rÉ   rÜ   r_   r   r   ú<module>rì      s¥  ðØ €€€Ø €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø #Ð #Ð #Ð #Ð #Ð #Ø Ð Ð Ð Ð Ð à €€€Ø €€€Ø Ð Ð Ð Ð Ð à 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4ð �C˜  c¤Ñ*Ð*Ô+€	à	ˆÔ	˜8Ñ	$Ô	$€ðA"ð A"ð A"ð A"ð A"ñ A"ô A"ð A"ðHmð mð mð mð mñ mô mð mð`~ð ~ð ~ð ~ð ~ñ ~ô ~ð ~ðB)5ð )5ð )5ð )5ð )5�ñ )5ô )5ð )5ðX3!ð 3!ð 3!ð 3!ð 3!ñ 3!ô 3!ð 3!ð 3!ð 3!r   