§
    [�’j’@  ã                   ó´   — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 dd	lmZ dd
lmZmZ ddlmZmZ ddlmZmZmZmZmZ  G d„ d¦  «        ZdS )é    N)ÚCallable)Únullcontext)Únné   )ÚContinuousBatchingConfigé   )ÚPagedAttentionCache)Ú%ContinuousBatchingLogitsProcessorList)ÚContinuousBatchingAsyncIOsÚContinuousBatchingIOs)ÚRequestStatusÚlogger)Úcreate_warmup_future_statesÚget_cuda_poolsÚmem_pool_ctxÚpad_to_intervalÚpad_to_pow2c                   óÀ  — e Zd ZdZdededeez  dede	de	dd	fd
„Z
dedede	deeef         fd„Zdej        de	fd„Zdej        dedd	fd„Zde	deee	f         fd„Zdedej        j        dd	fd„Zdej        dedej        dej        dej        dd	fd„Zdej        dej        dd	fd„Z ej        ¦   «         dej        dd	fd„¦   «         Zdej        deded	z  defd„Zd	S ) ÚModelRunnerzžThis class is the continuous batching entry point for running the model. As a rule of thumb, anything running on
    the device should happen from this class.Úlogit_processorÚ	cb_configÚinputs_and_outputsÚcacheÚ	do_sampleÚreturn_logprobsÚreturnNc                 ó¢  — || _         || _        || _        || _        || _        | j        j        \  | _        | _        || _        d | _	        | j        p| j        }| j        j
        d up| j        j        d u}|p|| _        | j        s| j        rt          ¦   «         \  | _        | _        nd\  | _        | _        d | _        | j        j
        �8t#          j        | j        fi | j        j
                             ¦   «         ¤Ž| _        d | _        | j        j        �:t#          j        | j        fi | j        j                             ¦   «         ¤Ž| _        d S d S )N)NN)r   r   r   r   r   Úcuda_graph_booleansÚuse_cuda_graph_varlenÚuse_cuda_graph_decoder   Ú_model_supports_logits_to_keepÚvarlen_compile_configÚdecode_compile_configÚ
pad_inputsr   Úmem_poolÚgraph_pool_idÚ_compiled_varlenÚtorchÚcompileÚ_forward_process_and_sampleÚto_dictÚ_compiled_decode)	Úselfr   r   r   r   r   r   Ú
cuda_graphr)   s	            úfC:\Projects\DSCons\.venv\Lib\site-packages\transformers\generation\continuous_batching\model_runner.pyÚ__init__zModelRunner.__init__!   sq  € ð  /ˆÔØ"ˆŒØ"4ˆÔà"ˆŒØ.ˆÔØAEÄÔAcÑ>ˆÔ" DÔ$>ØˆŒ
Ø;?ˆÔ+ð Ô/ÐM°4Ô3Mˆ
Ø”.Ô6¸dÐBÐvÀdÄnÔFjÐrvÐFvˆØ$Ð/¨ˆŒð Ô%ð 	;¨Ô)Cð 	;Ý0>Ñ0@Ô0@Ñ-ˆDŒM˜4Ô-Ð-à0:Ñ-ˆDŒM˜4Ô-ð !%ˆÔØŒ>Ô/Ð;Ý$)¤MØÔ0ð%ð %Ø48´NÔ4X×4`Ò4`Ñ4bÔ4bð%ð %ˆDÔ!ð
 !%ˆÔØŒ>Ô/Ð;Ý$)¤MØÔ0ð%ð %Ø48´NÔ4X×4`Ò4`Ñ4bÔ4bð%ð %ˆDÔ!Ð!Ð!ð <Ð;ó    Únum_q_tokensÚmax_kv_readÚuse_decode_fast_pathc                 óö   — | j         s||fS | j        j        }|sAt          || j        j        |¦  «        }t          || j        j        | j        j        ¦  «        }nt          || j        j	        ¦  «        }d}||fS )zZPads the input sizes for the next batch if it is needed. Often it is, for max performance.r   )
r$   r   Úmax_batch_tokensr   r   Úq_padding_interval_sizeÚkv_padding_interval_sizeÚ	num_pagesr   Úmax_requests_per_batch)r-   r2   r3   r4   r6   s        r/   Úmaybe_pad_inputszModelRunner.maybe_pad_inputsN   s†   € àŒð 	-Ø Ð,Ð,Øœ:Ô6Ðà#ð 	Ý*¨<¸¼Ô9_ÐaqÑrÔrˆLÝ)¨+°t´~Ô7^Ð`dÔ`jÔ`tÑuÔuˆKˆKõ ' |°T´^Ô5ZÑ[Ô[ˆLØˆKØ˜[Ð(Ð(r1   Úmodelc                 óp   — | j         €)t          |d¦  «        o|                     ¦   «         | _         | j         S )zJReturns True if the model accepts the logits_to_keep kwarg in its forward.NÚ_supports_logits_to_keep)r!   Úhasattrr>   )r-   r<   s     r/   Úsupports_logits_to_keepz#ModelRunner.supports_logits_to_keep]   s=   € àÔ.Ð6å˜Ð9Ñ:Ô:Ð_¸u×?]Ò?]Ñ?_Ô?_ð Ô/ð Ô2Ð2r1   Ú
batch_datac                 ó–  — | j                              ¦   «         \  }}}| j         j        }|                      |¦  «        r|d         |d<   |                      | j         j        ¬¦  «        \  }}|s[|�t          j                             |¦  «        nt          ¦   «         }	|	5   ||||||¦  «         ddd¦  «         dS # 1 swxY w Y   dS | j          
                    ¦   «         }
|
�Nt          j                             |¦  «        5  |
                     ¦   «          ddd¦  «         dS # 1 swxY w Y   dS |||||f} | j        ||g|¢R Ž  dS )züRuns the forward pass, processes the logits and samples the next tokens. It also handles which version of
        the forward pass to use (varlen or decode), whether to use CUDA graphs (with the eventual capture of the graph)
        and torch compile.Úlogits_indicesÚlogits_to_keep)Úuse_block_tableN)r   Úget_cb_kwargsÚcompute_streamr@   Ú_get_forward_fnrE   r(   ÚcudaÚstreamr   Ú	get_graphÚreplayÚ_capture_graph)r-   r<   rA   Úcarry_over_idsÚprev_output_idsÚ
output_idsrG   Ú
forward_fnÚuse_cuda_graphÚmaybe_streamÚgraphÚargss               r/   Úcompute_batchzModelRunner.compute_batche   s  € ð
 7;Ô6M×6[Ò6[Ñ6]Ô6]Ñ3ˆ˜¨àÔ0Ô?ˆð ×'Ò'¨Ñ.Ô.ð 	HØ+5Ð6FÔ+GˆJÐ'Ñ(ð &*×%9Ò%9È$ÔJaÔJqÐ%9Ñ%rÔ%rÑ"ˆ
�Nð ð 	GØ@NÐ@Z�5œ:×,Ò,¨^Ñ<Ô<Ð<Õ`kÑ`mÔ`mˆLØð [ð [Ø�
˜5 *¨n¸oÈzÑZÔZÐZð[ð [ð [ñ [ô [ð [ð [ð [ð [ð [ð [ð [øøøð [ð [ð [ð [ð [ð [ð
 Ô+×5Ò5Ñ7Ô7ˆEàÐ Ý”Z×&Ò& ~Ñ6Ô6ð #ð #Ø—L’L‘N”N�Nð#ð #ð #ñ #ô #ð #ð #ð #ð #ð #ð #ð #øøøð #ð #ð #ð #ð #ð #ð ˜z¨>¸?ÈJÐW�Ø#�Ô# J°ÐFÀÐFÐFÐFÐFÐFÐFs$   Â B=Â=CÃCÄD&Ä&D*Ä-D*rE   c                 ó€   — |r| j         €| j        n| j         }| j        }n| j        €| j        n| j        }| j        }||fS )zfHelper function to get the appropriate forward function based on the block table and compile behavior.)r,   r*   r    r'   r   )r-   rE   rQ   rR   s       r/   rH   zModelRunner._get_forward_fn‡   sY   € àð 	8Ø=AÔ=RÐ=Z˜Ô9Ð9Ð`dÔ`uˆJØ!Ô7ˆNˆNà=AÔ=RÐ=Z˜Ô9Ð9Ð`dÔ`uˆJØ!Ô7ˆNØ˜>Ð)Ð)r1   rQ   rG   c                 óÐ  — t           j                             |¦  «        5  t          | j        ¦  «        5   ||Ž  ddd¦  «         n# 1 swxY w Y   ddd¦  «         n# 1 swxY w Y   t           j                             ¦   «         }t           j                             ||| j        d¬¦  «        5   ||Ž  ddd¦  «         n# 1 swxY w Y   | j         	                    |¦  «         dS )zJHelper function to capture and store a graph for a given forward function.NÚthread_local)rJ   ÚpoolÚcapture_error_mode)
r(   rI   rJ   r   r%   Ú	CUDAGraphrT   r&   r   Ú	set_graph)r-   rQ   rG   rU   rT   s        r/   rM   zModelRunner._capture_graph‘   s¢  € õ ŒZ×Ò˜~Ñ.Ô.ð 	ð 	µ¸T¼]Ñ0KÔ0Kð 	ð 	ØˆJ˜ÐÐð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð 	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	õ ”
×$Ò$Ñ&Ô&ˆÝŒZ×ÒØ˜.¨tÔ/AÐVdð ñ 
ô 
ð 	ð 	ð ˆJ˜ÐÐð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð
 	Ô×)Ò)¨%Ñ0Ô0Ð0Ð0Ð0sE    AµA»AÁA	ÁAÁA	ÁAÁA"Á%A"Â/CÃCÃCrN   rO   rP   c                 óÜ  — | j                              |d         ||¦  «          |di |¤Žj        }|d         }d|vr|dd…|dd…f         }|                     ¦   «         }| j        j        ri|j        \  }}	}
|                     ||	z  |
¦  «        }|d         d|f         }|                      |||d         ¦  «        }|                     ||	|
¦  «        }n|}|                      ||¦  «         dS )z‚This function performs the forward pass, logits processing, and sampling. This is what is either captured
        and/or compiled.Ú	input_idsrC   rD   Nr   Úlogits_processor_args© )	r   Úcarry_over_tokensÚlogitsÚfloatr   Údo_processingÚshapeÚviewÚ_sample)r-   r<   rA   rN   rO   rP   rc   rC   Ú	dummy_dimÚ
num_logitsÚ
vocab_sizeÚ	logits_2dÚsliced_input_ids_2dÚscoress                 r/   r*   z'ModelRunner._forward_process_and_sampleŸ   s  € ð 	Ô×1Ò1°*¸[Ô2IÈ>Ð[jÑkÔkÐkð �Ð$Ð$˜Ð$Ð$Ô+ˆð $Ð$4Ô5ˆØ :Ð-Ð-Ø˜A˜A˜A˜~¨q¨q¨qÐ0Ô1ˆFà—’‘”ˆð ÔÔ-ð 
	à06´Ñ-ˆI�z :ØŸš I°
Ñ$:¸JÑGÔGˆIØ",¨[Ô"9¸!¸^Ð:KÔ"LÐà×,Ò,Ð-@À)ÈZÐXoÔMpÑqÔqˆIà—^’^ I¨z¸:ÑFÔFˆFˆFàˆFð 	�Š�V˜ZÑ(Ô(Ð(Ð(Ð(r1   rn   c                 ó®  — | j         s| j        r(t          j                             |d         d¬¦  «        }n|                     d¦  «        }| j         rt          j        |d¬¦  «        }nt          j        |dd¬¦  «        }| j        r>| 	                    d|¬¦  «                             d¦  «        }| 
                    ¦   «         }|                     d¦  «        }|                     d¦  «        }|dd	|…f                              |¦  «         | j        r?|dd	|…f                              |                     t          j        ¬
¦  «        ¦  «         d	S d	S )z5Private method to sample next tokens from the scores.r   éÿÿÿÿ)Údimr   )Únum_samplesT)rq   Úkeepdim)rq   ÚindexN)Údtype)r   r   r   Ú
functionalÚsoftmaxÚsqueezer(   ÚmultinomialÚargmaxÚgatherÚlogÚsizeÚcopy_rg   Úint32)r-   rn   rP   ÚprobsÚnext_tokensÚper_token_probsÚlogprobsÚtokenss           r/   rh   zModelRunner._sampleÆ   s`  € ð Œ>ð 	&˜TÔ1ð 	&Ý”M×)Ò)¨&°¬)¸Ð)Ñ<Ô<ˆEˆEà—N’N 1Ñ%Ô%ˆEð Œ>ð 	DÝÔ+¨E¸qÐAÑAÔAˆKˆKåœ, u°"¸dÐCÑCÔCˆKð Ôð 	-Ø#Ÿlšl¨q¸˜lÑDÔD×LÒLÈRÑPÔPˆOØ&×*Ò*Ñ,Ô,ˆHð "×)Ò)¨"Ñ-Ô-ˆð ×!Ò! !Ñ$Ô$ˆØ�1�g�v�g�:Ô×$Ò$ [Ñ1Ô1Ð1ØÔð 	Kð �q˜'˜6˜'�zÔ"×(Ò(¨¯ª½U¼[¨Ñ)IÔ)IÑJÔJÐJÐJÐJð	Kð 	Kr1   c                 óZ  — | j         sdS d}t          | j        t          ¦  «        rdnd}t	          |¦  «        D ]Ø}| j        j        }| j        j        | j        j        z  }||z  }||  	                    |||¬¦  «        z  }| j        j
        dk    rŒXd}	 ||  	                    ||d¬¦  «        z  }|| j        j        k    rnt          d|z  | j        j        ¦  «        }ŒJt          | j        t          ¦  «        r| j                             ¦   «          ŒÙt          j        d|d›d	�¦  «         dS )
zÖPre-capture CUDA graphs and/or trigger compile warmup for varlen and decode paths (if available). Unless the
        force_warmup flag is set, the warmup is only performed if the CUDA graphs or compile are enabled.Nr   é   r   )r<   r2   r3   TúWarmup completed in ú.2fÚs)r$   Ú
isinstancer   r   Úranger   r6   Ú
num_blocksÚ
block_sizeÚrun_one_warmupÚmax_blocks_per_requestr   r:   ÚminÚswap_io_pairsr   Úinfo)r-   r<   Útotal_durationÚ
iterationsÚ_r2   r3   Únum_requestss           r/   ÚwarmupzModelRunner.warmupä   s^  € ð
 Œð 	Ø�4ð ˆÝ$ TÔ%<Õ>XÑYÔYÐ`�Q�QÐ_`ˆ
Ý�zÑ"Ô"ð 	8ð 	8ˆAàœ:Ô6ˆLØœ*Ô/°$´*Ô2GÑGˆKØ˜<Ñ'ˆKØ˜d×1Ò1¸ÈLÐfqÐ1ÑrÔrÑrˆNð ŒzÔ0°AÒ5Ð5Øð ˆLð\Ø $×"5Ò"5¸EÐP\ÐjnÐ"5Ñ"oÔ"oÑo�Ø 4¤>Ô#HÒHÐHØÝ" 1 |Ñ#3°T´^Ô5ZÑ[Ô[�ð	\õ ˜$Ô1Õ3MÑNÔNð 8ØÔ'×5Ò5Ñ7Ô7Ð7øÝŒÐ@¨>Ð@Ð@Ð@Ð@ÑAÔAÐAÐAÐAr1   c           	      óÄ  — |du }|r5|}t           j        }d}| j        j        }t	          j        d|›d�¦  «         n)d}t           j        }t	          j        d|›d|›d�¦  «         t          ||||| j        ¦  «        }|s t	          j        d|›d|›d|›d�¦  «         d	S |  	                    ||z  ||¬
¦  «        \  }}	t          j        ¦   «         }
	 | j                             || j        |||	d¬¦  «         | j                             d¬¦  «        }|                      ||¦  «         t          j        ¦   «         |
z
  }t	          j        d|d›d�¦  «         n1# t"          $ r$}d	}t	          j        d|› d�¦  «         Y d}~nd}~ww xY w|D ]&}| j                             |j        j        ¦  «         Œ'n.# |D ]&}| j                             |j        j        ¦  «         Œ'w xY w|S )a.  Warms up the decode fast path (if max_kv_read is None) or varlen path (if max_kv_read is an int) for a
        specific number of query and cache-resident tokens. `max_kv_read` is the number of tokens already in cache,
        matching the terminology used by `prepare_batch_tensors` and the scheduler.Nr   z/Warming up decode fast path for num_requests = ú.z*Warming up varlen path for num_q_tokens = z, max_kv_read = z:Failed to warm up: no blocks allocated for num_requests = z, num_q_tokens = g        )r2   r3   r4   T)Úuse_paddingr‡   rˆ   r‰   zFailed to warm up: z-.
Graph pool may fragment and OOM under load.)r   ÚDECODINGr   r�   r   ÚdebugÚ
PREFILLINGr   Úwarningr;   ÚtimeÚperf_counterr   Úprepare_batch_tensorsr   Úget_model_kwargsrV   Ú	ExceptionÚfree_blocksÚstateÚ
request_id)r-   r<   r2   r3   r4   r–   ÚstatusÚfuture_statesÚpadded_qÚ	padded_kvÚstartrA   ÚdurationÚeÚfss                  r/   rŽ   zModelRunner.run_one_warmup  s�  € ð
  +¨dÐ2ÐØð 		]Ø'ˆLÝ"Ô+ˆFØˆLØœ*Ô/ˆKÝŒLÐN¸LÐNÐNÐNÑOÔOÐOÐOàˆLÝ"Ô-ˆFÝŒLÐ[°|Ð[Ð[È+Ð[Ð[Ð[Ñ\Ô\Ð\Ý3°LÀ&È,ÐXcÐeiÔeoÑpÔpˆØð 	ÝŒNØu¸lÐuÐuÐQ]ÐuÐuÐdoÐuÐuÐuñô ð ð �3ð #×3Ò3Ø%¨Ñ4Ø#Ø!5ð 4ñ 
ô 
Ñˆ�)õ Ô!Ñ#Ô#ˆð	<ØÔ#×9Ò9Ø˜tÔ3Ð5IÈ8ÐU^Ðlpð :ñ ô ð ð Ô0×AÒAÈdÐAÑSÔSˆJØ×Ò˜u jÑ1Ô1Ð1ÝÔ(Ñ*Ô*¨UÑ2ˆHÝŒLÐ?°Ð?Ð?Ð?Ð?Ñ@Ô@Ð@Ð@øõ ð 	dð 	dð 	dØˆHÝŒNÐb°ÐbÐbÐbÑcÔcÐcÐcÐcÐcÐcÐcøøøøð	døøøð $ð <ð <�Ø”
×&Ò& r¤xÔ':Ñ;Ô;Ð;Ð;ð<ø�mð <ð <�Ø”
×&Ò& r¤xÔ':Ñ;Ô;Ð;Ð;ð<øøøàˆs1   ÃBE ÅF2 Å
FÅ!F Å;F2 Æ FÆF2 Æ2+G) Ú__name__Ú
__module__Ú__qualname__Ú__doc__r
   r   r   r   r	   Úboolr0   ÚintÚtupler;   r   ÚModuler@   ÚdictrV   r   rH   r(   rI   ÚStreamrM   ÚTensorr*   rh   Úno_gradr—   rd   rŽ   ra   r1   r/   r   r      sn  € € € € € ð1ð 1ð+à>ð+ð ,ð+ð 2Ð4NÑNð	+ð
 #ð+ð ð+ð ð+ð 
ð+ð +ð +ð +ðZ)¨Sð )¸sð )ÐZ^ð )ÐchÐilÐnqÐiqÔcrð )ð )ð )ð )ð3¨R¬Yð 3¸4ð 3ð 3ð 3ð 3ð G 2¤9ð  G¸$ð  GÀ4ð  Gð  Gð  Gð  GðD*¨tð *¸¸hÈ¸nÔ8Mð *ð *ð *ð *ð1¨ð 1À5Ä:ÔCTð 1Ð`dð 1ð 1ð 1ð 1ð%)àŒyð%)ð ð%)ð œð	%)ð
 œð%)ð ”Lð%)ð 
ð%)ð %)ð %)ð %)ðNK˜eœlð K¸¼ð KÈð Kð Kð Kð Kð< €U„]�_„_ð B˜BœIð  B¨$ð  Bð  Bð  Bñ „_ð BðD2 B¤Ið 2¸Sð 2ÈsÐUYÉzð 2Ð^cð 2ð 2ð 2ð 2ð 2ð 2r1   r   )rŸ   Úcollections.abcr   Ú
contextlibr   r(   r   Úgeneration.configuration_utilsr   r   r	   Úcb_logits_processorsr
   Úinput_outputsr   r   Úrequestsr   r   Úutilsr   r   r   r   r   r   ra   r1   r/   ú<module>rÂ      s  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø "Ð "Ð "Ð "Ð "Ð "à €€€Ø Ð Ð Ð Ð Ð à FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø GÐ GÐ GÐ GÐ GÐ GØ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ +Ð +Ð +Ð +Ð +Ð +Ð +Ð +Ø jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jð\ð \ð \ð \ð \ñ \ô \ð \ð \ð \r1   