§
    ¹¯Åjà ã            	      óZ  — d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z d dlZd dlZd dlmZmZmZmZ d dlmZmZmZmZ ddlmZ ddlmZmZ ddlmZ dd	l m!Z!m"Z"  ej#        d
ej$        ¬¦  «          ej%        e&¦  «        Z' G d„ d¦  «        Z( G d„ de(¦  «        Z) G d„ de(¦  «        Z* G d„ de(¦  «        Z+ G d„ de(¦  «        Z, G d„ de(¦  «        Z- G d„ de(¦  «        Z.d„ Z/ G d„ d¦  «        Z0dDd!„Z1d"„ Z2 G d#„ d$¦  «        Z3 G d%„ d&¦  «        Z4 G d'„ d(¦  «        Z5d)„ Z6d*„ Z7d+„ Z8e&d,k    �rt e8¦   «         Z9e9j:        re' ;                    ej<        ¦  «         e9j=        Z>e9j?        Z@e!e9jA                 ZAe9jB        r eCe9jB        ¦  «        nd-ZBe9jD        r eCe9jD        ¦  «        ndZDejE         F                    e@¦  «        r(e' G                    d.e@› d/�¦  «          eHd.e@› d/�¦  «        ‚e9jI        r'e9jJ        d0k    re' K                    d1¦  «         d2e9_I         ejL        e>¦  «        ZMe9jJ        d0k    r e,e9jN        e9jO        eBeD¬3¦  «        ZP�n e9jJ        d4k    r' e-e9jN        e9jI        e9jQ        eAeBeDe9jO        ¬5¦  «        ZPnÎe9jJ        d6k    r e)eB¬7¦  «        ZPn¶e9jJ        d8k    r e*eB¬7¦  «        ZPnže9jJ        d9k    r e+e9jN        eB¬:¦  «        ZPn€e9jJ        d;k    rbeAe!jR        k    re' K                    d<¦  «         e!jS        ZAe>ZMe9jT        �e9jT        d=k    rd>ZTnd?ZTnd>ZT e.e9jU        e9jV        e9jW        eT¬@¦  «        ZPn eXdAe9jJ        › �¦  «        ‚ e5eMe9jO        e9jQ        e9jY        e9jZ        eP¬B¦  «        Z[e[ \                    ¦   «          e[jM         ]                    e@dC¦  «         dS dS )Eé    )ÚannotationsN)Ú
GraphProtoÚ
ModelProtoÚ	NodeProtoÚTensorProto)Úquantize_matmul_2bitsÚquantize_matmul_4bitsÚquantize_matmul_8bitsÚquantize_qdq_matmul_4bitsé   )ÚCalibrationDataReader)Úgptq_quantizeÚrtn_quantize)Ú	ONNXModel)ÚQuantFormatÚattribute_to_kwargz2%(asctime)s %(name)s [%(levelname)s] - %(message)s)ÚformatÚlevelc                  ó   — e Zd Z	 	 	 ddd„ZdS )ÚWeightOnlyQuantConfigNÚ	algorithmÚstrÚquant_formatr   Úop_types_to_quantizeútuple[str, ...] | NoneÚ
quant_axesú"tuple[tuple[str, int], ...] | NoneÚcustomized_weight_configúdict | Nonec                ó˜   — || _         || _        |rt          |¦  «        ndh| _        |rt	          |¦  «        ndddœ| _        || _        dS )aq  This is the Base class for Weight Only blockwise quantization Configuration.

        Args:
            algorithm:
                weight only quantize algorithm name.
            quant_format: QuantFormat{QOperator, QDQ}.
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
            op_types_to_quantize (optional):
                set of operator types to quantize. Default {MatMul}
            quant_axes (dict[str, int], optional):
                op:axis, which axis to quantize for an op. Default {MatMul: 0, Gather: 1}
            customized_weight_config:
                customized weight config for nodes if needed. It is dictionary with node name as key,
                and the value is a dict of customized config.
        ÚMatMulr   r   )r!   ÚGatherN)r   r   Úsetr   Údictr   r   )Úselfr   r   r   r   r   s         ú`C:\Projects\Historical\venv\Lib\site-packages\onnxruntime/quantization/matmul_nbits_quantizer.pyÚ__init__zWeightOnlyQuantConfig.__init__&   sb   € ð0 #ˆŒØ(ˆÔØAUÐ$e¥CÐ(<Ñ$=Ô$=Ð$=Ð\dÐ[eˆÔ!Ø.8ÐX�$˜zÑ*Ô*Ð*ÈÐVWÐ>XÐ>XˆŒØ(@ˆÔ%Ð%Ð%ó    )NNN)
r   r   r   r   r   r   r   r   r   r   )Ú__name__Ú
__module__Ú__qualname__r'   © r(   r&   r   r   %   s?   € € € € € ð
 8<Ø9=Ø04ðAð Að Að Að Að Að Ar(   r   c                  ó4   ‡ — e Zd Zdej        ddfdˆ fd„Zˆ xZS )ÚRTNWeightOnlyQuantConfigNr   r   r   r   c                óœ   •— |t           j        k    s
J d¦   «         ‚|€i }t          ¦   «                              d|||¬¦  «         || _        dS )ap  
        This is a class for round-to-nearest (RTN) algorithm Weight Only Quant Configuration.
        RTN is the most straightforward way to quantize weight using scale maps.

        Args:
            ratios:
                percentile of clip. Defaults to {}.
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
            customized_weight_config:
                customized weight config for nodes if needed. It is dictionary with node name as key,
                and the value is a dict of customized config.
        z"RTN only supports QOperator formatNÚRTN©r   r   r   r   ©r   Ú	QOperatorÚsuperr'   Úratios©r%   r5   r   r   r   Ú	__class__s        €r&   r'   z!RTNWeightOnlyQuantConfig.__init__F   sf   ø€ ð0 �{Ô4Ò4Ð4Ð4Ð6ZÑ4Ô4Ð4àˆ>ØˆFÝ‰Œ×ÒØØ%Ø!5Ø%=ð	 	ñ 	
ô 	
ð 	
ð ˆŒˆˆr(   ©r   r   r   r   ©r)   r*   r+   r   r3   r'   Ú__classcell__©r7   s   @r&   r.   r.   E   sS   ø€ € € € € ð Ø Ô*Ø7;Ø04ð"ð "ð "ð "ð "ð "ð "ð "ð "ð "ð "r(   r.   c                  ó4   ‡ — e Zd Zdej        ddfdˆ fd„Zˆ xZS )ÚKQuantWeightOnlyQuantConfigNr   r   r   r   c                óœ   •— |t           j        k    s
J d¦   «         ‚|€i }t          ¦   «                              d|||¬¦  «         || _        dS )aF  
        This is a class for k-quant algorithm Weight Only Quant Configuration.

        Args:
            ratios:
                percentile of clip. Defaults to {}.
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
        z&k-quant only supports QOperator formatNÚk_quantr1   r2   r6   s        €r&   r'   z$KQuantWeightOnlyQuantConfig.__init__l   sf   ø€ ð( �{Ô4Ò4Ð4Ð4Ð6^Ñ4Ô4Ð4àˆ>ØˆFÝ‰Œ×ÒØØ%Ø!5Ø%=ð	 	ñ 	
ô 	
ð 	
ð ˆŒˆˆr(   r8   r9   r;   s   @r&   r=   r=   k   sS   ø€ € € € € ð Ø Ô*Ø7;Ø04ðð ð ð ð ð ð ð ð ð ð r(   r=   c                  ó<   ‡ — e Zd Zddddddej        dfdˆ fd
„Zˆ xZS )ÚGPTQWeightOnlyQuantConfigNg{®Gáz„?é€   FTÚcalibration_data_readerúCalibrationDataReader | Noner   r   c	                óØ   •— |t           j        k    s
J d¦   «         ‚t          ¦   «                              d||¬¦  «         || _        || _        || _        || _        || _        || _	        dS )a  
        This is a class for GPTQ algorithm Weight Only Quant Configuration.
        GPTQ algorithm provides more accurate quantization but requires more computational resources.

        Args:
            calibration_data_reader:
                a calibration data reader. It enumerates calibration data and generates inputs for the original model.
            percdamp:
                percent of the average Hessian diagonal to use for dampening.
            block_size (int, optional):
                channel number in one block to execute a GPTQ quantization iteration.
            actorder (bool, optional):
                whether rearrange Hessian matrix considering the diag's value.
            mse (bool, optional):
                whether get scale and zero point with mse error.
            perchannel (bool, optional):
                whether quantize weight per-channel.
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
        z#GPTQ only supports QOperator formatÚGPTQ)r   r   r   N)
r   r3   r4   r'   rC   ÚpercdampÚ
block_sizeÚactorderÚmseÚ
perchannel)
r%   rC   rG   rH   rI   rJ   rK   r   r   r7   s
            €r&   r'   z"GPTQWeightOnlyQuantConfig.__init__Ž   s~   ø€ ðF �{Ô4Ò4Ð4Ð4Ð6[Ñ4Ô4Ð4å‰Œ×ÒØØ%Ø!5ð 	ñ 	
ô 	
ð 	
ð
 (?ˆÔ$Ø ˆŒØ$ˆŒØ ˆŒØˆŒØ$ˆŒˆˆr(   )rC   rD   r   r   r9   r;   s   @r&   rA   rA   �   sa   ø€ € € € € ð AEØØØØØØ Ô*Ø7;ð/%ð /%ð /%ð /%ð /%ð /%ð /%ð /%ð /%ð /%ð /%r(   rA   c                  ó8   ‡ — e Zd Zdddej        ddfd
ˆ fd	„Zˆ xZS )ÚHQQWeightOnlyQuantConfigrB   é   r   Nr   r   r   r   c                ó°   •— |t           j        k    s
J d¦   «         ‚t          ¦   «                              d|||¬¦  «         || _        || _        || _        dS )a  
        This is a class for HQQ algorithm Weight Only Quant Configuration.
        HQQ algorithm quant weight without needing calibrate data.

        Args:
            block_size (int, optional):
                channel number in one block to execute a HQQ quantization iteration.
            bits (int, optional):
                how many bits to represent weight.
            axis (int, optional):
                0 or 1. which axis to quantize. https://arxiv.org/pdf/2309.15531.pdf
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
            quant_axes (dict[str, int], optional):
                op:axis, which axis to quantize for an op. Default {MatMul: 0, Gather: 1}
        z"HQQ only supports QOperator formatÚHQQ©r   r   r   r   N)r   r3   r4   r'   rH   ÚbitsÚaxis)r%   rH   rR   rS   r   r   r   r7   s          €r&   r'   z!HQQWeightOnlyQuantConfig.__init__Á   sj   ø€ ð: �{Ô4Ò4Ð4Ð4Ð6ZÑ4Ô4Ð4å‰Œ×ÒØØ%Ø!5Ø!ð	 	ñ 	
ô 	
ð 	
ð %ˆŒØˆŒ	ØˆŒ	ˆ	ˆ	r(   )r   r   r   r   r9   r;   s   @r&   rM   rM   À   sY   ø€ € € € € ð ØØØ Ô*Ø7;Ø9=ð'ð 'ð 'ð 'ð 'ð 'ð 'ð 'ð 'ð 'ð 'r(   rM   c                  ó<   ‡ — e Zd Zdddej        ddddfdˆ fd„Zˆ xZS )ÚDefaultWeightOnlyQuantConfigrB   FNrN   rH   ÚintÚis_symmetricÚboolÚaccuracy_levelú
int | Noner   r   r   r   rR   Úchannel_wised_quantizec	                óÞ   •— t          ¦   «                              d|||¬¦  «         || _        || _        || _        || _        || _        |r|t          j        k    rt          d¦  «        ‚dS dS )a   
        This is a class for weight only affine quantization configuration.

        Args:
            block_size (int, optional):
                channel number in one block to execute an affine quantization iteration.
            is_symmetric (bool, optional):
                whether quantize weight symmetrically.
            accuracy_level (int, optional):
                Accuracy level of the 4-bit quantized MatMul computation.
                Refer to the MatMulNBits contrib op's 'accuracy_level' attribute for details.
                (https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#commicrosoftmatmulnbits)
            quant_format (QuantFormat{QOperator, QDQ}, optional):
                QOperator format quantizes the model with quantized operators directly.
                QDQ format quantize the model by inserting QuantizeLinear/DeQuantizeLinear on the tensor.
                Defaults to QuantFormat.QOperator.
            op_types_to_quantize (optional):
                set of operator types to quantize.
            quant_axes (dict[str, int], optional):
                op:axis, which axis to quantize for an op. Default {MatMul: 0, Gather: 1}
            bits (int, optional):
                number of bits per element after quantization. Default 4.
        ÚDEFAULTrQ   zAQuantFormat.QOperator is not supported channel_wised_quantize yetN)
r4   r'   rH   rW   rR   rY   r[   r   r3   ÚNotImplementedError)
r%   rH   rW   rY   r   r   r   rR   r[   r7   s
            €r&   r'   z%DefaultWeightOnlyQuantConfig.__init__ì   s’   ø€ õD 	‰Œ×ÒØØ%Ø!5Ø!ð	 	ñ 	
ô 	
ð 	
ð %ˆŒØ(ˆÔØˆŒ	Ø,ˆÔØ&<ˆÔ#Ø!ð 	k lµkÔ6KÒ&KÐ&KÝ%Ð&iÑjÔjÐjð	kð 	kÐ&KÐ&Kr(   )rH   rV   rW   rX   rY   rZ   r   r   r   r   rR   rV   r[   rX   r9   r;   s   @r&   rU   rU   ë   sj   ø€ € € € € ð Ø"Ø%)Ø Ô*Ø7;Ø9=ØØ',ð.kð .kð .kð .kð .kð .kð .kð .kð .kð .kð .kr(   rU   c                  ó2   ‡ — e Zd Z	 	 	 dˆ fd„	Zd„ Zd„ Zˆ xZS )ÚNVAWQWeightOnlyQuantConfigÚcnnú./cacheÚawq_litec                óÈ  •— 	 ddl }ddlm} || _         || _        n-# t          $ r  t	          d¦  «         t          d¦  «        d‚w xY w	 ddlm} || _        n-# t          $ r  t	          d¦  «         t          d¦  «        d‚w xY w	 dd	lm}m	}	 || _        |	| _	        n-# t          $ r  t	          d
¦  «         t          d¦  «        d‚w xY wt          ¦   «                              dt          j        dd¬¦  «         | j                              | j         j                             ¦   «         rdnd¦  «        }
|                      |||ddd|
ddddd¬¦  «        }|| _        || _        dS )a=  
        Configuration for the nvidia_awq quantization method.

        Args:
            tokenizer_dir (str): pathof the tokenizer dir.
            dataset_name (str): Name of the dataset.
            cache_dir (str): Directory for caching.
            calibration_method (str): calib method for nvidia_awq.
        r   N)Ú
DataLoaderzfError: The 'torch' library is required but not installed. Please install it using 'pip install torch'.z torch is not installed. Exiting.)Úload_datasetzlError: The 'datasets' library is required but not installed. Please install it using 'pip install datasets'.z#datasets is not installed. Exiting.)Ú
AutoConfigÚAutoTokenizerztError: The 'transformers' library is required but not installed. Please install it using 'pip install transformers'.z'transformers is not installed. Exiting.Ú
nvidia_awqrQ   ÚcudaÚcpué    r   i   TFrB   )Údataset_nameÚ
model_nameÚ	cache_dirÚ
calib_sizeÚ
batch_sizerH   ÚdeviceÚuse_fp16Úuse_buffer_shareÚadd_past_kv_inputsÚmax_calib_rows_to_loadÚadd_position_ids)ÚtorchÚtorch.utils.datare   ÚImportErrorÚprintÚdatasetsrf   Útransformersrg   rh   r4   r'   r   ÚQDQrr   rj   Úis_availableÚget_calib_inputsrC   Úcalibration_method)r%   Útokenizer_dirrm   ro   r�   rx   re   rf   rg   rh   rr   Úcalib_inputsr7   s               €r&   r'   z#NVAWQWeightOnlyQuantConfig.__init__  s  ø€ ð"
	LØˆLˆLˆLØ3Ð3Ð3Ð3Ð3Ð3àˆDŒJØ(ˆDŒOˆOøÝð 	Lð 	Lð 	LÝØxñô ð õ Ð@ÑAÔAÀtÐKð		Løøøð	OØ-Ð-Ð-Ð-Ð-Ð-à ,ˆDÔÐøÝð 	Oð 	Oð 	OÝØ~ñô ð õ ÐCÑDÔDÈ$ÐNð		Oøøøð		SØ>Ð>Ð>Ð>Ð>Ð>Ð>Ð>à(ˆDŒOØ!.ˆDÔÐøÝð 	Sð 	Sð 	SÝð Gñô ð õ ÐGÑHÔHÈdÐRð		Søøøõ 	‰Œ×ÒØ"Ý$œØ!%Øð	 	ñ 	
ô 	
ð 	
ð ”×"Ò"¨T¬Z¬_×-IÒ-IÑ-KÔ-KÐ#V 6 6ÐQVÑWÔWˆà×,Ò,Ø%Ø$ØØØØØØØ"Ø#Ø#&Ø!ð -ñ 
ô 
ˆð (4ˆÔ$Ø"4ˆÔÐÐs!   ƒ œ*AÁ
A Á*BÂB Â*Cc	           	     ól  — | j         }	|}
|}t          |t          ¦  «        r:|	                     |||	j        ¬¦  «        }
|	                     |||	j        ¬¦  «        }|
                     ¦   «         |                     ¦   «         dœ}|r[|                     ¦   «                              d¦  «        dz
  }|                     |dk    d¦  «         |                     ¦   «         |d<   |rÑ|r|	j	        n|	j
        }|
j        \  }}|j        }|j        |j        |j        z  }}t!          |j        ¦  «        D ]…}|	                     |||r|nd|||¬¦  «        }|	                     |||r|nd|||¬¦  «        }|                     d|› d�|                     ¦   «         d|› d	�|                     ¦   «         i¦  «         Œ†|S )
N)rr   Údtype)Ú	input_idsÚattention_maskéÿÿÿÿr   r   Úposition_idszpast_key_values.z.keyz.value)rx   Ú
isinstanceÚlistÚtensorÚint64Ú
contiguousÚlongÚcumsumÚmasked_fill_Úfloat16Úfloat32ÚshapeÚmax_position_embeddingsÚnum_key_value_headsÚhidden_sizeÚnum_attention_headsÚrangeÚnum_hidden_layersÚzerosÚupdate)r%   ÚconfigÚinput_ids_argÚattention_mask_argru   rr   rs   rt   rw   rx   r†   r‡   Úinputsr‰   Útorch_dtyperq   Úsequence_lengthÚmax_sequence_lengthÚ	num_headsÚ	head_sizeÚiÚpast_keyÚ
past_values                          r&   Úmake_model_inputz+NVAWQWeightOnlyQuantConfig.make_model_inputn  s  € ð ”
ˆà!ˆ	Ø+ˆå�m¥TÑ*Ô*ð 	`ØŸš ]¸6ÈÌ˜ÑUÔUˆIØ"Ÿ\š\Ð*<ÀVÐSXÔS^˜\Ñ_Ô_ˆNð #×-Ò-Ñ/Ô/Ø,×7Ò7Ñ9Ô9ð
ð 
ˆð
 ð 	?Ø)×.Ò.Ñ0Ô0×7Ò7¸Ñ;Ô;¸aÑ?ˆLØ×%Ò% n¸Ò&9¸1Ñ=Ô=Ð=Ø%1×%<Ò%<Ñ%>Ô%>ˆF�>Ñ"àð 	Ø+3ÐF˜%œ-˜-¸¼ˆKØ*3¬/Ñ'ˆJ˜Ø"(Ô"@ÐàÔ*ØÔ" fÔ&@Ñ@ð !ˆIõ ˜6Ô3Ñ4Ô4ð ð �Ø Ÿ;š;ØØØ+;ÐBÐ'Ð'ÀØØ!Ø%ð 'ñ ô �ð #Ÿ[š[ØØØ+;ÐBÐ'Ð'ÀØØ!Ø%ð )ñ ô �
ð —’à2¨1Ð2Ð2Ð2°H×4GÒ4GÑ4IÔ4IØ4¨1Ð4Ð4Ð4°j×6KÒ6KÑ6MÔ6Mðñô ð ð ð ˆr(   c                ó¤  — | j         }| j        }| j        }|                     |d|d¬¦  «        }|                     |d|d¬¦  «        }|                     ddi¦  «         |j        |_        ||k    s
J d¦   «         ‚d|v r1 |ddd	¬
¦  «                             t          |¦  «        ¦  «        }d}n'd|v r |dd¬¦  «        }d}nt          d|› d�¦  «        ‚||         d |…         }| 
                    |ddd|¬¦  «        }|                     |¦  «        }|d         }|d         }| j        } |||d¬¦  «        } |||d¬¦  «        }t          |j        ¦  «        t          |j        ¦  «        k    sJ ‚t          |¦  «        t          |¦  «        k    sJ ‚||z  }g }t          |¦  «        D ]%\  }}|                     |¦  «         ||dz
  k    r nŒ&g }t          |¦  «        D ]%\  }}|                     |¦  «         ||dz
  k    r nŒ&t#          d|› dt          |¦  «        › dt          |¦  «        › d�¦  «         g }t          |¦  «        D ]a} ||          }!||          }"|                      ||!|"|
|||	|¦  «        }#d„ |#                     ¦   «         D ¦   «         }#|                     |#¦  «         Œbt#          dt          |¦  «        › d�¦  «         |S ) NT)Úuse_auth_tokenro   Útrust_remote_codeÚ	pad_tokenz[PAD]z8calib size should be no more than max_calib_rows_to_loadra   Úcnn_dailymailz3.0.0Útrain)ÚnameÚsplitÚarticleÚpilezmit-han-lab/pile-val-backupÚ
validation)r±   Útextz	dataset "z" not supportedÚpt)Úreturn_tensorsÚpaddingÚ
truncationÚ
max_lengthr†   r‡   F)rq   Úshuffler   z/
--Quantize-Script-- number_of_batched_samples=z, batch-input-ids-list-len=z, batched_attention_mask=ú
c                ób   — i | ],\  }}||                      ¦   «                              ¦   «         “Œ-S r,   )rk   Únumpy)Ú.0Ú
input_nameÚtorch_tensors      r&   ú
<dictcomp>z?NVAWQWeightOnlyQuantConfig.get_calib_inputs.<locals>.<dictcomp>  s7   € ÐlÐlÐlÑAYÀÈ\�j ,×"2Ò"2Ñ"4Ô"4×":Ò":Ñ"<Ô"<ÐlÐlÐlr(   z0
--Quantize-Script-- number of batched inputs = )rg   rh   rf   Úfrom_pretrainedÚadd_special_tokensÚ	eos_tokenr­   Úselectr™   Ú
ValueErrorÚbatch_encode_plusÚtore   ÚlenÚdatasetÚ	enumerateÚappendr{   r©   Úitems)$r%   rm   rn   ro   rp   rq   rH   rr   rs   rt   ru   rv   rw   Úauto_configÚauto_tokenizerrf   r�   Ú	tokenizerÚdataset2ÚcolumnÚbatch_encodedÚbatch_encoded_input_idsÚbatch_encoded_attention_maskÚdata_loaderÚcalib_dataloader_input_idsÚcalib_dataloader_attention_maskÚnumber_of_batched_samplesÚbatched_input_idsÚidxÚdataÚbatched_attention_maskÚbatched_inputs_listr¦   r†   r‡   r    s$                                       r&   r€   z+NVAWQWeightOnlyQuantConfig.get_calib_inputs¯  sí  € ð  ”oˆØÔ+ˆØÔ(ˆà×,Ò,Ø t°yÐTXð -ñ 
ô 
ˆð #×2Ò2Ø t°yÐTXð 3ñ 
ô 
ˆ	ð 	×$Ò$ k°7Ð%;Ñ<Ô<Ð<Ø'Ô1ˆ	ÔàÐ3Ò3Ð3Ð3Ð5oÑ3Ô3Ð3à�LÐ Ð Ø#�| O¸'ÈÐQÑQÔQ×XÒXÕY^Ð_uÑYvÔYvÑwÔwˆHØˆFˆFØ�|Ð#Ð#Ø#�|Ð$AÈÐVÑVÔVˆHØˆFˆFåÐF¨ÐFÐFÐFÑGÔGÐGà˜FÔ# K Z KÔ0ˆØ!×3Ò3Ø T°4ÀDÐU_ð 4ñ 
ô 
ˆð &×(Ò(¨Ñ0Ô0ˆØ"/°Ô"<ÐØ'4Ð5EÔ'FÐ$ð ”oˆà%0 [Ð1HÐU_ÐinÐ%oÑ%oÔ%oÐ"Ø*5¨+Ø(°ZÈð+
ñ +
ô +
Ð'õ Ð-Ô5Ñ6Ô6½#Ð>]Ô>eÑ:fÔ:fÒfÐfÐfÐfÝÐ-Ñ.Ô.µ#Ð6UÑ2VÔ2VÒVÐVÐVÐVà$.°*Ñ$<Ð!àÐÝ"Ð#=Ñ>Ô>ð 	ð 	‰IˆC�Ø×$Ò$ TÑ*Ô*Ð*ØÐ0°1Ñ4Ò5Ð5Ø�ð 6ð "$ÐÝ"Ð#BÑCÔCð 	ð 	‰IˆC�Ø"×)Ò)¨$Ñ/Ô/Ð/ØÐ0°1Ñ4Ò5Ð5Ø�ð 6õ 	ðyÐ?Xð yð yÝ(+Ð,=Ñ(>Ô(>ðyð yÝY\Ð]sÑYtÔYtðyð yð yñ	
ô 	
ð 	
ð
 !ÐÝÐ0Ñ1Ô1ð 	/ð 	/ˆAØ)¨!Ô,ˆIØ3°AÔ6ˆNà×*Ò*ØØØØ"ØØØ Ø ñ	ô 	ˆFð mÐlÐ]c×]iÒ]iÑ]kÔ]kÐlÑlÔlˆFØ×&Ò& vÑ.Ô.Ð.Ð.åÐ^Å#ÐFYÑBZÔBZÐ^Ð^Ð^Ñ_Ô_Ð_Ø"Ð"r(   )ra   rb   rc   )r)   r*   r+   r'   r©   r€   r:   r;   s   @r&   r`   r`     su   ø€ € € € € ð ØØ%ðN5ð N5ð N5ð N5ð N5ð N5ð`?ð ?ð ?ðBa#ð a#ð a#ð a#ð a#ð a#ð a#r(   r`   c                óX   — t          |t          j        | |z  ¦  «        z  ¦  «        | k    S ©N)rV   ÚnpÚceil)Úval1Úval2s     r&   Úis_divisibleræ     s(   € Ýˆt•b”g˜d T™kÑ*Ô*Ñ*Ñ+Ô+¨tÒ3Ð3r(   c                  ó^   — e Zd Zdd„Ze	 	 	 ddd„¦   «         Zed„ ¦   «         Z	 dd„Zdd„ZdS ) ÚHQQWeightOnlyQuantizerr�   rM   c                ó   — || _         d S rá   ©r�   ©r%   r�   s     r&   r'   zHQQWeightOnlyQuantizer.__init__  ó   € ð ˆŒˆˆr(   r   NFÚmin_maxú	list[int]rS   rV   Ú
opt_paramsr   c                ó  ‡— dd l Š|€dddddœn|}|d         |d         |d	         |d
         f\  }}}	}
| j        r‰j        n‰j        }|                      |¦  «        }|                     |¦  «        }|                     |¦  «        }|fˆfd„	}d}t          |
¦  «        D ]ß}‰                     ||z  |z   ¦  «                             |d         |d         ¦  «        }||z
  |z  } |||z
  |¦  «        }‰                     |||z
  |z  z
  |d¬¦  «        }||	z  }t          ‰ 
                    ||z
  ¦  «                             ¦   «         ¦  «        }|r#t          |t          j        |d¦  «        ¦  «         ||k     r|}Œß ~~~~||fS )Nr   gffffffæ?g      $@g)\�Âõ(ð?é   )Úlp_normÚbetaÚkappaÚitersrò   ró   rô   rõ   c           
     ó¨  •— |dk    rN‰                      | ¦  «        ‰j        j                             ‰                     | ¦  «        d|z  z
  ¦  «        z  S ‰                      | ¦  «        ‰j        j                             ‰                     | ¦  «        d|z  ‰                     ‰                     | ¦  «        dz   |dz
  ¦  «        z  z
  ¦  «        z  S )Nr   ç      ð?g:Œ0âŽyE>)ÚsignÚnnÚ
functionalÚreluÚabsÚpow)Úxró   Úprx   s      €r&   Ú	shrink_opz:HQQWeightOnlyQuantizer.optimize_weights.<locals>.shrink_op8  sµ   ø€ Ø�AŠvˆvØ—z’z !‘}”} u¤xÔ':×'?Ò'?ÀÇ	Â	È!ÁÄÈsÐUYÉzÑ@YÑ'ZÔ'ZÑZÐZà—z’z !‘}”} u¤xÔ':×'?Ò'?Ø—I’I˜a‘L”L C¨$¡J°%·)²)¸E¿IºIÀa¹L¼LÈ4Ñ<OÐQRÐUVÑQVÑ2WÔ2WÑ#WÑWñ(ô (ñ ð r(   g     ˆÃ@r   T©rS   Úkeepdimé   )rx   Úis_cudar’   r“   rÉ   r™   ÚroundÚclampÚmeanÚfloatrü   r{   râ   )rŒ   ÚscaleÚzerorí   rS   rï   Úverboserò   ró   rô   rõ   r…   Úw_fr   Ú
best_errorr¦   Úw_qÚw_rÚw_eÚcurrent_errorrx   s                       @r&   Úoptimize_weightsz'HQQWeightOnlyQuantizer.optimize_weights  sÔ  ø€ ð 	ˆˆˆàR\ÐRd ¨c¸DÈ2ÐNÐNÐNÐjtˆ
à�yÔ!Ø�vÔØ�wÔØ�wÔð	'
Ñ#ˆ��u˜eð "(¤ÐB�”�°U´]ˆØ�iŠi˜ÑÔˆØ—’˜‘”ˆØ�wŠw�u‰~Œ~ˆà!(ð 	ð 	ð 	ð 	ð 	ð 	ð ˆ
Ý�u‘”ð 	ð 	ˆAØ—+’+˜c E™k¨DÑ0Ñ1Ô1×7Ò7¸À¼
ÀGÈAÄJÑOÔOˆCØ˜‘: Ñ&ˆCØ�)˜C #™I tÑ,Ô,ˆCØ—:’:˜c S¨3¡Y°%Ñ$7Ñ7¸dÈD�:ÑQÔQˆDØ�E‰MˆDå! %§)¢)¨C°#©IÑ"6Ô"6×";Ò";Ñ"=Ô"=Ñ>Ô>ˆMØð 5Ý�a�œ -°Ñ3Ô3Ñ4Ô4Ð4Ø˜zÒ)Ð)Ø*�
�
àà��c˜3à�dˆ{Ðr(   c                ó  — | j         d         |j         d         k    r|j        }| j        } |dv rO|                      ¦   «         dz  |z  }t          |¦  «        D ]#}| dd …xx         ||d |…         ||z  z  z  cc<   Œ$d S t	          d¦  «        ‚)Nr   ©é   rN   é   r  zOnly 2,4,8 bits are supported.)r”   ÚTÚelement_sizer™   r^   )Úpack_tensorÚori_int_tensorrR   Úcompress_ratioÚjs        r&   Úpack_on_row_fast_248bitz.HQQWeightOnlyQuantizer.pack_on_row_fast_248bitT  s»   € àÔ˜QÔ >Ô#7¸Ô#:Ò:Ð:Ø+Ô-ˆNØ%œ-ˆKØ�9ÐÐØ(×5Ò5Ñ7Ô7¸!Ñ;¸tÑCˆNÝ˜>Ñ*Ô*ð Uð U�Ø˜A˜B˜B��” >°!Ð2C°^Ð2CÔ#DÈÐQRÉÑ#TÑT��‘�ðUð Uõ &Ð&FÑGÔGÐGr(   rN   Té@   r   c                óö  — dd l }|                     ¦   «         }	|	j        }
||
|         |z  z
  |z  }|dk    r%|j        j                             |	d|fdd¦  «        }	n&|j        j                             |	ddd|fdd¦  «        }	|	j        }|�6|r4|dk    r|	                     d|g¦  «        n|	                     |dg¦  «        }	|du r+|	                     ¦   «         |	                     ¦   «         }}d}n:|	                     |d¬¦  «        d         }|	                     |d¬¦  «        d         }d|z  dz
  }d}||g}|||z
  z   	                    d	¬
¦  «        }||z
  }|dk     
                    ¦   «                              ¦   «         dk    r"|||dk    <   ||z   	                    d	¬
¦  «        }| |z  }|r|                     |¦  «        }|r|                      |	||||¬¦  «        \  }}|                     |	|z  |z   ¦  «         	                    |d         |d         ¦  «        }|                     |¦  «                             ¦   «         }d|z  }|dk    r9|                     |d         d¦  «        }|                     |d         d¦  «        }n8|                     d|d         ¦  «        }|                     d|d         ¦  «        }~	~~||                     |j        ¦  «        |                     |j        ¦  «        fS )Nr   r   Úconstantrˆ   FTr  r  g     ˆÓ@)Úmax)rŒ   r	  r
  rí   rS   r÷   )rx   r  r”   rù   rú   ÚpadÚreshapeÚminr!  r  ÚsumÚitemr  r  rV   rÉ   r…   )r%   rŒ   rR   Úchannel_wiseÚ
group_sizeÚoptimizeÚ
round_zerorS   rx   ÚweightÚ	ori_shapeÚpad_lenr”   Ú_minÚ_maxÚmax_vÚmin_vrí   r	  Úmin_max_axisr
  r  s                         r&   Úquantize_internalz(HQQWeightOnlyQuantizer.quantize_internala  s  € ð 	ˆˆˆà—’‘”ˆØ”Lˆ	à 	¨$¤°*Ñ <Ñ<À
ÑJˆØ�1Š9ˆ9Ø”XÔ(×,Ò,¨V°a¸°\À:ÈqÑQÔQˆFˆFà”XÔ(×,Ò,¨V°a¸¸A¸wÐ5GÈÐUVÑWÔWˆFØ”ˆð Ð"¨Ð"Ø:>À!º)¸)�V—^’^ R¨Ð$4Ñ5Ô5Ð5È&Ï.Ê.ÐZdÐfhÐYiÑJjÔJjˆFð ˜5Ð Ð ØŸš™œ v§z¢z¡|¤|�$ˆDØˆHˆHà—:’: 4°�:Ñ6Ô6°qÔ9ˆDØ—:’: 4°�:Ñ6Ô6°qÔ9ˆDà�4‘˜!‘ˆØˆØ˜%�.ˆð ˜$ ™+Ñ&×-Ò-°#Ð-Ñ6Ô6ˆà˜d‘{ˆØ˜AÒ×"Ò"Ñ$Ô$×)Ò)Ñ+Ô+¨aÒ/Ð/Ø.3ˆL˜¨Ò*Ñ+Ø˜\Ñ)×0Ò0°SÐ0Ñ9Ô9ˆEØˆu�u‰}ˆàð 	%Ø—;’;˜tÑ$Ô$ˆDð ð 	sØ×/Ò/°vÀUÐQUÐ_fÐmqÐ/ÑrÔr‰KˆE�4ð �kŠk˜& 5™.¨4Ñ/Ñ0Ô0×6Ò6°w¸q´zÀ7È1Ä:ÑNÔNˆØ�kŠk˜%Ñ Ô ×$Ò$Ñ&Ô&ˆà�e‘ˆØ�1Š9ˆ9Ø—M’M %¨¤(¨BÑ/Ô/ˆEØ—<’<  a¤¨"Ñ-Ô-ˆDˆDà—M’M " e¨B¤iÑ0Ô0ˆEØ—<’<  E¨"¤IÑ.Ô.ˆDà�D˜$à�E—H’H˜Vœ\Ñ*Ô*¨D¯GªG°F´LÑ,AÔ,AÐAÐAr(   Únoder   Úgraph_stackúlist[GraphProto]Úreturnúlist[NodeProto]c           	     ó	  — |j         dk    rt          d¦  «        ‚ddl}t                               d|j        › d�¦  «         |j        d         }t          ||¦  «        \  }}|€t                               d¦  «         |gS t          j	         
                    |¦  «        }t          |j        ¦  «        d	k    rt                               d
¦  «         |gS |                     |¦  «        }|j                             ¦   «         r|                     ¦   «         }| j        j        }	|                      |j        |	| j        j        ¬¦  «        \  }
}}|
                     ¦   «         }
|                     ¦   «         }|                     ¦   «         }d|	z  }|                     |
j        d         |
j        d         |z  f|j        |
j        ¬¦  «        }|                      ||
|	¦  «         |                     ¦   «                              ¦   «         }|                     ¦   «                              ¦   «         }|                     d¦  «        }|                     d¦  «        }|j        \  }}| j        j        }||z  }||z   dz
  |z  }|                     |||¦  «        }t          j	                             |                     ¦   «                              ¦   «         ¦  «        }|j        dz   t=          |	¦  «        z   |_        |j        D ])}|j        |k    r|j                             |¦  «          nŒ*t          j	                             |¦  «        }|j        dz   |_        |j          !                    ||g¦  «         |j        d         |j        |j        g}t          j	                             |¦  «        }|j        dz   |_        |j          !                    |g¦  «         | "                    |j        ¦  «         i }|j        \  }}||d<   ||d<   |	|d<   | j        j        |d<   t          j#        j$        	 d||j%        d         g|j        r|j        dz   t=          |	¦  «        z   ndddœ|¤Ž}t                               d|j        › d�¦  «         |gS )á½  
        Target node:        QOperator node:            QDQ nodes:
        MatMul              MatMulNBits                DeQuantizeLinear -> MatMul
        Gather              GatherBlockQuantized       Gather, Gather, Gather (optional) -> DequantizeLinear
        If the node is target node with fp32 or fp16 const weight, quantize the weight to int4 and
        return the new nodes.
        If QOperator format, return the corresponding QOperator nodes.
        If QDQ format, return the corresdponging QDQ nodes.
        Gather (quantized data) + Gather (scales) + Gather (optional, zero points) -> DequantizeLinear is
        not supported yet because Gather does not support int4 data.
        r"   z/Gather quantization is not supported yet in HQQr   Nústart to quantize ú ...r   ú2MatMul doesn't have const weight. Skip to quantizer  ú)MatMul weight is not 2D. Skip to quantize)rR   r(  r  )r…   rr   rˆ   Ú_QÚ_scalesÚ_zero_pointsÚKÚNrR   rH   ÚMatMulNBitsÚ úcom.microsoft©r    Úoutputsr°   Údomainúcomplete quantization of ©rD  )&Úop_typer^   rx   ÚloggerÚinfor°   ÚinputÚget_initializerÚonnxÚnumpy_helperÚto_arrayrÊ   r”   Ú
from_numpyrj   r   r�   rR   r3  r  rH   rŽ   r›   Úuint8rr   r  rk   r¾   r#  Ú
from_arrayr   ÚremoveÚinitializerÚextendrÍ   ÚhelperÚ	make_nodeÚoutput)r%   r4  r5  rx   Úinput_bÚb_pbÚbs_graphÚb_arrayÚb_array_torchrR   Úquant_weight_torchÚscales_torchÚzero_points_torchÚpacked_sizeÚpacked_torchÚscalesÚzero_pointsÚrowsÚcolsrH   Ú	blob_sizeÚk_blocksÚb_quantrO  Úscales_tensorÚinput_namesÚ	zp_tensorÚkwargsÚmatmul_q_nodes                                r&   ÚquantizezHQQWeightOnlyQuantizer.quantize¢  sz  € ð Œ<˜8Ò#Ð#Ý%Ð&WÑXÔXÐXàˆˆˆå�ŠÐ8¨¬Ð8Ð8Ð8Ñ9Ô9Ð9Ø”*˜Q”-ˆÝ(¨°+Ñ>Ô>‰ˆˆhØˆ<Ý�KŠKÐLÑMÔMÐMØ�6ˆMåÔ#×,Ò,¨TÑ2Ô2ˆÝˆwŒ}ÑÔ Ò"Ð"Ý�KŠKÐCÑDÔDÐDØ�6ˆMØ×(Ò(¨Ñ1Ô1ˆØŒ:×"Ò"Ñ$Ô$ð 	1Ø)×.Ò.Ñ0Ô0ˆMàŒ{ÔˆØ>B×>TÒ>TØŒO $°4´;Ô3Ið ?Uñ ?
ô ?
Ñ;Ð˜LÐ*;ð 0×:Ò:Ñ<Ô<ÐØ#×.Ò.Ñ0Ô0ˆØ-×8Ò8Ñ:Ô:Ðà˜4‘iˆà—{’{ØÔ% aÔ(Ð*<Ô*BÀ1Ô*EÈÑ*TÐUØ”+Ø%Ô,ð #ñ 
ô 
ˆð
 	×$Ò$ \Ð3EÀtÑLÔLÐLØ×!Ò!Ñ#Ô#×)Ò)Ñ+Ô+ˆØ'×+Ò+Ñ-Ô-×3Ò3Ñ5Ô5ˆà—’ Ñ#Ô#ˆØ!×)Ò)¨"Ñ-Ô-ˆØ"Ô(‰
ˆˆdØ”[Ô+ˆ
Ø +Ñ-ˆ	Ø˜:Ñ%¨Ñ)¨jÑ8ˆØ#×+Ò+¨D°(¸IÑFÔFˆåÔ#×.Ò.¨|×/?Ò/?Ñ/AÔ/A×/GÒ/GÑ/IÔ/IÑJÔJˆØ”y 4Ñ'­#¨d©)¬)Ñ3ˆŒØ”^ð 	ð 	ˆEØŒz˜WÒ$Ð$Ø”×%Ò% eÑ,Ô,Ð,Ø�ð %õ Ô)×4Ò4°VÑ<Ô<ˆØ!œY¨Ñ2ˆÔØÔ×#Ò# W¨mÐ$<Ñ=Ô=Ð=à”z !”} g¤l°MÔ4FÐGˆÝÔ%×0Ò0°Ñ=Ô=ˆ	Øœ ^Ñ3ˆ	ŒØÔ×#Ò# Y KÑ0Ô0Ð0Ø×Ò˜9œ>Ñ*Ô*Ð*àˆØ”]‰
ˆˆdØˆˆs‰Øˆˆs‰Øˆˆv‰Ø#œ{Ô5ˆˆ|ÑåœÔ-Øð
àØ”[ ”^Ð$Ø15´ÐB�”˜TÑ!¥C¨¡I¤IÑ-Ð-ÀØ"ð
ð 
ð ð
ð 
ˆõ 	�ŠÐ?°´	Ð?Ð?Ð?Ñ@Ô@Ð@àˆÐr(   )r�   rM   )r   NF)rí   rî   rS   rV   rï   r   )rN   Tr  TTr   ©r4  r   r5  r6  r7  r8  )	r)   r*   r+   r'   Ústaticmethodr  r  r3  rs  r,   r(   r&   rè   rè     s±   € € € € € ðð ð ð ð ð Ø"&Øð2ð 2ð 2ð 2ñ „\ð2ðh ð	Hð 	Hñ „\ð	Hð fgð?Bð ?Bð ?Bð ?BðB_ð _ð _ð _ð _ð _r(   rè   Ú
graph_pathr6  r7  útuple[TensorProto, GraphProto]c                ó˜   — t          t          |¦  «        dz
  dd¦  «        D ]'}||         }|j        D ]}|j        | k    r||fc c S ŒŒ(dS )Nr   rˆ   )NN)r™   rÊ   rX  r°   )r°   rv  ÚgidÚgraphrŒ   s        r&   rP  rP    sw   € Ý•S˜‘_”_ qÑ(¨"¨bÑ1Ô1ð %ð %ˆØ˜3”ˆØÔ'ð 	%ð 	%ˆFØŒ{˜dÒ"Ð"Ø˜u�}Ð$Ð$Ð$Ð$Ð$ð #ð	%ð ˆ:r(   c                ój  — ||z  }| dz	  dz  }| dz  }t          j        |t           j        ¬¦  «        }||dd d…<   ||dd d…<   |                     ||f¦  «        }|j        }|                     d¦  «        }	|	dd d…         dz  dz  |	dd d…         dz  z  } |                      t           j        ¦  «        S )	NrN   é   ©r…   r   r  r   rˆ   éð   )râ   ÚemptyrU  r#  r  Úastype)
Úpackedri  rj  ÚtotalÚhighÚlowÚ	int4_valsÚint4_matrixÚint4_matrix_transposedÚflats
             r&   Útranspose_packed_int4_matrixr‰    sÌ   € à�4‰K€EØ�a‰K˜4Ñ€DØ
�4‰-€CÝ”˜¥b¤hÐ/Ñ/Ô/€IØ€Iˆaˆd�ˆd�OØ€Iˆaˆd�ˆd�OØ×#Ò# T¨4 LÑ1Ô1€Kð )œ]Ðð "×)Ò)¨"Ñ-Ô-€DØ�A�D�q�DŒz˜Q‰ $Ñ&¨4°°°1°¬:¸Ñ+<Ñ=€FØ�=Š=�œÑ"Ô"Ð"r(   c                  ó–   — e Zd Zdd„Zd d„Zd!d„Zed"d„¦   «         Zed#d„¦   «         Zed$d„¦   «         Z	ed%d„¦   «         Z
d!d„Zd!d„ZdS )&ÚDefaultWeightOnlyQuantizerr�   rU   c                ó   — || _         d S rá   rê   rë   s     r&   r'   z#DefaultWeightOnlyQuantizer.__init__"  s   € ØˆŒˆˆr(   Ú
fp32weightúnpt.ArrayLiker7  ú)tuple[np.ndarray, np.ndarray, np.ndarray]c           
     óH  — | j         j        }d|z  }t          |j        ¦  «        dk    rt	          d¦  «        ‚|j        \  }}| j         j        }||z   dz
  |z  }| j         j        t          j        k    rû||z   dz
  |z  }||z  }	|	|z
  }
|
dk    rt          j
        |d|
fdfd¦  «        }t          j        |||fd¬	¦  «        }t          j        |||z   dz
  |z  fd¬	¦  «        }t          j        ||f|j        ¬	¦  «        }|dk    r!t          |||||||| j         j        ¦  «         nø|dk    r!t          |||||||| j         j        ¦  «         nÑt!          |||||||| j         j        ¦  «         n°| j         j        r|n| j         j        }||z   dz
  |z  }|d
k    s
J d¦   «         ‚t          j        ||z  dz   dz  d¬	¦  «        }t          j        ||z  dz   dz  d¬	¦  «        }t          j        ||f|j        ¬	¦  «        }t%          |||||||| j         j        ¦  «         |||fS )z54b/8b quantize fp32 weight to int4 using C++ kernels.r  r  z9Current int4 block quantization only supports 2D tensors!r   r   )r   r   r   rU  r}  rN   z+QDQ format only support 4 bits quantization)r�   rR   rÊ   r”   rÇ   rH   r   r   r3   râ   r"  r›   r…   r   rW   r
   r	   r[   r   )r%   r�  ÚqbitsÚkpackri  rj  rH   rl  rk  Úpadded_rowsr-  r�  Ú
zero_pointrg  s                 r&   Úqbits_block_quantz,DefaultWeightOnlyQuantizer.qbits_block_quant%  s¥  € ð ”Ô ˆØ�U‘
ˆÝˆzÔÑ Ô  AÒ%Ð%ÝÐXÑYÔYÐYØÔ%‰
ˆˆdà”[Ô+ˆ
Ø˜:Ñ%¨Ñ)¨jÑ8ˆàŒ;Ô#¥{Ô'<Ò<Ð<Ø# eÑ+¨aÑ/°EÑ9ˆIØ" ZÑ/ˆKØ! DÑ(ˆGØ˜Š{ˆ{ÝœV J°!°W°¸vÐ0FÈ
ÑSÔS�
õ ”X˜t X¨yÐ9ÀÐIÑIÔIˆFÝœ 4¨8°eÑ+;¸aÑ+?ÀEÑ*IÐ"KÐSZÐ[Ñ[Ô[ˆJÝ”X˜t XÐ.°jÔ6FÐGÑGÔGˆFØ˜ŠzˆzÝ%Ø˜J¨°
¸JÈÈdÐTXÔT_ÔTlñô ð ð ð ˜!’�Ý%Ø˜J¨°
¸JÈÈdÐTXÔT_ÔTlñô ð ð õ &Ø˜J¨°
¸JÈÈdÐTXÔT_ÔTlñô ð ð ð
 "&¤Ô!CÐ_˜˜ÈÌÔI_ˆJØ˜zÑ)¨AÑ-°*Ñ<ˆHà˜A’:�:�:ÐL‘:”:�:Ý”X˜t d™{¨Q™°1Ñ4¸GÐDÑDÔDˆFÝœ 4¨(¡?°QÑ#6¸1Ñ"<ÀGÐLÑLÔLˆJÝ”X˜x¨Ð.°jÔ6FÐGÑGÔGˆFÝ%Ø˜
 F¨J¸
ÀDÈ$ÐPTÔP[ÔPhñô ð ð ˜ 
Ð+Ð+r(   r4  r   r5  r6  r8  c                ó@  — | j         j        }|dk    r%| j         j        rt          j        nt          j        }n$| j         j        rt          j        nt          j        }|j        d         }t          ||¦  «        \  }}|€t                               d¦  «         |gS t          j        |¦  «                             ¦   «         }t          |j        ¦  «        dk    rt                               d¦  «         |gS |j        dk    }	|	r|                     t&          j        ¦  «        }|                      |¦  «        \  }
}}|	r|                     t,          j        ¦  «        }| j         j        t2          j        k    rzt          j                             t          j        |
|j        d|› �z   ¬	¦  «        ¦  «        }t          j                             t          j        ||j        d
z   ¬	¦  «        ¦  «        }n€t>          j          !                    |j        d|› �z   ||j        |
 "                    ¦   «         d¦  «        }t          j                             t          j        ||j        dz   ¬	¦  «        ¦  «        }| j         j        t2          j#        k    o| j         j$        o| j         j        }|r¯|j        \  }}tK          |
||¦  «        }
| &                    |df¦  «        }t>          j          !                    |j        d|› �z   |||g|
 "                    ¦   «         d¦  «        }t          j                             t          j        ||j        dz   ¬	¦  «        ¦  «        }|j        D ])}|j        |k    r|j         '                    |¦  «          nŒ*|j(         )                    ||g¦  «         g }| j         j        t2          j        k    �r|j        d         |j        |j        g}| j         j        s]t>          j*         +                    ||j        dz   ¦  «        }| ,                    |j        ¦  «         |j(         )                    |g¦  «         i }|j        \  }}||d<   ||d<   ||d<   | j         j-        |d<   | j         j.        r| j         j.        |d<   t?          j         j/        	 d#||j0        d         g|j        r|j        d|› �z   ndddœ|¤Ž}| ,                    |¦  «         �nÆ|j        |j        g}|j        dz   g}|d         g}|d         dz   g}|j        d         |r|d         n|d         g}|j0        d         g}| j         j        swt>          j          !                    |j        dz   ||j        | "                    ¦   «         d¦  «        }| ,                    |j        ¦  «         |j(         )                    |g¦  «         |j        \  }}|rdnd| j         j$        r|n| j         j-        dœ}t?          j         j/        	 d$|||j        r|j        d|› �z   nddœ|¤Ž}t>          j          /                    d|||j        r|j        d |› �z   nd¬¦  «        } |r>t>          j          /                    d!||ddg¬"¦  «        }!| )                    ||!| g¦  «         n| )                    || g¦  «         |S )%z˜
        Quantize weight B of MatMul node to int4 or int8.
        Currently only support 2D constant matrix and axis 0 blockwise quantization.
        r  r   Nr=  r  r>  Úbfloat16r?  )r°   r@  Ú_DQ_QTÚ
_DQ_scalesr   rA  rB  rC  rR   rH   rY   rD  rE  rF  rG  Ú_outputÚ_transposedÚ_DQ_zero_points)rS   rH   ÚDequantizeLinear)r    rH  r°   r!   Ú	_matmul_QÚ	Transpose)r    rH  ÚpermrK  )r�  )1r�   rR   rW   r   ÚINT8ÚUINT8ÚINT4ÚUINT4rO  rP  rM  rN  ÚirÚ
from_protor¾   rÊ   r”   r…   r€  râ   r“   r•  Ú	ml_dtypesr—  r   r   r3   ÚserdeÚserialize_tensorÚTensorr°   rQ  rZ  Úmake_tensorÚtobytesr~   r[   r‰  r#  rW  rX  rY  rR  rV  rÍ   rH   rY   r[  r\  )"r%   r4  r5  rR   Úqtyper]  Úb_tensorÚb_graphÚ	b_ndarrayr—  r�  rg  rh  rm  rn  Úqdq_opt_for_intel_npu_enabledri  rj  rO  Úoutput_nodesro  rp  rq  Úmatmul_qbit_nodeÚdq_input_namesÚdq_output_namesÚtp_input_namesÚtp_output_namesÚmatmul_input_namesÚmatmul_output_namesÚ	dq_kwargsÚdq_nodeÚmatmul_nodeÚtp_nodes"                                     r&   Úquantize_matmulz*DefaultWeightOnlyQuantizer.quantize_matmulW  sÓ  € ð
 Œ{ÔˆØ�1Š9ˆ9Ø(,¬Ô(@ÐW•KÔ$Ð$ÅkÔFWˆEˆEà(,¬Ô(@ÐW•KÔ$Ð$ÅkÔFWˆEØ”*˜Q”-ˆÝ+¨G°[ÑAÔAÑˆ�'ØÐÝ�KŠKÐLÑMÔMÐMØ�6ˆMå”M (Ñ+Ô+×1Ò1Ñ3Ô3ˆ	ÝˆyŒÑÔ 1Ò$Ð$Ý�KŠKÐCÑDÔDÐDØ�6ˆMà”? jÒ0ˆØð 	5Ø!×(Ò(­¬Ñ4Ô4ˆIà&*×&<Ò&<¸YÑ&GÔ&GÑ#ˆ�˜Øð 	7Ø—]’]¥9Ô#5Ñ6Ô6ˆFàŒ;Ô#¥{Ô'<Ò<Ð<Ý”h×/Ò/µ´	¸&ÀxÄ}ÐWbÐ\`ÐWbÐWbÑGbÐ0cÑ0cÔ0cÑdÔdˆGÝœH×5Ò5µb´iÀÈXÌ]Ð]fÑMfÐ6gÑ6gÔ6gÑhÔhˆMˆMå”k×-Ò-Ø” ¨  Ñ.°°y´ÈÏÊÑHXÔHXÐZ^ñô ˆGõ œH×5Ò5µb´iÀÈXÌ]Ð]iÑMiÐ6jÑ6jÔ6jÑkÔkˆMð ŒKÔ$­¬Ò7ð )Ø”Ô2ð)à”Ô(ð 	&ð
 )ð 	lØ"œ‰JˆD�$Ý1°&¸$ÀÑEÔEˆFØ—^’^ T¨1 IÑ.Ô.ˆFÝ”k×-Ò-Ø” ¨  Ñ.°¸¸d°|ÀVÇ^Â^ÑEUÔEUÐW[ñô ˆGõ œH×5Ò5µb´iÀÈXÌ]Ð]iÑMiÐ6jÑ6jÔ6jÑkÔkˆMà”]ð 	ð 	ˆEØŒz˜WÒ$Ð$Ø”×$Ò$ UÑ+Ô+Ð+Ø�ð %ð 	Ô×"Ò" G¨]Ð#;Ñ<Ô<Ð<àˆàŒ;Ô#¥{Ô'<Ò<Ñ<Øœ: aœ=¨'¬,¸Ô8JÐKˆKØ”;Ô+ð 8Ý Ô-×8Ò8¸ÀhÄmÐVdÑFdÑeÔe�	Ø×"Ò" 9¤>Ñ2Ô2Ð2ØÔ#×*Ò*¨I¨;Ñ7Ô7Ð7ØˆFØ"œ‰JˆD�$ØˆF�3‰KØˆF�3‰KØ!ˆF�6‰NØ#'¤;Ô#9ˆF�<Ñ ð Œ{Ô)ð FØ+/¬;Ô+E�Ð'Ñ(å#œ{Ô4Øð à"Øœ QœÐ(Ø04´	ÐA�T”Y  d  Ñ,Ð,¸rØ&ð ð  ð ð ð  Ðð ×ÒÐ 0Ñ1Ô1Ð1Ñ1à%œl¨MÔ,>Ð?ˆNØ&œ|¨iÑ7Ð8ˆOØ-¨aÔ0Ð1ˆNØ.¨qÔ1°MÑAÐBˆOà”
˜1”Ø&CÐ[� Ô"Ð"ÈÐYZÔI[ð"Ðð $(¤;¨q¤>Ð"2ÐØ”;Ô+ð 8Ý œK×3Ò3Ø”MÐ$5Ñ5°u¸f¼lÈK×L_ÒL_ÑLaÔLaÐcgñô �	ð ×%Ò% i¤nÑ5Ô5Ð5ØÔ#×*Ò*¨I¨;Ñ7Ô7Ð7Ø"œ‰JˆD�$à:ÐA˜˜ÀØ&*¤kÔ&HÐd˜d˜dÈdÌkÔNdðð ˆIõ ”kÔ+Ø"ðà%Ø'Ø37´9ÐD�T”Y ¨  Ñ/Ð/À"ð	ð ð
 ðð ˆGõ œ+×/Ò/ØØ)Ø+Ø7;´yÐH�T”YÐ!3¨TÐ!3Ð!3Ñ3Ð3Àbð	 0ñ ô ˆKð -ð 	<Ýœ+×/Ò/ØØ)Ø+Ø˜Q˜ð	 0ñ ô �ð ×#Ò# W¨g°{Ð$CÑDÔDÐDÐDà×#Ò# W¨kÐ$:Ñ;Ô;Ð;àÐr(   rÝ   ú
np.ndarrayútuple[np.ndarray, np.ndarray]c                ó¬  — t          j        | dd¬¦  «        }t          j        | dd¬¦  «        }t          j        t          j        |¦  «        t          j        |¦  «        k    ||¦  «        }|dz  }t          j        |dk    d| |z  ¦  «                             ¦   «                              dd¦  «                             t           j        ¦  «        }||fS )Nr   T©rS   Úkeepdimsg       Àr   iøÿÿÿé   )	râ   r!  r$  Úwhererü   r  Úclipr€  Úint8)rÝ   Úmax_valÚmin_valÚabs_maxr	  Úquantized_slices         r&   Úquant_slice_symmetricz0DefaultWeightOnlyQuantizer.quant_slice_symmetricß  s´   € å”&˜ A°Ð5Ñ5Ô5ˆÝ”&˜ A°Ð5Ñ5Ô5ˆÝ”(�2œ6 '™?œ?­R¬V°G©_¬_Ò<¸gÀwÑOÔOˆà˜$‘ˆÝœ( 5¨A¢:¨q°$¸±,Ñ?Ô?×EÒEÑGÔG×LÒLÈRÐQRÑSÔS×ZÒZÕ[]Ô[bÑcÔcˆà Ð%Ð%r(   c                óL  — t          j        |                      dd¬¦  «        d¦  «        }t          j        |                      dd¬¦  «        d¦  «        }||z
  dz  }t          j        |dk    d| |z  ¦  «                             ¦   «                              dd¦  «                             t           j	        ¦  «        }t          j        |dk    d| |z  |z   ¦  «                             ¦   «                              dd¦  «                             t           j	        ¦  «        }|||fS )Nr   TrÂ  r   g      .@r  r|  )
râ   Úminimumr$  Úmaximumr!  rÅ  r  rÆ  r€  rU  )rÝ   rÉ  rÈ  r	  r”  rË  s         r&   Úquant_slice_asymmetricz1DefaultWeightOnlyQuantizer.quant_slice_asymmetricê  sü   € å”*˜TŸXšX¨1°t˜XÑ<Ô<¸aÑ@Ô@ˆÝ”*˜TŸXšX¨1°t˜XÑ<Ô<¸aÑ@Ô@ˆà˜7Ñ" dÑ*ˆÝ”X˜e qšj¨!¨g¨X¸Ñ-=Ñ>Ô>×DÒDÑFÔF×KÒKÈAÈrÑRÔR×YÒYÕZ\ÔZbÑcÔcˆ
Ýœ( 5¨A¢:¨q°$¸±,ÀÑ2KÑLÔL×RÒRÑTÔT×YÒYÐZ[Ð]_Ñ`Ô`×gÒgÕhjÔhpÑqÔqˆà  zÐ1Ð1r(   c                óì   — |                       d¦  «        }t          |¦  «        dz  dk    rt          j        |d¦  «        }|ddd…         dz  |ddd…         dz  dz  z  }|                     d¦  «        S )	z2Pack int8 data to int4 and store in uint8 ndarray.rˆ   r  r   Nr|  r   rN   rU  )r#  rÊ   râ   rÍ   r€  )rÝ   Ú	data_flatÚquant_data_int4s      r&   Úpack_int8_to_int4z,DefaultWeightOnlyQuantizer.pack_int8_to_int4õ  s}   € ð —L’L Ñ$Ô$ˆ	Ýˆy‰>Œ>˜AÑ Ò"Ð"Ýœ	 )¨QÑ/Ô/ˆIØ$ S S q Sœ>¨CÑ/°Y¸q¸tÀ!¸t´_ÀsÑ5JÈqÑ4PÑQˆà×%Ò% gÑ.Ô.Ð.r(   Úquantize_axisrV   rH   rW   rX   ú0tuple[np.ndarray, np.ndarray, np.ndarray | None]c                óÔ  — d}| j         |         }d}t          | j         ¦  «        D ]\  }}||k     r||z  }Œ||k    r||z  }Œ||z   dz
  |z  }	t          | j         ¦  «        }
|	|
|<   |                      |||f¦  «        }t	          j        ||	|f| j        ¬¦  «        }|rt	          j        |||fd¬¦  «        }n2t	          j        |||fd¬¦  «        }t	          j        ||	|fd¬¦  «        }t          d||¦  «        D ]›}t          ||z   |¦  «        }|dd…||…dd…f         }|rt           
                    |¦  «        \  }}nt                               |¦  «        \  }}}||dd…||…dd…f<   ||z  }||dd…||dz   …dd…f<   |s||dd…||dz   …dd…f<   Œœt                               |¦  «        }d}|st                               |¦  «        }|                     |
¦  «        }|||fS )zXQuantize ndarray data to int4 using numpy, return (quantized data, scales, zero points).r   r}  rÇ  rU  r   N)r”   rÌ   r‹   r#  râ   r›   r…   r™   r$  r‹  rÌ  rÐ  rÔ  )rÝ   rÕ  rH   rW   ÚmÚkÚnr¦   Údimrl  Úscales_shapeÚdata_reshaperg  Úquant_data_int8Úzero_point_int8Úend_idxÚsliceÚquantized_slice_int8Úscale_sliceÚzero_point_slice_int8r  rÓ  Úzero_point_int4s                          r&   Úquantize_ndarrayz+DefaultWeightOnlyQuantizer.quantize_ndarrayÿ  sw  € ð ˆØŒJ�}Ô%ˆØˆÝ ¤
Ñ+Ô+ð 	ð 	‰FˆAˆsØ�=Ò Ð Ø�S‘��Ø�]Ò"Ð"Ø�S‘�øà˜
‘N QÑ&¨:Ñ5ˆÝ˜DœJÑ'Ô'ˆØ&.ˆ�]Ñ#à—|’| Q¨¨1 IÑ.Ô.ˆÝ”˜1˜h¨Ð*°$´*Ð=Ñ=Ô=ˆØð 	HÝ œh¨¨1¨a y¸Ð?Ñ?Ô?ˆOˆOå œh¨¨1¨a y¸Ð@Ñ@Ô@ˆOÝ œh¨¨8°QÐ'7¸wÐGÑGÔGˆOõ �q˜!˜ZÑ(Ô(ð 	Kð 	KˆAÝ˜!˜j™.¨!Ñ,Ô,ˆGØ     A g I¨q¨q¨q Ô1ˆEàð Ý4N×4dÒ4dÐejÑ4kÔ4kÑ1Ð$ k kõ /×EÒEÀeÑLÔLñ IÐ$ kÐ3Hð 0DˆO˜A˜A˜A˜q ˜y¨!¨!¨!˜OÑ,Ø�Z‘ˆAØ(3ˆF�1�1�1�a˜1˜q™5�k 1 1 1Ð$Ñ%Øð KØ5J�    1¨¨A© ;°°°Ð 1Ñ2øõ 5×FÒFÀÑWÔWˆØˆØð 	\Ý8×JÒJÈ?Ñ[Ô[ˆOØ—’ Ñ-Ô-ˆØ ¨Ð7Ð7r(   c                ó$  — | j         j        t          j        k    s
J d¦   «         ‚| j         j        rt
          j        nt
          j        }|j        d         }t          ||¦  «        \  }}|€t                               d¦  «         |gS t          j                             |¦  «        }t          |j        ¦  «        }| j         j                             dd¦  «        }	| j         j        }
|	|k     r|	| k    s
J d¦   «         ‚|
dk    r|
dz
  |
z  dk    s
J d	¦   «         ‚|	|z   |z  }	|                      ||	|
| j         j        ¦  «        \  }}}|j        D ])}|j        |k    r|j                             |¦  «          nŒ*t          j                             |j        d
z   ||j        |                     ¦   «         d¦  «        }t          j                             ||j        dz   ¦  «        }|j        |j        d         |j        g}|j                             ||g¦  «         | j         j        swt          j                             |j        dz   ||j        |                     ¦   «         d¦  «        }|                     |j        ¦  «         |j                             |g¦  «         	 t          j                             |d¦  «        }n# t>          $ r d}Y nw xY w||	|
dœ}t          j        j         	 d||j!        d         g|j        r
|j        d
z   ndddœ|¤Ž}|gS )z,Quantize weight data of Gather node to int4.z0Gather only supports QOperator format currently.r   Nz4Gather doesn't have const weight. Skip quantization.r"   r   z&Invalid quantize axis for Gather node.é   z#Invalid block size for Gather node.Ú_Q4Tr@  rA  rS   )Úgather_axisrÕ  rH   ÚGatherBlockQuantizedrE  rF  rG  )rë  )"r�   r   r   r3   rW   r   r£  r¤  rO  rP  rM  rN  rQ  rR  rS  rÊ   r”   r   ÚgetrH   ræ  r°   rW  rZ  r«  r¬  rV  rX  rY  rÍ   Úget_node_attr_valuerÇ   r[  r\  )r%   r4  r5  r­  Údata_argÚdata_tensorprotoÚdata_graphprotoÚdata_ndarrayÚ	data_rankrÕ  rH   Úquantized_datarg  rh  rO  Úquantized_data_tensorprotoÚscales_tensorprotoro  Úzp_tensorprotorê  rq  Úgather_q4_nodes                         r&   Úquantize_gatherz*DefaultWeightOnlyQuantizer.quantize_gather7  sc  € àŒ{Ô'­;Ô+@Ò@Ð@Ð@ÐBtÑ@Ô@Ð@à$(¤KÔ$<ÐS•Ô Ð Å+ÔBSˆØ”:˜a”=ˆÝ,;¸HÀkÑ,RÔ,RÑ)Ð˜/ØÐ#Ý�KŠKÐNÑOÔOÐOØ�6ˆMåÔ(×1Ò1Ð2BÑCÔCˆÝ˜Ô*Ñ+Ô+ˆ	ØœÔ.×2Ò2°8¸QÑ?Ô?ˆØ”[Ô+ˆ
à˜yÒ(Ð(¨]¸y¸jÒ-HÐ-HÐ-HÐJrÑ-HÔ-HÐHØ˜RÒÐ j°1¡n¸
Ñ%BÀaÒ%GÐ%GÐ%GÐJoÑ%GÔ%GÐHà&¨Ñ2°iÑ?ˆØ.2×.CÒ.CØ˜-¨°T´[Ô5Mñ/
ô /
Ñ+ˆ˜ ð %Ô*ð 	ð 	ˆEØŒz˜XÒ%Ð%ØÔ%×,Ò,¨UÑ3Ô3Ð3Ø�ð &õ &*¤[×%<Ò%<ØÔ! EÑ)¨5°,Ô2DÀn×F\ÒF\ÑF^ÔF^Ð`dñ&
ô &
Ð"õ "Ô.×9Ò9¸&ÐBRÔBWÐZcÑBcÑdÔdÐØ1Ô6¸¼
À1¼ÐGYÔG^Ð_ˆØÔ#×*Ò*Ð,FÐHZÐ+[Ñ\Ô\Ð\ØŒ{Ô'ð 	AÝ!œ[×4Ò4Ø Ô%¨Ñ6¸¸v¼|È[×M`ÒM`ÑMbÔMbÐdhñô ˆNð ×Ò˜~Ô2Ñ3Ô3Ð3ØÔ'×.Ò.°Ð/?Ñ@Ô@Ð@ð	Ýœ+×9Ò9¸$ÀÑGÔGˆKˆKøÝð 	ð 	ð 	ØˆKˆKˆKð	øøøð 'Ø*Ø$ð
ð 
ˆõ œÔ.Ø"ð
àØ”[ ”^Ð$Ø&*¤iÐ7�”˜UÑ"Ð"°RØ"ð
ð 
ð ð
ð 
ˆð ÐÐs   Ê J> Ê>KËKc                ól  — t                                d|j        › d�¦  «         | j        j        }|j        dk    rT|dk    r7| j        j        t          j        k    rt            	                    d¦  «         |gS |  
                    ||¦  «        }nu|j        dk    rD| j        j        dk    rt            	                    d¦  «         |gS |                      ||¦  «        }n&t            	                    d	|j        › d
�¦  «         |gS t                                d|j        › d| j        j        › d�¦  «         |S )r:  r;  r<  r!   r  z>MatMul only supports QOperator format for 8 bits quantization.r"   rN   z)Gather only supports 4 bits quantization.zUnsupported operator z1 for weight only quantization. Skip quantization.rJ  z with z	 bits ...)rM  rN  r°   r�   rR   rL  r   r   r~   Úerrorr¾  rø  )r%   r4  r5  rR   Úresultss        r&   rs  z#DefaultWeightOnlyQuantizer.quantizew  s-  € õ 	�ŠÐ8¨¬Ð8Ð8Ð8Ñ9Ô9Ð9àŒ{ÔˆØŒ<˜8Ò#Ð#Ø�qŠyˆy˜Tœ[Ô5½¼ÒHÐHÝ—’Ð]Ñ^Ô^Ð^Ø�v�Ø×*Ò*¨4°Ñ=Ô=ˆGˆGØŒ\˜XÒ%Ð%ØŒ{Ô 1Ò$Ð$Ý—’ÐHÑIÔIÐIØ�v�à×*Ò*¨4°Ñ=Ô=ˆGˆGå�LŠLÐp°´ÐpÐpÐpÑqÔqÐqØ�6ˆMå�ŠÐ\°´	Ð\Ð\ÀÄÔAQÐ\Ð\Ð\Ñ]Ô]Ð]Øˆr(   N)r�   rU   )r�  rŽ  r7  r�  rt  )rÝ   r¿  r7  rÀ  )rÝ   r¿  r7  r�  )rÝ   r¿  r7  r¿  )
rÝ   r¿  rÕ  rV   rH   rV   rW   rX   r7  rÖ  )r)   r*   r+   r'   r•  r¾  ru  rÌ  rÐ  rÔ  ræ  rø  rs  r,   r(   r&   r‹  r‹  !  s   € € € € € ðð ð ð ð0,ð 0,ð 0,ð 0,ðdFð Fð Fð FðP ð&ð &ð &ñ „\ð&ð ð2ð 2ð 2ñ „\ð2ð ð/ð /ð /ñ „\ð/ð ð58ð 58ð 58ñ „\ð58ðn> ð > ð > ð > ð@ð ð ð ð ð r(   r‹  c                  ó   — e Zd Zd
d„Zdd„Zd	S )ÚNVAWQWeightOnlyQuantizerr�   r`   c                ó   — || _         d S rá   rê   rë   s     r&   r'   z!NVAWQWeightOnlyQuantizer.__init__š  rì   r(   ÚmodelúModelProto | strr7  r   c                ó   — 	 ddl m} n-# t          $ r  t          d¦  «         t          d¦  «        d‚w xY wt                               d¦  «         | j        j        } ||| j        j        |¬¦  «        }t                               d¦  «         |S )	zé
        Perform nvidia_awq quantization using ModelOpt's int4 quantize function.

        Args:
            model (ModelProto): The ONNX model to quantize.

        Returns:
            ModelProto: The quantized ONNX model.
        r   )rs  zlPlease ensure that the 'modelopt' package is installed. Please install it using pip install nvidia_modelopt.zXmodelopt is not installed. Please install it using pip install nvidia_modelopt. Exiting.Nz#Starting nvidia_awq quantization...)r�   rC   ú"Completed nvidia_awq quantization.)	Úmodelopt.onnx.quantization.int4rs  rz   r{   rM  rN  r�   rC   r�   )r%   rÿ  Úquantize_int4rƒ   Úquantized_models        r&   Úquantize_awqz%NVAWQWeightOnlyQuantizer.quantize_awq   sÈ   € ð	ØQÐQÐQÐQÐQÐQÐQøÝð 	ð 	ð 	ÝØ~ñô ð õ Øjñô àðð		øøøõ 	�ŠÐ9Ñ:Ô:Ð:ð ”{Ô:ˆð (˜-ØØ#œ{Ô=Ø$0ð
ñ 
ô 
ˆõ 	�ŠÐ8Ñ9Ô9Ð9ØÐs   ‚	 ‰*3N)r�   r`   )rÿ  r   r7  r   )r)   r*   r+   r'   r  r,   r(   r&   rý  rý  ™  s<   € € € € € ðð ð ð ð!ð !ð !ð !ð !ð !r(   rý  c                  óV   — e Zd ZdZddddddej        ddddfdd„Zd d„Zd„ Zd„ Z	d„ Z
dS )!ÚMatMulNBitsQuantizera¼  
    Target node:        QOperator node:            QDQ nodes:
    MatMul              MatMulNBits                DeQuantizeLinear -> MatMul
    Gather              GatherBlockQuantized       Gather, Gather, Gather (optional) -> DequantizeLinear

    Perform 2/4/8 bits quantization of constant weights for target nodes.
    If algo_config.quant_format is QOperator:
      - nodes are replaced by the corresponding QOperator nodes.
      - quantized weights are stored in the contrib ops.
    If algo_config.quant_format is QDQ:
      - the quantized weight is stored in a standard onnx node. For MatMul, it is DequantizeLinear. For Gather,
        it is the three Gathers, one for quantized data, one for scales and one for optional zero points.
      - The nodes are replaced by the corresponding QDQ nodes.
      - currently Gather is not supported in QDQ because Gather does not support int4 yet.
    Note:
      - for quantized gather, the memory usage of "DequantizeLinear + Gather" is the same as the original Gather
        during runtime. Therefor it is not recommended.
      - when a node is in nodes_to_exclude, and the node configuration in algo_config.customized_weight_config will be ignored.
    rN   rB   FNrÿ  r   rR   rV   rH   rW   rX   rY   rZ   Únodes_to_includeúlist[str] | Noner   r   r   r   r[   Úalgo_configúWeightOnlyQuantConfig | Nonec           
     óú  — |€g }t          |t          ¦  «        r!t          t          j        |¦  «        ¦  «        nt          |¦  «        | _        t          |t          ¦  «        r|nd | _        || _        || _        || _	        || _
        t          |¦  «        | _        |rt          |¦  «        nd | _        d | _        |€t          |||||	|
||¬¦  «        }|| _        t#          | j        d¦  «        r| j        j        dv s
J d¦   «         ‚|j        dk    rt'          | j        ¦  «        | _        d S |j        dk    rt)          | j        ¦  «        | _        d S |j        dk    rt+          | j        ¦  «        | _        d S d S )N)rH   rW   rY   r   r   r   rR   r[   rR   r  z(Only support 2, 4 or 8 bits quantizationrP   r]   ri   )rŠ   r   r   rQ  Úloadrÿ  Ú
model_pathrR   rH   rW   rY   r#   Únodes_to_excluder	  Únode_quantizerrU   r  Úhasattrr   rè   r‹  rý  )r%   rÿ  rR   rH   rW   rY   r  r	  r   r   r   r[   r  s                r&   r'   zMatMulNBitsQuantizer.__init__Ù  s¢  € ð Ð#Ø!ÐÝ4>¸uÅcÑ4JÔ4JÐ`•Y�tœy¨Ñ/Ô/Ñ0Ô0Ð0ÕPYÐZ_ÑP`ÔP`ˆŒ
Ý#-¨eµSÑ#9Ô#9ÐC˜%˜%¸tˆŒØˆŒ	Ø$ˆŒØ(ˆÔØ,ˆÔÝ #Ð$4Ñ 5Ô 5ˆÔØ9IÐ S¥Ð$4Ñ 5Ô 5Ð 5ÈtˆÔØ"ˆÔàÐÝ6Ø%Ø)Ø-Ø)Ø%9Ø%ØØ'=ð	ñ 	ô 	ˆKð 'ˆÔÝ�4Ô# VÑ,Ô,ð 	bØÔ#Ô(¨IÐ5Ð5Ð5Ð7aÑ5Ô5Ð5àÔ  EÒ)Ð)Ý"8¸Ô9IÑ"JÔ"JˆDÔÐÐØÔ" iÒ/Ð/Ý"<¸TÔ=MÑ"NÔ"NˆDÔÐÐØÔ" lÒ2Ð2Ý":¸4Ô;KÑ"LÔ"LˆDÔÐÐð 3Ð2r(   r5  r6  c                óˆ  — g }|d         }|j         D �]ë}d„ |j        D ¦   «         }|�ri }|j        D ]å}|j        t          j        j        k    r7|                     |j        ¦  «         |j        |  	                    |¦  «        i}n}|j        t          j        j
        k    rTg }	|j        D ]@}
|                     |
¦  «         |	                     |  	                    |¦  «        g¦  «         ŒA|j        |	i}nt          |¦  «        }|                     |¦  «         Œæt          j        j        |j        |j        |j        fd|j        i|¤Ž}g }|j        | j        v r't*                               d|j        › d�¦  «         |g}nj| j        r|j        | j        v s|j        | j        j        v r| j                             ||¦  «        }n&t*                               d|j        › d�¦  «         |g}|                     |¦  «         �Œí|                     d¦  «         |j                              |¦  «         |                     ¦   «          |S )	Nrˆ   c                óz   — g | ]8}|j         t          j        j        k    s|j         t          j        j        k    ¯6|‘Œ9S r,   )ÚtyperQ  ÚAttributeProtoÚGRAPHÚGRAPHS)r¿   Úattrs     r&   ú
<listcomp>z:MatMulNBitsQuantizer._process_subgraph.<locals>.<listcomp>  sG   € ð ð ð àØ”9¥Ô 3Ô 9Ò9Ð9¸T¼YÍ$ÔJ]ÔJdÒ=dÐ=dð à=dÐ=dÐ=dr(   r°   zexclude to quantize z$ as specified by nodes_to_exclude...zskip to quantize r<  r4  )r4  Ú	attributer  rQ  r  r  rÍ   Úgr°   Ú_process_subgraphr  ÚgraphsrY  r   rœ   rZ  r[  rL  rO  r\  r  rM  rN  r	  r  r   r  rs  Ú
ClearFieldÚpop)r%   r5  Ú	new_nodesrz  r4  Úgraph_attrsrq  r  ÚkvÚvalueÚsubgraphÚ	out_nodess               r&   r  z&MatMulNBitsQuantizer._process_subgraph  sw  € Øˆ	Ø˜B”ˆà”Jð %	(ñ %	(ˆDðð à œNðñ ô ˆKð
 ñ Ø�Ø œNð &ð &�DØ”y¥DÔ$7Ô$=Ò=Ð=à#×*Ò*¨4¬6Ñ2Ô2Ð2Ø"œi¨×)?Ò)?ÀÑ)LÔ)LÐM˜˜Øœ¥dÔ&9Ô&@Ò@Ð@Ø "˜Ø(,¬ð Pð P˜Hà'×.Ò.¨xÑ8Ô8Ð8Ø!ŸLšL¨$×*@Ò*@ÀÑ*MÔ*MÐ)NÑOÔOÐOÐOØ"œi¨Ð/˜˜å/°Ñ5Ô5˜Ø—M’M "Ñ%Ô%Ð%Ð%Ý”{Ô,Ø”L $¤*¨d¬kðð Ø@DÄ	ðØMSðð �ð ˆIØŒy˜DÔ1Ð1Ð1Ý—’Ðb°4´9ÐbÐbÐbÑcÔcÐcØ!˜F�	�	ØÔ'ð #¨D¬I¸Ô9NÐ,NÐ,NØ” Ô 0Ô EÐEÐEà Ô/×8Ò8¸¸{ÑKÔK�	�	å—’Ð?°´	Ð?Ð?Ð?Ñ@Ô@Ð@Ø!˜F�	Ø×Ò˜YÑ'Ô'Ð'Ñ'à×Ò˜Ñ Ô Ð ØŒ
×Ò˜)Ñ$Ô$Ð$Ø�ŠÑÔÐØˆr(   c                óz  ‡ — i }‰ j         j         j        j        D ] }|j        dv r•t	          ˆ fd„|j        D ¦   «         ¦  «        sud‰ j        ‰ j        rdnddœ}‰ j        j	        rK|j
        ‰ j        j	        v r8‰ j        j	        |j
                                      ¦   «         D ]\  }}||v r|||<   Œ|||j
        <   Œ¡|S )z3Generate weight only quant configuration for nodes.©r!   c              3  óP   •K  — | ] }‰j                              |¦  «        d u V — Œ!d S rá   )rÿ  rP  )r¿   r¦   r%   s     €r&   ú	<genexpr>z@MatMulNBitsQuantizer._generate_q4_node_config.<locals>.<genexpr>@  s8   øè è € ÐUÐUÀQ˜4œ:×5Ò5°aÑ8Ô8¸DÐ@ÐUÐUÐUÐUÐUÐUr(   rN   ÚsymÚasym)rR   r(  Úscheme)rÿ  rz  r4  rL  ÚallrO  rH   rW   r  r   r°   rÎ   )r%   Úq4_node_configr4  Útemplate_config_q4Úkeyr$  s   `     r&   Ú_generate_q4_node_configz-MatMulNBitsQuantizer._generate_q4_node_config;  sü   ø€ àˆØ”JÔ$Ô*Ô/ð 	Cð 	CˆDØŒ|˜zÐ)Ð)ÝÐUÐUÐUÐUÈ$Ì*ÐUÑUÔUÑUÔUð Cà !Ø&*¤oØ+/Ô+<Ð"H % %À&ð*ð *Ð&ð Ô(ÔAð@à œI¨Ô)9Ô)RÐRÐRà*.Ô*:Ô*SÐTXÔT]Ô*^×*dÒ*dÑ*fÔ*fð @ð @™J˜C Ø"Ð&8Ð8Ð8Ø:?Ð 2°3Ñ 7øØ0B�N 4¤9Ñ-øØÐr(   c                ó´  ‡ — ˆ fd„}i }‰ j         �
‰ j         |d<   ‰                      ¦   «         }‰ j        j        }t                               d|› d�¦  «         |dv rQ‰ j        j        |d<   ||d<   	 ‰ j        D ]}d	||<   Œt          d‰ j	        �‰ j	        n‰ j
        j
        |d
œ|¤Ž‰ _
        n�|dk    r‡‰ j        j        |d<   ‰ j        j        |d<   ‰ j        j        |d<   ‰ j        j        |d<   ‰ j        j        |d<   d|d<    |¦   «         }t!          d‰ j	        �‰ j	        n‰ j
        j
        ||dœ|¤Ž‰ _
        t                               d|› d�¦  «         dS )u  4b quantize a model with RTN or GPTQ algorithm. Please refer to
        https://github.com/intel/neural-compressor/blob/master/docs/source/quantization_weight_only.md
        for more details on weight only quantization using IntelÂ® Neural Compressor.
        c               3  ó^   •K  — t          j        ‰j        j        ¦  «        } | D ]}|d fV — Œ	d S rá   )ÚcopyÚdeepcopyr  rC   )Údata_readerrÝ   r%   s     €r&   Úinc_dataloaderz<MatMulNBitsQuantizer.int4_quant_algo.<locals>.inc_dataloaderV  sH   øè è € Ýœ-¨Ô(8Ô(PÑQÔQˆKØ#ð !ð !�Ø˜D�jÐ Ð Ð Ð ð!ð !r(   NrY   zstart to quantize model with z algorithm...)r0   r?   r5   r   Úfp32)rÿ  Úweight_configrF   rG   Ú	blocksizerI   rJ   rK   rˆ   Ú	n_samples)rÿ  r:  Ú
dataloaderz$complete quantization of model with z algorithm.r,   )rY   r2  r  r   rM  rN  r5   r  r   r  rÿ  rG   rH   rI   rJ   rK   r   )r%   r8  rq  Úweight_only_node_configr   rÚ  r=  s   `      r&   Úint4_quant_algoz$MatMulNBitsQuantizer.int4_quant_algoP  sÚ  ø€ ð	!ð 	!ð 	!ð 	!ð 	!ð
 ˆØÔÐ*Ø'+Ô':ˆFÐ#Ñ$Ø"&×"?Ò"?Ñ"AÔ"AÐàÔ$Ô.ˆ	Ý�ŠÐL°IÐLÐLÐLÑMÔMÐMØÐ*Ð*Ð*Ø#Ô/Ô6ˆF�8ÑØ"+ˆF�;Ñðð Ô*ð 4ð 4�Ø-3Ð'¨Ñ*Ð*å%ð Ø)-¬Ð)D�d”o�oÈ$Ì*ÔJZØ5ðð ð ðð ˆDŒJˆJð
 ˜&Ò Ð Ø!%Ô!1Ô!:ˆF�:ÑØ"&Ô"2Ô"=ˆF�;ÑØ!%Ô!1Ô!:ˆF�:ÑØ Ô,Ô0ˆF�5‰MØ#'Ô#3Ô#>ˆF�<Ñ Ø"$ˆF�;ÑØ'˜Ñ)Ô)ˆJå&ð Ø)-¬Ð)D�d”o�oÈ$Ì*ÔJZØ5Ø%ðð ð ð	ð ˆDŒJõ 	�ŠÐQ¸9ÐQÐQÐQÑRÔRÐRÐRÐRr(   c                óÐ  — | j         j        dv �r| j                             ¦   «         g}| j         j        t
          j        k    r| j                             dd¦  «         | j         j        t
          j        k    sd| j         j	        v rl| j         
                    ¦   «         }|D ]P}|j        dv rE|j        dk     r:t                               d¦  «         | j                             |j        d¦  «         ŒQ|                      |¦  «         | j                             ¦   «          d S | j         j        dk    rŸt                               d	¦  «         | j                             | j        €| j        j        n| j        ¦  «        | _        t                               d
¦  «         t+          | j        ¦  «        | _        | j                             ¦   «          d S |                      ¦   «          d S )N)rP   r]   rF  r   r"   )Nzai.onnxrE  é   zžThe opset of the input model is under 21 and doesn't support int4 data type. Force to update it to opset 21, but the generated model may not be a valid model.ri   z%Processing nvidia_awq quantization...r  )r  r   rÿ  rz  r   r   r3   Úset_opset_importr~   r   Úopset_importrI  ÚversionrM  Úwarningr  Úclean_initializersrN  r  r  r  r   r?  )r%   r5  rC  Úopsets       r&   ÚprocesszMatMulNBitsQuantizer.process‚  sÏ  € ØÔÔ%Ð);Ð;Ñ;àœ:×+Ò+Ñ-Ô-Ð.ˆKð ÔÔ,µÔ0EÒEÐEØ”
×+Ò+¨O¸QÑ?Ô?Ð?àÔÔ,µ´Ò?Ð?À8ÈtÔO_ÔOtÐCtÐCtØ#œz×6Ò6Ñ8Ô8�Ø)ð Fð F�EØ”|Ð'<Ð<Ð<ÀÄÐQSÒASÐASÝŸšðpñô ð ð œ
×3Ò3°E´LÀ"ÑEÔEÐEøà×"Ò" ;Ñ/Ô/Ð/ØŒJ×)Ò)Ñ+Ô+Ð+Ð+Ð+ØÔÔ'¨<Ò7Ð7å�KŠKÐ?Ñ@Ô@Ð@ØÔ,×9Ò9Ø$(¤OÐ$;�”
Ô Ð ÀÄñô ˆDŒJõ �KŠKÐ<Ñ=Ô=Ð=Ý" 4¤:Ñ.Ô.ˆDŒJØŒJ×)Ò)Ñ+Ô+Ð+Ð+Ð+ð × Ò Ñ"Ô"Ð"Ð"Ð"r(   )rÿ  r   rR   rV   rH   rV   rW   rX   rY   rZ   r	  r
  r   r   r   r   r[   rX   r  r  )r5  r6  )r)   r*   r+   Ú__doc__r   r3   r'   r  r2  r?  rH  r,   r(   r&   r  r  Ä  s©   € € € € € ðð ð. ØØ"Ø%)ØØ-1Ø Ô*Ø7;Ø9=Ø',Ø48ð0Mð 0Mð 0Mð 0Mð 0Mðd.ð .ð .ð .ð`ð ð ð*0Sð 0Sð 0Sðd #ð  #ð  #ð  #ð  #r(   r  c                ó.   — |                       ¦   «         dv S )N)ÚtrueÚ1)Úlower)r$  s    r&   Úort_convert_str_to_boolrN  ¥  s   € Ø�;Š;‰=Œ=˜MÐ)Ð)r(   c                óT   — |                       d¦  «        \  }}|t          |¦  «        fS )Nú:)r±   rV   )Úsr1  r$  s      r&   Úparse_key_value_pairrR  ª  s%   € Ø—’˜‘”�J€CˆØ•�E‘
”
ˆ?Ðr(   c            
     ó´  — t          j        d¬¦  «        } |                      ddd¬¦  «         |                      ddd¬¦  «         |                      d	d
dt          d¬¦  «         |                      ddt          g d¢d¬¦  «         |                      ddt          d¬¦  «         |                      dd
dddt
          dd
gd¬¦  «         |                      dd
t          d¬¦  «         |                      ddd
d ¬!¦  «         |                      d
¬"¦  «         |                      d#d$t          d
g d%¬&¦  «         |                      d'd$t          d
d(¬)¦  «         |                      d*d+t          d+d,gd-¬¦  «         |                      d.t          d$d/d0gd1¬2¦  «         |                      d3t          d$d
d4¬5¦  «         |                      d6d7¦  «        }|                     d8t          d9d:¬;¦  «         |                     d<t          d
d=¬>¦  «         |                     d?t          d
d@dAgdB¬C¦  «         |                     dDt          dEdF¬;¦  «         |  	                    ¦   «         S )GNa
  Blockwise int4 quantization for MatMul 2D weight matrices.

A weight matrix is partitioned into into blocks, where each block is a
continguous subset inside each column. Each block is quantized into a
set of 4b integers with a scaling factor and an optional offset.
)Údescriptionz--input_modelTzPath to the input model file)ÚrequiredÚhelpz--output_modelzPath to the output model filez--block_sizeFrl   zBlock size for quantization)rU  Údefaultr  rV  z--quant_methodrW  )rW  ÚhqqÚrtnr?   Úgptqri   uW   the algorithm used to quantize weight, 
rtn and gptq leverage IntelÂ® Neural Compressor)rW  r  ÚchoicesrV  z--bitsrN   z#the target bits to represent weight)rW  r  rV  z--symmetricú?zWIndicate whether to quantize the model symmetrically, symmetric is not supported by hqq)rU  rW  ÚconstÚnargsr  r[  rV  z--accuracy_levelzíAccuracy level of the 4-bit quantized MatMul computation. Refer to the MatMulNBits contrib op's 'accuracy_level' attribute for details (https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#commicrosoftmatmulnbits).)rU  r  rV  z-vz	--verboseÚ
store_true)rU  Úaction)r  z--nodes_to_excludeú+zBSpecify the nodes to be excluded from quantization with node names)r^  r  rU  rW  rV  z--nodes_to_includezKSpecify the specific nodes to be included from quantization with node names)r^  r  rU  rV  z--quant_formatr3   r~   z±QuantFormat {QOperator, QDQ}QOperator format quantizes the model with quantized operators directly.QDQ format quantize the model by inserting DeQuantizeLinear before the MatMul.z--op_types_to_quantizer!   r"   zPop_types_to_quantize {MatMul, Gather}. Operators to quantize. Default is MatMul.)r  r^  r[  rV  z--quant_axeszªKey-value pairs in op_type:axis_to_quantize separated by space.Specify the axis to quantize for an op. Default {MatMul:0, Gather:1}Example: --quant_axes MatMul:0 Gather:1)r  r^  rU  rV  ri   z-Arguments specific to nvidia_awq quantizationz--calib_dataset_namera   z/Name of the calibration dataset for nvidia_awq.)r  rW  rV  z--tokenizer_dirzPath of the tokenizer dir.)r  rU  rV  z--calibration_methodÚawqÚawq_clipz<Support two options, awq implementation and weight clipping.)r  rU  r[  rV  z--cache_dirrb   z%Cache directory for calibration data.)
ÚargparseÚArgumentParserÚadd_argumentrV   r   rN  Úset_defaultsrR  Úadd_argument_groupÚ
parse_args)ÚparserÚnv_awq_configs     r&   ri  ri  ¯  s<  € ÝÔ$ððñ ô €Fð ×Ò˜°$Ð=[ÐÑ\Ô\Ð\Ø
×ÒÐ(°4Ð>]ÐÑ^Ô^Ð^Ø
×Ò˜°ÀÍÐSpÐÑqÔqÐqØ
×ÒØØÝØJÐJÐJØgð ñ ô ð ð ×Ò˜¨!µ#Ð<aÐÑbÔbÐbØ
×ÒØØØØØÝ$Ø�u�Øfð ñ 	ô 	ð 	ð ×ÒØØÝðqð	 ñ ô ð ð ×Ò˜˜k°EÀ,ÐÑOÔOÐOØ
×Ò ÐÑ&Ô&Ð&Ø
×ÒØØÝØØØQð ñ ô ð ð ×ÒØØÝØØZð ñ ô ð ð ×ÒØØÝØ˜eÐ$ðYð ñ ô ð ð ×ÒØ ÝØØ˜8Ð$Ø_ð ñ ô ð ð ×ÒØÝ!ØØð2ð ñ ô ð ð ×-Ò-¨lÐ<kÑlÔl€MØ×ÒØÝØØ>ð	 ñ ô ð ð ×ÒØÝØØ)ð	 ñ ô ð ð ×ÒØÝØØ˜
Ð#ØKð ñ ô ð ð ×ÒØÝØØ4ð	 ñ ô ð ð ×ÒÑÔÐr(   Ú__main__r(  zfile z already existsrX  zAsymmetric is not supportted by hqq, will force to symmetric=FalseF)rH   rR   r   r   rW  )rH   rW   rY   r   r   r   rR   rY  )r   r?   rZ  )rH   r   ri   zFQOperator is not applicable to nvidia_awq. overriding the value to QDQrb  rc   rc  )rm   r‚   ro   r�   z!Unsupported quantization method: )rÿ  rR   rY   r  r	  r  T)rv  r6  r7  rw  )^Ú
__future__r   rd  r5  ÚloggingÚosr§  r¾   râ   Únumpy.typingÚtypingÚnptrQ  Úonnx_irr¥  Úonnx.onnx_pbr   r   r   r   Úonnxruntime.capi._pybind_stater   r	   r
   r   Ú	calibrater   Úneural_compressorr   r   Ú
onnx_modelr   Úquant_utilsr   r   ÚbasicConfigÚINFOÚ	getLoggerr)   rM  r   r.   r=   rA   rM   rU   r`   ræ   rè   rP  r‰  r‹  rý  r  rN  rR  ri  Úargsr  ÚsetLevelÚDEBUGÚinput_modelÚinput_model_pathÚoutput_modelÚoutput_model_pathr   r   Útupler   ÚpathÚexistsrú  Ú	ExceptionÚ	symmetricÚquant_methodrE  r  rÿ  rH   rR   Úquant_configrY   r3   r~   r�   Úcalib_dataset_namer‚   ro   rÇ   r  r	  ÚquantrH  Úsave_model_to_filer,   r(   r&   ú<module>rŽ     s  ðð #Ð "Ð "Ð "Ð "Ð "à €€€Ø €€€Ø €€€Ø 	€	€	€	à Ð Ð Ð Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ Gðð ð ð ð ð ð ð ð ð ð ð ð -Ð ,Ð ,Ð ,Ð ,Ð ,Ø :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ø !Ð !Ð !Ð !Ð !Ð !Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8à €Ô ÐOÐW^ÔWcÐ dÑ dÔ dÐ dØ	ˆÔ	˜8Ñ	$Ô	$€ðAð Að Að Að Añ Aô Að Að@#ð #ð #ð #ð #Ð4ñ #ô #ð #ðLð ð ð ð Ð"7ñ ô ð ðD0%ð 0%ð 0%ð 0%ð 0%Ð 5ñ 0%ô 0%ð 0%ðf(ð (ð (ð (ð (Ð4ñ (ô (ð (ðV/kð /kð /kð /kð /kÐ#8ñ /kô /kð /kðds#ð s#ð s#ð s#ð s#Ð!6ñ s#ô s#ð s#ðl4ð 4ð 4ðjð jð jð jð jñ jô jð jðZð ð ð ð#ð #ð #ð&uð uð uð uð uñ uô uð uðp(ð (ð (ð (ð (ñ (ô (ð (ðV^#ð ^#ð ^#ð ^#ð ^#ñ ^#ô ^#ð ^#ðB*ð *ð *ð
ð ð ð
lð lð lð^ ˆzÒÑØˆ:‰<Œ<€DØ„|ð 'Ø�Š˜œÑ&Ô&Ð&àÔ'ÐØÔ)ÐØ˜tÔ0Ô1€LØ?CÔ?XÐi˜5˜5 Ô!:Ñ;Ô;Ð;Ð^iÐØ+/¬?ÐD���t”Ñ'Ô'Ð'À€Jà	„w‡~‚~Ð'Ñ(Ô(ð DØ�ŠÐ?Ð.Ð?Ð?Ð?Ñ@Ô@Ð@ØˆiÐBÐ 1ÐBÐBÐBÑCÔCÐCà„~ð ˜$Ô+¨uÒ4Ð4Ø�ŠÐZÑ[Ô[Ð[ØˆŒàˆDŒIÐ&Ñ'Ô'€EØÔ˜EÒ!Ð!Ø/Ð/Ø”¨T¬YÐMaÐnxð
ñ 
ô 
ˆ‰ð 
Ô	˜iÒ	'Ð	'Ø3Ð3Ø”ØœØÔ.Ø%Ø!5Ø!Ø”ð
ñ 
ô 
ˆˆð 
Ô	˜eÒ	#Ð	#Ø/Ð/ÐEYÐZÑZÔZˆˆØ	Ô	˜iÒ	'Ð	'Ø2Ð2ÐH\Ð]Ñ]Ô]ˆˆØ	Ô	˜fÒ	$Ð	$Ø0Ð0¸D¼OÐbvÐwÑwÔwˆˆØ	Ô	˜lÒ	*Ð	*Ø˜;Ô0Ò0Ð0Ø�NŠNÐcÑdÔdÐdØ&œ?ˆLà ˆØÔ"Ð.ØÔ&¨%Ò/Ð/Ø%/Ð"Ð"à%/Ð"Ð"à!+Ðà1Ð1ØÔ0ØÔ,Ø”nØ1ð	
ñ 
ô 
ˆˆð ˆjÐP¸TÔ=NÐPÐPÑQÔQÐQà Ð ØØŒYØÔ*ØÔ.ØÔ.Ø ðñ ô €Eð 
‡M‚M�O„O€OØ	„K×"Ò"Ð#4°dÑ;Ô;Ð;Ð;Ð;ðQ Ðr(   