jddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddl m!Z!Gd d eZ"dS) N)onnx_pb) BaseQuantizerQuantizationParams) TensorData) ONNXModel)TENSOR_NAME_QUANT_SUFFIXQuantizationModeQuantizedValueQuantizedValueType __producer__ __version__add_infer_metadataattribute_to_kwargcompute_scale_zpcompute_scale_zp_float8 find_by_nameget_qmin_qmax_for_qTypeget_qrange_for_qType ms_domainquantize_onnx_initializer&save_and_reload_model_with_shape_infertensor_proto_to_array)CreateOpQuantizerc^eZdZ d*dZdZdZdZdZdZdZ d Z d+d Z d Z d Z dZdZd,dZ d-dZdZ d*dZdejdejdedejdedeeejdzff dZdedejddfdZd.dZdZd+d Z d/d"Z d0d$Z!d1d%Z" d2d&Z#d'Z$d(Z%d)Z&dS)3 ONNXQuantizerNc tj|||||||| | | | |s|jt |jj}d|jjD|_|jd|jj D|jd|jj Dt||_||_ ||_ |jdk|_d|jvo |jd|_g|_d|_i|_|jd|jj D|jd|jj D|jjjjD]:} |jt,| j d ;|j t0vrt3d |j ||_d |_d |_d |_d|_i|_ |j!|_"dS)Nci|] }|j| Sname).0vis XC:\Projects\Historical\venv\Lib\site-packages\onnxruntime/quantization/onnx_quantizer.py z*ONNXQuantizer.__init__..HsMMMMMMci|] }|j| Srr r"ots r$r%z*ONNXQuantizer.__init__..Is$N$N$NRRWb$N$N$Nr&ci|] }|j| Srr r"its r$r%z*ONNXQuantizer.__init__..Js$M$M$MRRWb$M$M$Mr& MatMulConstBOnly/ci|] }|jd Srr r(s r$r%z*ONNXQuantizer.__init__..Vs!J!J!J"'1!J!J!Jr&ci|] }|jd Sr1r r+s r$r%z*ONNXQuantizer.__init__..Ws!I!I!I"'1!I!I!Ir&rzunsupported quantization mode fixed_quantization_range_uint8fixed_quantization_range_int8 fixed_zero fixed_zero_zp)#r__init__modelreplace_gemm_with_matmulrgraph value_info value_infosupdateoutputinputrmodestatic opset_versionfuse_dynamic_quant extra_optionsq_matmul_const_b_only new_nodes graph_scope tensor_namesnodedictfromkeysr ValueErrorcalculate_quantization_paramsquantization_paramsfixed_qrange_uint8_namefixed_qrange_int8_namefixed_zero_namefixed_zero_zp_namequantized_value_mapget_non_initializer_inputsgenerated_value_names)selfr8 per_channel reduce_ranger@rA weight_qTypeactivation_qType tensors_rangenodes_to_quantizenodes_to_excludeop_types_to_quantizerDrIs r$r7zONNXQuantizer.__init__'s`             * J / / 1 1 1:4:;KLLEMMek6LMMMD    # #$N$N5;;M$N$N$N O O O   # #$M$M5;;L$M$M$M N N N"5))DJ  "&"4r"9%74;M%M%xRVRdewRx"   !J!Ju{7I!J!J!JKKK   !I!Iu{7H!I!I!IJJJJ$*/ D DD   $ $T]]4;%B%B C C C C 9, , ,IdiIIJJ J#'#E#E#G#G (H$&E#+"1$& &*Z%J%J%L%L"""r&ctj|d|jjj}t |t ||j|j|j |j |j |j |j |j|j|j|j }||_|j|d|_||jjjS)z generate submodel for the subgraph, so that we re-utilize current quantization implementation. quantize the submodel update subgraph and set it back to node onnx-quantizer) producer_name opset_importsr/)onnxhelper make_modelr8 opset_importrrrWrXr@rArYrZr[r\r]r^rDparentrGquantize_modelr:)rVsubgraph graph_key warped_model sub_quantizers r$quantize_subgraphzONNXQuantizer.quantize_subgraphps {-- ***7.  <(((%      I K    !    "  !  %      $ '+'7$E$E$E$E !$$&&&"(..r&cd|jD}t|dkr|S|jr|jn|jdt|j}i}|jD]}|jt jjkr-|j| |j |d|ji}n|jt jj krZg}|j D]F}| | ||d|jdt|gG|j|i}nt|}||t jj|j|j|jfd|ji|S)z| Check subgraph, if any, quantize it and replace it. return new_nodes added for quantizing subgraph czg|]8}|jtjjks|jtjjk6|9Sr)typercAttributeProtoGRAPHGRAPHS)r"attrs r$ z>ONNXQuantizer.quantize_node_with_sub_graph..sG   yD/555dFYF`9`9` 9`9`9`r&r _node_count_:r!) attributelenr!op_typerFrprcrqrrrmgrsgraphsextendrr=rd make_noder?r>) rVrI graph_attrs node_namekwargsrtkvvalueris r$quantize_node_with_sub_graphz*ONNXQuantizer.quantize_node_with_sub_graphs      {  q K!%bDII4<0b0bSQUQ_M`M`0b0b N  DyD/555i!7!79@Z@Zty@Z@Z![![\d1888 $ HLL 22 (#, G Gty G G3u:: G Gi''-- MM"    {$T\4:t{eeQUQZe^deeer&cbtd|jDS)zQ Detect if model already has QuantizeLinear or DequantizeLinear. c3BK|]}|jdkp |jdkVdS)QuantizeLinearDequantizeLinearN)rzr"rIs r$ z.ONNXQuantizer.has_QDQ_nodes..sH  W[DL, , R @R0R      r&)anyr8nodes)rVs r$ has_QDQ_nodeszONNXQuantizer.has_QDQ_nodessA  _c_i_o_o_q_q      r&ct||jdS|j|j|SdS)NTF)rr8 initializerrgfind_initializer_in_path)rVinitializer_names r$rz&ONNXQuantizer.find_initializer_in_pathsJ ($**@*@*B*B C C O4 ; ";778HII Iur&c|j||D]&}|jD]}|j|'dSN)rFr}r>rUadd)rVrrI output_names r$ add_new_nodeszONNXQuantizer.add_new_nodessa e$$$ < .s%bbbe XaHaHaEHaHaHar&c(g|]}|jdk |S)z com.microsoft)rrs r$ruz0ONNXQuantizer.quantize_model..s$ZZZ4;/;Y;Y;Y;Y;Yr&r) rloggingwarningr8renable_subgraph_quantizationrryrFrquantizeranger>rUr_dequantize_outputsr: ClearFieldrIr}rgclean_initializers RuntimeErrorstrr rarproducer_versionrfversionrr) rVrInumber_of_existing_new_nodes op_quantizerir_initializers_not_foundms_opsetms_nodesrs r$rhzONNXQuantizer.quantize_modelsG       On    J$$&& @ @D0 ?88>>+.t~+>+> (,T488L  ! ! # # #7T^9L9LMM @ @#'>!#4#;@@K.22;????@ @   """ %%f--- &&t~666 ; (, (E(E(G(G %A%)**Q.."#UX[\rXsXs#sttt)5 &,7 )bbtz'7'Dbbb )ZZZZZH ) (599;; ! ( zr&cd|jvr.tjd||jd|jdStd|d)NDefaultTensorTypezDget_tensor_type returns DefaultTensorType for tensor name %r, use %dz)Unable to find data type for weight_name=a7. shape_inference failed to return a type probably this node is from a different domain or using an input produced by such an operator. This may happen if you quantize a model already quantized. You may use extra_options `DefaultTensorType` to indicate the default weight type, usually `onnx.TensorProto.FLOAT`.)rDrinforrV tensor_names r$_get_default_tensor_typez&ONNXQuantizer._get_default_tensor_typess $"4 4 4 LV"#67    %&9: : J  J J J   r&FcPt||j}||jS||jvrd|j|}|jdr=|r*|jjjdkr| |S|jjjS|j r|j |r| |SdS|j |}||S|j r%|j r|j |}||S|r| |SdS)N tensor_typer)rr8r data_typer<rpHasFieldr elem_typerrrgis_valid_quantize_weightget_tensor_type)rVr mandatoryweightr#otyperess r$rzONNXQuantizer.get_tensor_typesFk4:+A+A+C+CDD  # # $* * *!+.Bw .. 5F!4!>!!C!C88EEEw*441 t{7J B44[AAA4 44[AA  L  ,  +--k::C  >00== =tr&c||r||S||jvr~|j|}|jdr5|jjjtjj tjj fvrdStj d|d|jddS|j r!|jr|j|Stj d|ddS) NrTz00== = $* * *!+.Bw .. 273F3P&,&.U44t Or{rrhjhorrr   5  , < <;..{;; ; 7K 7 7 7   ur&c|tjjkr||||S|tjjkr||||St d|d)a Create nodes for dynamic quantization of input and add them to nodes_list. parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. parameter qType: type to quantize to. parameter initial_type: type to quantize from return: scale_name, zero_point_name, scale_shape, zero_point_shape. zUnexpected value for qType=r)rrINT8+_get_dynamic_input_quantization_params_int8UINT8,_get_dynamic_input_quantization_params_uint8rL)rV input_name nodes_listqType initial_types r$&_get_dynamic_input_quantization_paramsz4ONNXQuantizer._get_dynamic_input_quantization_params6st J*/ / /CCJPZ\hii i J*0 0 0DDZQ[]ijj j?u???@@@r&ctjj}|dz}|dz}tjd|g|dzg|d}|||dz}tjd|g|dzg|d} || |d z} tjd |jdg| dzg| } || |d z} tjd | jdg| dzg| } || |d z}tjd | jd| jdg|dzg|}||tj|j |gt|d z g}|j ||dz}tjd|jd|j g|g|}||tj|j |gdg}|j |||j ggfS)az Create nodes for dynamic quantization of input to int8 and add them to nodes_list parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. parameter initial_type: initial weight type (FLOAT or FLOAT16) return: scale_name, zero_point_name, scale_shape, zero_point_shape. _scale _ReduceMin ReduceMin:0rkeepdims _ReduceMax ReduceMax_AbsAbs_Abs_MaxMax@ scale_DivDiv)rrrrcrdr~appendr> make_tensorrPrr8add_initializerrR)rVrrrrinput_scale_namereduce_min_namereduce_min_nodereduce_max_namereduce_max_nodereduce_min_abs_namereduce_min_abs_nodereduce_max_abs_namereduce_max_abs_node abs_max_name abs_max_nodeinitializer_divscale_div_namescale_div_nodeinitializer_zps r$rz9ONNXQuantizer._get_dynamic_input_quantization_params_int8Es&+&0$|3+//  L t # $  0   /***$|3+//  L t # $  0   /***.6"k33   #A & ' 4 ' (     -...-6"k33   #A & ' 4 ' (     -...!J. {,,  ' *,?,Fq,I J D !    ,'''+11  '   !% ( (3 . /    ""?333#k1..    #T%@ A       .)))001H%QSVWUXYY "">222!8"b@@r&ctjj}|dz}|dz}|dz}tjd|g|dzg|d}|||dz} tjd |g| dzg| d} || tj|j|gt|g} |j | tj|j |gd g} |j | |d z} tjd | j d|j dg| dzg| }|||d z}tjd|j d|jg|g|}|||dz}tjd |j |j dg|dzg|}|||dz}tjd|j d|g|dzg|}|||dz}tjd|j |dzg|}|||dz}tjd|j |g||}||||ggfS)a{ Create nodes for dynamic quantization of input to uint8 and add them to nodes_list parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. parameter initial_type: initial weight type (FLAOT or FLOAT16) return: scale_name, zero_point_name, scale_shape, zero_point_shape. r _zero_pointrrrrrrr _scale_SubSub _scale_Divr_zero_point_Sub_zero_point_Div_zero_point_FloorFloor_zero_point_CastCast)to)rrrrcrdr~rrrOrr8rrQr>)rVrrrrr input_zp_namerrrrinitializer_qrangeinitializer_qvaluescale_sub_namescale_sub_noderr zp_sub_name zp_sub_node zp_div_name zp_div_node zp_floor_name zp_floor_node zp_cast_name zp_cast_nodes r$rz:ONNXQuantizer._get_dynamic_input_quantization_params_uint8sb&,%0"]2 $|3+//  L t # $  0   /***$|3+//  L t # $  0   /***"[44  (   !% ( ( )    ""#5666![44T5Iq(A B d " #     .)))#l2..   "1 %t'C D       .)))!#44 k++   !?#9!#< = 4     +&&& #44 k++    "$4 5 4     +&&&"%88  --g{7IM\`L`Kacpqq -(((!$66 {,,V]5IM?\hmr,ss ,'''B66r&c"|j}||R|j ||jvrtjd|ddS|j|}t |t s#t dt|d|d|t|dkrtd |d |tj |d g}t|d d r%|d j tjtjfvr(tdt|d d|tj |d g}|j tjksJ|d}nttj |g}tj |g}|j|}d |vr"|d j }||}|j tjksJg} |dz} g} |dz} t$j| || |} |j| |j tjkrt2jj}nA|j tjkrt2jj}ntd|j d|t$j| || |d}|j|d| | | | fS)a\ Create initializers and inputs in the graph for zero point and scale of output. Zero point and scale values are obtained from self.quantization_params if specified. parameter param_name: Name of the quantization parameter. return: result, scale_name, zero_point_name, scale_shape, zero_point_shape. Nz$Quantization parameters for tensor:"z" not specified)FrrrUnexpected type  for rzbQuantization parameters should contain zero point, scale, quant type. Specified values for output z: zero_pointscaledtypez and param_name= quant_typerrzUnexpected dtype=z for param_name=)T)rZrNrr isinstancer TypeErrorrpryrLnparrayhasattrrfloat32float16float64astypercrdrraveltolistr8rrrrrreshape)rV param_name use_scale use_zeropointzero_point_typeparamszero_point_values scale_valuesrzero_point_shapezero_point_name scale_shape scale_nameinit_zp scale_type init_scales r$_get_quantization_paramsz&ONNXQuantizer._get_quantization_paramss5/   5'/:TE]3]3] _J___```,,-j9Ff&899 W U4<< U Uj U U UVVV~V!1!1 J3=JJAGJJ !#&*>)? @ @ 6'?G44 kw8MVXV`bdblUm8m8m !iD4I4I!i!i[e!i!ijjj8VG_$566L%3333$\2OO "- 9 9 8YK00L-j9F&  w-+22599 %3333$}4 (* +)) _.>@Q@W@W@Y@Y@`@`@b@b   ""7+++   + +#/5JJ  2: - -#/7JJc1CccU_ccdd d[,,Z[R^RfRfglRmRmRtRtRvRvww  "":...Z+?OOOr&c |j|}|dks Jd|tz}|dz} | |d||} } } n||\} } } } } g}| r'tjd|| | g|g| }n|jrdS|jrF|tj j kr1|dz} |dz} tjd |g|| | g| }n\|Jd |d |d |d || ||||\} } }}tjd|| | g|g| }t||| | ||j |<g||S)a Given an input for a node (which is not a initializer), this function - add nodes to compute zero point and scale for this input if they don't exist. - add new QuantizeLinear node to quantize the input. :param node: node being quantized in NodeProto format. :param input_index: index of input in node.input. :param qType: type to quantize to. :param given_scale_name: if those inputs need to be quanitzed using this scale tensor. :param given_zp_name: if those inputs to be quantized using this zeropoint tensor. :param initial_type: type of the weight to quantize :return: List of newly created nodes in NodeProto format. rz*Cannot access undefined variable in graph._QuantizeLinearNTrrrDynamicQuantizeLinearzCCannot quantize input without knowing the initial type, input_name=z, input_index=z, qType=z, node=r)r?r r2rcrdr~rArCrrrrr rS)rVrI input_indexrgiven_scale_name given_zp_namerrr ql_node_name data_foundr.zp_namerr qlinear_noder-zp_shapes r$_get_quantize_input_nodesz'ONNXQuantizer._get_quantize_input_nodes2s "Z , R!M #;; !$55  (}/H/35E}G JJ484Q4QR\4]4] 1J GQ % ;00 Z1  LL{ t& 5J4J4P+P+P'(2 $}4#{44+L *g6    $//h",hh>IhhSXhhaehh0//?? ESXgs?tt #{44$W5 M    0>j+Wacjlq/r/r ,%% %%r&ct||jvr |j|S|j|j|SdSr)rSrgfind_quantized_value)rVrs r$rAz"ONNXQuantizer.find_quantized_valuexsA 1 1 1+J7 7 ; ";33J?? ?tr&c &tj|} |d| zz|z } tj|tj} tj|tj} | | z}|| kr|dkrz| |z }| |z}| 6t jd|d|d|dd tj||fSt jd | d |d |d|d d ||fSd |fS)zHAdjust a single weight scale to ensure the int32 bias does not overflow.rrrNzIncreasing scale for weight `z` by the ratio z to ensure bias `z` has a valid scale.TzIncreased scale[z] for weight `z ` by ratio F)rabsritemrrrr )rVbias_val input_scale weight_scaleweight_scale_dtype weight_name bias_nameqrangemultiplicative_epsilonidxabsmaxbias_smallest_valid_scaleinput_scale_fp64weight_scale_fp64bias_candidate_scaleratio new_scales r$$adjust_single_weight_scale_if_neededz2ONNXQuantizer.adjust_single_weight_scale_if_neededs!!$:cFl$Kf$T!8K$4$4$6$6bjIIIH\%6%6%8%8 KKK/2CC #< < | |fS) zOChecks if the bias scale is too small and increases the weight scale if needed.)FNgqh?rCrFrT)rN)sizerriinfoint32rmaxrminrminimummaximumrDrVr!shaperyr)rVrGrHrJrWrXbias_float_data int32_inforMrLrIupdatedrminrmaxrOchangedrUrs r$#_adjust_weight_scale_for_int32_biasz1ONNXQuantizer._adjust_weight_scale_for_int32_biass  ;/88Xbh'' !'*. ;;;bhz~XYGYacak>l>l>ll)/ #:o1133RXarz5R5R5RSSD:o1133RXarz5R5R5RSSDZt bfTll;;F!%!J!J" & " " GY (    #C (:$;$;q$@$@<-a011 # #%)%N%N#A& O&L*&O & &"#&/LO"G $$r&rUc||jvrdS|j|}t||j}t|j|j}t|j|j}t|j|j}||||dS|j||j|tj |}|j } tj |tj|j} tj | |j|j} |j| t+||j|| | |j} |j| dS)zCRe-quantizes the given weight initializer using the provided scale.NrC)quant_weight_name)rSrr8rr.r<q_nameremove_initializerrc numpy_helperto_arrayaxisrasarrayrdtensor_dtype_to_np_dtyper from_arrayr#dimsrrrY) rVrJrUqv weight_tp scale_initzp_init q_weight_initweight_zero_pointrpscale_npnew_scale_init new_q_weights r$_requantize_weightz ONNXQuantizer._requantize_weights d6 6 6 F  %k 2 dj.D.D.F.FGG !"-1G1G1I1IJJ rz4:+A+A+C+CDD$RY 0F0F0H0HII   2goI^ F %%j111 %%m444 -66w??w:it{/S/ST]Tg/h/hiii*55h6F6Fz6W6WY[Yfgg "">2221       i     ""<00000r&?c ||jvr|j|jS|j|j}t||j}t |}||jvr|j|j}n8||jvr||\} }} } } ntd|dt||j} t | } |j|j } t| |j} | tj | nd}|j}||jr|sv|jt$jjfvr]t||j}|| ||||\}}|r||||}||| ||\}}}}}}||jvsJt1||||t2j|jdkrdnd||}||j|<|S)z] Quantized the bias. Zero Point == 0 and Scale == Input_Scale * Weight_Scale z Expected z5 to be in quantized value map for static quantizationNrr) node_type node_qtype)rSrlr.rr8rrrNr2rLr<rcrnrorWr[rrYrrrrir~quantize_bias_static_implr r Initializer)rVrKrrJbetaweight_scale_nameweight_initializerrHrrinputscale_initializerrGweight_zp_nameweight_zp_initrzrXbias_initializer did_updatenew_weight_scalequantized_bias_namequantized_bias_scale_namequantized_bias_zp_namebias_scale_datarrquantized_values r$quantize_bias_staticz"ONNXQuantizer.quantize_bias_statics 0 0 0+I6= =!4[AL)*;TZ=S=S=U=UVV,-?@@  1 1 1#7 CN   43 3 3+/+H+H+T+T (AAqqjjjjkk k!-.> @V@V@X@X!Y!Y+,BCC 1+>F%ndj6L6L6N6NOOJXJdD-66~FFFjn)  )!& *%))++ *!j&<&A%CCC+Itz7M7M7O7OPP +/+S+S  ,, (J( 0'' 5EFFF/   * *9k +""r&c8||jvp||jvp||jvS)zq only check for value info and newly generated tensor names, initializers are checked separately )r<rHrUrs r$contains_tensorzONNXQuantizer.contains_tensorJs4 D, , ;t00 ;t99 r&c :|||dddd|S)NFrrIindicesinitializer_use_weight_qTyperXop_level_per_channelrp from_subgraph_ONNXQuantizer__quantize_inputs)rVrIrrs r$quantize_activationz!ONNXQuantizer.quantize_activationTs4%%).!&'&   r&rc :|||d||||S)NTrr)rVrIrrXrrprs r$quantize_weightzONNXQuantizer.quantize_weightas6%%)-%!5'&   r&Tc g}g} g} g} |D]4} |j| } | |jvr\|j| }||j| |j| |ju| s@| d|d| dt | |j}||j r2|r0| |j |r|j n|j ||\}}}n)|||r|j n|j |\}}}| || ||||| r|j| dz|j|j}||j| }||jvrj|j|}|dsJd|d|jdsJd|d|jjj}n$||jvsJd |d |j|}||| |j | }|d S|r||n| ||d }|jdkr\| |j||jd| |jd9| |jd||jd| |jd|jz|j|| g||||d\}}}}| |d||d| |dtAd| d|j!| | || fS)a Given a node, this function quantizes the inputs as follows: - If input is an initializer, quantize the initializer data, replace old initializer with new initializer - Else, add QuantizeLinear nodes to perform quantization parameter node: node being quantized in NodeProto format. parameter indices: input indices to quantize. return: (List of quantized input names, List of zero point names used for input quantization, List of scale names used for input quantization, List of new QuantizeLinear nodes created) rNr4rpz value_info=z has no type.rz is not a tensor.zshape inference failed for zF and attribute 'tensor_names' does not have any value for this tensor.r6)NNNNrrrrT)rrXrrprz!Invalid tensor name to quantize: z @graph scope)"r?rSrr.r<rlrr8rrWquantize_weight_per_channelr!rYrZquantize_initializerrfind_node_by_namerFr:r<rrprrrHr?rr}rzr>rgrrLrG)rVrIrrrXrrpr scale_nameszero_point_namesquantized_input_namesrr7 node_inputrr q_weight_namer<r.r=rr;rquantize_input_nodesparent_quantized_input_namesparent_zero_point_namesparent_scale_namesrs r$__quantize_inputszONNXQuantizer.__quantize_inputsts.  ""c rc rKK0JT555"&"::"F""?#=>>> ''(?@@@%,,_-CDDD %,,R000""2&&& ''+++&z4:3I3I3K3KLLK&#(< 88#(-Id))tOd$  %" :>9R9R#-Id))tOd$::6M7J &,,];;; ''000"":....%%j11; r#z;;!22DNDJDTDTDVDV    '!%K!8J!T%555%)%5j%A )226::cc>>>,*,,, ?>> (,'8'D +/+I+Ik4+@|,J,,(,3777$;**+?@@@@ %9:::#7#;L'+;;;)001DEEE&&|'9!'<===$++L,>q,ABBBB)001DQ1GHHH&&|':1'=>>>$++L,?,BCCCC( K11 M1M!-)="&2 0+&&,,-I!-LMMM""#5a#8999 ''(?(BCCCC!!pZ!p!p^b^n!p!pqqq$&6 UJJr&c|j|jvr&|j|j}|j|j|jfS|||||\}}}t |j|||tjd}||j|j<|||fS)a :param weight: TensorProto initializer :param qType: type to quantize to :param keep_float_weight: Whether to quantize the weight. In some cases, we only want to qunatize scale and zero point. If keep_float_weight is False, quantize the weight, or don't quantize the weight. :return: quantized weight name, zero point name, scale name N) r!rSrlr<r.quantize_initializer_implr r r) rVrrrXkeep_float_weightrrr<r.s r$rz"ONNXQuantizer.quantize_initializers ;$2 2 2"6v{CO&'*  .2-K-K E<):. . * w ) K     *    1@ -gz11r&c||jvr!|j|}|j|j|jfS||||||\}}} t ||| |t jd}||j|<||| fSr)rSrlr<r. quantize_weight_per_channel_implr r r) rVrJrY channel_axisrXrrrr<r.s r$rz)ONNXQuantizer.quantize_weight_per_channels $2 2 2"6{CO&'*  .2-R-R |\CT. . * w )      *    1@ -gz11r&cf||jvr&||jvr|j|}t|j|j}|jjjdks|jjjdkr.|,|*tj |j dksJ|dz}|j ||j |j }|9|j|j|jg}tjd||g|}|S||jdksJdS)a Given a value (input/output) which is quantized, add a DequantizeLinear node to dequantize it back to float32 or float16 parameter value_name: value to dequantize parameter new_nodes_list: List of new nodes created before processing current node return: None if there is already a DequantizeLinear node that dequantizes it A DequantizeLinear node otherwise r`Nr_DequantizeLinearrr)rSrUrr.r8rrarcrnror[rrFr:rlr<rdr~r>)rV value_namerrw dqlinear_name dqlinear_nodedqlinear_inputsdequantize_nodes r$_dequantize_valuezONNXQuantizer._dequantize_value8sK $2 2 24Ke9e9e"6zBO&o&@$*BXBXBZBZ[[Jz-1AAA  .2BBBzG]")T->-G-G -S-S-X\]-]-]-]]&)<rr!rFr)rVr>rs r$rz!ONNXQuantizer._dequantize_outputs_s`j&&((/ 7 7F"44V[AAO*%%o666 7 7r&c Z|jdS|i}|jD]}|j|}t|ts#t dt |d|d|j|i}|j}d|vr |dj }d|vrd|vr|d|d}}n|tj j krt||jd\}}n|d |jd }|d |jd} |d |j} |d d} t%|| | \} } t'|| | | | |j\}}t+|||||<|S)Nrrr) default_valrrrrrfrrg symmetricrXF)rXr)rrr)r[adjust_tensor_rangesrrrrptensor_quant_overridesget_per_tensor_overridesrZrrcr FLOAT8E4M3FNravg_stdget range_valueis_activation_symmetricrrmin_real_ranger)rVrNrtdquant_overridesrzerorrfrgrrXqminqmaxs r$rMz+ONNXQuantizer.calculate_quantization_paramsks   %4 !!### - w wK#K0Bb*-- T R488 R R+ R R RSSS"9RRS^lnRooO.J..,\:F /))lo.M.M-l;_W=Uet/<<<5j"*Q-PP ee&**62>!3DEE&**62>!3DEE+// T=YZZ .22>5II 4Zlfoppp d.tT4yRVReff e/ATY^ku/v/v/v  , ,""r&r)F)NN)NNN)r)FFrF)TFFrF)FF)TF)'__name__ __module__ __qualname__r7rmrrrrrhrrrrrrr2r?rArVrndarrayrrcrbooltuplerir~rrrrrrrrrrMrr&r$rr&sFMFMFMFMR///> f f fD   <<< + + + Z   "22 A A ARARARAh\7\7\7|9P9P9P9PxaeD&D&D&D&L" ########J6%Z6%j6% 6% ! 6%  6% tRZ$&& '6%6%6%6%p$1c$1bj$1T$1$1$1$1LF#F#F#F#P        ""     .&*" AKAKAKAKF2222L 2222@%%%N 7 7 7 # # # # #r&r)#rnumpyrrconnx.numpy_helperrrbase_quantizerrr calibrater onnx_modelr quant_utilsr r r r r rrrrrrrrrrrrregistryrrrr&r$rs  &&&&&&========!!!!!!!!!!!!&('''''e#e#e#e#e#Me#e#e#e#e#r&