jF ddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZmZddlZddlmZddlmZddlmcmZddlmcmZddlmZmZddl m!Z!ddl"m#Z#m$Z$gdZ%d e&d ee&e&ffd Z'd e ed ej(de e&e ffdZ)d ej(de e&e fdejj*fdZ+d+dejj*d ejj*fdZ,dej*d ej*fdZ-dej*de ej(de ej(de ej(fdZ.ej/ej0ej1ej2ej3ej4ej5ej6ej7ej8ej6ej9ej:g Z;ej<ej=gZ>ej/ej?ej0ej@ej1diZAde ej(de e&ej*ffdZBde ej(de e&ej*fde ej*ej*ffdZCGdd ZDd,d#ZEd$eDd eFfd%ZGGd&d'ZHdejIfdejj*d(ee e&e fd)eejId ejj*fd*ZJdS)-N) defaultdict)Enum)AnycastDictIterableListOptionalTupleType)ArgumentTarget) ShapeProp)fuse_conv_bn_evalfuse_linear_bn_eval) matches_module_patternreplace_node_modulefuseremove_dropoutextract_subgraphmodules_to_mkldnn reset_modules MklSubgraphgen_mkl_autotuneruse_mkl_length UnionFindoptimize_for_inferencetargetreturncP|dd^}}|r|dnd|fS)zp Splits a qualname into parent path and last atom. For example, `foo.bar.baz` -> (`foo.bar`, `baz`) .r)rsplit)rparentnames SC:\Projects\Historical\venv\Lib\site-packages\torch/fx/experimental/optimization.py _parent_namer($s3 MM#q))MVT &6!99B ,,patternnodemodulescjt|jdkrdS|jd|f}t||D]x\}}t|tjsdS|jdkrdSt|jtsdS|j|vrdSt||j|urdSydS)NrF call_moduleT) lenargszip isinstancefxNodeoprstrtype)r*r+r,nodes expected_type current_nodes r'rr.s 49~~u"&)A,!5E'*7E':':  # |,00 55 ?m + +55,-s33 55  g - -55  +, - -] B B55 C 4r) new_modulect|jtsJt|j\}}|||j<t ||||dSN)r2rr6r(setattr)r+r,r; parent_namer&s r'rrBsY dk3 ' ''' '$T[11K%GDK GK $ 33333r)Fmodelctjtjftjtjftjtjftjtjfg}|stj |}|rt|tj j stj|}n|}t|}tj |j}|D]}|jD]}t'|||rt)|jdjdkr8||jdj} ||j} | jse|dtjtjtjfvrt3| | } nt5| | } t7|jd|| ||jd||tj ||S)z Fuses convolution/BN and linear/BN layers for inference purposes. Will deepcopy your model by default, but can modify the model inplace as well. rr")nnConv1d BatchNorm1dConv2d BatchNorm2dConv3d BatchNorm3dLinearcopydeepcopyr2torchr3 GraphModulesymbolic_tracedict named_modulesgraphr8rr/r0usersrtrack_running_statsrrrreplace_all_uses_with erase_node) r@inplaceno_tracepatternsfx_modelr, new_graphr*r+ first_layerbn fused_layers r'rrKs BN# BN# BN# BN# H % e$$ :eUX-ABB$U++8))++,,G hn--I++O + +D%gtW== +ty|)**Q..%dil&9: T[)-1:")RY !BBB"3K"D"DKK"5k2"F"FK#DIaL';GGG**49Q<888$$T*** + >(I . ..r)ctj|}Gddtjj}||S)z5 Removes all dropout layers from the module. cPeZdZdedeedfdeeefdeffd Z xZ S)&remove_dropout..DropoutRemoverrr0.kwargsrct|j|tjrt |dksJ|dSt |||S)Nr"r)r2 submodulesrBDropoutr/superr.)selfrr0ra __class__s r'r.z2remove_dropout..DropoutRemover.call_modulezsY$/&12:>> A4yyA~~~~Awww**64@@@r)) __name__ __module__ __qualname__rr r rr6rr. __classcell__)rgs@r'DropoutRemoverr`ys} A  A(-hm(< AFJ3PS8n A  A A A A A A A A A Ar)rl)r3rNrL Transformer transform)r@rYrls r'rrssi ''HAAAAA-AAA >( # # - - / //r) orig_moduler8inputsoutputscPtj}i|D]!}||j}||<"|D] }||fd}||<!|fd|D|tj||S)z Given lists of nodes from an existing graph that represent a subgraph, returns a submodule that executes that subgraph. c|Sr=)xenvs r'z"extract_subgraph..s s1vr)c g|] }| Srtrt).0outputrvs r' z$extract_subgraph..s888fc&k888r))r3Graph placeholderr& node_copyrzlintrM) ror8rprqrZinputnew_noder+rvs @r'rrs I"$C((44E &&t-=-=-=-=>>D  8888888999 NN >+y 1 11r)c*tj|Sr=) th_mkldnnMkldnnBatchNorm)a_s r'rwrws!:1!=!=r)ci}|D]}|jdkrt|jtsJ||j}t |t vrqt t ||t j}t|tj sJtj |||<t||||S)z For each node, if it's a module that can be preconverted into MKLDNN, then we do so and create a mapping to allow us to convert from the MKLDNN version of the module to the original. r.) r5r2rr6r7 mkldnn_maprLfloatrBModulerJrKr)r8r, old_modulesr+ cur_moduler;s r'rrs /1K?? 7m # #dk3// / // -JJ:--'Z(8(89*ekRR !*bi88888*.- *C*C J'#D':>>> r)rc|D]Q}|jdkrDt|jtsJ||j}||vrt ||||RdS)za Maps each module that's been changed with `modules_to_mkldnn` back to its original. r.N)r5r2rr6r)r8r,rr+rs r'rrsuLL 7m # #dk3// / // -J[((#D';z3JKKK LLr)c$eZdZdejfdZdS)rfx_graphc>||_g|_g|_g|_dSr=)rr8 start_nodes end_nodes)rfrs r'__init__zMklSubgraph.__init__s#  $& *,(*r)N)rhrirjr3r|rrtr)r'rrs1+++++++r)r r"cHdddtdtffd }|S)aW This generates a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by running it with the example_inputs. Example usage: heuristic = gen_mkl_autotuner(example_inputs, iters=10) fast_model = optimization.optimize_for_inference(model, heuristic) NrQrc|j} :|jj |jj t d|Dt ttj d|j D}t |j || fd}|fd}tjj t |fd}||kS)Nc@g|]}tj|jSrt)rLrandnshaperyr+s r'r{z@gen_mkl_autotuner..use_mkl_heuristic..s$IIITTZ00IIIr)c(g|]}|jdS)r)r0rs r'r{z@gen_mkl_autotuner..use_mkl_heuristic..s*T*T*TD49Q<*T*T*Tr)ctD] }| tj}tD] }| tj|z Sr=)rangetime)frbeginiterswarmups r' benchmarkz?gen_mkl_autotuner..use_mkl_heuristic..benchmarksc6]]  IKKE5\\  9;;& &r)c6ddDDS)Nc6g|]}|Srt)to_denseryis r'r{zRgen_mkl_autotuner..use_mkl_heuristic....s-!" r)c6g|]}|Srt) to_mkldnnrs r'r{zRgen_mkl_autotuner..use_mkl_heuristic....s 1W1W1WA!++--1W1W1Wr)rt sample_inputs submodulesr'rwz>gen_mkl_autotuner..use_mkl_heuristic..s7&/i1W1W1W1W1W&Xr)cSr=rtrsr'rwz>gen_mkl_autotuner..use_mkl_heuristic.. s =(Ar))rr owning_modulerr propagaterr r3r4rrr8rrQrOrP) rQ input_nodes output_argsrmkl_time no_mkl_timerrexample_inputsrYrrrs @@r'use_mkl_heuristicz,gen_mkl_autotuner..use_mkl_heuristics8'  ~3H.4K h   ) ). 9 9 9II[III 4=*T*TEO*T*T*TUU $Xu{KUU  ' ' ' ' ' '9         O !4 (?(?(A(A#B#BK    i A A A A ABB +%%r))rbool)rrrrrYrs``` @@r'rrs[HK&&&&&&&&&&&&> r)rQc2t|jdkS)z This is a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by checking if there are more than 2 nodes in it )r/r8)rQs r'rrs u{  a r)c@eZdZdZdefdZdedefdZdedefdZd S) rc2dg|z|_dg|z|_dS)Nrr%size)rfns r'rzUnionFind.__init__s ,06A: !sQw r)vc.||j|<d|j|<dS)Nr"r)rfrs r'make_setzUnionFind.make_sets A ! r)rc|j|}||kr|S|J|||j|<tt|j|Sr=)r%findrint)rfrpars r'rzUnionFind.find#sOk!n 88H3 ACQ(((r)rbc||||}}||kr|S|j||j|kr||}}||j|<|j|xx|j|z cc<dSr=)rrr%)rfrrs r'joinzUnionFind.join+szyy||TYYq\\1 66H 9Q<$)A, & &aqA A !  ! $ r)N)rhrirjrrrrrrtr)r'rrs'''#)c)c))))%c%c%%%%%%r)r pass_configtracercdddtid}|i}|||drt|}|drt|}|ddur|St |dt st d d|dvrt d |dd}|}|tj |tj |j t | }Gd d t}tjD]}|j} |jd kr||j} t)| t*vrq|j} t/| d} | F| jt4jks Jd| jt5jdks Jdn6|jdkr+|jt*vr|j} n|jt:vr|j} | |jkr| |jkr t?d|j Ds!|5tj"|j fd} dddn #1swxYwYtGtHtj%j&| |_ '|5(dd|f} |)| |f| _ dddn #1swxYwYtUtj|}|_+jD]}|jdkr|jdkr|j d}t|j,}|D]B}|jdkr5|jdkr*|)|-|Ct]|j,dkr-|t]j}t_|fdtajD] \}}|jdkr(|jdkr||_12|9|jdkr?|jdkr4|j dJ|j d|_3fd|j4D}t]|dkrt?d|DrJtk|}|d|_6|ddD]}7|d| tqfd}jD]}ts|dr8|:|j6j;|ts|dr8|:|j1j<;|ts|dr8|:|j3j=;||>D]l}||s_|j<|j=zD]9}|j d}|)|-|:t|j||md}jD]}|jdks |jdkr|dz }tjAtCd |Dtj |}|S)!a Performs a set of optimization passes to optimize a model for the purposes of inference. Specifically, the passes that are run are: 1. Conv/BN fusion 2. Dropout removal 3. MKL layout optimizations The third optimization takes a function `use_mkl_heuristic` that's used to determine whether a subgraph should be explicitly run in MKL layout. Note: As FX does not currently handle aliasing, this pass currently assumes nothing aliases. If that isn't true, use at your own risk. T heuristic) conv_bn_fusermkldnn_layout_optimizeNrrrFz+mkldnn_layout_optimize config is not a dictz4Heuristic not found in mkldnn_layout_optimize configceZdZdZdZdZdS)*optimize_for_inference..MklSupportr"rN)rhrirjNOYESUNKNOWNrtr)r' MklSupportras r)rr.z)this pass is only for torch.float modulescpuz!this pass is only for CPU modules call_functionc3,K|]}|jdkVdS)rN)r)ryargs r' z)optimize_for_inference..s)II3:3IIIIIIr)c2d|fS)Nr) call_method)rrs r'rwz(optimize_for_inference..s)=)=kA4)P)Pr)rrrrct|dr|jSt|dr|jSdS)Ncolor start_color)hasattrrrr)rufs r' get_colorz)optimize_for_inference..get_colorsT 1g   $7717## # 1m $ $ *771=)) )tr)cpg|]2}t|tjr|'|3Sr=)r2r3r4)ryrrs r'r{z*optimize_for_inference..sQa))9Q<<+ ! ,++r)c3K|]}|duV dSr=rtrs r'rz)optimize_for_inference..s&9919999999r)r"c"tSr=)r)rsr'rwz(optimize_for_inference..s H@U@Ur)rr end_colorzmkldnn conversions: %s)Erupdaterrr2rO RuntimeErrortracerJrKr3rMrootrPrlistr8rr5rr7mkldnn_supportedrnext parametersdtyperLrdevicemkldnn_supported_unknownranyr0inserting_beforemap_argrr r+r inserting_after create_noderTrrrRrUr/r enumeraterrrall_input_nodessortedrrrrrappendrrvaluesrlogging getLoggerrhinfor)r@rrdefault_pass_configr cur_tracerr,rr+supports_mkldnnrsample_parameter mkldnn_argsdense_xrprv_noderRuser num_nodescur_idx cur_colors other_color mkldnn_graphsrQprvmkldnn_conversionsresultrrrs @@@r'rr5sc&#."?  {+++>*U +,&u%%34== )*BCT J JJHIII-.FGGGQRRR+,DEkRJ e 4 455HN:?H---$()<)<)>)>$?$?GTX^$$"'"'$- 7m # # -JJ#333",.#' (=(=(?(?#F#F #/(.%+===B>==+2el77:W ' '{...",. 888","4 jm + +*"444IItyIIIII**400   jIPPPP                 U27#34kBBDI))$// ' '"..}j4'RR**7333 $w  ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' $D$8$8'BBK&H * * 7m # # z(A(Ay|H$$E . .7m++ {0J0J..x888''---4:!####D)))HN##I 9  B$#8>2244  7m # # {(B(B&D  KK W % %$+*C*C9TYq\**666&Yty|44DNN-J:!##99j99999 9 99 ++J#ADJ)!""~ 4 4  1 {3333 4-88U8U8U8U,V,VMJJ 4 ! ! B "''$*-- . 4 ; ;D A A A 4 ' ' N "''$"233 4 @ G G M M M 4 % % J "''$.11 2 < C CD I I I%%''==  '' =)EO; * *il**3///##D)))) %+w < < <$$ ;+ % % )B)B ! #  h$$%=?QRRR MMOOO ^E8 , ,F Ms$4JJ" %J" -6L//L3 6L3 )FF)rr")KrJroperatorr collectionsrenumrtypingrrrrr r r r rLtorch.fxr3torch.nnrBtorch.nn.functional functionalFtorch.utils.mkldnnutilsmkldnnr torch.fx.noder rtorch.fx.passes.shape_proprtorch.nn.utils.fusionrr__all__r6r(r4rrrrrrrErIrFReLU MaxPool2d AvgPool2dAdaptiveAvgPool2drelu transposesigmoid avg_pool2dadaptive_avg_pool2draddmulr MkldnnConv2d MkldnnLinearrrrrrrrrTracerrrtr)r'r-s  ######IIIIIIIIIIIIIIIIIIII &&&&&&&&&********000000HHHHHHHH    --sCx---- d^#%759#s(^(4 '4 cN48=4444%/%/%/58?%/%/%/%/P0")0 0000&22 =2 M2"'] 2222.IINGLL J O MFL&%L(,7Iy%Iy%N== T"']T#ry.5I&L =L #ry. !Lbi*+LLLL"++++++++++++\ + $    %%%%%%%%:-1 ipp 8?p$sCx.)p Op X_ ppppppr)