j8ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z ddl Z ddl Z ddlmZddlmZddlmcmZddl mZddlmZddlmZdd lmZmZmZdd lm Z dd l!m"Z"m#Z#m$Z$ej%e&Z'd Z(ed Z)edej*de fdZ+d"dZ,dZ-edej*de fdZ.Gddej/Z0edej*de fdZ1edZ2dZ3e j4j5Z5e5j6e5j7e5j8e5j9e5j:e5j;e5j<e5j=e5j>e5j?e5j@e5jAe5jBe5jCjDe5jCjEe5jFe5jGe5jHe5jIe5jJe5jKe5jLhZMeeMZMedZNde e ejOffdZPdZQdaRdZSdZTd#d!ZUdS)$N)contextmanager)partial)CallableUnion)SymInt)get_decompositions) bind_symbols) aot_function aot_modulemake_boxed_compiler)strip_overloads)default_partition draw_graph#min_cut_rematerialization_partitionc|jdtjjjD]}tjjj|_||S)N call_functionoptarget) graph find_nodestorchopsaten_to_copytor recompile)fx_gnodes KC:\Projects\Historical\venv\Lib\site-packages\torch/_functorch/compilers.py _canonicalizer"$s[ %% 59>#:&((in' NN Kc#Ktjd} dVtj|dS#tj|wxYw)NF)r_C_jit_set_autocast_mode)old_jit_autocast_flags r!_disable_jit_autocastr(-sd!H;;EBB?  ''(=>>>>>''(=>>>>s A!A)rreturnct5t||jdtjjjD]V}t|j dkrzts_compile..`s2MM1:a!2!=>>MMMMMMr#N)r(rrrrrrrlenargskwargsrrnodesitemsr.devicetypelintrjitscriptr%_jit_pass_remove_mutationfreezeevaloptimize_for_inferenceany)rinpsr new_kwargskvfs r! ts_compilerH6s  J))uy~'>*   0 0D49~~""s4;'7'71'<'<DKAWAW#in/ J$ % %DJ ))++ " "1a..A ! 1 $DKK   I  T " " **17333 I  QVVXX & & I , ,Q / /MMMMMMM  AtHH9: HsGG--G14G1TcRt|jt||||S)N) clear_meta)printcoder)r_namerJs r!_draw_graph_compilerOes, $)tTj1111 Kr#cHttt|S)NrN)r rrOrQs r!draw_graph_compilerRks w':FFF G GGr#c|S)z Returns the :attr:`fx_g` Fx graph module as it is. This is a no-op compiler and can be used to check accuracy. .. warning:: This API is experimental and likely to change. rrMs r!noprVos  Kr#c(eZdZfdZfdZxZS)DebugInterpretercbt|jg|R|_tj|dSr-)r modulesymbol_mappingsuperrun)selfr5 __class__s r!r]zDebugInterpreter.run}s6*4;>>>> Tr#cP fd fd fd fd}t|}d|jvrtj|jd\}}tj|\}}t |t |ks)Jt |dt |t tt |||D]2\ }} t| tj s!||| fd3|S)Nct|ts|Stj|jjj}|js J|t|Sr-) r.rsympyexpandr exprxreplacer[ is_numberint)nirr^s r! subst_symintz/DebugInterpreter.run_node..subst_symints^b&))   RW\2243FGGHHA; ! ! ! !;q66Mr#c:tfd|DS)Nc3.K|]}|VdSr-rT)r1rhrjs r!r3zHDebugInterpreter.run_node..subst_symint_tuple..s-88bb))888888r#)tuple)nisrjs r!subst_symint_tuplez5DebugInterpreter.run_node..subst_symint_tuples&8888C88888 8r#c,|dkrqt|jD]\}||||kr%||dkrdS]dS)Nrr FT)numelrangendimstridesize)abidxrjs r!check_significant_stridesz.check_significant_stridess|AGGII&&** ==%%C$ QXXc]]33qxx}}DD(L5599$uu4r#c t|sJ|j|jks$J|d|jd|j||ks\J|d|d|d|||}|s\J|d|d|d|dS)Nz:  != z aka )callabler+rurt)nvrvdesc same_stridesryros r!checkz(DebugInterpreter.run_node..checks\D>> ! !>8rx'''DDFF)N)Nbh)N)NBH)N)N'''""27799--:::$&&ZZBGGIIZZ,>,>rwwyy,I,IZZrwwyyZZ;::44R<z+DebugInterpreter.run_node..s&O&O&O$:M&O&Or#) r\run_nodemetapytree tree_flattenr4ziprrr.rTensor)r^nrrin_valsn_specr_valsr_specr}r~ryrrjror_s` @@@@r!rzDebugInterpreter.run_nodes{      9 9 9 9 9      a a a a a a GG  Q   AF??#0??NFF#033NFFv;;#f++---#f++/P/P3v;;/P/P--- s6{{!3!3VVDD Q Q 2r!"el33b"OOOOOPPPPr#)__name__ __module__ __qualname__r]r __classcell__)r_s@r!rXrX|sQ/////////r#rXc*t|jS)z Returns a (slow) interpreter over the FX graph module that also checks various debugging properties (e.g., that tracing strides matched real strides.) )rXr]rUs r! debug_noprs D ! ! %%r#ct|tj|}tj|}|Sr-)rrr<r=r?r@)rrMrGs r!simple_ts_compilersGD A ""A Hr#c,t|tSr-)r r)rGs r!nnc_jitrs , - --r#c.t|j|Sr-)rKrLrUs r! print_compilers $) Kr#fnc ttttd}||t |t jjr t|fi|St|fi|S)a Wrapper function over :func:`aot_function` and :func:`aot_module` to perform memory efficient fusion. It uses the :func:`min_cut_rematerialization_partition` partitioner to perform efficient recomputation. It uses NVFuser to compile the generated forward and backward graphs. .. warning:: This API is experimental and likely to change. Args: fn (Union[Callable, nn.Module]): A Python function or a ``nn.Module`` that takes one ore more arguments. Must return one or more Tensors. **kwargs: Any other overrides you want to make to the settings Returns: Returns a ``Callable`` or ``nn.Module`` that retains the eager behavior of the original :attr:`fn`, but whose forward and backward graphs have gone through recomputation optimizations, and the graphs have been compiled with nvfuser.  fw_compiler bw_compiler partition_fndecompositions) rHrdefault_decompositionsupdater.rnnModuler r )rr6configs r!memory_efficient_fusionrsr6"!;0 F  MM&"eho&&*"'''''B))&)))r#c|dtdd|Ddddlm}||t ||S)NfooaQ ############################################################## # To minimize FX graph, copy and paste the below and run it # ############################################################## import torch import torch.fx as fx from functorch.compile import minifier, check_nvfuser_subprocess, check_nvfuser_correctness_subprocess inps = c*g|]}|j|jfSrT)shaper+)r1rs r! z!debug_compile..s!***!'17 ***r#a? inps = [torch.ones(shape, dtype=dtype, device='cuda') for (shape, dtype) in inps] from foo import FxModule mod = FxModule().cuda() with torch.jit.fuser("fuser2"): # check_nvfuser_subprocess can be replaced with check_nvfuser_correctness_subprocess minifier(fx.symbolic_trace(mod), inps, check_nvfuser_subprocess) r)FxModule) to_folderrKrrcudarH)rrCrs r! debug_compilersNN5   +*T***   (HHJJOOt dD ! !!r#c 6g}t|d5}tj|}g}|D]}t|dkr|}|t j}n|\}}}} } | t jt jt j t j t jt j tthvrt j dd|| | }nt j|| | }|| dddn #1swxYwY|S)zZ Return a random input for the given inputs meta generated from _save_fx_default. rbr r)r+r9N)openpickleloadr4randomrandrrgint32int64booluint8floatrandintappend) input_data_pathinputsrG inputs_metarr:inputrrtr+r9s r! get_inputsr3sRF ot $ $!k!nn  ! !D4yyA~~V[]]++592eVUFIKKJIK   "M!QU6RRREE!JuE&IIIE MM% ' !!!!!!!!!!!!!!!!. MsC.DDDc| ddlm} fd  fd fd} fd} fd}||||||tS) aO The forward, backward, and joint computation graph will be stored in {folder_name}/{current_name}/{current_name}_forward_{graph_index}, {folder_name}/{current_name}/{current_name}_backward_{graph_index}, and {folder_name}/{current_name}/{current_name}_joint_{graph_index} respectively. The input shape of the graphs will be stored in the .input files. These files can be loaded with pickle, and is a list of format (type, shape, stride, dtype, device). In the case of type = int or float, it is just (type,). For joint graph input, it is a nested list [[],[]] where the two inner lists have the same format. If dump_example_input is True, example_inputs will be stored in .pt file. Since each function might produce multiple graphs, the graph_index is used to distinguish difference graphs r)aot_module_simplifiedcg}t|dkrEt|dtr*||dz }||dz }|S|D]}t|tkst|t kr$|t|fV|t||j||j |j f|S)Nrr ) r4r.rmr:rgrrrrtr+r9)r5 input_metaargget_input_metas r!rz(_save_fx_default..get_input_metads t99q==ZQ77= ..a11 1J ..a11 1J   CCyyC499#5#5!!499,////!!#YY 3::<<CJOr#ct|jjdkr/ttjd|t dStj|}|j tj j | |}tjdd|ddd|dt t#j|t'ddd|dt dd|dt ddr9tj|ddd|dt dd|dt d dSdS) Nrz!No nodes in graph {%s}_{%s}_{%s}./T)exist_okrMz.inputwbz.pt)r4rr7logloggingWARNING graph_indexcopydeepcopy set_codegenrfxCodeGenrosmakedirsrrdumprsave) gm_to_saver5 type_namegmr current_namedump_example_input folder_namers r!graph_saver_helperz,_save_fx_default..graph_saver_helperssO z% & &! + + GG3     F ]: & & UX^3355666 #^D))  {33\33dCCCC  T T\ T TL T T9 T T{ T T      HHHH HHyHH;HHYeHHhqHHtHHH        JEEEE EEyEE;EEYeEEhqEEtEEE       r#c"||d|S)NforwardrT)rfw_argsrs r!graph_saver_forwardz-_save_fx_default..graph_saver_forwards2w 222 r#c6||dtdz a|S)Nbackwardr )r)rbw_argsrs r!graph_saver_backwardz._save_fx_default..graph_saver_backwards'2w 333q  r#c>||dt||S)Njoint)r)r joint_argsrs r!graph_saver_jointz+_save_fx_default..graph_saver_joints(2z7333 Z000r#r)functorch.compilerr) rrrrexample_inputsrrrrrrs ``` @@r!_save_fx_defaultrRs 877777     !!!!!!!!F 11111 !  '(&-    r#Fc4datt|||S)as Dump the forward, backward, and joint computation graph. Example Usage: save_fx_func = graph_dumper_aot(current_name, folder_name, dump_example_input = False) optimize_ctx = torchdynamo.optimize( save_fx_func ) with torch.enable_grad(): with optimize_ctx: result = forward_and_backward_pass(model, example_inputs) r)rrr)rrrs r!graph_dumper_aotrsK #\;@R S SSr#)T)F)Vrrrrr contextlibr functoolsrtypingrrrbrtorch.fxrtorch.nnrtorch.utils._pytreeutils_pytreerr torch._decompr%torch.fx.experimental.symbolic_shapesr aot_autogradr r r compile_utilsr partitionersrrr getLoggerrrr"r( GraphModulerHrOrRrV InterpreterrXrrrrrdetach gelu_backwardleaky_relu_backwardsigmoid_backwardthreshold_backwardhardtanh_backwardhardsigmoid_backwardhardswish_backward tanh_backward silu_backward elu_backwardcudnn_batch_normcudnn_batch_norm_backward masked_fillScalarrelu leaky_reluhardtanh hardswish hardsigmoid conj_physical is_same_sizerrrrrrrrrrTr#r!rs  %%%%%%"""""""" $$$$$$$$$,,,,,,>>>>>>GGGGGGGGGG******g!! ???+ R^+ h+ + + + \ HHH bn H    44444r~444n&BN&(&&&&   ... y~K"HOMN-2,+,BCC $* h !"$*$*$*$*N""": >YYYzTTTTTTr#