j=/ddlZddlmcmZddlmZddlZdZdZ dZ dZ dZ ej dZd Zd Zd Zd Zd ZdS)N) namedtuplec4tfd}|S)Nc dr"dj}||i|SdsdrYdrdnd}|dkrdnd}|j}t d|dd|d|d |i|S) Nautogradsave_for_backwardbackwardz We found a 'z' registration for  at z but were unable to find a 'z' registration. To use the CustomOp API to register a backward formula, please provide us both a backward function and a 'save for backward' function via `impl_backward` and `impl_save_for_backward` respectively.) _has_impl _get_implfunclocation RuntimeError)argskwargskernelmissingfoundlocautograd_fallback custom_ops JC:\Projects\Historical\venv\Lib\site-packages\torch/_custom_op/autograd.pyinnerz*autograd_kernel_indirection..inners)   z * * +((449F64*6** *   2 3 3 ;y7J7J:7V7V ;'0':'::'F'F ## ,3j+@+@''jE%%e,,5C:u::::::4;:::;; ;! $1&111)autograd_not_implemented)rrrs` @rautograd_kernel_indirectionrs70;;222222. Lrcfd}|S)Nctjr'tjd||frt dtj5|i|cdddS#1swxYwYdS)NcDt|tjo|jSN) isinstancetorchTensor requires_gradxs rz:autograd_not_implemented..kernel..4sjEL11Eaorz.Autograd has not been implemented for operator)r!is_grad_enabledpytreetree_anyr_C_AutoDispatchBelowAutograd)rrrs rrz(autograd_not_implemented..kernel2s  " " Qv E Ef~( (  QOPP P X 0 0 2 2 . .9d-f-- . . . . . . . . . . . . . . . . . .sA//A36A3)rrs` rrr1s#..... Mrc |t|ts|f}n|}t|t|ksJg}tt ||D]\}\}}t|t jr|s||:t|tr|s| |g|r&td|d|dt|d|r|j |dSdSdS)NzWith output_differentiability=z . At idx z , we received an object of type za that is not a Tensor, so it cannot have be marked as differentiable in output_differentiability.) r tuplelen enumeratezipr!r"appendlistextendrtypemark_non_differentiable)ctxoutputoutput_differentiability tuple_outputnon_differentiable_tensorsidxdifferentiableouts rr6r6<so  +&%(( ""9LL!L+,,L0A0AAAAA%'"*3C8PR^4_4_*`*` 2 2 &C&.##u|,, %;.55c:::#t$$ %;.55c::: 2"15M11!11CG99111222 2 & E 'C ')C D D D D/,+, E Erc&fd}|S)Nctj|\}d  fd} fd}t jdz||}|j|}Jtjt |S)Nc|dtjt| }tj5|}dddn #1swxYwYt tjt|}t |} ||}t|||ft|| tj |\} t|S)NT)set_materialize_gradsr(tree_unflattenr3r!r*r+namedtuple_argstree_mapr5save_pytree_for_backwardr6 tree_flattenr.)r7 flat_argsrr8 args_infosave_for_backward_fn_inputsto_save flat_output op_overloadout_specr9save_for_backward_fnschemaspecs rforwardz9construct_autograd_kernel..apply..forwardhs8  % %d + + +(i$??D4466 , ,$d+ , , , , , , , , , , , , , , ,(d3355I+:&$*G*G '**+FOOG $S7I*> ? ? ? #C1I J J J%+$7$?$? !K%% %sA))A-0A-c Jtjt| }t|\}}t }t |t s|f}||g|R}t||t||Sr) r(rCr3 unpack_savedobjectr r.validate_grad_inputs_dictgrad_inputs_dict_to_flat_tuple) r7flat_grad_outputgradssavedrI inner_ctxgrad_inputs_dict backward_fnrrNs rrz:construct_autograd_kernel..apply..backward|s''')$/?*@*@(KKE+C00 E9IeU++ !*{9eDeDDD  &&6 9 M M M12BINN Nr _customop)r(rGgen_autograd_function_opnameapplyrCr3)rrHrRr generated_clsrLrNrQr]rrMr9rOrPs @@rraz(construct_autograd_kernel..applyds -d33 4 & & & & & & & & & &( O O O O O O O .   +Wh@@ *m)95 ###$T+%6%6AAArr,)rPr9rrMrOr]ras`````` rconstruct_autograd_kernelrc\sL-B-B-B-B-B-B-B-B-B-B\ Lrct|tjjft |t |d}|S)N)rRr)r5r!rFunction staticmethod)namerRrrbs rr_r_sG  "#G,,$X..  M rcd|jjD}t|jdz}t ||}|S)Ncg|] }|j Sr,)rg.0args r z'namedtuple_args_cls..s===Csx===r_args) argumentsflat_allstrrgr)rPattribsrg tuple_clss rnamedtuple_args_clsrtsD==6#3#<===G v{  g %D4))I rcXt|tsJt|}||Sr)r r.rt)rPrrss rrDrDs2 dE " """ "#F++I 9d rc fd}t|ts|dt|djjjD}|}||kr|d|d|d|D]\}}t||}t|tr t|ttfs|d|dt|d t|t|ks.|d|d t|d t|tt||D]w\} \} } |  t| tjs!|d|d t| d | t!| tjs|d|d| d| d|xL|Pt|tjs|dt|d|dt!|tjs|d|d|d|ddS)Nchd}tdd|jd|)Nrz%In the backward function defined for r z using the CustomOp API, )r rr )whatr forward_ops rerrorz(validate_grad_inputs_dict..errors`'' 33 BJ B B  B B;? B BCC CrzBexpected the output of the backward function to be a dict but got cNh|]"}|j|j#Sr,)r5is_tensor_likergrjs r z,validate_grad_inputs_dict..s<333#//113SX333rz3expected the returned grad_input dict to have keys z but got z. The backward function must return a gradient (can be None) for each arg to the CustomOp that may be a Tensor or Sequence[Tensor]. Args declared to be non-Tensor-like types should not appear in the grad_input dictz for input 'zR' expected the grad_input dict to hold a list of gradients but got object of type .z1' expected the grad_input dict to hold a list of z gradients but got z\' expected the grad_input dict to hold a list of None or Tensor gradients but got object of z at index z(', got a Tensor as the gradient for the z(-th value but expected None because the z(-th value was not a Tensor (it was type zgot object of type z as the gradient for input 'z:', but expected the gradient to be either None or a Tensorz(got a Tensor as the gradient for input 'z3' but expected None as the gradient because input 'z ' was not a Tensor (it was type z).)r dictr5_schemarorpkeysitemsgetattrr3r.r/r0r1r!r" issubclass) r\ryrIrz expected_keys actual_keysrggradarg_infor<ginfos ` rrVrVsCCCCC & - -/ .*++.. / / /33);)E)N333M"''))K ## ((()4((( ) ) )',,..#A#A d9d++ h % % dUDM22 ('D''d'''(((t99H --%D%%(+H %%T%%&&&#,Ch,?,?"@"@ . .Ya9!!U\22AE@@@'+Aww@@:=@@AAA"$ 55.E---%(--!$--#+--...  < $ -- N EMT MMMMM N N N(EL11 A E@T@@BF@@3;@@@ A A AC#A#Arc2g}|D]N\}}||vr*|tjd|3|||Ot tj|S)NcdSrr,r$s rr&z0grad_inputs_dict_to_flat_tuple..sDr)_asdictrr2r(rEr. tree_leaves)r\rIresultrgrs rrWrWs F#++--3355..h ' ' ' MM&/..(CC D D D  &t,---- #F++ , ,,rc@tj|\}}t|}dt|D}dt|D}d|D}d|D}||_||_|j|||_||_||_ dS)NcLg|]!\}}t|tj|"Sr,r r!r"rkr<things rrmz,save_pytree_for_backward..s<777:3  5573777rcLg|]!\}}t|tj|"Sr,rrs rrmz,save_pytree_for_backward..s<???zsE( ==?s???rcFg|]}t|tj|Sr,rrkrs rrmz,save_pytree_for_backward..s)PPP 5%,0O0OPuPPPrcFg|]}t|tj|Sr,rrs rrmz,save_pytree_for_backward..s)XXXU 5%,8W8WX5XXXr) r(rGr/r0rQnum_eltsr tensor_idxssaved_non_tensorsnon_tensor_idxs) r7stuff flat_stuffrQrrrtensors non_tensorss rrFrFs*511J:H77:)>)>777K??Yz-B-B???OPP*PPPGXXjXXXKCHCLC7##!CO'C)Crcdg|jz}t|j|jD] \}}|||< t|j|jD] \}}|||< t j||j}|Sr) rr1 saved_tensorsrrrr(rCrQ)r7rtensorr< non_tensorrs rrTrT s#,&J3,co>>!!  3s4c6IJJ%% C$ 3  !*ch 7 7E Lr)r!torch.utils._pytreeutils_pytreer( collectionsr functoolsrrr6rcr_ lru_cachertrDrVrWrFrTr,rrrs $$$$$$$$$"""""">EEE@666r     9A9A9Ax---***&r