jjbUddlZddlZddlZddlZddlZddlZddlZddlmZm Z m Z m Z m Z m Z ddlZddlmZddlmZddlmZddlmZmZddlZddlmZ gdZdd d Zhd Zd Z d.d edejedejfdZ iZ!ej"edfe#d<GddZ$ej%GddZ&dede fdZ'deddfdZ(deddfdZ)d edej*eeffdZ+deddfdZ,d ej-de.fd!Z/ded"ejddfd#Z0d$ej1d%eddfd&Z2d'Z3d(Z4d/d*Z5d+Z6d0d-Z7dS)1N)FunctionSchema OperatorName SchemaKindBaseTypeListTypeBaseTy)get_ctx)autograd_kernel_indirectionconstruct_autograd_kernel) infer_schema) custom_opCustomOpr CPUCUDA)cpucuda>atatenprimprimstorchpytorchc:tjdtdS)Nzwtorch._custom_op is deprecated and will be removed in PyTorch 2.6, please use the equivalent torch.library API instead.)warningswarnDeprecationWarningFC:\Projects\Historical\venv\Lib\site-packages\torch/_custom_op/impl.pywarn_deprecatedr!-s- M 89KMMMMMrqualname manual_schemareturnc2tfd}|S)L This API is deprecated, please use torch.library.custom_op instead c tj|stdt|t \}}t ||j|krtd d|d|jd t|dn }||}tj |}t| t||tj |d}||t||j}t#|||||d }|j|_|j|_|j|_tj||jd t-t/j|t2j|t9jt<t/j||S) NzDcustom_op(...)(func): Expected `func` to be a Python function, got: zcustom_op(qualname='z-', ...)(func): expected `func` to have name 'z ' but got 'zX'. Please either change the name of `func` or the qualname that is passed to `custom_op`r) mutates_argsFRAGMENTT_private_accessAutograd)inspect isfunction ValueErrortypeparse_qualnamevalidate_namespace__name__r rparsevalidate_schema validate_function_matches_schemalibraryLibrarydefinefind_ophandle_or_thrownamer __module____doc__impl_opnamer weakrefproxyr_C#_dispatch_set_report_error_callback functoolspartialreport_error_callback) funcnsr;schema schema_strfunction_schemalibophandleresultr#r"s r innerzcustom_op..inner;s!$'' /"&t**//  "(++D2 =D ,x,,!%,,26-,,, 9F8Md4444S`&f&& (.z::(((  $ ,_d C C Cob*-- :)"o.BCC#r?D(TXYYY- O5 S&.*55 ' f(=(= > >    44 i'(=w}V?T?TUU    r)r!)r"r#rOs`` r rr3s8 ))))))V Lrrglobal_registryceZdZdZddfd ZdZddZdZd Zd Z d Z d Z dd e j ee jefde jfdZdZde jfdZdde jfdZdZdZdZdZddZddZxZS)rr&Fr*c0tt|std|d|}||_||_||_||_||_||_ d|_ i|_ d|_ |t|j <dS)Nz|The CustomOp constructor is private and we do not guarantee BC for it. Please use custom_op(...) to create a CustomOp object::F)super__init__r! RuntimeError_schema_cpp_ns_lib _ophandler? _qualnamer3_impls'_registered_autograd_kernel_indirectionrP) selfrLcpp_nsrI operator_namerMr+r; __class__s r rUzCustomOp.__init__ws  S ++M++  %( 5=) " KM 7<4*.'''rc|jrJ|j|jt t j|dd|_dS)Nr,T)r]rYr>r?r r@rAr^s r %_register_autograd_kernel_indirectionz.CustomOp._register_autograd_kernel_indirectionsM???? t|%@tATAT%U%UWabbb7;444rc @||r9|j|}|J|j}td|d|jd|d|d t jtj|}|j d|j }t|||j|<dS)NzAttempting to register a z impl for operator z that already has a z impl registered from Python at z. This is not supported.:) _has_implr\locationrVr[r- getframeinfosys _getframefilenamelinenoFuncAndLocation)r^kindrG stacklevelfunc_and_locationriframes r _register_implzCustomOp._register_impls >>$    $ D 1 $000(1H6D66T^66&*66666  $S]:%>%>??n55u|55+D(;; Drc|j|SNr\r^rps r _get_implzCustomOp._get_impls{4  rc||jvSrvrwrxs r rhzCustomOp._has_implst{""rc|`ttj|j}t ||jrt||jt|j =dSrv) rYgetattrropsrXhasattrr?delattrrPr[)r^ opnamespaces r _destroyzCustomOp._destroysR Iei66 ; - - / K . . . DN + + +rcd|jdS)Nz)r[rcs r __repr__zCustomOp.__repr__s33333rc:tj|jg|Ri|}|Srv)rB_dispatch_call_boxedrZ)r^argskwargsrNs r __call__zCustomOp.__call__s+(I$III&II r device_typesr$crttrgD]}t|fd}|S)T This API is deprecated, please use torch.library.custom_op instead ctD]e}|||t|}t jjj||f|S)Nrq)set_check_doesnt_have_library_implrtSUPPORTED_DEVICE_TYPE_TO_KEYr7r>rYr?)f device_type dispatch_key _stacklevelrr^s r rOzCustomOp.impl..inners"<00 G G 44[AAA##K{#KKK;KH C TY lCCAFFFFHr) isinstancestrvalidate_device_type)r^rrrrOs``` r r>z CustomOp.implsj lC ( ( *(>L' . .K  - - - -        rc||rdSt|}tj|j|rt d|d|jddS)Nzimpl(..., device_types=z): the operator zs already has an implementation for this device type via a pre-existing torch.library or TORCH_LIBRARY registration.)rhrrB._dispatch_has_computed_kernel_for_dispatch_keyr[rV)r^rkeys r rz(CustomOp._check_doesnt_have_library_impls >>+ & &  F*;7 rYr?)rr^s r rOz$CustomOp.impl_factory..inners@    1 - - - BGLDL/ B B1 E E EHrr)r^rOs` r impl_factoryzCustomOp.impl_factorys#       rcfd}|S)rc,ddjjt jfd}jj |dS)Nabstractrcfd}tjj|5|i|cdddS#1swxYwYdS)Nc.tdd)Nz.inner..f_with_ctx..error_on_ctxs6&)'))'))   r)r_library fake_implset_ctx_getter)rrrrrir"s r f_with_ctxz9CustomOp.impl_abstract..inner..f_with_ctxs      ^-<<\JJ..1d-f--..................sAAAMeta) $_check_doesnt_have_library_meta_implrtryrir[rDwrapsrYr>r?)rrrir"rr^s` @@r rOz%CustomOp.impl_abstract..inners  5 5 7 7 7    A+  F F F~~j11:H~H_Q   . . . . . .  . INN4<V < < <Hrrr^rrOs`` r impl_abstractzCustomOp.impl_abstracts)       8 rcfd}jtjkr |dj}js |dt |dksJt d|D}|r |dttj dttj dttj d ttj d ttj d tttj dd i}jD]B}|j|vr |d t!|d|jdCdS)Nc>td|djd)NzCCannot use torch._custom_ops APIs to register backward formula for z. Got operator z with schema: )rVr[)detailrIr^s r errorz4CustomOp._check_can_register_backward..errorsC:%::>::17:: rznon-functional operatorzoperator with no returnsrc3BK|]}|jduo |jj VdSrv) annotationis_write).0rs r z8CustomOp._check_can_register_backward..#sG# # GHAL $ BQ\-B)B# # # # # # rzoperator that returns viewsintSymIntboolfloatTensorz List[Tensor]zoperator with return not in z (got ))rWrpr functionalreturnslenanyrrrrrrrrr0listvalues)r^rretsis_non_mutating_viewallowed_return_typesretrIs` @r _check_can_register_backwardz%CustomOp._check_can_register_backwards       ;;==J1 1 1 E+ , , ,~~ . E, - - -4yy1}}}}"# # LP# # #     1 E/ 0 0 0 VZ % V] # #X V[ ! !6 V\ " "G V] # #X Xfm,,d 3 3^  > i iCx/// Eg6J6Q6Q6S6S1T1Tgg\_\dggg h h h h i irc|jrdStj|jdrt d|jddD]7}tj|j|rt d|jd|d8dS)NCompositeImplicitAutogradz3impl_backward/impl_save_for_backward: the operator a3 already has an implementation for this device type via a pre-existing registration to DispatchKey::CompositeImplicitAutograd.CompositeImplicitAutograd operators do not need an autograd formula; instead, the operator will decompose into its constituents and those can have autograd formulas defined on them.)r, AutogradCPU AutogradCUDAz; already has an Autograd kernel registered to DispatchKey::z vi a pre-existing torch.library or TORCH_LIBRARY registration. Please either remove those registrations or don't use the torch._custom_ops APIs)r]rB%_dispatch_has_kernel_for_dispatch_keyr[rV)r^rs r (_check_doesnt_have_library_autograd_implz1CustomOp._check_doesnt_have_library_autograd_impl7s  7  F 3DND_ ` ` @?dn???@@ @? [ [C7LL ["Z$(NZZ25ZZZ[[[ [ [ [rch|drdStj|jdrtj|jdsdStj|jdrt d|jdtj|jdrt d|jddS)NrCompositeExplicitAutogradrrz!impl_abstract(...): the operator a- already has an implementation for this device type via a pre-existing registration to DispatchKey::CompositeImplicitAutograd.CompositeImplicitAutograd operators do not need an abstract impl; instead, the operator will decompose into its constituents and those can have abstract impls defined on them.z already has an DispatchKey::Meta implementation via a pre-existing torch.library or TORCH_LIBRARY registration. Please either remove that registration or don't call impl_abstract.)rhrBrr[rVrcs r rz-CustomOp._check_doesnt_have_library_meta_implOs >>* % %  F  4T^E` a a  " " NN. / / 4 NN: & & + -- J/////rcfd}|S)zyRegister a function that tells us what to save for backward. Please see impl_backward for more details. cjsd|drdSdS)Nrrr)rrr]rdrtrhr)rrr^s r rOz.CustomOp.impl_save_for_backward..inners  - - / / /  9 9 ; ; ;? =::<<<    3Q;  O O O~~j)) 1..00000 1 1rrrs`` r impl_save_for_backwardzCustomOp.impl_save_for_backwards)  1 1 1 1 1 1 rNc|fd}tts |D]!}t|ts |"tjjtkr |fd}|S)rNc(td)Nzimpl_backward(output_differentiability): expected output_differentiability to be a list of bools with length equal to the number of outputs of this CustomOp got: r)output_differentiabilitysr yellz$CustomOp.impl_backward..yells)"7577888rc"jsd|_drdSdS)Nrrr)rrr]rdrtrrhr)rrrr^s r rOz%CustomOp.impl_backward..inners  - - / / /  9 9 ; ; ;? =::<<<    A+  F F F-ED *~~122 1..00000 1 1r)rrrrrWr)r^rrrdiffrOs``` r impl_backwardzCustomOp.impl_backwards $ / 8 8 8 8 86== 0  !$--DFFF4<'((C0H,I,III 1 1 1 1 1 1 1 r)re)Nre)r3r< __qualname__r=rUrdrtryrhrrrtypingUnionrIterableCallabler>rrrrrrrrr __classcell__)ras@r rrrsY^///////4<<< < < < <!!!### , , ,444RS"Lfoc.B)BC *NNNfo!!fo!!!!F#i#i#iJ[[[0"X"X"XR 0 0 0    rc.eZdZUejed<eed<dS)rorGriN)r3r<rrr__annotations__rrrr roros' /MMMMMrror_r`cz|jdn|j}tj|dt|j|S)NrS) overload_namerB_dispatch_find_schema_or_throwrr;)r_r`rs r r:r:sN)1}7R  , ..S+,,..   rrHcrd|vrtd|d|tvrtd|d|ddS)N.zcustom_op(..., ns="zC"): expected ns to not contain any . (and be a valid variable name)zcustom_op(..., ns='z'): 'z9' is a reserved namespace, please choose something else. )r/ RESERVED_NS)rHs r r2r2s| byy $" $ $ $    [ ." . .2 . . .   rrIctjj|st d||jjt d|dS)Nzcustom_op only supports functional operators (ops that do not mutate any inputs, do not return views of the inputs, and has at least one return). Got the following non-functional schema: zUcustom_op does not support arguments named 'self'. Please rename your argument. Got: )rrutilsis_functional_schemar/ argumentsself_arg)rIs r r5r5s >  4 4V < <  A9? A A    , 3*0 3 3   -,rc|dd}t|dkrtd|dd|dvrtd||d|dfS) NrSr rez$Expected there to be a namespace in z;, i.e. The operator name should look something like ns::foorzThe torch.custom_ops APIs do not handle overloads, i.e. operator names with '.' in them. Please name your operator something like ns::foo. Got: r)splitrr/)r"namess r r1r1s NN4 # #E 5zzQMMMMNN N eAh,"*,,-- - 8U1X rrcr|tvr-td|dtddS)NzCustomOp.impl(device_types=[z(, ...]): we only support device_type in r)rr/keys)rs r rrsW666 9; 9 9.3355 9 9 9   76rparamcT|jtjjtjjfvSrv)rpr- ParameterPOSITIONAL_OR_KEYWORD KEYWORD_ONLY)rs r supported_paramrs( :/& rrGcvtj|tdjDst dt djDsjtjj urt ddjD}djD}fdfdfd }||j j ||j j dS) Nc3:K|]\}}t|VdSrv)rr_ps r rz3validate_function_matches_schema..s.EEdaq!!EEEEEErzcustom_op(..., manual_schema)(func): positional-only args, varargs, and kwargs are not supported. Please rewrite `func` to not have them. Got `func` with signature: c3LK|]\}}|jtjjuV dSrv)rr-remptyrs r rz3validate_function_matches_schema..sG  1 L 1 7 7      rzcustom_op(..., manual_schema)(func): When passing in a manual schema, we expect `func` to have no type annotations to avoid ambiguity. Got `func` with signature: cPg|]#\}}|jtjjk||f$Sr)rpr-rrrr;rs r z4validate_function_matches_schema..s= D% :*@ @ @ u @ @ @rcPg|]#\}}|jtjjk||f$Sr)rpr-rrr s r r z4validate_function_matches_schema..s= D% :*7 7 7 u 7 7 7rc.tdd)Nzcustom_op(..., manual_schema)(func): When passing in a manual schema, we expect `func`'s signature to match `manual_schema` (aside from type annotations). func's signature: , manual_schema: r/rIsigsr rz/validate_function_matches_schema..error= @"% @ @8> @ @   rc.tdd)Nzycustom_op(..., manual_schema)(func): neither func nor manual_schema should have default arguments. Got func's signature: rrrsr error_default_argsz.error_default_args'rrc t|t|kr t||D]F\\}}}||jkr |jtjjus|j GdSrv)rzipr;defaultr-rr )sig_args schema_argsr;rargrrs r comparez1validate_function_matches_schema..compare/s x==C ,, , , EGGG"%h "<"< % % MT53sx}G$5$;;;s{?V""$$$  % %r) r- signatureall parametersitemsr/rreturn_annotation Signaturer rflat_positionalflat_kwarg_only)rIrG positional kwargonlyrrrrs` @@@r r6r6s  D ! !C EEcn.B.B.D.DEEE E E  B//11J >//11I             %%%%%% GJ(8999 GIv'788888rrrc |dkrt|d|dkrt|d|dvr/|}t|d|d|d|d t|d |d ) N Undefineda: There were no Tensor inputs to this operator (e.g. you passed an empty list of Tensors). If your operator is a factory function (that is, it takes no Tensors and constructs a new one), then please use CustomOp.impl_factory to register an implementation for itrz: when running with device='Meta' tensors: there is no abstract impl registered for this CustomOp. Please register one via CustomOp.impl_abstract to get this CustomOp to work with Meta tensors)rrz: when running with device='z' tensors: there is no zW impl registered for this CustomOp. Please register one via CustomOp.impl(device_type='z')z%: No implementation for dispatch key z. It is likely that we have not added this functionality yet, please either open an issue or if you're feeling adventurous, use the low-level torch.library API)NotImplementedErrorlower)rrdevices r rFrF<s k! ( ( (    f}}! U U U    o! 5 5f 5 5 5 5*0 5 5 5      3     rcZ|j}tj|d}|dd}t |j}|dd}tj |}t|||||dS)Nr)rSTr*) namespacerr7r8r;rrrWrr4r)oprHrLr;rJrIs r custom_op_from_existingr1Zs B -  J / /C 7799??4  $DRZJ!!$''+J  !* - -F CVT2t D D DDrcDfd}t\}}ttj|s |t tj|}t||s |t ||}t|ds ||jS)Nc*tdd)NzCould not find the operator z~. Please make sure you have already registered the operator and (if registered from C++) loaded it via torch.ops.load_library.r)r"sr error_not_foundzget_op..error_not_foundfs+ 58 5 5 566 6rr)r1r~rr}r|r)r"r4rHr;rpackets` r rres66666 h''HB 59b ! !%)R((K ; % % [$ ' 'F 69 % % >rFc|tvr t|S|std|dt|}t|}|S)NzCould not find custom op "z5". Did you register it via the torch._custom_ops API?)rPrVrr1)r"also_check_torch_libraryoverloadrNs r _find_custom_opr9xsi?""x(( #+ * * * *++ +hH $X . .F Mrc|tjjjvrdStjjj|}|dS|dsdS|djS)Nr)r _custom_opr>rPrhryrG)r"rs r get_abstract_implr<sku',<<<t %5h?It   z * *t   z * * //rTc @|d\}}||}tj|}t||rtjjjgng}tj |d}| ||t||j } t||||| d} | tj| t!jt$t'j| t+|S)NrSr))tagsTr*)rrr4r5rrBTagneeds_fixed_stride_orderr7r8r9r:r;rrdrCrDrErFr@rAr) r"rIr@rHr;rJrKr>rLrMrNs r _custom_op_with_schemarAs ~~d##HB"&""J$*:66OO$$$6N VEHL 1 2 2TVD /"j ) )CJJzJ%%%%b/*>??H c2hPT U U UF 00222 H00)#$97=;P;PQQ (  rrv)F)T)8 dataclassesrDr-rkrr@rtorchgen.modelrrrrrrrtorch._CrB torch.libraryr7r rr r torch._library.infer_schemar __all__rrr!rOptionalrrrPDictrr dataclassror:r2r5Tupler1rrrrr6AnyrFr1rr9r<rArrr rMs ________________ !!!!!!LLLLLLLL""""444444 / . .     MMM:>333"(/#"63 _3333x13S*_-222~~~~~~~~B   3|  3  4      N t    " S V\#s(%;     c d    7,>9 >9"(/>9 >9>9>9>9BVZcd<EEE&    000r