EjIbhddlmZddlZddlmZmZmZddlZddlm Z ddlm Z ddl m Z er ddl mZmZmZdNd ZdOdZdPdZdQdZdQdZ dRdSd!ZdTd&Z dRdUd(Z dVdWd.ZdXd0ZdYd2ZGd3d4e jZGd5d6e jZ dRdZd9Z dRd[d;Z d\d?Z!d]dBZ" d^d_dKZ# d^d`dMZ$dS)a) annotationsN)AnyNoReturn TYPE_CHECKING)Tensor)NamedMemberAccessor)CallableIterableSequencereturnrc td)Na$make_functional(module): we don't yet support models that do parameter tying (also sometimes known as weight sharing). Please try to rewrite your model by replacing all instances of the tied parameter with another and/or comment your support in https://github.com/pytorch/functorch/issues/446) RuntimeErrorNC:\Projects\DSCons\.venv\Lib\site-packages\torch\_functorch\make_functional.pyraise_parameter_tying_errorrs  :  r named_params0dict[str, Tensor] | Iterable[tuple[str, Tensor]]tied_named_paramsdict[str, list[str]]cBt|}t|}t|}t|}||st d|d|i}|D] \}}|gf||< |D]=\}}||vrt d|d||d|>t|S)a[ named_params is a dictionary of tensors: {'A': A, 'B': B} tied_named_params is another dictionary of tensors {'A': A, 'B': B, 'B_tied': B} with potentially tied (or 'duplicated') tensors This function creates a mapping from the names in named_params to the names in tied_named_params: {'A': ['A'], 'B': ['B', 'B_tied']}. ztensors_dict_keys z+ is not a subset of tied_tensors_dict_keys ztensor for key 'z ' not found in tensor_to_mapping)dictsetkeysissubsetAssertionErroritemsappendvalues) rrnamed_params_dicttied_named_params_dicttensors_dict_keystied_tensors_dict_keystensor_to_mappingkeytensors rcreate_names_mapr($sb\**!"344-224455 !7!!>??  % %&< = =  ?!2 ? ?&< ? ?   >@(..00.. V%("I&!!-335511 V * * * H3HHH  &!!$++C0000 !((** + ++rmod nn.Module named_members+Callable[..., Iterable[tuple[str, Tensor]]]subclassCallable[[Tensor], Tensor]@tuple[tuple[Tensor, ...], tuple[str, ...], dict[str, list[str]]]ct|d}t|d}t||}i}t|}|D]I\}} | |vr"|tj| d|| <|| } ||| Jt |dkrd\} } n t|\} } | | |fS)NF)remove_duplicateTmeta)devicerrr)tupler(rtorch empty_like set_tensorlenzip) r)r+r-all_named_membersunique_named_members names_mapmemoaccessornamep replacementnamesparamss r_extract_membersrEGs mmUCCCDD !E!E!EFF !57HIII D"3''H$//a D==hu/&AAABBDG1g D+....   A%% vv12 v 5) ##rcBt||jtjS)aZ This function removes all the Parameters from the model and return them as a tuple as well as their original attribute names. The weights must be re-loaded with `load_weights` before the model can be used again. Note that this function modifies the model in place and after this call, mod.parameters() will be empty. )rEnamed_parametersnn Parameterr)s rextract_weightsrKas C!5r| D DDrc0t||jdS)Nc|SNr)xs rz!extract_buffers..rsar)rE named_buffersrJs rextract_buffersrRos C!2KK @ @@rFrC Sequence[str]rDSequence[Tensor] as_paramsboolNoneclt|}|r d|D}|||dS)a  Reload a set of weights so that `mod` can be used again to perform a forward pass. Note that the `params` are regular Tensors (that can have history) and so are left as Tensors. This means that mod.parameters() will still be empty after this call. c6g|]}tj|Sr)rHrI).0rAs r z load_weights..s 222a",q//222rNr set_tensors)r)rCrDrUr?s r load_weightsr^usH#3''H3226222 '''''rr=elemsIterable[Tensor] list[Tensor]c4g}t|}t||D]c\\}}}t|D]K\}} |dkr*||| |5|| |Ld|S)Nr)rr:r enumerater swap_tensorr8) r)r=r_resultr?_ attr_nameselemi attr_names r _swap_staterksF"3''H!$Y__%6%6!>!>55J%j11 5 5LAyAvv h229dCCDDDD##It4444  5 MrbufferscPt|}|||dSrNr\)r)rCrlrUr?s r load_buffersrns- #3''H (((((rrmodelweights weight_names buffer_namesct|t|kr/tdt|dt|t|||t|dkr`t|t|kr/tdt|dt|t||||S)zload_state(model, weights, weight_names, buffers=(), buffer_names=()) -> model load_state takes `weights` and `buffers` and assigns them to the model. This is the inverse operation of `make_functional_deprecated_v1`. zlen(weight_names)=z != len(weights)=rzlen(buffer_names)=z != len(buffers)=)r9rr^rn)rorprqrlrrs r load_staterts <CLL(( S\!2!2 S SS\\ S S    g... 7||a |  G , , WS%6%6WWWWW  UL'222 Lr>tuple[tuple[Tensor, ...], Callable[..., Any], tuple[str, ...]]ct}t|dkrtdt \}}d fd }||fS) amake_functional_deprecated_v1(model) -> weights, func, weight_names Given an nn.Module, make_functional_deprecated_v1 extracts the state (weights) and returns a functional version of the model, `func`. This makes it so that it is possible use transforms over the parameters of `model`. `func` can be invoked as follows: ``` x = torch.randn(4, 3) model = nn.Linear(3, 3) weights, func, _ = make_functional_deprecated_v1(model) func(weights, (x,)) ``` And here is an example of applying the grad transform: ``` x = torch.randn(4, 3) model = nn.Linear(3, 3) weights, _, func = make_functional_deprecated_v1(model) grad_weights = grad(func)(weights, (x,)) ``` To put the state back into a model, use `load_state`. rzmake_functional_deprecated_v1(model): `model` has buffers. Please use make_functional_with_buffers_deprecated_v1(model) instead.rptuple[Tensor, ...]datatuple[Any, ...]r rcXtj}t||||SrN)copydeepcopyr^)rprx mutable_model descriptorsros rfunz*make_functional_deprecated_v1..funs1 e,, ]K999}d##r)rprwrxryr r)listrlr9rrK)rorlrprfrr~s` @rmake_functional_deprecated_v1rs85==??##G 7||a I   .e44G[!$$$$$$$ C $$rctuple[tuple[Tensor, ...], tuple[Tensor, ...], Callable[..., Any], tuple[str, ...], tuple[str, ...]]crt\}}t\}}d fd }|||fS) a`make_functional_with_buffers_deprecated_v1(model) -> weights, buffers, func, weight_names, buffer_names Given an nn.Module, make_functional_with_buffers_deprecated_v1 extracts the state (weights and buffers) and returns a functional version of the model, `func`. `func` can be invoked as follows: ``` x = torch.randn(4, 3) model = nn.Linear(3, 3) weights, buffers, func, _, _ = make_functional_with_buffers_deprecated_v1(model) func(weights, buffers, (x,)) ``` And here is an example of applying the grad transform: ``` x = torch.randn(4, 3) model = nn.Linear(3, 3) weights, buffers, func, _, _ = make_functional_with_buffers_deprecated_v1(model) func(weights, buffers, (x,)) grad_weights = grad(func)(weights, buffers, (x,)) ``` To put the state back into a model, use `load_state`. rprwrlrxryr rcztj}t||t||||SrN)r{r|r^rn)rprlrxr}buf_descriptorsroweight_descriptorss rrz7make_functional_with_buffers_deprecated_v1..funsF  e,, ]$6@@@]OW===}d##r)rprwrlrwrxryr r)rKrR)rorprfrlrrrs` @@r*make_functional_with_buffers_deprecated_v1rsmB&5U%;%;"G "1%"8"8G_a$$$$$$$$ GS"4o EErcHeZdZdZdfd Ze dddZddZxZS)FunctionalModuleWithBufferszW This is the callable object returned by :func:`make_functional_with_buffers`. stateless_modelr* param_namestuple[str, ...]rrparam_names_maprbuffer_names_mapr rWct||_||_||_t ||_|j|dSrN)super__init__rrrrr all_names_mapupdate)selfrrrrrr __class__s rrz$FunctionalModuleWithBuffers.__init__sa .&(!/22 !!"233333rFrodisable_autograd_trackingrVJtuple[FunctionalModuleWithBuffers, tuple[Tensor, ...], tuple[Tensor, ...]]ctj|}t|\}}}t|\}}}|r|D]} | dt |||||||fSNF)r{r|rKrRrequires_grad_r) ror model_copyrDrrrlrrrparams r _create_fromz(FunctionalModuleWithBuffers._create_from*s ]5)) />z/J/J, _2A*2M2M// $ , , ,$$U++++ 'KHX       rrDr`rlargsrkwargsct|j|jt|t|z} |j|i|t|j|j|S#t|j|j|wxYwrN)rkrrr5)rrDrlrr old_states rforwardz#FunctionalModuleWithBuffers.forward=s    &MME'NN *   M'4'888 ,d.@) L L L LK,d.@) L L L Ls A""A?) rr*rrrrrrrrrr rWFror*rrVr r) rDr`rlr`rrrrr r __name__ __module__ __qualname____doc__r staticmethodrr __classcell__rs@rrrs444444  func, params Given a ``torch.nn.Module``, :func:`make_functional` extracts the state (params) and returns a functional version of the model, ``func``. This makes it so that it is possible use transforms over the parameters of ``model``. ``func`` can be invoked as follows: .. code-block:: python import torch import torch.nn as nn from functorch import make_functional x = torch.randn(4, 3) model = nn.Linear(3, 3) func, params = make_functional(model) func(params, x) And here is an example of applying the grad transform over the parameters of a model. .. code-block:: python import torch import torch.nn as nn from functorch import make_functional, grad x = torch.randn(4, 3) t = torch.randn(4, 3) model = nn.Linear(3, 3) func, params = make_functional(model) def compute_loss(params, x, t): y = func(params, x) return nn.functional.mse_loss(y, t) grad_weights = grad(compute_loss)(params, x, t) If the model has any buffers, please use :func:`make_functional_with_buffers` instead. Args: model (torch.nn.Module): Input model. disable_autograd_tracking (bool): Flag to disable gradients tracking for output parameters. The returned params are unrelated to the set of params from the original model. If False (default), the params will have ``requires_grad=True`` on them (aka they will be trackable with regular PyTorch autograd), matching the requires_grad-ness of the params from the original model. Otherwise, the returned params will have ``requires_grad=False``. Default, False. If you plan on using regular PyTorch autograd (e.g., if you want to call ``.backward()`` or ``torch.autograd.grad()``, then set ``disable_autograd_tracking=False``. Otherwise, if you're only planning on using functorch's gradient transforms, then please set ``disable_autograd_tracking=True`` to avoid unnecessarily tracking history with PyTorch autograd. rzdmake_functional(model): `model` has buffers. Please use make_functional_with_buffers(model) instead.r)rrlr9rrr)rorrls rmake_functionalrwsgz5==??##G 7||a ;     ( ( )B )  rrc:t||S)amake_functional_with_buffers(model, disable_autograd_tracking=False) -> func, params, buffers Given a ``torch.nn.Module``, make_functional_with_buffers extracts the state (params and buffers) and returns a functional version of the model ``func`` that can be invoked like a function. ``func`` can be invoked as follows: .. code-block:: python import torch import torch.nn as nn from functorch import make_functional_with_buffers x = torch.randn(4, 3) model = nn.Linear(3, 3) func, params, buffers = make_functional_with_buffers(model) func(params, buffers, x) And here is an example of applying the grad transform over the parameters of a model: .. code-block:: python import torch import torch.nn as nn from functorch import make_functional_with_buffers, grad x = torch.randn(4, 3) t = torch.randn(4, 3) model = nn.Linear(3, 3) func, params, buffers = make_functional_with_buffers(model) def compute_loss(params, buffers, x, t): y = func(params, buffers, x) return nn.functional.mse_loss(y, t) grad_weights = grad(compute_loss)(params, buffers, x, t) Args: model (torch.nn.Module): Input model. disable_autograd_tracking (bool): Flag to disable gradients tracking for output parameters. The returned params are unrelated to the set of params from the original model. If False (default), the params will have ``requires_grad=True`` on them (aka they will be trackable with regular PyTorch autograd), matching the requires_grad-ness of the params from the original model. Otherwise, the returned params will have ``requires_grad=False``. Default, False. If you plan on using regular PyTorch autograd (e.g., if you want to call ``.backward()`` or ``torch.autograd.grad()``, then set ``disable_autograd_tracking=False``. Otherwise, if you're only planning on using functorch's gradient transforms, then please set ``disable_autograd_tracking=True`` to avoid unnecessarily tracking history with PyTorch autograd. r)rr)rors rmake_functional_with_buffersrs(t ' 3 3 )B 4  rtuple_of_tuple_of_tensorstuple[tuple[Tensor, ...], ...]rwcdtt|}td|D}|S)Nc3bK|]*}tj|V+dSrNr6stackdetachrZshardss r z"transpose_stack..sI)/ F""$$r)r5r:)rresultss rtranspose_stackrsJ!&c+D&E F F3LG NrmodelsSequence[nn.Module]ct|dkrtdtd|Ds(td|Dstdt|dtfd|Dstdt d|D\}}}t |}t |}|d||fS) a(combine_state_for_ensemble(models) -> func, params, buffers Prepares a list of torch.nn.Modules for ensembling with :func:`vmap`. Given a list of ``M`` ``nn.Modules`` of the same class, stacks all of their parameters and buffers together to make ``params`` and ``buffers``. Each parameter and buffer in the result will have an additional dimension of size ``M``. :func:`combine_state_for_ensemble` also returns ``func``, a functional version of one of the models in :attr:`models`. One cannot directly run ``func(params, buffers, *args, **kwargs)`` directly, you probably want to use ``vmap(func, ...)(params, buffers, *args, **kwargs)`` Here's an example of how to ensemble over a very simple model: .. code-block:: python num_models = 5 batch_size = 64 in_features, out_features = 3, 3 models = [torch.nn.Linear(in_features, out_features) for i in range(num_models)] data = torch.randn(batch_size, 3) fmodel, params, buffers = combine_state_for_ensemble(models) output = vmap(fmodel, (0, 0, None))(params, buffers, data) assert output.shape == (num_models, batch_size, out_features) .. warning:: All of the modules being stacked together must be the same (except for the values of their parameters/buffers). For example, they should be in the same mode (training vs eval). This API is subject to change -- we're investigating better ways to create ensembles and would love your feedback how to improve this. rz?combine_state_for_ensemble: Expected at least one model, got 0.c3$K|] }|jV dSrNtrainingrZms rrz-combine_state_for_ensemble..4s$++q ++++++rc3&K|] }|j V dSrNrrs rrz-combine_state_for_ensemble..4s&2R2Raqz>2R2R2R2R2R2RrzTcombine_state_for_ensemble: Expected all models to have the same training/eval mode.c3:K|]}t|uVdSrN)type)rZr model0_typs rrz-combine_state_for_ensemble..:s.55tAww*$555555rzHcombine_state_for_ensemble: Expected all models to be of the same class.c,g|]}t|Sr)rrZros rr[z.combine_state_for_ensemble..?s! B B B% &u - - B B Br)r9rallrr:r)rfuncsrDrlrs @rcombine_state_for_ensemblersP 6{{a M    ++F+++ + + s2R2R62R2R2R/R/R  0   fQiJ 5555f555 5 5  V   ! B B6 B B BE67V $ $Fg&&G 8VW $$rcpu model_classtype[nn.Module]ensemble_shapetuple[()] | tuple[int, ...]r3torch.types.DeviceMCallable[..., tuple[tuple[Tensor, ...], Callable[..., Any], tuple[str, ...]]]cdfd }|S)Nrrrr ruc*t dkrtdt dkr* i }t|S d}|dkrtd|dt  fdt |D}t i\}}}t d|D}t t |}t d|D}|||fS) N,NYI: ensemble_shape with more than 1 elementr num_models  should be > 0c3NK|]}iV dSrNtorZrfrr3rrs rrz3functional_init..wrapped..WQ  89KK ( ( ( + +F 3 3      rc3@K|]}t|dVdS)rN)rrs rrz3functional_init..wrapped..[s0TTE5e<.wrapped..]8KK F++2244KKKKKKr)r9 ValueErrorrrr5ranger:) rrro num_modelsrrffnrCrpr3rrs `` rwrappedz functional_init..wrappedKs_ ~  ! # #KLL L ~  ! # #K00033F;;E077 7#A& ??E:EEEFF F       =B:=N=N     5[[$5Q&5Q5QRR 2uTTVTTTTTW &&KK7KKKKKE!!r)rrrrr rurrrr3rs``` rfunctional_initrFs4 """"""""* NrCallable[..., tuple[tuple[Tensor, ...], tuple[Tensor, ...], Callable[..., Any], tuple[str, ...], tuple[str, ...]] | tuple[tuple[Tensor, ...], Callable[..., Any], tuple[str, ...]]]cdfd }|S)Nrrrr tuple[tuple[Tensor, ...], tuple[Tensor, ...], Callable[..., Any], tuple[str, ...], tuple[str, ...]] | tuple[tuple[Tensor, ...], Callable[..., Any], tuple[str, ...]]ct dkrtdt dkr* i }t|S d}|dkrtd|dt  fdt |D}t  i\}}}}}tt d|D\} } t t| } t d| D} t t| } t d | D} | | |||fS) Nrrrrrc3NK|]}iV dSrNrrs rrz@functional_init_with_buffers..wrapped..rrc3DK|]}t|ddVdS)Nr)rrs rrz@functional_init_with_buffers..wrapped..sG;5AA"1"Erc3bK|]*}tj|V+dSrNrrs rrz@functional_init_with_buffers..wrapped..rrc3bK|]*}tj|V+dSrNrrs rrz@functional_init_with_buffers..wrapped..rr)r9rrrr5rrr:)rrrorrrfrrqrrrprlr3rrs`` rrz-functional_init_with_buffers..wrappedrs ~  ! # #KLL L ~  ! # #K00033F;;E077 7#A& ??E:EEEFF F       =B:=N=N      7{{D7SF7S7S T T     #  W &&KK7KKKKKW &&KK7KKKKK\<??r)rrrrr rrrs``` rfunctional_init_with_buffersrcs=)@)@)@)@)@)@)@)@V Nr)r r)rrrrr r)r)r*r+r,r-r.r r/)r)r*r r/r) r)r*rCrSrDrTrUrVr rW)r)r*r=rr_r`r ra) r)r*rCrSrlrTrUrVr rWr4) ror*rprTrqrSrlrTrrrSr r*)ror*r ru)ror*r rrr)rrr rw)rrr r)rr)rrrrr3rr r)rrrrr3rr r)% __future__rr{typingrrrr6torch.nnrHr%torch.nn.utils._named_member_accessorrcollections.abcr r r rr(rErKrRr^rkrnrtrrModulerrrrrrrrrrrrs#""""" ////////// EEEEEE=<<<<<<<<<< , , , ,F$$$$4 E E E EAAAA ((((("    $ )))))!#"$ 4)%)%)%)%X.F.F.F.Fb9M9M9M9M9M")9M9M9Mx#I#I#I#I#Iry#I#I#IN9>EEEEER9><<<<<~;%;%;%;%@35!&>35!&:::::::r