UjpJddlmZddlZddlZddlZddlZddlZddlm Z m Z m Z m Z ddl Z ddlmZmZmZmZmZmZddlmZddlmZe rddlmZddlmZdd lmZed e fZ!ed e fZ"ej#e$Z%Gd d ej&Z'e'j(de'j)de'j*de'j+de'j,de'j-de'j.dddiZ/de0e/DZ1Gddej&Z2de2DZ3ej4dddGdd Z5ej4GddZ6ej4dddGd d e5Z7dBd%Z8dCd'Z9dDd)Z:dEd-Z;Gd.d/Z<Gd0d1Z=Gd2d3Z>ej4Gd4d5Z?Gd6d7Z@Gd8d9ZAej4Gd:d;ZBej4Gd<d=ZCGd>d?ZDGd@dAZEdS)F) annotationsN)AnycastLiteral TYPE_CHECKING) _EventType_ExtraFields_Allocation_ExtraFields_TorchOp_ProfilerEvent_TensorMetadata RecordScope) _element_size)_utils)Iterator)FunctionSchema)_ProfilerResultKey TensorKeyceZdZejZejZejZejZejZ ejZ ejZ dS)CategoryN) __name__ __module__ __qualname__enumautoINPUT TEMPORARY ACTIVATIONGRADIENTAUTOGRAD_DETAIL PARAMETEROPTIMIZER_STATEMC:\Projects\DSCons\.venv\Lib\site-packages\torch\profiler\_memory_profiler.pyrr%si DIKKE IJty{{HdikkO IdikkOOOr$r darkgreen goldenrodblack mediumpurplered mediumblue royalbluegreyci|]\}}|| Sr#r#).0ics r% r2:sFFFtq!aFFFr$ceZdZejZejZejZejZdS)ActionN) rrrrr PREEXISTINGCREATEINCREMENT_VERSIONDESTROYr#r$r%r4r4=sC$)++K TY[[F! dikkGGGr$r4ci|] }||j Sr#valuer/r0s r%r2r2Ds///1Aqw///r$TF)eq unsafe_hashfrozenceZdZUded<dS)r torch.devicedeviceN)rrr__annotations__r#r$r%rrGsr$c@eZdZUdZded<ded<ddZdd Zdd Zd S)_StoragezBundle storage pointer and id. All profiling logic should use `allocation_id`, however it is useful to print storage pointers for debugging and unit tests sometimes look up values using the storage data pointer of a live Tensor.intptr allocation_idreturnstrcBt|jdd|jdS)Nz>18 ())hexrGrHselfs r%__repr__z_Storage.__repr__Ws'dh--<< > > QK a ' ? x{(+215 G G ?I))(1+*;A*>*Ea*HIIIIIIII A*+ + +z!}   *|/E/Q !JKK K   * , 3 > N N 1f++A.. 0E0Ef0M0MMMMMM  ! - , 6 A N N 61++A.. 0E0Ef0M0MMMMMM , + . - N Nr$Iterator[TensorKey]c#@Kt|D] \}}||V dSrVr)r~r_p_grads r%extract_parametersrs:7== 7 =GGGr$,Iterator[tuple[TensorKey | None, TensorKey]]c#DKt|D] \}}|||fVdSrVr)r~rrs r%extract_gradientsrsA7t<< 6  V)OOOr$event_ProfilerEvent | Nonetuple[RecordScope, ...]cg}|rI|jdtjkr%||jdj|j}|It |SNrr)rrrappendrparenttuple)rscopess r% get_scopesrs] F  ;q>Z/ / / MM%+a.. / / /   ==r$creZdZdZeddZeddZedd ZeddZ dS) SchemaMatcheraLookup operator schema based on profiled name. When profiling we record the operator's name but not the schema. However some analysis requires that information. Fortunately we can look up registered schema from the recorded name. We do not, however, record the overload and so we must compare the profiled arguments with all overloads to determine viable matches. Note: Once https://github.com/pytorch/pytorch/issues/78871 is completed this code will be obsolete. rrr rItuple[bool | None, ...]c d}||D]S}|pd|jD}t|jD])\}}||xxt|jddzcc<*Tt |pd|jDS)aDetermine which inputs may have mutated based on function schema. Note that we don't need to resolve down to a single schema to perform this analysis. An input is mutable if it is mutable in any overload. In practice, however, it is overwhelmingly common to match a single overload. If we cannot find any valid schema then we must be conservative and assume all inputs are mutable. Ncg|]}dS)Fr#r/rs r% z4SchemaMatcher.inputs_are_mutable..s!B!B!BA%!B!B!Br$is_writeFc3K|]}dVdSrVr#rs r% z3SchemaMatcher.inputs_are_mutable..s" 8 8! 8 8 8 8 8 8r$) match_schemas arguments enumerategetattr alias_inforr)rprrmutableschemar0args r%inputs_are_mutablez SchemaMatcher.inputs_are_mutables&*''** I IFB!B!B1A!B!B!BG#F$455 I I3 gcnj%HHH  IW8 8 8qx 8 8 8999r$tuple[FunctionSchema, ...]ctd|jDdfd tfd|jpdDS)Nc3K|]V}t|trt|n"t|tr d|Dn|VWdS)cBg|]}t|Sr#)rrv)r/js r%rz9SchemaMatcher.match_schemas...s&666q)''**666r$N)rWr rrvlistr<s r%rz.SchemaMatcher.match_schemas..s   !_-- I ! !! $ $ $ !T""66A6666       r$rIrTct|jtko/tfdt|jdDS)Nc3TK|]"\}}||jV#dSrV) _types_matchry)r/observed schema_argrps r%rz?SchemaMatcher.match_schemas..matches..sUCC(Hj  :?;;CCCCCCr$Tstrict)lenrallzip)rrp signatures r%matchesz,SchemaMatcher.match_schemas..matchesszv'((C NN:sCCCC,/v/---CCC@@ r$c32K|]}| |VdSrVr#)r/srs r%rz.SchemaMatcher.match_schemas.. s0OO1GGAJJOQOOOOOOr$r#rIrT)rrlookup_schemasr)rprrrrs` @@r%rzSchemaMatcher.match_schemass   X             OOOO 2 216 : : @bOOOOOOr$rTc ~t|tjjr.|}|dup|||St|tjjrdS|tjj r.t|totd|DStjj tftjjtdftjjt"ftjjt&ftjjt*ftjjt.ftjjt"t&t*t.fff}|D]'\}}t||rt||cS(|duS)NTc3@K|]}t|tVdSrV)rWrr<s r%rz-SchemaMatcher._types_match..s=66-. 1i((666666r$)rWtorch_C OptionalTypegetElementTyperAnyType isSubtypeOfListType ofTensorsrr TensorTyperNoneTyperyBoolTyperTIntTyperF FloatTypefloat ComplexTypecomplex NumberType)rpr schema_typetype_mapjit_typepy_typess r%rzSchemaMatcher._types_matchs k58#8 9 9 O%4466Kt#Ns'7'7+'N'N N k58#3 4 4 4  " "58#4#>#>#@#@ A A h--#662:66633  X ) , X T + X  % X s # X  ' X !7 + X 4eW"= >E #+ 6 6 Hh+x00 6!(H55555 64r$rrJ!tuple[FunctionSchema, ...] | Nonec d|vrdSttj|S#t$rYdSwxYw)Nz::)rrr_jit_get_schemas_for_operator RuntimeErrorrs r%rzSchemaMatcher.lookup_schemas1sY 4t??EEFF F   44 s4+4 AAN)rrr rIr)rrr rIrr)rrJrIr) rrrr_r|rrrr{rr#r$r%rrs  :::[:$PPP[P2   [ B\r$rc6eZdZd dZd dZed d Zd S)OpTreeresultrrINonec||_tt|d|_dS)Nc|jSrV start_time_nsxs r%z!OpTree.__init__..JsAOr$key)experimental_event_tree _root_nodesrsorteddfs _sorted_nodesrPrs r%__init__zOpTree.__init__HsE!99;;"6$((**:S:S#T#T#TUUr$Iterator[_ProfilerEvent]c/JKtj|jg|Ri|Ed{VdSrV)r traverse_dfsrrPargskwargss r%rz OpTree.dfsLsB&t'7I$III&IIIIIIIIIIIr$tuple[_ProfilerEvent, ...]c|jSrV)rrOs r% sorted_nodeszOpTree.sorted_nodesOs !!r$NrrrIr)rIrrIr)rrrrrr}rr#r$r%rrGsdVVVVJJJJ"""X"""r$rc>eZdZddZddZedd ZddZdS)SizeMapop_treerrIrc i|_|jD]A}|jdtjkr9||jdD]}||W|jdtjkr|jd}|j|j td|j=|jj D]0\}}}||||1|j Z|j j D]M\}}}|||||D]\}}||NCi} |jD]}|jdtj kr{|jd} t| } | rRt| j} | | | } | | kr"| d| }t$d||j| dS)Nrrrz vs. z(Mismatch between allocation and free: %s)_valuesrrrr_flat_tensor_inputs_update_valuesrrrrr Allocationrrqabs alloc_size setdefaultlogwarningupdate)rPr r~rrrrrrstate allocations alloc_fieldsrnew_size prior_sizedeltas r%rzSizeMap.__init__UsJ-/ ( 3 3Dz!} 22211$*Q-@@++A''****+A*"333#z!}  '3$.:()RSSS&2(4(;(F44 1f++A...++F3333)5,8,B,M33(65++A...++F333$)33DAq //22223-/ ( W WDz!} 555#z!} // == W"<#:;;H!,!7!7X!F!FJ"X--#- > >H > > $NPUVVV K(((((r$rrrsct|}|||jtjkrt dt |jpdg|jpdgdD}|t|j z}|dkrtd|t |j |d||j |<dSdSdSdS)Nc38K|]}|d|dzVdSrrNr#r<s r%rz)SizeMap._update_values..s? !!qt r$rTrrz$num_bytes must be non-negative, got )rrvlayoutrstridedmaxrsizesstridesrdtyperrget)rPrrrn num_bytess r%rzSizeMap._update_valuess##A&& ?q}U]1J1J%(QC9IqcRV%W%W%WAM!'222I1}}$%WI%W%WXXX #DL$4$4S!$<$.s4SSSdaA4;--a001SSSr$c3(K|] \}}||kVdSrVr#)r/r0rs r%rz(DataFlowNode.__init__..s*88da16888888r$zversion mismatch: z, ) _eventr?_determine_edges_edgesitemsr4r6bumpoutputsrvaluesr)rPrr;redgeversionss` r%rzDataFlowNode.__init__s  595J5J5L5L **,, & &IC| &D$6 &   %%%TSSSdl>P>P>R>RSSS88hoo&7&788888 Q !Oh!O!O$+!O!OPP P Q Qr$dict[TensorKey, DataFlowEdge]c ttj|jg}i}d|DD]}t |jt |dD]\}}t|trPt |}| |t|jt|trT|D]Q}t |}| |t|RԐt!jt$}|D]T\}} |M|r|j|nd||_d| vpt| dk} | ||_U|D]} | jdt2jkr| jdjdkrrt| jd}||} || jt;d|d| _|r|j|nd| _|D]c} | jdt2jkrF| jdjdkr0d|t| jd_dt=t?d |DS) Nc3fK|],}|jdtjk|jdV-dSr )rrrr<s r%rz0DataFlowNode._determine_edges..s;SS!!'!* @R2R2R171:2R2R2R2RSSr$TrrVrrzDouble delete: c3(K|] \}}|||fVdSrVr#r/rArBs r%rz0DataFlowNode._determine_edges..s*MMdaq}Aq6}}}}MMr$) rrrrDrrrrrWr rrvrsetaddr collections defaultdictr2rGr?r@r3r4rrrrrqrdictr) rPsubtreemutable_by_keyr*op_inputrr op_input_iedges mutable_setr4r0rKs r%rEzDataFlowNode._determine_edgess+T[M::;;DFSSwSSS K KB%( =;;B??&&& K K!'h88K#//99C"--c35599==gFFFF $//K&.KK '33J??&11#suu==AA'JJJJ K' 55 . 4 4 6 6 - - CFI+Q4;+=+=c+B+B+Brc (  ;.RE+4F4F'4Q%,c " L LAwqzZ222qwqz7Lq7P7P// ;;Sz?t|';()@3)@)@AAA# @C%KT[%7%7%<%<%<" R RAwqzZ222qwqz7Lq7P7PMQi// ;;<JFMMekkmmMMMMMNNNr$!dict[TensorKey, tuple[bool, int]]cHd|jDS)Nci|];\}}|j |t|jtt|jf.sS   1?   QYc1?!;!;<   r$rFrGrOs r%rzDataFlowNode.inputss3   ))++     r$dict[TensorKey, int]cHd|jDS)Ncfi|].\}}|jr|jr|j||jdn |jdz/Sr)r6r8r4r3rRs r%r2z(DataFlowNode.outputs..s`   1 )*  ;<) AO+qq11D   r$rarOs r%rIzDataFlowNode.outputss1   ))++    r$tuple[TensorKey, ...]cbtd|jDS)Nc3<K|]\}}|j |j|VdSrV)r6r8rRs r%rz-DataFlowNode.intermediates..sK  !Q EF]       r$)rrFrGrOs r% intermediateszDataFlowNode.intermediatess>  +++--      r$rFc|jjSrV)rDrrOs r% start_timezDataFlowNode.start_times {((r$N)rr r;r<rIr)rIrM)rIr^)rIrb)rIrer^) rrrrrEr}rrIrhrjr#r$r%r:r:s Q Q Q Q2O2O2O2Oh   X    X    X )))X)))r$r:c~eZdZddZeddZddZedd Zedd Z ddZ ddZ ddZ dS)r<r rrIrc|_|_i_fdjD_jddS)Nc0g|]}t|Sr#)r:)r/erPs r%rz*DataFlowGraph.__init__..s#LLLaLD11LLLr$c|jSrV)rjrs r%rz(DataFlowGraph.__init__..sALr$r)_op_tree_extract_leaf_events _leaf_events_active_version leaf_events _flow_nodessortvalidate)rPr s` r%rzDataFlowGraph.__init__ sv  55g>><>LLLL4;KLLL "8"8999 r$tuple[DataFlowNode, ...]c*t|jSrV)rrurOs r% flow_nodeszDataFlowGraph.flow_nodessT%&&&r$c Pt}|jD][}t|j}||z}|r't d|jjd|jd|||z}\i}|jD]}|jD]9\}\}}| |d} | |krt d| d|:|jD];\}}| ||} || krt d|d| |||< TorchOp: "My annotation" TorchOp: zero_ TorchOp: fill_ TorchOp: zero_ TorchOp: fill_ The recursive structure of operator calls makes data flow unwieldy. In order to simplify analysis we would like to select the highest level ops to represent in the graph. In this case those are the `zero_` ops; the fact that `fill_` is called is an implementation detail. We also do not want to group everything under "My annotation" as this could create overly coarse bundles and lose critical semantics. To address this issue we walk over the graph and select the topmost torch ops ** which match at least one operator schema **. These form the leaves of the first pass through the op tree. (As well as any allocations or frees which do are not part of a kernel.) These events form the logical nodes in our data flow graph. rnr rIrTc|jdtjkoQ|jdjtjkp1t t|jdSr) rrrrr rrTrr)rns r%leaf_opz3DataFlowGraph._extract_leaf_events..leaf_op^sV71:!33  K$AAA 33AGAJ??@@ r$c|s|jtjkr|gS|jSrV)tagrrrr)rnrtrs r% children_fnz7DataFlowGraph._extract_leaf_events..children_fndsCwqzz QUj&;;;""1%%% : r$)rc|jSrVrrs r%rz4DataFlowGraph._extract_leaf_events..nsqr$r)rnr rIrT)rnr )rrr)r rrrtrs @@r%rqz"DataFlowGraph._extract_leaf_events;sB-/             55  A VK-F-FGGGHHHr$rrrFcf|j|d}|td|d|S)Nrzversion for key  is Nonersrr)rPrrs r%r@zDataFlowGraph.lookupps>&11#q99 ? !AC!A!A!ABB Br$c|j|d}|td|d|dz|j|<dS)Nzprior_version for key rr)rsr'r)rPrrs r%rHzDataFlowGraph.bumpvsR,00d;;   !G#!G!G!GHH H$1A$5S!!!r$cv|j|dtd|dd|j|<dS)Nrzcannot delete key z, already deletedrr.s r%deletezDataFlowGraph.delete|sH   * *3 2 2 : !Lc!L!L!LMM M$(S!!!r$Nr0)rIrxrIrr )r rrIr)rrrIrF)rrrIr) rrrrr}rzrwrtr{rqr@rHrr#r$r%r<r< s'''X'////<!!!X!2I2I2I\2Ih 6666 ))))))r$r<ceZdZUdZded<ejeZded<ejeZ ded<eje Z d ed <dS) CategoryElementNCategory | Noneby_iddefault_factoryzdict[TensorKey, Category]by_keyzdict[TensorAndID, Category] by_versionzset[TensorKey] _by_id_keyset) rrrrrC dataclassesfieldrWrrrSrr#r$r%rrs!E!!!!(9 (9$(O(O(OFOOOO.?k.?PT.U.U.UJUUUU%6K$5c$J$J$JMJJJJJJr$rcfeZdZUejdZded<dd Zdd ZddZ ddZ ddZ dS) CategoryDictc4tjtSrV)rUrVrr#r$r%rzCategoryDict.s 7 H Hr$rz-collections.defaultdict[int, CategoryElement]rrrcategoryrrIrc||j|j_|j|jj|dSrV)rrarrrTrPrrs r% set_by_idzCategoryDict.set_by_ids9%- SV" SV*..s33333r$c:||j|jj|<dSrV)rrarrs r% set_by_keyzCategoryDict.set_by_keys+3 SV#C(((r$rrFc>||j|jj||f<dSrV)rrarrPrrrs r%set_by_versionzCategoryDict.set_by_versions":B SV'g777r$c`|j|jj||f|dSrV)rrarrrs r%setdefault_by_versionz"CategoryDict.setdefault_by_versions0 SV'22C>8LLLLLr$rrct|trt|tsdS|j|j}|jp7|j|dp|j||fdSrV) rWrrrrarrr'r)rPrrelements r%r'zCategoryDict.gets{ c3    3 (B(B 4,sv& M <~!!#t,, <!%%sGnd;; r$N)rrrrrIr)rrrrFrrrIr)rrrrFrIr) rrrrrrrCrrrrr'r#r$r%rrs=N[=NHH>>>G44444444CCCCMMMM       r$rc~eZdZddZeddZdd Zdd Zdd ZddZ ddZ ddZ ddZ ddZ ddZddZdS) MemoryProfilerrrIrct||_t|j|_t |j|_t |_|| | | | | |dSrV)rrpr<_data_flow_graphr  _size_mapr _categories_set_gradients_and_temporaries#_set_parameters_using_python_tracer _set_inputs_set_parameters_using_data_flow_set_activations_set_optimizer_state_set_autograd_detailrs r%rzMemoryProfile.__init__sv -dm < < //'>> ++--- 00222  ,,...  !!### !!#####r$-tuple[tuple[int, Action, KeyAndID, int], ...]c g}ii}jD]?}|jdtjkr |jd}|j}|dk}|j}t|}||||f<dt|j } |j | j f} |rU| |vr&| |tj| df|fd|| <| |tj| df|f| |tj| df| f|| ds&| dtj| df| fA} t)t+| } fd| D} jjD]}|jD]\} }|jr/| df}| |tj| dffnR|jrK|jj}|j}|t?d| | |tj| |ff|j r4| df}| |tj| | | ffɌ|!fd| D|"d tG|S) NrrTFrPcPg|]"\}}|dfv |dkdtj||ff#S)TrrP)r4r5)r/rrallocation_timess r%rz*MemoryProfile.timeline..sK9 9 9 WT{"222w!||#c7^ 47C||r$zinput_version is None for key c3LK|]\}}\}}||||fj|fVdSrV)r)r/timeactionrrrPs r%rz)MemoryProfile.timeline..sU  ,fnsG6C>4>#+> ?      r$c,|d|djfSrr:rs r%rz(MemoryProfile.timeline..s1Q41"4r$r)$rprrrrrrrrqrrBrGrr4r7r6r8popr5_category_snapshotrWrkeysrrzrFrGr6r4rDr3rr8extendrvr)rPoutput live_unknownrrrr6rrtkeyrptr_and_devicesnapshot last_versioneventsr~rKrrs` @r%timelinezMemoryProfile.timelinesb:<>@FH ]&&((  E{1~!666${1~ )4 *Q ' 00>>#>?$dM%:;;l122C&2&6 %CN$ )\99"MM!"F$????\Q 1A"0G,-Sc-S-STTTMM1f&>g"OPPP#Q(#u6AMM1fnsL.s.'T'T91fq!A'T'T'T'T'T'Tr$c3K|]}|dfV dSrNr#r/rs r%rz3MemoryProfile._category_snapshot.. s&&N&NCQx&N&N&N&N&N&Nr$c3K|]}|dfV dSrr#rs r%rz3MemoryProfile._category_snapshot.. s&&K&KCQx&K&K&K&K&K&Kr$cRi|]#\}}||fj||$Sr#rr')r/rrrPs r%r2z4MemoryProfile._category_snapshot..sF   W'ND,00g>>   r$) rSrrzrrrGrhrIrrrJrr)rPall_tensor_versionsr~r0s` r%rz MemoryProfile._category_snapshots03)4 = =D  & &'T'T @Q@Q@S@S'T'T'T U U U  & &&N&N4;M&N&N&N N N N  & &t|'9'9';'; < < < <!)0022 L LA  & &&K&K1?&K&K&K K K K K    &': ; ;    r$set[int]cht t}jjD]p}t fd|jD}|r9|d|jDqt|krS)a\Extract IDs of Tensors which depend or will depend on a gradient. Note that this weakened definition of "depends" requires us to loop over the data flow graph multiple times because it allows dependency information to flow backward through edges and removes the guarantee that nodes are topologically sorted. (Or indeed, even that a valid topological order exists.) Put another way, we have converted an acyclic data flow graph into a cyclic graph and we are attempting to partition cycles involving a gradient from the rest of the graph. Tc3K|]M\}\}}j||tjtjfvs |jvD|jVNdSrV)rr'rrr!ra)r/rrrdepends_on_gradientrPs r%rzAMemoryProfile._any_version_depends_on_gradient..#sw)\a'++C99 )8+=>??v!444 F5444 r$c3$K|] }|jV dSrVrars r%rzAMemoryProfile._any_version_depends_on_gradient...s$.N.N#sv.N.N.N.N.N.Nr$) rSrrrzrrrGrrI)rP start_sizer~idsrs` @r% _any_version_depends_on_gradientz.MemoryProfile._any_version_depends_on_gradients), +011J-8 O O-1[->->-@-@O'..s333'...N.N.N.N.NNNN&'':55**- +r$c.|jD]<}t|D]*\}}|j|t j+=|jjD]1}|j D]'}|j |t j (2dS)z>Mark Tensors which are unambiguous and simple to reason about.N) rprrrrrrrrzrhrr)rPrrrr~r0s r%rz,MemoryProfile._set_gradients_and_temporaries8s ]&&(( F FE.u55 F F 6 **683DEEEE F )4 C CD' C C ++Ax/ABBBB C C Cr$c|jD];}t|D])}|%|j|t j*z,MemoryProfile._set_inputs..ns%SSS*;#|7W~SSSr$c3nK|]/}jj|tjtjfvp|vV0dSrV)rr'rrr!)r/r0produces_gradientrPs r%rz,MemoryProfile._set_inputs..psd% $a(X->@R,SS*))r$N)rrSreversedrrzrrGrIanycopyr rrrDrarrrr)rPrr~tensorsinput_candidatesrrrs` @r%rzMemoryProfile._set_inputsOsw2#CCEE/2eeT2=>> - -DSSt{?P?P?R?RSSSG t|))++ +G! - "W,!-1133)4 > >D, 4;0G0GGG C (:(:(<(<$=$== , U ULCv000 66sGX^TTT U Ur$c\ }t}d|D}jjD]}d|jD}t jt|j vrtfd|Ds{tfd|j DsI| |r4||j z}|| |z}t tjjD]k}t fd|j Dr6 d|jDl| d|D}|z}|D]3\}}|j|vr%j|t*j4dS) aDeduce which Tensors are parameters. Consider the following code for the step of SGD with momentum (nesterov=False), where `d_p` is the gradient of `param` and `buf` is the momentum buffer. ``` buf.mul_(momentum).add_(d_p, alpha=1 - dampening) d_p = buf param.add_(d_p, alpha=-lr) ``` Both `param` and `buf` take a gradient and perform an in-place update. The python tracer will inspect calls to `nn.Module.forward` and `optim.Optimizer.step` to extract parameter and optimizer state respectively (including parameters), so this is generally a non-issue. However as a fallback we can also exploit several properties of parameters to distinguish them from other model state. First, they are directly used in the forward pass. (At this point we haven't established which parts of the graph correspond to the forward pass but we can deduce enough to suffice.) Some mutable state such as batch norm moving averages also contribute to the forward pass, but optimizer state does not. Second, a parameter is by definition used to compute at least one gradient and depends on at least one gradient. c8h|]\}}|tjk|Sr#)rr)r/r0rs r%rz@MemoryProfile._set_parameters_using_data_flow..s/3 3 3 !XX5O5OA5O5O5Or$c"h|] \}\}}||f Sr#r#r/rrr;s r%rz@MemoryProfile._set_parameters_using_data_flow..$NNNsJQsElNNNr$c3,K|]}j|VdSrVrr/r0rPs r%rz@MemoryProfile._set_parameters_using_data_flow..s.BBa-D-q1BBBBBBr$c3,K|]}j|VdSrVrrs r%rz@MemoryProfile._set_parameters_using_data_flow..s.PPa-D-q1PPPPPPr$c34K|]}j|p|vVdSrVr)r/r0rPused_for_gradients r%rz@MemoryProfile._set_parameters_using_data_flow..sL"!1%?.?)?r$c3*K|]\}\}}||fVdSrVr#rs r%rz@MemoryProfile._set_parameters_using_data_flow..s?))'8sLQS'N))))))r$c"h|] \}}|j Sr#r)r/rrs r%rz@MemoryProfile._set_parameters_using_data_flow..sDDDVS!#&DDDr$N)rrSrGrrzrr rrrDrrI intersection differencerrintersection_updaterrarrrr!) rPrcandidate_parameterscandidate_fwd_tensorsr~rparameter_keysrrrs ` @r%rz-MemoryProfile._set_parameters_using_data_flows:**,, 253 3 !)!1!13 3 3 )4 Q QDNN$+:K:K:M:MNNNF-Z 5L5LLLBBBB6BBBBBMPPPP4<;M;M;O;OPPPPPM *66v>> M&););)=)==%$(9(9:O(P(PP$/2eeT2=>>  D++-- "(())<@K.rr$c.h|]}jj|Sr#rrs r%rz1MemoryProfile._set_activations..s'IIIQ 4 0 4a 8IIIr$N)rrrr!rrrzrrGr rrrDrIrr)rPrequired also_allowedr~rinput_categoriesr0s` r%rzMemoryProfile._set_activationssNH$78 *H,>? )4 T TDNN$+:K:K:M:MNNNFIIII&III "H, T)X -DE T  1DK9P9PPP++--TTA:D$:SASx?RSSSSS T Tr$c|jD]}|jdtjkr|jdjr|jdjj}tj d|DD]F\}}t |}|%|j |tjGdS)Nrrc3"K|] \}}}|V dSrVr#)r/rrs r%rz5MemoryProfile._set_optimizer_state..s(99{q!UU999999r$)rprrrrrritchain from_iterablerrvrrrr")rPrrrrrrs r%rz"MemoryProfile._set_optimizer_states]&&(( R RE{1~!222u{1~7O2"[^5@ H2299j999RRDAq$//22C(2238PQQQ R Rr$cVdtjh}|jjD]}tjt |jvrk|j D]Q\}}|dks |j ||dz |vr&|j ||tjRdSr) rr rrzr rrrDrIrGrr'r)rPpriorr~rrs r%rz"MemoryProfile._set_autograd_detailsx/0)4  D, 4;0G0GGG$(L$6$6$8$8LC!||t'7';';C1'M'MQV'V'V(>>(*B   r$Nr)rIrr)rIr)rIrr)rrrrr}rrrrrrrrrrrr#r$r%rrs8 $ $ $ $BBBXBHJJJJ    "+"+"+"+HCCCC"FFFF 1U1U1U1UfHDHDHDHDTTTTT& R R R Rr$rc8eZdZd dZdZd dZd dZ d d d ZdS) MemoryProfileTimelinerIrc6|j|_|j|_dS)aThe minimum representation of the memory profile timeline includes the memory timeline and categories. The timeline consists of [timestamp, action, (TensorKey, version), numbytes] elements, to denote any actions (pre-existing, create, destroy, or increment_version) that occurred to a specific Tensor for a chunk of memory. The categories help map each (TensorKey, version) pair into a category.N)rr categories)rPmemory_profiles r%rzMemoryProfileTimeline.__init__s'/ (4r$cV tj|}g}g d fd }d jD]k\}}\}}} |j|kr|dkrt|dz } dks | kr|dkr| t |dkr>|| dgdt DznN||dkrB||  d|tj tj fvr||||  |tj kr |||| |||dz| ;|tj kr|||| [td | fd |D}| fS) zConvert the memory timeline and categories into a memory plot consisting of timestamps and their respective sizes by category for a given device. Input: device Output: [timestamps, sizes by category] rIrct|trj||nd}t|dz}d|xxt |z cc<dS)NrrP)rWrrr'_CATEGORY_TO_INDEXrF)rrrrrzrPr$s r%rz8MemoryProfileTimeline._coalesce_timeline..update ssc9--##C111  'x014E "Ie   E *     r$rPircg|]}dSrr#rs r%rz.&s#B#B#B!A#B#B#Br$rUnknown action: c$g|] }|dkrn| Srr#)r/rrt_mins r%rz.:s%6661!a%%Q666r$r)rrBrrFrrrrr4r5r6r7r8 ValueError) rP device_strrBtimesrrrrrrnumbytesr$rs ` @@r%_coalesce_timelinez(MemoryProfileTimeline._coalesce_timelines j))!# + + + + + + +37=! >! > /Av~WxzV##BwwDMM{{q5yyQUU5zzQ Q aS#B#B/A#B#B#BBCCCCeBi Q U2Y^^--...&,fm<<<sGX....6333sGhY///sGaK22226>))sGhY////!!.get_category_indexPsCc9--##C111  &h/ /r$rrrNr#)rrBrr4r5r6r_ACTION_TO_INDEXr7r8rr$r%r&) rPr'rrB raw_eventsr,rrrrrr r$r(s ` r%export_memory_timeline_rawz0MemoryProfileTimeline.export_memory_timeline_rawHsH j))68  0 0 0 0 048=/ >/ > /Av~WxzV##&,fm<<<!!(0 **388 6333!!(0! **388 !!(0 **3! << 6>))!!(0! **388 !!.s"TTT1IITTTr$z Time (ms)z Memory (GB)z zMax memory allocated: z.2fz GiB Max memory reserved: z GiBzw+bz.png)suffixpng)formatzutf-8z failed to encode image as base64z} GPU Memory Timeline HTML r#)encoding)&importlib.utilutil find_specprintbase64r5tempfiler6matplotlib.pyplotpyplotnumpyr!arraymincumsumrrBcudamax_memory_allocatedmax_memory_reservedfiguregca_CATEGORY_TO_COLORSrGr fill_betweenlegend set_xlabel set_ylabeljoin set_titlesavefigseekreaddecoderr%write)rPr'rr9title importlibmatplotlib_specr5r6pltnpmtrr$rstackedrBrPrQfigaxesrr;r0tmpfileencodedhtmlr(s r%export_memory_timeline_htmlz1MemoryProfileTimeline.export_memory_timeline_htmls #.22<@@  " V    F$$$$$$//////''''''  $ $Z 0 0xx1AuE  ))E)**W4j))$z>>vFF#j<r|ss"""""" 444444444444 '&&&&&!!!!!!3((((((''''''222222  K$% g!!"""""ty"""   k NG   | k& GFyy1D'E'EFFFTY0////$E$???@? ((((((((($D>>>.X.X.X.X.X.X.X?>.Xb(N(N(N(NV ooooooood " " " " " " " "G!G!G!G!G!G!G!G!T $ $ $ $ $ $ $ $])])])])])])])])@s)s)s)s)s)s)s)s)l KKKKKKKK         >EEEEEEEEP ]]]]]]]]]]r$