jrddlmZddlZddlZddlZddlZddlmZddlm Z m Z m Z m Z ddl Z ddlmZddlmZmZddlmZdd lmZmZmZmZmZmZmZejeZe j !ed Z"e rdd l#m$Z$d,dZ%d,dZ&d,dZ'd-dZ(d.dZ)d/dZ*dZ+dZ,d,dZ-d0d"Z.d1d$Z/d%Z0d2d+Z1dS)3) annotationsN) defaultdict)DictListSet TYPE_CHECKING)StorageWeakRef)configir)WeakDep)contains_collective contains_waitfind_recursive_deps_of_nodefind_recursive_users_of_node is_collectiveis_fallback_opis_waitoverlap)BaseSchedulerNodesnodesList[BaseSchedulerNode]returnc(t|dddS)z7 Greedily schedules waits as late as possible. FT raise_comms sink_waitsreorder_for_overlap_schedule_for_commrs FC:\Projects\Historical\venv\Lib\site-packages\torch/_inductor/comms.pyrr#s$ Ed   c(t|dddS)z8 Greedily schedules comms as early as possible. TFrrr!s r"rr,s$ DU   r#c(t|dddS)a This achieves the following overall scheduling procedure: Step 1: Given that we've currently scheduled comm N, we now schedule all compute nodes that are required for comm N + 1 but do not depend on comm N, to run at the same time with comm N. Step 2: If all those compute nodes are sufficient to overlap comm N, we're done. Otherwise, we now need to look elsewhere to find compute that overlaps with comm N. We prioritize compute nodes that are needed sooner. Step 3: We schedule the compute nodes dependent on comm N and required for comm N + 1. Step 4: We schedule comm N + 1. Repeat this for subsequent comm nodes. Trrr!s r"reorder_compute_for_overlapr&5s$ DTt   r#rboolrrcBi}iiiict|D]\}}|D]}|||<|D]}||<||<|} tj| <d| <|| <d} |D]}|rit |rZ| |<|jD]5} | } t| | | <6| dz } m|r&t|rd|<Gfddd|Dgttd|D D]V\}} t| dkrtj|| D]}||Wgfdfdfd }trPtjj}|rt |r ||n |tP D]%\}} t| dks Jd &S) a Schedule `snodes` for various comm optimization objectives. Args: snodes: the nodes to be scheduled. raise_comms: whether to greedily schedule collectives as early as possible sink_wait: whether to greedily schedule waits as late as possible reorder_compute_for_overlap: whether to reorder compute nodes to optimize for compute/communication overlapping. Returns: The new schedule order. Some notes on the synergy between different options: - `raise_comms` provides more overlapping oppurtunies for `reorder_compute_for_overlap`. - When both `raise_comms` and `sink_waits` is `True`, `raise_comms` is prioritized. rr c(eZdZdfd ZdZdS)$_schedule_for_comm..RunnablerNonec||_tt|}|}|||f|_dSN)snodenextiterget_operation_namesget_namescore)selfr.name fused_namename_to_fused_nodescores_0scores_1scores_2s r"__init__z-_schedule_for_comm..Runnable.__init__sfDJU668899::D+D1::<.Runnable.__lt__s: + +r#N)rr+)__name__ __module__ __qualname__r;r?)r7r8r9r:sr"Runnabler*sQ          , , , , ,r#rCc2i|]}|d|jDS)ch|] }|j S)r5).0deps r" z0_schedule_for_comm...s===S===r#)unmet_dependenciesrGr.s r" z&_schedule_for_comm..s9555BG==E$<===555r#c.i|]}|t|SrF)estimate_op_runtimerKs r"rLz&_schedule_for_comm..s#KKK5U/66KKKr#c||D]_}|D]T}||t|dkrt j|U`dS)zU Schedules `snode` and put all unblocked nodes onto the ready queue. rN)appendget_buffer_namesremovelenheapqheappush)r.buf_namerC buffer_usersready scheduled unmet_depss r"schedulez$_schedule_for_comm..schedules ..00 ; ;H%h/ ; ;5!((222z%())Q..N5((5//::: ; ; ;r#cjdD}t|dkrdSt|dS)zh Return the next node in the ready queue that's neither a collective or a wait. cbg|],}t|jt|j*|-SrF)rr.rrGxs r" zI_schedule_for_comm..get_overlapping_candidate..sM   &qw// 9Fag8N8N    r#rNc|jSr-r=r_s r"zG_schedule_for_comm..get_overlapping_candidate..sQWr#key)rSmin) candidatesrXs r"get_overlapping_candidatez5_schedule_for_comm..get_overlapping_candidatesP      z??a  4:#4#45555r#c*t|sJ||}|dkrSx}G||j||jz}|dkr x}GtjdS)z Schedules collective node `snode`, along with one or more compute nodes to overlap with it. The strategy is described in the comment of `reorder_compute_for_overlap`. rN)rrRr.rTheapify)r.collective_cost candidaterhrXr[ snode_to_costs r"schedule_collective_for_overlapz;_schedule_for_comm..schedule_collective_for_overlaps #5)))))'. a  77999F LL # # # HY_ % % % }Y_= =O a  77999F  er#z;Detected unscheduled nodes. Nodes with unmet dependencies: ) enumeraterQr1r2sysmaxsizer ancestorsrfrrsetitemsrSrTrUaddheappopr.)rrrrbuf_name_to_snodeidxr.rVop_name node_namecomm_idxancanc_fused_namedepsrHrnrCrWrhr7rXr[rYr8r9r:rmrZs @@@@@@@@@@@@r"r r HsL#%r2 Hh'' " " U..00 0 0H*/ h ' '0022 0 0G*/ w ' '/45>>++,NN$$ !k!H++  +.u55 +)1HU^^%% & S S!3C!8!A!A!C!C+.x/G+R+R(( MHH  +M%00 +)*HU^^%% & , , , , , , , , , , , , ,55KQ555JE6A#6F6FLKKFKKKM!'')))) t t99>> N5((5// 2 2 2 ) )C   ! !% ( ( ( ( )I ; ; ; ; ; ; ; ; ; 6 6 6 6 6& e** e$$*  #6u#=#=  + +E 2 2 2 2 HUOOO e**"''))  t4yyA~~~ ;.8 ; ;~~~ r#nodesctjs|Std|Drt |||}d|D}t dt |D]}tt|| }||dz  D],}|| t||-|S)z Decide global ordering of comms, by just enforcing the ordering that's in the input graph (might not be the same ordering as the eager mode program). TODO: Come up with a better approach c3K|]M}t|jtjjjjtjjjjhVNdSr-)rnodetorchopsfsdpall_gather_copy_indefault chunk_catr^s r" z2decide_global_ordering_of_comms..sf     F 19 (0         r#c0g|]}t||SrF)r)rGns r"r`z3decide_global_ordering_of_comms..s&===&9!&<&<=!===r#r  mutating_buf) r distributed is_availableanyenforce_comm_ordering_for_fsdprangerSr/r0rQ add_fake_depr )r name_to_bufr7 comm_nodesirbufs r"decide_global_ordering_of_commsrs   ) ) + +         W/ukCUVV==U===J 1c*oo & &PPDA!?!?!A!ABBCC a!e$5577 P PC qM & &ws'N'N'N O O O O P Lr#r.rfloatctjdkr|}n/ttjsJtj|}|S)z: Returns estimated op runtime in nanoseconds (ns) r)r rNget_estimated_runtimecallable)r.runtimes r"rNrNsQ!Y..--//233333,U33 Nr#c`d}t|jtjrd|jjd}d}|j}t|tjrd|jd|jd}|j pd}|jj j ||d|dS)Nz ()z (size=z , stride=) isinstancerr ExternKernelOutpython_kernel_nameget_output_specLayoutsizestridemaybe_get_name __class__r@)r.detailout_tensor_infolayoutrzs r" node_summaryrs F%*b01176ej3666O Z ' ' ) )F&")$$KJFKJJ&-JJJ ))++1rIj"+ TV T_ T T T T TTr#cd}d}|D]}|t|r|t|z }|j}n5t|jrt d|t|z }t t|t|rt dt|jr+t t|d}t dt|t d|dz dz dS)Ngz8Wait is not expected when there is no collective runningzkFound two collectives running at the same time. `visualize_overlap` needs to be updated to handle this casez| zEst. runtime (ms): i)rrNrrAssertionError overlap_logdebugr)ordertotal_est_runtime cur_comm_noder.s r"visualize_overlaprsr"M>>  "5)) @!%8%?%??! % $$ @$N"%8%?%??!   e!4!46 7 7 7 7"5)) >$R$$ >!!\%%8%8":;;; $ !!"<|E':':"<"<====?/$6=??r#c|}tjD]g}t|tr$|t vrt |}t jdkrmt d|d t|n>#t$r1}t t|Yd}~nd}~wwxYw||}t jdkrot d|d t|(#t$r2}t t|Yd}~_d}~wwxYwi|S)Nrz.==== Visualize overlap before reordering pass z ====z-==== Visualize overlap after reordering pass ) r 'reorder_for_compute_comm_overlap_passesrstrglobalsrrget_rankrrr Exception)rrpes r"$reorder_compute_and_comm_for_overlapr<s E  ;** a   !wyy.. ! A   % % ' '1 , ,   IIII    *!%(((( * * *!!#a&&)))))))) *%   % % ' '1 , ,   HHHH    *!%(((( * * *!!#a&&)))))))) * - Ls0 B C&'CC&D77 E3'E..E3graphtorch.fx.Graphc *t|jtttttD]\}}|jdkr|jt jjj j kr|j djdksJd|d|j dd|j d}|j d}|dkr| || |fd}tt}tD]\}}|jdkru|jt jj jj krQ|}|j djdksJd d |d|r| |d }d D]y}|jdkrlt|jt jjrH|jjjr7||s,||rJd |dz|D]\} t| D]\} } | }|j dusJ|j \} | dz} | t-| dz kr | | dznt-dz }| |}t/fd|DrJdd|d|d|D]b}|jdkrU|j vrL|jt jjj j kr(t1fd|j D}||_ c|D]7\} t| D]"\} } | }||#8D]U}|jdkrH|jt jjj j kr$|j d|vr||VdS)a This FX graph pass replaces uses of FSDP2 unsharded params with their corresponding graph intermediates that were fsdp.copy_ into the unsharded params in the original graph. NOTE: Can only apply this pass to any of the FSDP2 unsharded params that have this pattern (or repetition of): `resize_(full) -> copy_ -> resize_(0)`. Because of this, for partial-graph case where `resize_(full) -> copy_` is in one graph and `resize_(0)` is in another graph, we can't remove these resize and copy ops and thus we will have worse performance there. In other words, "do we try to remove all the resize_(full) -> copy_ -> resize_(0) nodes for this unsharded param" is actually a per-unsharded-param decision, since for each unsharded param, we look at its resize sequence pattern (in `check_resize_pattern()`) to determine if its set of resize and copy nodes can be removed. call_functionr placeholderz1Resize can only operate on graph inputs, but got z# which is resizing non-graph-input  r c|g}|g}t|t|ks@td|dt|dt|ddSt ||D]D\}}||kr9td|d|d|d |d|d dSEd S) NzH Unequal number of resize-to-full and resize-to-0 nodes for graph input z: z vs. zK. Skipping `remove_fsdp2_unsharded_param_graph_input_usage` FX graph pass. Fz For graph input z: resize-to-full node z at index z happens after resize-to-0 node zd. Skipping `remove_fsdp2_unsharded_param_graph_input_usage` FX graph pass for that unsharded param. T)getrSlogwarningzip) graph_inputresized_to_full_idxesresized_to_0_idxesresize_to_full_idxresize_to_0_idx&graph_input_to_resized_to_0_node_idxes)graph_input_to_resized_to_full_node_idxes node_lists r"check_resize_patternzLremove_fsdp2_unsharded_param_graph_input_usage..check_resize_pattern|sm !J M M ! ! DGG UWXX())S1C-D-DDD KKHS"%&8"9"9   547 !#54 4   / "_44 4=>P4Q]o )/ :FUuu5tr#z\ Assumed all FSDP2 `unsharded_param`s to be graph input, but it's not true! Offending node: z . Graph: c|jtjjjjkp#|jtjjjjkSr-)targetrrrcopy_rinductorresize_storage_bytes_)rs r"is_allowed_mutationzKremove_fsdp2_unsharded_param_graph_input_usage..is_allowed_mutations6 K59>/7 7 O{ei0FNN r#ctjtjjr(dt jjjDng}fd|D}d|D}t||zdkS)Nc>g|]\}}|j |jj|Sr-) alias_infois_write)rGrr_s r"r`zyremove_fsdp2_unsharded_param_graph_input_usage..is_node_mutating_unsharded_param_or_its_alias..s8   Aq<+ 0E++++r#c~h|]9}tj|jd:Sval)r argsmetauntyped_storage)rGrrs r"rIzxremove_fsdp2_unsharded_param_graph_input_usage..is_node_mutating_unsharded_param_or_its_alias..sK% % %  49Q<,U3CCEE F F% % % r#cfh|].}t|jd/Sr)r rr)rGunsharded_params r"rIzxremove_fsdp2_unsharded_param_graph_input_usage..is_node_mutating_unsharded_param_or_its_alias..sD( ( (  ?/6FFHH I I( ( ( r#r) rrr_ops OpOverloadro_schema argumentsrS)runsharded_paramsmutated_arg_idxesmutated_node_arg_storagesstorages_of_unsharded_paramss` r"-is_node_mutating_unsharded_param_or_its_aliaszeremove_fsdp2_unsharded_param_graph_input_usage..is_node_mutating_unsharded_param_or_its_aliass$+uz'<==    %dk&9&CDD      % % % % &% % % !( ( #3( ( ( $,/KKLLqPPr#zdUser mutation on FSDP2 unsharded param is not allowed when Traceable FSDP2 is used. Violating node: c32K|]}|gVdSr-rF)rGrrrs r"rzAremove_fsdp2_unsharded_param_graph_input_usage..sG>=d_DUVVr#z(Assumed no ops mutating unsharded param z in subgraph z, but it's not true! Graph: c3(K|] }|urn|V dSr-rF)rGarg replacementrs r"rzAremove_fsdp2_unsharded_param_graph_input_usage..sE%%(+o'='= 3%%%%%%r#N)listrrrooprrrrrrrrPrrrrrr is_mutablekeysrtrSrtuple erase_node)rrxrrnew_sizer'unsharded_param_to_fsdp_copy_node_idxesfsdp_copy_noderfsdp_copy_node_idxesrfsdp_copy_node_idx_subgraph_start_idxsubgraph_end_idxsubgraph_nodesnew_argsrrrrrrs @@@@@@r".remove_fsdp2_unsharded_param_graph_input_usagerXsU[!!I1->*y))PP T G & & uy1GOOO ! =00026[_[def[g100)A,Ky|H!||9+FMMcRRRR6{CJJ3OOO"""""""J/:$.?.?+y)) U U T 7o % %$+9M9U*U*U!N"ilO"m333 +0433 $#O44 U7HOOPSTTT   QQQ,   G & &4; (=>> ' #. '('-- ' ED=BBDD  ei  , 1 6 6 8 8")") %./C%D%D ) ) !A!&'9:N!&q)_<<<<+0NA{!3a!7 s/001444%QU++^^a'  ''9:J'JKN*  )8GU   ' ) )G..'4944 uy'9'O'WWW$%%%%%#'9%%%  H!)DI )) )H 1 6 6 8 8-- %./C%D%D - - !A!&'9:N   ^ , , , , - ## G & & uy1GOOO ! GGG   T " " " ##r#r+c ddl jsJ jjjr jjjsJn#tttf$rYdSwxYwddl m }m }m }m}m} fd}|}|| jjjj|t"j| jjjj|d|d|d|d|d |d |d |d |d |d|dd fd }||||dS)Nrr ) CallFunction KeywordArgMatchPatternMatcherPassregister_graph_patternct|j}|D]e}|jtjkrN|jdjjjjj ur&|jddkr| |fdS)Nrr ) rrroperatorgetitemrrrrrr)grrrs r"remove_unused_getitemz8reinplace_fsdp_all_gather..remove_unused_getitemAsvMM   AH,,,F1I$ (I(QQQF1INN Q   r#all_gather_inputsinp_split_sizesall_gather_input_numel world_sizerankdtypedeviceitem_idx group_size group_namec$|jddkS)Nrr)kwargs)matchs r"rcz+reinplace_fsdp_all_gather..cs%,z":a"?r#) pass_dict extra_checkrrc fd}|||d|d|d|d|d|d|d|d |d g dS) Nc|dd}|d}|d}jjjj|}|d}|d}jjj||||}|S)Nrr )out)rrrr_c10d_functionalall_gather_into_tensor_out) r copy_in_argsrrrr  getitem_1all_gather_into_tensorrs r"replzEreinplace_fsdp_all_gather..reinplace_all_gather..replfs 9LbJbJ!J!B!J" )+G*1-I *EMMZN # * )r#rrrrrrrrr)replace_by_example)rrrr'rs r"reinplace_all_gatherz7reinplace_fsdp_all_gather..reinplace_all_gatherNs0 * * * * *$   *+()/0|$vwx |$|$  r#)rr)5torch.distributed.fsdp._fully_shard._fsdp_collectivesrrrr"r&r# ImportErrorAttributeErrorrpattern_matcherrrrrrrr r rrapply) rrrrrrr  graph_passr)rs @r"reinplace_fsdp_all_gatherr0sM DDDD --///// I & =  *E   F   8      $#%%J I & = E L  IN5=J233J011J788J|,,Jv&&Jw''Jx((   :&&   J| $ $ J| $ $#  &??+.      /.  D%   UsAAA"!A"ct|tjjjtjjjfrJt |ddS)N)rr _inductor schedulerFusedSchedulerNodeGroupedSchedulerNodeintr2)r.s r" get_op_idxr8s_ O % 8 O % :   u~~# $ $$r#1List[torch._inductor.scheduler.BaseSchedulerNode]r4Dict[str, torch._inductor.scheduler.SchedulerBuffer]r7Dict[str, BaseSchedulerNode]c   !ddlm g}t}d}d}i}i}i! !fd} |D]} t| jt jjjj rtfd| j Drd}| } t} t| | |t jjjj t jjj j t jjjj ht!| | | fd t#| d } t%| }d }t't%| D]G}| |}t)|jt jjjj r|dz }|dkr|}nH| d|} d}t't%| dz D]1}t+| |dzjt,jr|dz}n2|J| | d|}| | |d}|||<t)| jt jjjj rd}| }t}t!|||t#|d }d}t't%|dz D]1}t+||dzjt,jr|dz}n2|J| |d|}| ||d}|||<t%!d ksJ|rt%|d ksJ|rt%|d ksJ|D]a} | !vr!| } | |vr7|| || bd}|D]\}}|{t;t=|}| D]8}|!tE||9|}d}|D]\}}|{t;t=|}| D]8}|!tE||9|}|S)Nr )r4Fcj|}|D]}||<||<|Sr-)r6creater2)snodes_to_group group_noder.r4snode_name_to_final_snodes r"_create_group_nodez:enforce_comm_ordering_for_fsdp.._create_group_nodes_3::?KK $ E EE:D %enn&6&6 7 7;E!*"5"5"7"78r#)rc3K|]8}t|jtjjjjV9dSr-)rrrrrrr)rGr_r7s r"rz1enforce_comm_ordering_for_fsdp..sZ   "1%*EIN,M,U        r#Tcvt|jp"t|jo |jjv Sr-)rNopKernelSchedulerNodeExternKernelSchedulerNoder op_overload)r_ allowed_opsr4s r"rcz0enforce_comm_ordering_for_fsdp..sAq)"BCC"1i&IJJ>F.+= 'r#) criteria_cbc t|Sr-r8rbs r"rcz0enforce_comm_ordering_for_fsdp.. JqMMr#rdrc t|Sr-rKrbs r"rcz0enforce_comm_ordering_for_fsdp.. rLr#r)#rr4rsrrrrr"r#rrrrr wait_tensorrsplit_with_sizes_copyrsortedrSrrrr _WaitKernelrr2rPrurtr/r0rQ get_outputsrr )"rrr7 new_orderrY ag_exists rs_exists$ag_grouped_node_to_wait_grouped_node$rs_grouped_node_to_wait_grouped_noderBr.ag_snodeag_related_snode_setag_related_snodesend_idx_of_current_ag_blockcopy_out_countr cur_snode wait_node_idx ag_group_nodeag_wait_group_noders_snoders_related_snode_setrs_related_snodes rs_group_noders_wait_group_node prev_ag_waitwait_group_nodero prev_rs_waitrHr4rAs" ` @@@r"rrsC )+IIII+-(+-( "lUlU  J595PX   j U    _      j UIHEHUU  ($"     *EM *6> 4<K )$"    !'$*A*A!!!  +..?*@*@ 'N301122  -a0 !NEIN$H$P(#a'N!A%%23/E&!22N3N2N O !M3011A566  /A6;R^LL$%EME!,,,../@-/PQQM"4!34Emnn4U!V!V BT 0 ? ?EJ (@(H I I UIHFIUU ($"    !'$*A*A!!!  !M3011A566  /A6;R^LL$%EME!,,,../@-/PQQM"4!34Emnn4U!V!V BT 0 ? ( ) )A - - - -=7881<<<<=7881<<<< >>  8 8 8-enn.>.>?E I    eL*N*T*T*V*V''&   #]%C%C%E%E F FGGL!--//  **AJJLL|DDD' L*N*T*T*V*V''&   #]%C%C%E%E F FGGL!--//  **AJJLL|DDD' r#)rrrr) rrrr'rr'rr'rr)rrrr)r.rrr)rr)rrrr+)rr9rr:r7r;rr9)2 __future__rrTloggingr rp collectionsrtypingrrrrr torch.multiprocessing.reductionsr rr r dependenciesr utilsrrrrrrr getLoggerr@r_logginggetArtifactLoggerrr4rrrr&r rrNrrrrr0r8rrFr#r"rtso#"""""  ######111111111111 ;;;;;;!!!!!!g!!n..xCC -,,,,,,&WWWWt    F     U U U>8A#A#A#A#Hllll^%%%llllllr#