jv `ddlZddlZddlZddlZddlZddlmZmZddlm Z m Z ddl m Z ddl mZmZmZmZmZmZmZmZmZmZddlZddlmZddlmZmZddlm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&erdd lm'Z'gd Z(ej)e*Z+Gd de Z,e,j-Z-e,j.Z.e,j/Z/e,j0Z0e,j1Z1e,j2Z2e,j3Z3e,j4Z4e,j5Z5e,j6Z6e-Z7e.Z8e/Z9e6Z:ej;dZ<GddeZ= dBdee>eee=fdee>de?fdZ@GddeZAdBdeejBdee?fdZC dBdeejBdee?dee>ej'ffdZDGddeAZEGddeEZFGd d!eEZGGd"d#eEZH dCd%eee=d&e>dee=fd'ZId%eee=dee=fd(ZJd%ee>ee=fd)ee>ge>fd*e>dee>ee=ffd+ZKd,ee>eee=fd-e>d*e>d.e>fd/ZLGd0d1eAZMGd2d3eMZNGd4d5eMZO dDd7ZPGd8d9eMZQGd:d;eMZRGd<d=eMZSd>e?fd?ZTd)ee>ge>fd*e>fd@ZUdAZVdS)EN)ABCabstractmethod)Counter defaultdict)Enum) AnyCallableDictList NamedTupleOptionalSetTuple TYPE_CHECKINGUnion) FSDPModule UnshardHandle)record_function) merge_chunkssplit_args_kwargs_into_chunksTensorChunkSpec)_PipelineStageBase)Work) get_schedule_classPipelineScheduleSinglePipelineScheduleMulti Schedule1F1B ScheduleGPipeScheduleInterleaved1F1BScheduleLoopedBFSScheduleInterleavedZeroBubbleScheduleZBVZeroBubblecReZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z ed Zd S)_ComputationTyper ctjdtjdtjdtjdtjdtjdtjdtjdtj d tj d i }||S) NFIWUNSHARDRESHARDSEND_FRECV_FSEND_BRECV_BB) r%FORWARDBACKWARD_INPUTBACKWARD_WEIGHTr3r4r5r6r7r8 FULL_BACKWARD)selfstr_maps WC:\Projects\Historical\venv\Lib\site-packages\torch/distributed/pipelining/schedules.py__str__z_ComputationType.__str__Ase  $c  +S  ,c  $i  $i  #X  #X  #X  #X  *C  t}c|dkr tjS|dkr tjS|dkr tjS|dkr tjS|dkr tjS|dkr tjS|dkr tjS|dkr tjS|d kr tj S|d kr tj Std |) Nr0r1r2r3r4r5r6r7r8r9zInvalid computation type ) r%r:r;r<r3r4r5r6r7r8r= RuntimeError)actions r@from_strz_ComputationType.from_strPs S==#+ + s]]#2 2 s]]#3 3 y #+ + y #+ + x  #* * x  #* * x  #* * x  #* * s]]#1 1C6CCDD DrBN)__name__ __module__ __qualname__r:r;r<r3r4r5r6r7r8r=rA staticmethodrFrBr@r%r%4suGNOGG F F F FM   EE\EEErBr%z?(\d+)(F|I|B|W|UNSHARD|RESHARD|SEND_F|RECV_F|SEND_B|RECV_B)(\d*)c`eZdZUeed<eed<dZeeed<dZe de fdZ dS)_Action stage_indexcomputation_typeNmicrobatch_indexct|j}|t|jz }|j|t|jz }|SN)strrNrOrP)r>reprs r@__repr__z_Action.__repr__sJ4#$$ D)***  , C-.. .D rB action_stringcp|}t|x}rl|\}}}t t |t |t|rt |ndS|dkrdStd|d)z Reverse of __repr__ String should be formatted as [stage][action type][(microbatch)] e.g. `2F0`, `1UNSHARD`, `3SEND_F1` NzInvalid action string: zD, should be formatted as [stage][action type][(microbatch)] e.g. 2F0) strip _action_regexmatchgroupsrMintr%rFlenrD)rVr[rNrOrPs r@rFz_Action.from_strs&++-- !'' 66 65 >Cllnn ;K)+;K   ))*:;;),-=)>)>H$%%%D  b 4 ym y y y   rB) rGrHrIr]__annotations__r%rPr rUrJrSrFrKrBr@rMrMsy&&&&&*hsm***    \   rBrMpipeline_ordererror_step_numberreturncD tjD]@}tt|D]}|| d||<At dD fdt D} fdt D}ttj |ddi}t}dt|D}dt|g|RD d t|d d zzd fd t|Dz}  fd t||D} | dzd | zdz} | S)z Formats the pipeline order in a timestep (row) x rank (column) grid of actions and returns the formatted string. If `error_step_number` is passed in, an additional label will be added to signify which step that it is erroring on. NrXc34K|]}t|VdSrR)r^).0actionss r@ z)_format_pipeline_order..s(HHWCLLHHHHHHrBc g|]D}dt|ttdz zES)zStep r)rSzfillr^)rei num_stepss r@ z*_format_pipeline_order..sQ<=#a&&,,s3y1}#5#566777rBcDg|]}|dgzS)rX)get)rekeyrkr`s r@rlz*_format_pipeline_order..s=693y 011rB fillvaluec2g|]}dt|zS)zRank rS)rerjs r@rlz*_format_pipeline_order..s#>>>7SVV#>>>rBc@g|]}td|DS)c3VK|]$}|tt|ndV%dS)Nr)r^rS)reitems r@rgz4_format_pipeline_order...s7 F F$d.CD NNNA F F F F F FrB)max)recols r@rlz*_format_pipeline_order..s?  F F# F F FFFrB rr&c38K|]\}}|d|VdS.sJ<<)1E5$;q>$$$<<<<<<rBc g|]k\}}|ddfdt|Dz-t|dkrdndzlS)z: rxc3RK|]!\}}t|d|V"dSrzrr)rerjrur}s r@rgz4_format_pipeline_order...s?RR4c$ii3+a.333RRRRRRrBNrz <-- ERROR HERErX)join enumerater]split)rer|rowrar}s r@rlz*_format_pipeline_order..s    E3  ((RRRR9S>>RRR R R S!,EKKMM!$%%):::       rB ) copydeepcopyranger^rvvaluessortedlist itertools zip_longestziprr)r`rarankrj step_labels rank_actionstransposed_actions num_ranks rank_labels header_rowformatted_rowsformatted_tabler}rks`` @@r@_format_pipeline_orderrs9]>22N--s>$/0011 - -Ad#A&.*,t$Q' - HH0E0E0G0GHHHHHIAFyAQAQK=CN=S=SLi3\PRPPQQN##I>>U9-=-=>>>K{8%7888K KN++a/0388<<<<5>{5K5K<<<44J     k+=>>   N!4'$))N*C*CCdJO rBc"eZdZ ddedeedejfdeee dfdee e e fdee e e e fee ff dZd Zd Zd Ze dd eed eedeedeefdZeddddeefdZ dd eed eedeedeefdZdZ ddee dfdee e e ffdZdee de fdZdS)_PipelineScheduleNn_microbatchesloss_fn.args_chunk_speckwargs_chunk_specoutput_merge_specc||_||_||_||_||_ |jdu|_g|_td|j j dS)NzUsing %s) _n_microbatches_loss_fn_args_chunk_spec_kwargs_chunk_spec_output_merge_spec _has_backward_internal_lossesloggerinfo __class__rG)r>rrrrrs r@__init__z_PipelineSchedule.__init__sj . /"3"3 "]$657 J 788888rBc|jr?|jr:||||}|j|dSdSdSrR)is_lastr _compute_lossrappend)r>stageoutput target_mbsmb_indexlosss r@_maybe_compute_lossz%_PipelineSchedule._maybe_compute_losssa = /T/ /%%fj.BCCD  ! ( ( . . . . . / / / /rBcd|cxkot|jknc}|jr|jr|r |j|St|jdkr|st d|d|jdS)NrzLoss for microbatch z6 is not available. Available losses for microbatches: )r^rrrrD)r>rr valid_indexs r@_maybe_get_lossz!_PipelineSchedule._maybe_get_losss8@@@@c$*?&@&@@@@@ = T/ K (2 2 & ' '1 , ,[ ,NxNN6:6KNN  4rBct|ts|g}td|D}|ry|wt|j|jkr,t d|jdt|j|||j|jdS)zB Update the losses to those in the internal state c3$K|] }|jV dSrRrrers r@rgz3_PipelineSchedule._update_losses..s$!D!DE%-!D!D!D!D!D!DrBN Expecting z losses but got ) isinstanceranyr^rrrDclearextend)r>stageslossescontains_last_stages r@_update_lossesz _PipelineSchedule._update_lossess &$'' XF!!D!DV!D!D!DDD  16#54())T-AAA"c!5ccs4K`GaGacc LLNNN MM$/ 0 0 0 ##%%%%%rBarg_mbs kwarg_mbsrrct)z Run one iteration of the pipeline schedule with list of microbatches. Will go through all the microbatches according to the schedule implementation. Args: microbatches: list of microbatch args. NotImplementedError)r>rrrrs r@_step_microbatchesz$_PipelineSchedule._step_microbatches(s "!rBtargetrct) Run one iteration of the pipeline schedule with *whole-batch* input. Will chunk the input into microbatches automatically, and go through the microbatches according to the schedule implementation. args: positional arguments to the model (as in non-pipeline case). kwargs: keyword arguments to the model (as in non-pipeline case). target: target for the loss function. losses: a list to store the losses for each microbatch. r)r>rrargskwargss r@stepz_PipelineSchedule.step:s "!rBcdtffd }| ||dn dgjz}| ||dn igjz}| ||d|4t|tst dt |||fS) z* Pre-process/check inputs namec t|ts!t|dt|t |jkr*t djd|dt |dS)Nz must be a list but got a rrxz but got )rr TypeErrortyper^r ValueError)mbsrr>s r@check_type_and_lenz;_PipelineSchedule._check_inputs..check_type_and_lenSsc4(( P4 N N499 N NOOO3xx4/// Q!5QQQQs3xxQQ0/rBNrrKrrz losses must be a list but got a )rSrrrrr)r>rrrrrs` r@ _check_inputsz_PipelineSchedule._check_inputsHs #          w 2 2 2 2dT11G   y+ 6 6 6 6t33I  !  z< 8 8 8  fd++ S Q4<< Q QRRR !!rBc.|||SrR)r)r>rrs r@rz_PipelineSchedule._compute_lossns}}VV,,,rBrrc|s|r)t|||j|j|j\}}||fSdg|jzig|jzfS)zj Splits a full-batch input into chunks (i.e. microbatches) and returns the chunks rK)rrrr)r>rr args_split kwargs_splits r@ _split_inputsz_PipelineSchedule._split_inputsqso  L6 L'D$%' (( $J |+ +4$..t7K0KK KrB output_chunksrbc,t||jS)z Merge output chunks back to a batch state. If output_merge_spec is None, the utility will merge output chunks by dimension 0 (batch dim). )rr)r>rs r@_merge_outputsz _PipelineSchedule._merge_outputss    #   rBNNNNrR)rGrHrIr]r r torchTensorrrr rSrrrrrrrr rrrrrrrKrBr@rrsd:>AEBFIM 999(3 #4569"%(<"=> 9 $Do)=$>? 9 $E$sCx.%**D$EF 9999:///   &&&.#'$(%)!% ""$"D>"TN "  """^""!% " " "x~ " " "^ "#'$(%)!% $"$"$$"D>$"TN $"  $"$"$"$"L--- ,0LLCHoLc3h(LLLL. DI #      rBrp2p_opsdescct|dkrdS|r|dnd}td||tj|S)zt Simple wrapper over batch_isend_irecv from torch.distributed, which just adds a descriptive logger on top. rNz, rXzbatch_p2p %s%s)r^rdebugdistbatch_isend_irecvpop)rrdesc_strs r@ _batch_p2prsb 7||qt"*${{{{H LL!8W555  !' * * . . 0 00rBctt}i}t|dkr|S|D]"}||j|#t |D]\}}t||||<|S)z Sorts the list of P2P ops by the peer rank, and then calls batch_isend_irecv. Return a dictionary of works by peer rank. This function helps us avoid hangs in case of skip connections. rr)rrr^peerrritemsr)rr ops_by_peer work_by_peeropropss r@_sorted_batch_p2prs0;4/@/@K)+L 7||q((BG##B''''K--//0088 c'$777 T rBceZdZdZ ddededeedeee dfdee e e fd ee e e e fee ff fd Zd Zddd d eefdZxZS)rz Base class for single-stage schedules. Implements the `step` method. Derived classes should implement `_step_microbatches`. Nrrrr.rrct|||||||_|j|_|j|j_d|_dS)NrrrrrF)superr_stage num_stages _num_stagesr has_backward_stage_initialized)r>rrrrrrrs r@rzPipelineScheduleSingle.__init__sd )+//      +#'#5  "'rBc|j|j|||jr|j|jd|_dSNT)r_prepare_forward_infrarr_prepare_backward_infrar)r>rrs r@_initialize_stagez(PipelineScheduleSingle._initialize_stagesR **4+?vNNN   F K / /0D E E E"&rBrrcH|j|||\}}|(tt j||j}nd}||||||jjr| |jj SdSrN) rclear_runtime_statesrrr tensor_splitrrrrr)r>rrrrrr targets_splits r@rzPipelineScheduleSingle.steps ((***$(#5#5dF#C#C L   !3FD ( $Do)=$>? ($E$sCx.%**D$EF((((((0''' "&x~rBrc deZdZdZ ddeedeedeedeefdZdS) _ScheduleForwardOnlyzo The forward-only schedule. Will go through all the microbatches and perform only the forward pass Nrrrrc\||td|||||\}}|js"||d|dg}t |jD]#}t d|5|j|}t|d}| D]} | |j ||||||j |}t|d}|| dddn #1swxYwYtd|jj|%|D]} | dS) z< Run one iteration of the pipeline schedule Nz7Forward-only schedule does not support loss computationrForward fwd_recvrfwd_send[%s] Forwarded microbatch %s)rDrrrrrrrget_fwd_recv_opsrrwaitforward_one_chunkget_fwd_send_opsrrrrN) r>rrrrfwd_sends_to_waitrjrworksworks r@rz'_ScheduleForwardOnly._step_microbatchess  !V%7I "//JPVWW& =  " "71:y| < < <.0t+,, U UA A00 9 9k22155)#J???!LLNN  DIIKKKK --aYq\JJJk22155)#J???!((888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 LL79PRS T T T T &  D IIKKKK  sCE  E$ 'E$ rrGrHrIr r r rrKrBr@rrs{#'$(%)!% **$*D>*TN *  ******rBrc deZdZdZ ddeedeedeedeefdZdS) rz^ The GPipe schedule. Will go through all the microbatches in a fill-drain manner. Nrrrrc |||||\}}|js"||d|dg}t|jD]@}t d|5|j|}t|d}| D]} | |j |||||} |j |}t|d}| | dddn #1swxYwYtd|jj|||j| ||B|D]} | |jsdSg} t|jD]?}t d|5|j|}t|d }| D]} | ||j|} |j|| ||jd z k |j|}t|d }| | dddn #1swxYwYtd |jj|A||j|| D]} | dS)z Run one iteration of the pipeline schedule with list of microbatches. Will go through all the microbatches according to the GPipe schedule. Args: microbatches: list of microbatch args. rrrrrNrz Backward bwd_recvrr last_backwardbwd_sendz[%s] Backwarded microbatch %s)rrrrrrrrrrrrrrrrrNrrget_bwd_recv_opsrbackward_one_chunkget_bwd_send_opsr) r>rrrrrrjrrrrbwd_sends_to_waitrs r@rz ScheduleGPipe._step_microbatches9s"//JPVWW& =  " "71:y| < < <.0t+,, I IA A00 9 9k22155)#J???!LLNN  DIIKKKK66q'!*iPQlSSk22155)#J???!((888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 LL79PRS T T T  $ $T[&*a H H H H &  D IIKKKK!  F.0t+,, V VA Q11 9 9k22155)#J???!LLNN  DIIKKKK++DK;; ..DT5IA5M0M/k22155)#J???!((888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 LL8$+:QST U U U U DK000&  D IIKKKK  s&0CE  E E *C-K##K' *K' rrrKrBr@rr3s#'$(%)!% LL$LD>LTN L  LLLLLLrBrc deZdZdZ ddeedeedeedeefdZdS) rzo The 1F1B schedule. Will perform one forward and one backward on the microbatches in steady state. Nrrrrc|||||\}}|js"||d|dt|j|j|jjz }d}d}d}g} t|D]} |j |} t| dx} r| |j |||||} |r| |j |} ||dz krt| d}||j| |||dz } |j|}t| |zdx}r| ||j|}|j||||jdz k |j|}|dz }||jkrn|j |} t|| zd x}r| |j |||||} ||j| |||j |} |dz }]t|d }||jkr|j|}t|d x} r| ||j|}|j||||jdz k |r| |j|}t|d }|dz }||jk|r| ||j|dS) z Run one iteration of the pipeline schedule with list of microbatches. Will go through all the microbatches according to the 1F1B schedule. Args: microbatches: list of microbatch args. rNrrrrTfwd_send_bwd_recvr bwd_send_fwd_recvr"r)rrrminrrrrNrrrrrrrr#rr$r%r)r>rrrr warmup_chunks fwd_mb_index bwd_mb_index send_work fwd_sends_ fwd_recvs recv_workr bwd_recvs fuse_workr bwd_sendss r@rzSchedule1F1B._step_microbatchessO"//JPVWW& =  " "71:y| < < <   t{6 6      }%%  A 44\BBI&yzBBBBy !   [22<AVXabnXoppF  !    44\BBI}q000&yzBBB  $ $T[&*l S S S A LL '  44\BBI'y9'>>I A L+T1110   NN    DK00000rBrrrKrBr@rrs#'$(%)!% I1I1$I1D>I1TN I1  I1I1I1I1I1I1rBrr'compute_actionsmax_active_stagesc dtdtttdttfd}t g dtf fd }dtf fd }t |D]\}}|||||d t t fd  }t t fd  }|D] } || |D] } ||  | S) aRGiven a basic schedule involving only compute actions (F,B,W), add UNSHARD/RESHARD actions for FSDP. UNSHARD refers to fetching the full contents of an FSDP-sharded layer, requiring an all-gather operation. RESHARD does the opposite, releasing memory (but doing no commmunication) We abandon the "timestep lock" during lowering max_active_stages controls how many prefetches we allow. It should be measured in mb and tuneable but in practice 3 stages is probably the thing we want? (to account for having one f and one b active, and something else prefetching?) count next_actionsrbct}g}|D]V}|R|j|vrI||j||jt ||krnW|S)zdRemove duplicates (same stage, different microbatch), find next 'count' stages that will do compute.)setrNaddrr^)r:r;seenretas r@next_stage_indicesz0_add_unshard_reshard..next_stage_indices*sv  A}d!:!:''' 1=)))s88u$$E rBrNc|t|tddSrR)r>rrMr3rN active_stagesfsdp_aware_actionss r@_unshardz&_add_unshard_reshard.._unshard<s=+&&&!!'+w"E"EFFFFFrBc|t|tddSrR)removerrMr4rDs r@_reshardz&_add_unshard_reshard.._reshard@s=[)))!!'+w"E"EFFFFFrBNc |vSrRrK)srEs r@z&_add_unshard_reshard..Ks a}&<rBc |vSrRrK)rLnext_ns r@rMz&_add_unshard_reshard..Ms avorB) r]r r rMr=rrfilterr) r7r8rBrGrJrjrEfetchevictrrErFrOs @@@r@_add_unshard_reshardrSs  "&x'8"9  c    "eeM(*GcGGGGGGGGcGGGGGGG//** 6 > $#$5qrr7JKKV<<< /"" #q7I(I 'R    " " "/"" #q7I(I 'R  #~ 5 5',??"k&===';+GGG  ! !*M6;RSS       " " " "  ! !& ) ) )+ *, rB stage_to_rankrc B d|D}d|D}dtdtffd dtdtttff fd }dttdttdtffd }|rd }t |D]r}t ||d ks#Jd |d t ||||d } || ||s`| ||| |||  | r|| \} } ||| ||| || j | || j | || d t ||d kr||=d }t|s Jd||S)Nci|]}|gSrKrKrers r@ z"_add_send_recv..s-S-S-S4dB-S-S-SrBc,i|]}|tSrKr=r[s r@r\z"_add_send_recv..s,U,U,UTT355,U,U,UrBrErbc|jtkr3|jdz ko$|jdz|jkS|jttfvr0|jdko$|jdz |jkSdS)NrrF)rOr0rNr;r=)rErrXs r@ _has_commsz"_add_send_recv.._has_commss  "a ' '%a73MM"Q&==v122=3 3 $(G G G%*3}}"Q&00v12203 3urBc0|s J|d|j}|j}|j}t||tkrt nt |}|tkr|dzn|dz }t||tkrtnt|}||fS)Nz is not a valid comm actionr) rNrOrPrMr0r5r7r6r8)rE stage_idxctypemb_idxsendrecv_stage_idxrecvr`s r@ _get_commsz"_add_send_recv.._get_commssz&!!IIf#I#I#III!& '(yEQJJ&&FFKK*/1**Q)a-~!vvPPTzrB prev_actionsc|dS|jtkrX|jdksMt|jt|j|vrdSt|jdz t|j|vrdSdS|jt tfvr|jdz kstt|jt|j|vrdSt|jdzt |j|vrdSt|jdzt|j|vrdSdSdS)aWe don't put our own recv ops in the schedule, we let a sender on another rank put our recv ops in place. This helps ensure a sane (non-hanging) ordering of sends and recvs. But it also means we might not be able to schedule our next compute action yet. NTrrF) rOr0rNrMr6rPr;r=r8)rErirs r@_ready_to_schedulez*_add_send_recv.._ready_to_schedules< >4  $ ) )&2D2I2I*FF4KLL  t*Q.63JKK  t5  # 'F F F&*q.88*FF4KLL  t*Q.@WXX  t*Q. v?VWW  t54rBFrrank=z, len(compute_actions[rank])=Tz6Malformed compute schedule, can't schedule sends/recvs) rMboolrr rrr^rr>rNr) r7rXr comm_actionsrirhrkprogressrrErergr`s `` @r@_add_send_recvrps .T-S?-S-S-SL,U,U_,U,U,UL 7 t       7uWg-='>*!*14W* ******X R?++  DOD)**Q...8$883t45588/..$T*1-F%%fl4.@AA !T"))&111T"&&v...:f%%L!+F!3!3JD$!&--d333 &**4000 t/?!@!@AHHNNN t/?!@!@AEEdKKK D ! % %a ( ( (?4())Q..#D)HHQQQQQx9 R: rBrf pp_group_sizenum_microbatchesc t||ksJd|dt|t|D]}||vs Jd|dt|D}|D]}||D]x}|t|tsJd|d|j}|j}|j} |tkr'||t| p|tkrM| ||tvsJd|d| d ||t| |tkrN| ||tvsJd |d| d ||t| !|tkrL| ||tvsJd |d| d ||t| z|D]}t||t} t||t} t||t} t||t} | |ksJd | dtd|d|| | | zdzz|ksJd|d|d| d| d| dS)Nz2Schedule has incorrect number of ranks - expected z , actual z%Schedule is missing actions for rank c i|]Q}|ttttttttiRSrK)r0r=r9r1r2)restage_ids r@r\z&_validate_schedule..sS===   suu suu suu suu  ===rBzGot an invalid action: z, expected instance of _Actionz Running Full Backward for stage z , microbatch z without first running Forwardz!Running Backward Input for stage z"Running Backward Weight for stage z% without first running Backward InputzGot rxz microbatches for stage z , expected r&z(Invalid backward microbatches for stage z : expected z( total backwards, but got B=z, I=z, W=) r^rrrMrNrOrPr0r>r9r1r2)rfrqrrrr stage_actionsrEs_idrcmb_idf_mbb_mbi_mbw_mbs r@_validate_scheduler}s G %%%bMbbTWX_T`T`bb &%%m$$OOw N N N==j))===M22dm 2 2F~ P POOOO P P%D+E+Ezzd#A&**51111!]403333ndnnnnn433d#A&**51111!]403333otoo%ooo433d#A&**51111!]403333www5www433d#A&**51113 26 2 2=&q)**=&q)**=&q)**=&q)** $ $ $ $ W$ W W W WD W WEU W W % $ $ D4KA% %)9 9 9 9 2d 2 2GW 2 2 2 2!% 2 2+/ 2 2 : 9 9 9 2 2rBceZdZdZ ddeededeedee e dfdee e e fd ee e e efe efd ee eefd eeffd Zd e edffdZdZddZddddeefdZ ddeedeedeedeefdZxZS)rzQ Base class for multi-stage schedules. Implements the `step` method. Nrrrr.rrstage_index_to_group_rankuse_full_backwardc  t|||||||_|dj|_|dj|_|dj|_||jD] } || _ |dj |_ |jD]} |j | _ d|_ |j du fd|_i|_|t ddSdS)NrrFc|joSrRr)rhas_losss r@rMz0PipelineScheduleMulti.__init__..Zs%-2LHrBzDeprecation warning: 'use_full_backward' is no longer supported. Simply stop passing it, and everything should still work fine.)rr_stagesrr group_sizerq group_rankrrrr_stages_initializedr_should_compute_lossr`rwarning) r>rrrrrrrrrrrs @r@rzPipelineScheduleMulti.__init__5s& )+//     !!9/#AY11I( $ 0 L L2K//)/)L&\ 4 4E!%!3E  #( d2$L$L$L$L!CE  ( NNQ      ) (rBrct}|jD]c}|jr||j||}n||j||}|jr||jdd|_dSr)tupleris_firstrrrrr)r>rrnext_stage_argsrs r@_initialize_stagesz(PipelineScheduleMulti._initialize_stageses,177\ D DE~ "'">">($###(">">(/6##! D--d.BCCC#'   rBct|dd5}tj|}|jD]"}||j|# ddddS#1swxYwYdS)zQDump a CSV representation of the schedule into a file with the provided filename.wrXnewlineN)opencsvwriterr`writerowr>filenamecsvfilerrs r@ _dump_csvzPipelineScheduleMulti._dump_csvws (C , , , ;Z((F+ ; ; 3D 9:::: ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;s?A  A$'A$ compute_onlyc0|dksJt|d5}tj|}t|D]\}}d|D|j|< dddn #1swxYwYt |j|j|j|jdS)zLoad a CSV representation of the schedule from a file with the provided filename. This API will most likely get renamed/refactored so is marked as internal for now. format must be "compute_only" for PipelineScheduleMulti rrXrcBg|]}t|SrKrMrFrerLs r@rlz3PipelineScheduleMulti._load_csv..s&,N,N,NQW-=-=a-@-@,N,N,NrBN) rrreaderrr`r}rqrr)r>rformatrrrrs r@ _load_csvzPipelineScheduleMulti._load_csv~s '''' (B ' ' ' O7Z((F&v.. O O c,N,N#,N,N,N#D)) O O O O O O O O O O O O O O O O              s>A%%A),A)rrcV|jD]}||||\}}|(tt j||j}nd}||||||jD]%}|jr| |j cS&dSr) rrrrrr rrrrr) r>rrrrrrrr s r@rzPipelineScheduleMulti.steps\ ) )E  & & ( ( ( ($(#5#5dF#C#C L   !3FD????? @trBrrrc  |||||\}}|js"||d|dd|jD}t }t }|D]\}|dkr#||j|dz ||jdz kr#||j|dz]t} t|j |j D]#\} } g} | | j } | j}| j}| Jd| t jkrm||}||||||}|||||| ||n`| t jkr||}|||}| |xxdz cc<|||d| ||jk| ||n| t jkr`||}|||}|||dd| ||na| t jkr?||}| |xxdz cc<||| ||jk nt=d | |D]}|j |}d}| t?|kr|| }||j } |j}|j}| Jd| t jkr;|dz|vr3||dz}| | || t,t6t8fvrt=d | |D]}|j |}d}| t?|kr|| }||j } |j}|j}| Jd| t"t8fvr`| t6t,fvr;|dz |vr3||dz }| |!|t=d | | r!tE| ##tH$rc}tJ&d |j |j'j(| | tJ&d tS|j | |d}~wwxYw|*|j|dS) Operate on the microbatches for looped schedules (multiple stages on each rank). TODO: Does not use sorted_batch_isend_irecv(). As a result, this schedule does not support models with skip connections. rci|] }|j| SrKrNrs r@r\z.*? ? ? ).E u? ? ? rBrNzCAll currently supported action types require valid microbatch_indexTr full_backwardr!Fr!zUnknown computation type zy[Rank %s] pipeline schedule %s caught the following exception at time_step %s when running action %sz%sra)+rrrrr=keysr>rrrrr`rrOrPrNr%r:rrrrr=rr$rr%r;r<backward_weight_one_chunkrr^rr#rr ExceptionrerrorrrGrr)r>rrrrstage_index_to_stageall_prev_ranksall_next_ranksrNbackward_counter time_steprErrOrrrr prev_rank prev_rank_opsprev_rank_action next_rank next_rank_opsnext_rank_actiones r@rz(PipelineScheduleMulti._step_microbatchesss"//JPVWW' >  # #GAJ ! = = =? ? 26,? ? ?  $'55#&55/4466 T TKQ""4#A+PQ/#RSSST-111""4#A+PQ/#RSSS)0!*4+>ty+I!J!JE E  IvD (*%'-'>$%6H"("4K ,,\-,,'+;+CCC 4[ A!&!8!8$gh&789L""00 HUUU 5#9#9(#C#CDDDD)-=-KKK 4[ A#33E8DD(555:55500$!%*.*:;*G#3+4 1 5#9#9(#C#CDDDD)-=-LLL 4[ A#33E8DD00$!%*/*/ 1  5#9#9(#C#CDDDD)-=-MMM 4[ A(555:55577$*:;*G#3+48 ))WEU)W)WXXX"0I$($7 $BM'+$ 3}#5#555+8+C('3+;+L(#3#D&6&B $00`100,/?/GGG*Q2FFF)=[1_(M # 5+A+A(+K+K L L L-)*+2 !", NB,0..'(..(# . "%(<"=> . $Do)=$>? .$E$sCx.%**D$EF.$,DcN#;.$D>......`(uS#X(((($;;;    $"&   x~    H#'$(%)!% k2k2$k2D>k2TN k2  k2k2k2k2k2k2k2k2rBrc eZdZdZ ddeeeeefde fdZ dde de ffd Z de fdZ d Z dd eed eed eedeefdZxZS)_PipelineScheduleRuntimea% Provides a simple runtime that requires a 'schedule IR' including specified communication operations. Can be instantiated directly by creating _PipelineScheduleRuntime and calling load_csv, or can be subclassed and the subclass can be responsible for creating a schedule IR. rrfrcj Jdi_|dkr@|D];}gj|<||D]&}|Jj||'s r@rMz8_PipelineScheduleRuntime._load_actions..s(Fq(IrB)rXrformat= is not implemented)rpipeline_order_with_commsrrSrprr)r>rfrrrEs` r@ _load_actionsz&_PipelineScheduleRuntime._load_actionshs2  * 6 6 N 7 6 6CE& _ $ $ H H79.t4%dmHHF!---248??GGGGH H H ~ % %  7KDM88.t44.<.IIII+...D * * * &&E&E&E&EFF FrBrc|dkr=t|||jdS|dkr}i}t |d5}t j|}t|D]\}}d|D||<|||ddddS#1swxYwYdStd|d ) aLoads a csv in simple format and then lowers it to include comunication actions format must be either "compute_only" or "compute_comms". If compute_only, the lowering passes will automatically be run to generate a compute_comms schedule. rrrXrcBg|]}t|SrKrrs r@rlz6_PipelineScheduleRuntime._load_csv..s&$F$F$FQW%5%5a%8%8$F$F$FrB)rNrr) rrrr`rrrrr) r>rrrfrrrrrs r@rz"_PipelineScheduleRuntime._load_csvsC ^ # # GG  h ' ' '   t2 3 3 3 3 3  & &Gh+++ ;wG,,!*6!2!2GGID#$F$F#$F$F$FGDMM""76":::  ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; &&E&E&E&EFF FsAB;;B?B?c|j Jdt|dd5}tj|}|jD]"}||j|# ddddS#1swxYwYdS)zaDump a CSV representation of the compute + comms schedule into a file with the provided filename.Nz6Must initialize compute_comms schedule before dump_csvrrXr)rrrrrrs r@rz"_PipelineScheduleRuntime._dump_csvs  * 6 6 C 7 6 6 (C , , , FZ((F6 F F >t DEEEE F F F F F F F F F F F F F F F F F F Fs?A11A58A5c@tjfdjS)Ncj|SrRrrs r@rMz4_PipelineScheduleRuntime._simulate..sd4Q7rB)_simulate_comms_computerr)r>s`r@ _simulatez"_PipelineScheduleRuntime._simulates-&  * 7 7 7 7     rBNrrrrc |||||\}}|js"||d|dd|jD}|j Jdi}i}g}it dt ffd } t} t|j|j D]E\} } | j } | j | j nd}|dks| ttfvsJd| d | j}||}t|jt"}|d z|v}|d z |v}t$d | | | t(kr7|t-||n| t0kr7|t-||n| t4kr9||f|vs Jd t-|||||f<n| t8kr9||f|vs Jd t-|||||f<nQ| tkr8|r4|vr|vsJd|d|jd|<n| tkrA|r=|vsJd|d|vsJd|d|jn| t@kr|r | ||j!s?|s=||f|vsJd| d|"||f#|$|||||}|%|||||r||d z&||n| tNkr|r | ||j(s?|s=||f|vsJd| d|"||f#|)||}| |xxd z cc<|*||d| ||j+k|r2||d z ,|-||n$| t\kr|r | ||j(s?|s=||f|vsJd| d|"||f#|)||}|*||dd|r2||d z ,|-||nb| t^krD|r | || |xxd z cc<|0|| ||j+kntcd| d#td$rF}t$3d| | titk|j| |d}~wwxYwtm|r5|"#tm|5tmdks Jd|7|j|dS)rrci|] }|j| SrKrrs r@r\z?_PipelineScheduleRuntime._step_microbatches..rrBNz=Must call _load_actions() before calling _step_microbatches()rbc|vr2||=||vs Jd|dS)zQIf an unshard is active for `stage_idx`, wait() it and mark `stage_idx` unshared.z*Attempted to compute on sharded stage_idx=N)rr>)rb unshard_opsunsharded_stagess r@_assert_unshardedzF_PipelineScheduleRuntime._step_microbatches.._assert_unshardedslK''I&++--- * $$Y///---->)>>.----rBzaction=z missing mb_indexrz8_PipelineScheduleRuntime running time_step %d, action %szARecv twice for {stage_idx=} {mb_index=} without executing forwardzBRecv twice for {stage_idx=} {mb_index=} without executing backwardzUnsharding the same stage_idx=z twiceT)async_opzResharding stage_idx=z without unshardingz before finishing unshardzComputing action=z before receiving inputz Attempted to run compute action=rFrz is unknown or unsupportedz\_PipelineScheduleRuntime caught exception at step %s when running action %s. Full Schedule:rzUnused unshard operations)8rrrrrr=r]rrrrOrPr3r4rNrsubmodrrrr5rrrr7r%r6rr8r#unshardreshardr:rrrrrset_local_fwd_inputr=rrr$rset_local_bwd_inputget_local_bwd_outputr;r<rrrrprintrr^r)r>rrrrr bwd_recv_ops fwd_recv_opssend_opsrrrrE comp_typerrbrstage_uses_fsdpis_next_stage_on_this_rankis_prev_stage_on_this_rankrrrrrs @@r@rz+_PipelineScheduleRuntime._step_microbatchess "//JPVWW' >  # #GAJ ! = = =? ? 26,? ? ?   * 6 6 J 7 6 657 46  "13 55 ? ? ? ? ? ? ? ?*1!*4+I$)+T!U!Ui i  Ivh "3 .:++  1}} 6)))1f000))#. ,Y7",U\:"F"F-6]>R-R*-6]>R-R* N&&OOJu/E/Eh/O/O$P$PQQQQ&((OOJu/E/Eh/O/O$P$PQQQQ&((! ****,o***;E..x88;;L)X!677&((! ****,p***;E..x88;;L)X!677'))&U%-=== ) < < <D)DDD!= <=271E1Et1E1T1T I.'))&/%)9999HHHH:99&[888NNNN988 ,,...'))&5)))444"N G!; G &$ * * * *,Y+X+X+X * * *%(()X)>??DDFFF"44 '("3Yx5HF,,UFJQQQ2,Y];OO"H-//&5)))444"M G!; G &$ * * * *YXXX  * * * %(()X)>??DDFFF//x@@D$Y///14///,, !&*&6y&A/'0 -2,Y];OO!66x@@(.00&5)))444 =G1KG%$ * * * *YXXX  * * * %(()X)>??DDFFF//x@@D,, !&+&+ -2,Y];OO!66x@@(/11&5)))444$Y///14///33 &6y&A/'04 %%K%K%K%KLLL    r,T-K_hiiijjj (mm " LLNN   ! ! !(mm ";1$$$&A$$$ DL&11111sS+V.. W>8AW99W>rr)rGrHrIr r r]r r rMrSrrrrrr r s@r@rr`sH%!G!Gc4 1223!G!G!G!G!GFGG#GsGGGGGG* F# F F F F   #'$(%)!% d2d2$d2D>d2TN d2  d2d2d2d2d2d2d2d2rBrc eZdZdZ d deededeedee e e e fe e fffd ZdZxZS) r!ai Breadth-First Pipeline Parallelism. See https://arxiv.org/abs/2211.05953 for details. Simliar to Interleaved 1F1B, Looped BFS supports multiple stages per rank. What is different is that when microbatches are ready for multiple local stages, Loops BFS will prioritizes the earlier stage, running all available microbatches at once. Nrrrrct||||i|_t|jD]!}||}||j|<"dS)N)rrrr)rrr`rrq!_calculate_single_rank_operations)r>rrrrrrank_opsrs r@rzScheduleLoopedBFS.__init__s )/    CE$,-- 1 1D==dCCH(0D  % % 1 1rBc t|j}t||j|z|j}dt|D}|D]5|fdt|jD6d|jdz |z z}|dg|zt |D]B|fdt t|jDC|S)Ncg|]}dSrRrKrer1s r@rlzGScheduleLoopedBFS._calculate_single_rank_operations..,G,G,GaT,G,G,GrBc3NK|]}ttj|V dSrR)rMr%r:rerrNs r@rgzFScheduleLoopedBFS._calculate_single_rank_operations..sG %5%=xHHrBr&rc3NK|]}ttj|V dSrR)rMr%r=rs r@rgzFScheduleLoopedBFS._calculate_single_rank_operations..sG %5%CXNNrB)r^rrrqrrreversed)r>rn_local_stages stage_indicesrpost_warmup_opsrNs @r@rz3ScheduleLoopedBFS._calculate_single_rank_operationssNT\** $$~5t7I  -H,G5;;,G,G,G(  K OO %d&: ; ;    t1A5<=0111#M22  K OO (t/C)D)D E E    rB)NN)rGrHrIr r rr]r r rr rSrrrrr r s@r@r!r!s'+IM 11'(11(# 1 $E$sCx.%**D$EF 111111.rBr!Fc tt} tt} tt} dt|D} ||zd|dz |z zz||zz }| r||z dz }||z|z}g}d}| rtnt}t|D]Q}||krl||}| |x}dz| |<| t |tj|||dz kr| dg|zu||cxkr ||zkrnn||}| |x}dz| |<| t |tj|||}| |x}dz| |<| t |||||| r]||z |krT|||}| |x}dz| |<| t |tj ||dz }| s| d||}| |x}dz| |<| t |||||| r]||z |krT|||}| |x}dz| |<| t |tj ||dz }S| r||t|kri|||}| |x}dz| |<| t |tj ||dz }| r|t|ki| S)Ncg|]}dSrRrKrs r@rlz&_get_1f1b_rank_ops..s(C(C(C!(C(C(CrBr&rr) rr]rr;r=rrMr%r:rr<r^)rrq warmup_ops fwd_bwd_ops cooldown_opsrforward_stage_indexbackward_stage_indexnum_1f1b_microbatchesenable_zero_bubblefwd_stage_mb_indexbwd_stage_mb_indexweight_stage_mb_indexrr total_opsbackward_op_idsweight_op_countFULL_BACKWARD_OR_BACKWARD_INPUTrfwd_stage_indexrr-bwd_stage_indexr.weight_stage_indexweight_mb_indexs r@_get_1f1b_rank_opsr s*5S)9)9)4S)9)9,7,<,<)D(CuT{{(C(C(CH &ma.?$.F)GG d O3'$.2[(<7IOO-?-$IM%M% ??11"55O/??3  / OO)9)A8LL   Z!^## 8999 2 8 8 8 8 [ 8 8 8 8 8 811"55O 2? CC 3  / OO)9)A<PP   32266O 2? CC 3  / OO)H,WW     " "2 & & &! %b:o9N&N&N%9%9#O4&&"(==O'PPO=%&89*(8' 1$ & &%%%22266O 2? CC 3  / OO)H,WW     " "2 & & &! %b:o9N&N&N%9%9#O4&&"(==O'PPO=%&89*(8' 1$  33G3G!G!G11//2RSS45GH HO 501  "$4$Do     1  33G3G!G!G OrBceZdZdZ d deededeedee e dfdee e e fd ee e e efe eff fd Zd eeefd ZxZS)r a The Interleaved 1F1B schedule. See https://arxiv.org/pdf/2104.04473 for details. Will perform one forward and one backward on the microbatches in steady state and supports multiple stages per rank. When microbatches are ready for multiple local stages, Interleaved 1F1B prioritizes the earlier microbatch (also called "depth first"). This schedule is mostly similar to the original paper. It differs by being relaxing the requirement of num_microbatch % pp_size == 0. Using the flex_pp schedule, we will have num_rounds = max(1, n_microbatches // pp_group_size) and it works as long as n_microbatches % num_rounds is 0. As a few examples, support 1. pp_group_size = 4, n_microbatches = 10. We will have num_rounds = 2 and n_microbatches % 2 is 0. 2. pp_group_size = 4, n_microbatches = 3. We will have num_rounds = 1 and n_microbatches % 1 is 0. Nrrrr.rrc|dj|_t||||||t ||_|dj|_td||jz|_ ||j z|_ ||j zdkrtd|j d|di|_ t|jD]!}||}||j |<"dS)Nrrrrrrrrz_Interleaved 1F1B requires the number of microbatches to be a multiple of the number of rounds ( ), but got .)rrqrrr^rrrrvnumber_of_roundsmicrobatches_per_roundrr`rr r>rrrrrrrrrs r@rz ScheduleInterleaved1F1B.__init__ss.$AY1 )+//    "&kk1I( #A~9K'K L L&48M&M# D1 1Q 6 6-595J--)--- CE$,-- 1 1D==dCCH(0D  % % 1 1rBrbc  fd}| jjz}| z }||z } |z|z}td |||fd} fd}t jj ||||S)Ncjdz jz}d}||jdz |z zz}t|jjzS)Nrr&rrrqr+rrwarmups_ops_last_stagemultiply_factorrr>s r@get_rank_warmup_opszVScheduleInterleaved1F1B._calculate_single_rank_operations..get_rank_warmup_opsc#a'+&, " O//#a'4/3J z4#7$:M#MNN NrB=rank %s, warmup_ops %s, 1f1b %s, cooldown_ops %s total_ops %scB|jzjz}|jzzSrRrrrqr local_indexrr>s r@rzVScheduleInterleaved1F1B._calculate_single_rank_operations..forward_stage_index+4#>>$BUUK$"44< rs r@rzWScheduleInterleaved1F1B._calculate_single_rank_operations..backward_stage_indexK#:%$*EE%&&   $"44< rrmicrobatch_opsrrrrrrs `` @r@rz9ScheduleInterleaved1F1B._calculate_single_rank_operationss O O O O O)(.. ,t/CC$z1 % 3 ,|;  K          = = = = = =  = = = = = = ="             rBrrGrHrIr r rr]r r rrr rSrrrrMrr r s@r@r r as*'+AEBFIM"1"1'("1"1(# "1 "%(<"=> "1 $Do)=$>? "1$E$sCx.%**D$EF"1"1"1"1"1"1H9 hw>O9P9 9 9 9 9 9 9 9 rBr ceZdZdZ ddeededeedee e dfdee e e fd ee e e efe eff fd Zd eeefd Zd ZxZS)r"aw The Interleaved Zero Bubble schedule. See https://arxiv.org/pdf/2401.10241 for details. Will perform one forward and one backward on inputs for the microbatches in steady state and supports multiple stages per rank. Uses the backward for weights to fill in the pipeline bubble. In particular this is implementing the ZB1P schedule in the paper. Nrrrr.rrc8|dj|_t||||||t ||_|dj|_td||jz|_ ||j z|_ ||j zdkrtd|j d|di|_ t|jD]!}||}||j |<"||j|jz|_ dS)Nrr rzZZero bubble requires the number of microbatches to be a multiple of the number of rounds (r r)rrqrrr^rrrrvrrrr`rr_add_bubbles_to_actionsrs r@rz&ScheduleInterleavedZeroBubble.__init__sR$AY1 )+//    "&kk1I( #A~9K'K L L&48M&M# D1 1Q 6 6-595J--)--- CE$,-- 1 1D==dCCH(0D  % % #::  $"4 4  rBrbc   fd}| jjz}| z }||z } |z|z}td |||fd} fd}} t jj ||||| d S)Ncjdz jz}d}||jdz |z zz}t|jjzSr!rrs r@rz\ScheduleInterleavedZeroBubble._calculate_single_rank_operations..get_rank_warmup_ops rrBrcB|jzjz}|jzzSrRrrs r@rz\ScheduleInterleavedZeroBubble._calculate_single_rank_operations..forward_stage_index,rrBc^jdz |z jzjzz }|jzzSr!r"r#s r@rz]ScheduleInterleavedZeroBubble._calculate_single_rank_operations..backward_stage_index1r$rBT)rr%) r>rrr&rrrrrrrs `` @r@rz?ScheduleInterleavedZeroBubble._calculate_single_rank_operations s O O O O O)(.. ,t/CC$z1 % 3 ,|;  K          = = = = = =  = = = = = = =!%!          !#    rBcb|j}d}t}i}i}i}d}t|jD]} g|| <d|| <d|| < d} t} t|jD]} || } | t || kr%d} || | || | } | J| \}}}||||||sR|| || | || |||f|| xxdz cc<|| d|| xxdz cc<|| xxdz cc<|| d|| | rnE|dkrt d|||S)Nc|tjkr|dkr |dz ||f|vrdSn4|tjkr$||dz kr|tj|f|vS|dz||f|vSdS)NrrTF)r%r:r=)rr microbatchnum_stages_globalseen_opss r@ need_bubblezJScheduleInterleavedZeroBubble._add_bubbles_to_actions..need_bubbleLs%---A::519b*"=X"M"M4'555-111!#3#;ZHPXXX 2z2(BB5rBrTFrz?Non zero bubbles added: total_bubbles_added=%s bubbles_added=%s) r`r=rrqr^rr>updaterr)r>r2rfr4r3result next_pointer bubbles_addedtotal_bubbles_addedr should_stop temp_seen_ops timestamp temp_actionrNrr1s r@r*z5ScheduleInterleavedZeroBubble._add_bubbles_to_actionsIsF%    ) $Do)=$>? ) $E$sCx.%**D$EF) ) ) ) ) ) V> hw>O9P> > > > @@@@@@@@rBr"ceZdZdZ ddeededeedee e dfdee e e fd ee e e efe efd ee eefffd Zd eeefd ZxZS)r#a The Zero Bubble schedule (ZBV variant). See https://arxiv.org/pdf/2401.10241 Section 6 for details. This schedules requires exactly two stages per rank. This schedule will perform one forward and one backward on inputs for the microbatches in steady state and supports multiple stages per rank. Uses backward with respect to weights to fill in the pipeline bubble. This ZB-V schedule would have the "zero bubble" property only if time forward == time backward input == time backward weights. In practice, this is not likely true for real models so alternatively a greedy scheduler could be implemented for unequal/unbalanced time. Nrrrr.rrrc |dj|_t|||||||t ||_|jdkrt d|jd|dj|_|dj |_ i|_ t|jD]!}| |} | |j |<"dS)Nr)rrrrrrrr&z0ZBV requires exactly 2 stages per rank, but got r) rrqrrr^rrrrrr`rr) r>rrrrrrrrrrs r@rzScheduleZBVZeroBubble.__init__s$AY1 )+//&?    "&kk  ! # #*&***  1I(  ). CE$,-- 1 1D==dCCH(0D  % % 1 1rBrbctdjzdz j}dt|D}d\}}}}dj|z zdz }|} jdz |z } t|D]1} |t | t||dz }2|} t| D]`} |t | t||dz }|t | t||dz }aj|z } t| D]} |t | t||dz }|t | t||t | t||dz }||ks||kr$||kr/|t | t||dz }|t | t||t | t||dz }|t | t||dz }|t | t||t | t||dz }||k||k$||}}|}t|D]`} |t | t||dz }|t | t||dz }aj|z }t|D]`} |t | t||dz }|t | t||dz }a||kr5|t | t||dz }||k5||kr5|t | t||dz }||k5||kr||ksJ||kr||ksJfd|D}|S)Nr&rcg|]}dSrRrKrs r@rlzKScheduleZBVZeroBubble._calculate_single_rank_operations..rrB)rrrr)rOrPcJg|]}||j|jjkr|nd SrR)rPr)rerEr>s r@rlzKScheduleZBVZeroBubble._calculate_single_rank_operations.., sT     %+7+d.BBB    rB) rvrqrrrrrMr0r1r2)r>rn_microrf0_cntf1_cntb0_cntb1_cnt warmup_n1stage_id_chunk0stage_id_chunk1r1 warmup_n2 warmup_n3w0_cntw1_cnt cooldown_n1 cooldown_n2s` r@rz7ScheduleZBVZeroBubble._calculate_single_rank_operationssna$,,q0$2FGG,G,G5;;,G,G,G*4&+d23a7 /A-4y!!  A OO!fUUU    aKFF y!!  A OO!fUUU    aKF OO!fUUU    aKFF&- y!!  A OO!fUUU    aKF OO!fUUU    OO!fUUU    aKFFvoo'!1!1'!f !  OO!fUUU    OO!fUUU    aKF OO!fUUU    aKF OO!fUUU    OO!fUUU    aKF5voo'!1!18  {##  A OO!fUUU    aKF OO!fUUU    aKFF(4/ {##  A OO!fUUU    aKF OO!fUUU    aKFFvoo OO!fUUU    aKF voo voo OO!fUUU    aKF voo Ff$4$4$44Ff$4$4$44     #    rB)NNNNNr'r s@r@r#r#s!  &'+AEBFIM>B$1$1'($1$1(# $1 "%(<"=> $1 $Do)=$>? $1$E$sCx.%**D$EF$1$,DcN#;$1$1$1$1$1$1Lthw>O9PttttttttrBr# schedule_namec Xtttttt t td}d|D}| }||vr4td|dt||||S)z Maps a schedule name (case insensitive) to its corresponding class object. Args: schedule_name (str): The name of the schedule. )1F1BInterleaved1F1BGPipe LoopedBFSInterleavedZeroBubblerr ZBVZeroBubblec8i|]}||SrK)lower)reks r@r\z&get_schedule_class..J s"@@@qaggii@@@rBzUnknown schedule name 'z'. The valid options are ) rr rr!r"rrr#rrZrr)rQ schedule_maplowercase_keyslowercase_schedule_names r@rr9 s2&!>"8!6.  LA@L,=,=,?,?@@@N+1133n44 im i idS_SdSdSfSfNgNg i i    '>? @@rBc fdtDdtD d D dtdttf fd }dttdtf fd }rd }tD]m}t |d kr|d }||r,| ||||d d }a||d ntd D]}t |d kr|=tD]}t |d kr |d+|d }||rC|&| |d< |||d td D]}t |d kr|=|sRtdt D]#}td|d|d $td͉ S)aThis function dry-run simulates the actions in the schedule from the perspective of all ranks, and flags any deadlocks caused by missing or misordered communications. It also simulates any bubbles in time where a rank can not execute any action due to waiting for unmet dependencies. The total number of simulator steps can be used as a metric for unit tests involving IR optimization passes as reordering and merging of IR can reduce the number of simulated steps. The simulation is not high-fidelity and does not model overlapping of compute and communication, or cuda streams. Future work may be to enhance this and model the compute time, comms overlap, and even memory. c6i|]}|d|DS)cg|]}||SrRrK)rerAs r@rlz6_simulate_comms_compute...` s@@@Q!-q---rBrK)rerr`s r@r\z+_simulate_comms_compute.._ s?  @@..@@@rBci|]}|gSrKrKr[s r@r\z+_simulate_comms_compute..c s'222b222rBc,i|]}|tSrKr^r[s r@r\z+_simulate_comms_compute..g s.Q.Q.QttSUU.Q.Q.QrBrrEc||||||dSdSrR)rr>)rrE_prev_ops_rank _schedules r@add_to_schedulez0_simulate_comms_compute..add_to_schedulei sH$v&&&   4 $ $V , , , , ,  rBrbc|dS|j} |}|jtkrZ|jdkrdSt|jt|j|vrdSt|jdz t|j|vrdSdS|jt tfvr|j dz krdSt|jt|j|vrdSt|jdzt |j|vrdSt|jdzt|j|vrdSdS|jtkrdS|jtkr$t|jt|j}||vS|jtkr3|dz }t|t|j}| |vS|jtkrHt|jt |j}t|jt|j}||vp||vS|jtkr3|dz}t|t|j}| |vStd|)NTrrFzUnsupported action type ) rNrOr0rMr6rPr;r=r8r<r5r7r) rErbprev_ops expected_fpeer_stage_idx expected_send expected_b expected_bwrerrXs r@rkz3_simulate_comms_compute.._ready_to_schedulen s >4& !-- ":":;  "a ' '!Q&&t*FF4KLLPXXXt*Q.63JKKxWWt5  $(G G G!Z!^33tv)663JKKxWWt*Q.@WXXt*Q. v?VWWt5  $ 7 74  $ . . !3Q8OPPJ) )  $ . .&]N#NFFrrr) r`rXrrgrkrorrErjrerfs ``` @@r@rrS sJ>**N22#N33222I/R.Qy.Q.Q.QN-c-8G+<------- 8B8G#48B8B8B8B8B8B8B8B8Bt +<>** , ,D>$'((A--#D)!,F!!&)) ,%#OD&111t$((+++d++++555 & &A>!$%%**"1%>** , ,D>$'((A--r".#D)!,F!!&)) ,%*0IdOB'"4(,,V444t$((+++555 & &A>!$%%**"1% < )+A)+L+L M M M& I IGGGnT.B1.EGGHHHH:;; ;W +rsP  ########,,,,,,,,                         <<<<<<<<******TTTTTTTTTT%%%%%%'&&&&&&     8 $ $3E3E3E3E3Et3E3E3El  "!0"2  "  "         .   F      j    J(,;;d8G#4556;}; ;;;;|n n n n n n n n b11TZ(1 11116: $* %-c] #ty.6DDDDD.DDDN000001000fRRRRR*RRRjO1O1O1O1O1)O1O1O1hCC(7+,CC ']CCCCL (7+,  ']    Ff#tG},-fSE3J'ff #tG}  ffffR@2 #tHW-.. /@2@2@2 @2@2@2@2Fn2n2n2n2n2-n2n2n2b w2w2w2w2w24w2w2w2t ;;;;;-;;;NFFFFRo o o o o 3o o o dvvvvv$9vvvrjjjjj1jjjZAcAAAA4B#+SE3J#7BEHBBBBJ!.!.!.!.!.rB