Oj4XUddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z mZmZddlZddlmcmZddlmcmcmZddlmZddlmZddlmZdd lmZm Z m!Z!m"Z"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.dd l/m0Z0m1Z1ddl2m3Z3ej4e5Z6da7e8dze9d<da:e8e9d<de0de;fdZej=dSde8fdZ?ej@ddGddZAej@ddGddeAZBdeAd eCe,d!eDeEeCeEfddfd"ZFd#e'd$eGeEd%fde'dzfd&ZHd#e'd$eGeEd%fde'dzfd'ZIeJZKeJeGeEeGeEd%feLfe9d(<d)e'd$eGeEd%fd*eGe,d%fd+eGe,d%fd,eEd-eLd.eLddfd/ZMd0eCeAd)e'd*eGe,d%fd+eGe,d%fdeCeAeBzf d1ZNiZOeDeGejPe'e#fd2fe9d3<d)e'd4e#dd2fd5ZQdTd6ZRGd7d2ZS dUd8e d9e d:e8dzdeCeAfd;ZTe dUd8e d9e d:e8dzdeCeAfd<ZUdddd=d>ejVd?e d@e dAe8d:e8dzdBe8dejVfdCZWddDdEdFdGe dHejXdIejXdAe8f dJZYGdKdLeZZGdMdNeZ[GdOdPej\j]Z^GdQdRej\j]Z_dS)VN) defaultdict)Sequence)cache)Anycast TypedDict)_are_we_tracing) _MeshLayoutone_step_redistribute_cost)_StridedShardNotDecodableError DTensorSpec ShardOrderShardOrderEntry TensorMeta)assert_no_mixed_partial_types) DeviceMesh)_is_shard_like _StridedShardPartial Placement ReplicateShard) FloatLikeType IntLikeType)get_active_debug_mode#_FORCE_MIN_COST_REDISTRIBUTION_PLANF,_DISABLE_REDISTRIBUTE_TRANSFORM_OPTIMIZATIONcostreturnct|tr|St|tjr |jj}nt|S|j}|st|S|jjj}| |r"t| |St|jj |dS)N)fallback) isinstancefloattorchSymFloatnodeexpr free_symbols shape_envvar_to_hint_overrideissubsetxreplaceoptimization_hint)rr(r)hint_overridess TC:\Projects\DSCons\.venv\Lib\site-packages\torch\distributed\tensor\_redistribute.py_redistribute_cost_sort_keyr18s$ $''y~T{{$L T{{Y(=N^,,4T]]>22333 $66td6KK L LLTenabledc#8Kt}|a dV|adS#|awxYw)uX Context manager to control the redistribution planning strategy for DTensor operations. This context manager allows you to choose between two algorithms for computing the sequence of collective operations needed to redistribute a DTensor from one placement to another: - **Graph-based**: Uses Dijkstra's algorithm to find the minimum-cost path through all possible placement transformations. This approach considers the global cost of all collective operations and finds the optimal sequence. Best for complex redistribution patterns where reducing communication cost and memory overhead is critical. - **Greedy**: Uses a heuristic approach that makes locally optimal choices at each step. This is faster to compute but may not produce the globally optimal transformation sequence. Best for simple redistribution patterns or when planning speed is more important than optimal communication. **Default Behavior (without this context manager):** When this context manager is NOT used, the algorithm selection follows this priority: 1. **Non-default shard orders** → Always use graph-based algorithm (required for correctness) 2. **Explicit `use_graph_based_transform` parameter** to `_gen_transform_infos_non_cached` → Use the specified algorithm (True = graph-based, False = greedy) 3. **No explicit parameter** (default case) → Use greedy algorithm for faster planning **Behavior with this context manager:** This context manager overrides the default selection by setting the global flag `_FORCE_MIN_COST_REDISTRIBUTION_PLAN`, which takes precedence over the explicit `use_graph_based_transform` parameter (but not over non-default shard order requirements). **Cache Considerations:** The redistribution planner caches transform info for performance via the `@cache` decorator on `_gen_transform_infos`. If you need to change the algorithm selection for the same input specs, clear the cache using `_gen_transform_infos.cache_clear()` to ensure the new setting takes effect and doesn't reuse cached results from a previous run. Args: enabled (bool): If True, forces the use of the graph-based algorithm. If False, forces the use of the greedy algorithm. Default: True N)r)r3 old_values r0 use_min_cost_redistribution_planr6Ks@j4I*1'8 .7+++i+7777disabledc#8Kt}|a dV|adS#|awxYw)a Context manager to disable the transform optimization pass that merges consecutive same-type collectives into single flattened operations. When the optimization is disabled, ``_optimize_transform_infos`` becomes a no-op and returns the original list of ``_TransformInfo`` objects unchanged. This is useful for debugging or isolating issues related to the flattened collective merging logic. The flag can also be set directly:: torch.distributed.tensor._redistribute._DISABLE_REDISTRIBUTE_TRANSFORM_OPTIMIZATION = True Args: disabled (bool): If True (default), disables the optimization. If False, explicitly enables it (the normal default). N)r)r8r5s r0+disable_redistribute_transform_optimizationr:s@*=I3;0A 7@444y4@@@@r7frozenslotscbeZdZUeed<eeefed<eeed<dZ de dzfdZ dS)_TransformInfomesh_dimsrc_dst_placements logical_shapec|jdkrt|jd|jdkrtddS)NrzQTransformInfo should only be created if it is an op with some effect, not a no-op)r@AssertionErrorrAselfs r0 __post_init__z_TransformInfo.__post_init__sN =1    "1 %)@)C C C c  D Cr2r NcB|j\}}|r|rdS|rt|rdSt|r|rdSt|rt|rdSdS)z Return a key for grouping transforms by communication type. Returns None for local ops (no communication needed), or a string that identifies the collective type for potential grouping/merging. all_reducereduce_scatter all_gather all_to_allN)rA is_partial is_replicater)rGsrcdsts r0_comm_type_keyz_TransformInfo._comm_type_keys*S >>    0 0 2 2 < ^^   ."5"5 ## C  S%5%5%7%7 < C  ^C%8%8 <4r2) __name__ __module__ __qualname__int__annotations__tuplerrrrHstrrRr2r0r?r?stMMMi23333K((((d r2r?cTeZdZUdZeed<eedfed<dZedzed<d dZ dS) _FlattenedTransformInfoa4 Represents a flattened transform that combines multiple mesh dimensions into a single collective operation using a flattened DeviceMesh. Note: inherits the fields from _TransformInfo. Gets an __init__ with parent fields, followed by child fields, and runs parent validation (post_init) mesh.original_mesh_dimsN avg_scaler ct||j"|jdkstd|jdSdS)NrDz"avg_scale must be > 1 if set, got )r?rHr_rErFs r0rHz%_FlattenedTransformInfo.__post_init__sX$$T*** > %>A%%$III & %%%r2r N) rSrTrU__doc__rrWrXrVr_rHrZr2r0r\r\sl c3h'''!IsTz   r2r\transform_infocurrent_placementsshard_order_dictc |j\}}t|tr|j}n|jf}t|t t zr|j}t}|D]c}t||dkrtd|d|d|d|| || dt||ks+tdt|d|d|d|d| t|t t zr0|j} | |vrg|| <|D]} ||  | |D]} ||| <dS) zw Update current_placements and shard_order_dict in-place to reflect the effect of a single transform step. rz?Invalid shard_order update. No entries left to pop for src_dim z. transform_info=z, current_placements=z, shard_order=z^Mismatch between expected and removed mesh dims during shard_order update. Expected to remove z, but removed N)rAr#r\r^r@rrdimsetlen ValueErroraddpopappend) rcrdre src_placement dst_placement mesh_dimssrc_dim removed_dim_dst_dimr@s r0"_update_shard_order_and_placementsrus$2#D M=."9::/"5 #,. -!677#ee  = =A#G,--22 6661?66*<66$466 OO,W599;; < < < <9~~,,2y>>221<22#122'9 22 0 22 -!6777# * * *(* W %! 7 7H W % , ,X 6 6 6 655'48$$55r2r]rp.c@|}|jdStfd|D}||}t |g}|jD]}|j|kr|cSdS)z Query for an explicitly created flattened mesh using layout comparison. Searches root_mesh._flatten_mapping for a mesh whose layout matches the expected flattened layout for the given dims. Pure Python layout math. Nc3(K|] }|V dSNrZ).0imesh_dim_namess r0 z5_get_flattened_mesh_by_layout_impl..%s(;;AnQ';;;;;;r2) _get_root_meshr{rX_get_slice_mesh_layoutr collapse_flatten_mappingvalues_layout)r]rp root_mesh dim_names sliced_layoutexpected_layoutflattened_meshr{s @r0"_get_flattened_mesh_by_layout_implrs##%%I(Nt;;;;;;;;;I// ::M!=#9#9#;#;"<==O$4;;==""  !_ 4 4! ! ! ! 5 4r2ctr`tjjjrJt ||dSddlm}tjj |t|St ||S)aB Query for an explicitly created flattened mesh using layout comparison. When tracing with compile_on_one_rank, delegates to a custom op so the flattened mesh appears as a call_function node derived from mesh (a graph input) rather than as a get_attr constant holding an unpicklable ProcessGroup. Nr) device_mesh) r r% distributedconfigcompile_on_one_rankrtorch.distributed._opsrops_get_flattened_submeshlist)r]rprss r0_get_flattened_mesh_by_layoutr4s~SU.5IS .dI > > F4;;;;;;y$;;D$y//RRR -dI > >>r2_warned_flatten_issuesrsrc_placementsdst_placementsnum_ops comm_typereasonc t|||f}|tvrdSt||j . fd|D}dd|D} n%ddd|Dd|} d} |d krd | d } n(|d krd } n|dkrd|d} nt d|t | ||||| dS)a- Warn once per (mesh, dims, reason) about inability to flatten operations. Args: device_mesh: The device mesh being used mesh_dims: Tuple of mesh dimensions that could not be flattened src_placements: Source placements for the redistribution dst_placements: Target placements for the redistribution num_ops: Number of sequential operations that will be performed comm_type: Type of collective operation (e.g., "reduce_scatter") reason: Either "no_flattened_mesh" or "uneven_tensor_shape" Nc g|] }| SrZrZ)rydr{s r0 z;_warn_flatten_optimization_not_possible..ks:::1^A&:::r2z, c3"K|] }d|dV dS)"NrZ)rynames r0r|z:_warn_flatten_optimization_not_possible..ls*??T[[[[??????r2zdims c34K|]}t|VdSrx)rY)ryrs r0r|z:_warn_flatten_optimization_not_possible..ns($?$?SVV$?$?$?$?$?$?r2z of a%While redistributing from %s to %s, %d sequential %s operations will be performed. This is suboptimal: multiple collective operations have higher latency (separate kernel launches and synchronization points) and may give inconsistent results between ranks due to different reduction orders. %sno_flattened_meshz&To optimize, flatten mesh dimensions [z0] so DTensor can use a single operation instead.uneven_tensor_shape)z Unfortunately, because the tensor dimension is not evenly divisible by the product of the mesh dim sizes that would need to be flattened for the optimization to work, it can not be optimized.non_ascending_mesh_dimsz0it is not possible to merge non-ascending order z operations.zUnexpected reason: )hashrrkr{joinrEloggerwarning) rrprrrrr cache_keyrdims_strcommon_warning reason_msgr{s @r0'_warn_flatten_optimization_not_possiblerOs`*k""Iv6I***y))) /N!:::: ::: 99??Y?????R499$?$?Y$?$?$???RR[RR `$$$xhxxx ( ( (  , , , Vy V V V  ;6;;<<< NNJr2transform_infosc :t|dkr|Str|Sthddtdzdtffd }dt t t fdt t t fdtfd d ttdt tdztdzfffd }g}d }|t|kr||}| } || s| ||d z }V|j } |g} |d z} | t|kr|||  r|| j | rm| || | d z } | t|kr:|||  r|| j | m|| \} }| | | na| | |dvrHt td| D}t|||t| | || }|t|kt d|||S)a Optimize transform infos by merging consecutive same-type collectives into a single flattened operation when a matching flattened DeviceMesh exists. Merging requirements: - Operations must be consecutive in the transform list (no reordering). Notably, redistributing from P, P, P -> R, S, R is not optimized here and cannot be optimized due to optimization needing to fuse non-contiguous reductions, leaving this pattern vulnerable to numerics issues and suboptimal perf - Operations must have the same comm type (e.g., all allgather or all reduce_scatter) - Operations must have identical src_dst_placements (e.g., can't merge Partial->Shard(0) with Partial->Shard(1)) - A flattened mesh covering the relevant dimensions must exist - For reduce_scatter, tensor dim must be evenly divisible by flattened mesh size For nested sharding, the merged operation uses the logical_shape from the outermost mesh dimension (smallest mesh_dim index) which represents the global tensor shape needed for correct padding/unpadding. TODO: - all_to_all operations are excluded from merging, but it may be possible to merge them in some cases. >rLrJrKkeyNr c |vS)z:Check if a comm type key represents a mergeable operation.rZ)rMERGEABLE_COMM_TYPESs r0 is_mergeablez/_optimize_transform_infos..is_mergeables***r2p1p2c ||krdS|\}}|\}}||krdS|r|sdStt|}tt|}ddh}|j|vo|j|vS)z Check if two src_dst_placements can be merged. Allows merging of Partial("sum") and Partial("avg") since they can be combined: perform sum reduction, then scale by avg mesh dims afterward. TFsumavg)rNrr reduce_op) rrsrc1dst1src2dst2partial1partial2mergeable_reduce_opss r0are_placements_mergeablez;_optimize_transform_infos..are_placements_mergeables 884 d d 4<<5!! doo&7&7 5&&&& %u~  "6 6 ;"&:: r2infosc"t|dkrdS|dj|d}tfd|Dst dt d|D}t t |}|dkr ||krdSn|d kr||d d d krdSt|}|d S\}}|d kr/tt|j t|fd }n|dkrgtt|j t|fd}|j }tjfd|D} || zdkrdSn!|dkr |d}ntd|d } |} |r7tjfd|D} | dkr| } t#d} | |f} t%d| |j ||| d fS)a Try to create a flattened transform from 2+ same-type transforms. Returns (result, failure_reason) where: - result is the FlattenedTransformInfo if successful, None otherwise - failure_reason is None if successful, or one of: - "too_few_transforms": Less than 2 transforms provided - "no_flattened_mesh": No flattened mesh exists for the required dimensions - "uneven_tensor_shape": For reduce_scatter, tensor dim not evenly divisible r)Ntoo_few_transformsrc3:K|]}|jVdSrx)rA)ryinforfirst_placementss r0r|zJ_optimize_transform_infos..try_create_flattened..sH   % $T%<>N O O      r2z5All transforms must have mergeable src_dst_placementsc3$K|] }|jV dSrxr@)ryrs r0r|zJ_optimize_transform_infos..try_create_flattened..s$::D$-::::::r2rK)NrrLN)Nrc|jSrxrBx affected_dims r0zI_optimize_transform_infos..try_create_flattened..aol6Sr2)rc|jSrxrrs r0rzI_optimize_transform_infos..try_create_flattened..rr2c3LK|]}|jVdSrx)sizer@ryrrs r0r|zJ_optimize_transform_infos..try_create_flattened..sD2248   //222222r2)NrrJz0Unsupported comm type for try_create_flattened: c3K|]G}tt|jdjdk+|jVHdS)rrN)rrrArrr@rs r0r|zJ_optimize_transform_infos..try_create_flattened..)s_!8!;<<F%OO  //OOOOr2rDr)r@rArBr]r^r_)rirArRallrErXsortedrrrrgmaxrBmathprodNotImplementedErrorrNrr\)rrrpsorted_mesh_dimsrrPrQoutermost_infotensor_dim_sizeeffective_shard_mesh_sizer_ merged_srcscalemerged_placementsrrrrs @@r0try_create_flattenedz7_optimize_transform_infos..try_create_flatteneds u::>>--!86!H++--            !G ::E:::::   !2!233 ( ( (,,,66- , & &,TTrT222666{DTUU  !,,$S  $ $s++/L ,S,S,S,STTTNN * * *s++/L ,S,S,S,STTTN,:<HO(, 2222>   ,I!E qyy! $U^^ '- $#4,:##3#       r2rrD)rrrc3$K|] }|jV dSrxr)rygs r0r|z,_optimize_transform_infos..ks$(C(C(C(C(C(C(C(Cr2z5_optimize_transform_infos original: %s, optimized: %s)rir frozensetrYboolrXrrr?r\rRrmrAextendrrrdebug)rrrrrrresultrzr current_keyrdgroupj flattenedfailure_reasonrprrs ` @@r0_optimize_transform_infosrsZ: ?a3%%S%S%STT+#*+++++++ )Y& ' -29i3G-H     @i N#i &-sTz9 :i i i i i i i X>@F A c/"" " "q!))++ |K((  MM$    FA  "4'+f E O$$ $ $ _Q/>>@@AA %(("57I % LL+ , , , FA O$$ $ $ _Q/>>@@AA %(("57I %%9$8$?$?! >  MM) $ $ $ $ MM% " "&(C(CU(C(C(C"C"CDD 7""JJ" e c/"" " "f LL?RX Mr2DTensorRedistributePlanner_planner_cache dtensor_metactrt||Stj||f}|tvrt||}|t|<t|S)a Factory function to get or create a DTensorRedistributePlanner instance. This function provides transparent caching of planner instances based on device mesh and dtensor meta. Multiple calls with the same parameters will return the same cached instance for better performance. Args: device_mesh: The device mesh for the planner dtensor_meta: TensorMeta of the DTensor to redistribute Returns: A DTensorRedistributePlanner instance (potentially cached) )r rweakrefrefr)rrrplanners r0get_redistribute_plannerrsfE)+|DDD[))<8I&&,[,GG$+y! ) $$r2c8tdS)z8Clear the cache of DTensorRedistributePlanner instances.N)rclearrZr2r0 clear_redistribute_planner_cachersr2c.eZdZdZejddGddZdZede e e e fde fd Z ede de e e e ffd Ze d&d edeedeedfde d zdedef dZdededd fdZ d'dZdeedfde de deffdZdedede dfdZddde d ee dfde efd!Zd"ed#ed ee dfde efd$Z d"ed#ede efd%Z!d S)(ra This class is used to plan the collective calls to transform the local shard of the DTensor from its current spec to the target spec. Suppose there are N tensor dimensions and M mesh dimensions, the total possible state size will be (N+2)*M*M!. Note: Use get_redistribute_planner() factory function instead of direct instantiation for automatic caching. Tr;ceZdZUeedfed<eed<ejddddZ e dzed<dZ d Z d Z d e fd Zd e fd Zded efdZdS)$DTensorRedistributePlanner.DistState. placementstensor_dim_to_mesh_dimNF)defaultinitreprcompare_hashc@tj|j|jSrx)rformat_shard_order_strrrrFs r0__str__z,DTensorRedistributePlanner.DistState.__str__s"5+ r2c*|Srx)r rFs r0__repr__z-DTensorRedistributePlanner.DistState.__repr__s<<>> !r2cbt|d|dS)Nr)object __setattr__ _compute_hashrFs r0rHz2DTensorRedistributePlanner.DistState.__post_init__s9   ""$$     r2r cF|j|jn|Srx)rrrFs r0__hash__z-DTensorRedistributePlanner.DistState.__hash__s!!%!74::T=O=O=Q=Q Qr2c8t|j|jfSrx)rrrrFs r0rz2DTensorRedistributePlanner.DistState._compute_hashs$O/ r2otherct|tjsdS|j|jkrdS|j|jf|j|jfkSNF)r#r DistStaterrr)rGrs r0__eq__z+DTensorRedistributePlanner.DistState.__eq__s^e%?%IJJ uzU[((u+ , r2)rSrTrUrXrrWr dataclassesfieldrrVr r rHrrr rrrZr2r0rrs)S.)))) ****-K-u5%   sTz        " " "    Rc R R R R 3      4      r2rcxt|ttzrtfd|DS|S)z.s/<<$--<<<<<.sS  U seEll C C C      r2)rXritems)rs r0_dict_to_ShardOrderz.DTensorRedistributePlanner._dict_to_ShardOrdersB  $QWWYY//      r2cptt}|D]}t|j||j<|S)z1Convert ShardOrder to dict with tensor dim as key)rrrpr )rtensor_mesh_dim_dictentrys r0_ShardOrder_to_dictz.DTensorRedistributePlanner._ShardOrder_to_dictsD +400 K KE59%/5J5J !1 2 2##r2NFr]rrn.src_shard_order use_strided_shard_as_shard_orderct||jkr'tdt|d|j|rtj||d\}}|tj|}t |}t|}t t||}|g}g} |D]} t| t} t| ||t t|t|} || | | t!|dg} t#| D]K\}} | rdnd}| || t!||d zLd | S) a Generate a string representation of the sequence of state transitions (placements and shard orders) as described by the given transform_info. Args: mesh: The DeviceMesh used for the redistribution. transform_infos: A sequence of _TransformInfo objects describing each transformation step. src_placement: The initial tuple of Placement objects. src_shard_order: (Optional) The initial ShardOrder representing the mapping of tensor dimensions to mesh dimensions. If None, the default shard order is computed from src_placement and mesh. use_strided_shard_as_shard_order: If True, normalize _StridedShard placements into regular Shard placements with an explicit shard_order before stringifying. Returns: A string showing the sequence of DistState transitions, separated by '->'. zdimensions mismatch z vs Tr)Nrz-->z->rD)rindimrEr&_normalize_placements_into_shard_ordercompute_default_shard_orderrrr'rrXr#r\rur#rmrY enumerater)r]rrnr(r) cur_placementre cur_state state_listis_flattened_listrc is_flattened new_state trace_partsrz separators r0stringify_transform_infosz4DTensorRedistributePlanner.stringify_transform_infoss6 }   * * Js='9'9JJtyJJ  , B!4$ +M?  ")EmTTO]++ 5II   /88 - /    )+- 3 3N%n6MNNL . /?   3<<m$$*>>?OPPI   i ( ( (  $ $\ 2 2 2 2:a=))* ():;; 7 7OA|!-74I   y ) ) )   s:a!e#455 6 6 6 6ww{###r2rrc||_|st|t||_t |j|_t|_t|_ | dS)z Initialize DTensorRedistributePlanner. Args: device_mesh: The device mesh for this planner dtensor_meta: TensorMeta of the DTensor to redistribute N) r_is_current_rank_part_of_meshrErrishapetensor_dimensionrh"strided_shard_placements_in_targetpartial_reduce_ops_in_targetsetup_cost_callbacks)rGrrs r0__init__z#DTensorRedistributePlanner.__init__@s}'88:: !   ( #L$6 7 7FIee/69ee) !!#####r2cTdtjdtffd fd}|_dS)z Set up the cost function for different collective operations. Uses communication time estimation based on actual tensor sizes and mesh topology for accurate cost modeling. stater cTtj|jj|jdS)NF)r]r tensor_meta shard_orderr))rrrrr)rCrGs r0 state_to_speczFDTensorRedistributePlanner.setup_cost_callbacks..state_to_spec`s7% + -!816  r2cHt||Srxr ) src_state dst_staterGs r0 cost_functionzFDTensorRedistributePlanner.setup_cost_callbacks..cost_functionks.- i((-- *B*B r2N)rrrrK)rGrKrGs` @r0r@z/DTensorRedistributePlanner.setup_cost_callbacksWs^ -7              +r2rtensor_mesh_dim_tuplerci}t|}||||}|D]7}|j}||d}t ||t s4t|jD]} || kr || } ||  | t|} t | | | <||| t |} | || || <|| | ||  9|D]}|j}||d}t ||t s3|| } t|} t| | <||| t |} || | | || || <t|D]}\}} t | t st|} t| |<||| |} | || || <~t|D]\}} t | tst|jD]} t|} t | | |<||  |||| t |} | || || <||  t|D]\}} t | t st|jD]} t|} t | | |<||  |||| t |} | || || <||  t|D]\}} t | ts|jD]}t|} t!|| |<d| D}t%|dkr |ddhkrK||| |} | || || <|D]}|j}||d}t ||t&s3|| } t|} t| | <||| t |} || | | || || <|js|St|D]\}} t | ts|jD]}|j} t|} || |<||  |||| t |} | || || <||  |S)NrcFh|]}t|t|jSrZ)r#rrryps r0 z.@s9&&&$%:a;Q;Q&K&&&r2rDrr)rr'rrr r#rranger=rlrmrr#rKrr0rr?rirr>rg)rGrrLall_next_stater%cur_dist_stater&src_tensor_dim src_mesh_dimdst_tensor_dim move_mesh_dimnew_placements dist_state placementr@rpartial_reduce_opsstrided_shard_objs r0get_next_statez)DTensorRedistributePlanner.get_next_staters HMO9MM !   NN: & & !  + ; ;E"-N/?CLj6>> "'(=">"> ; ;!^33!5^ D H H J J $^4;;MJJJ!%j!1!105n0E0E}-!^^NN>22.BB, .2-?-?"..z* %^4;;MJJJ$^488::::+ ;6+  E"-N/?CLj6>> 0@DDFFM!*--N,5KKN= )~..*>>?STTJ ! 0 7 7 F F F)-););**N: & &(1'<'<   #L)i11 !*--N+4;;N< (~..0EJ*.););**N: & &$-Z#8#8 ; ; Hii33 "'(=">"> ; ;!%j!1!1+0+@+@x($^4;;HEEE!^^NN>22.BB, .2-?-?"..z*%^488:::: ;&$-Z#8#8 ; ; Hii11 "'(=">"> ; ;!%j!1!1+0+@+@x($^4;;HEEE!^^NN>22.BB, .2-?-?"..z*%^488:::: ;*$-Z#8#8   Hii33 !>   !%j!1!1+29+=+=x(&&)7&&&")**Q..3E%QV3W3W!^^NN>224I .2-?-?"..z** 4+  E"-N/?CLj6 FF 0@DDFFM!*--N,5KKN= )~..*>>?STTJ ! 0 7 7 F F F)-););**N: & & 6 "! !$-Z#8#8 ; ; Hii33 %)%L ; ;!!2!6!%j!1!1+<x($^4;;HEEE!^^NN>22.BB, .2-?-?"..z*%^488::::! ;$r2rIrJc ddl}d}d|d||gfg}t}|r||\}}}} } | |kr| S| |vr*|| || j| j} | D]T\} } | |vrKtt|| z}| | gz}|dz }| |t|||| |fU|td|d|)aB Find the min cost path from src_state to dst_state using Dijkstra's algorithm. Args: src_state: The source state dst_state: The destination state Returns: A list of states representing the min cost path from src_state to dst_state rNgrDzNo path found from src_state z to dst_state ) heapqrhheappoprkr^rrr"rrheappushr1rE)rGrIrJr`counterpqvisitedrsr current_statepath next_states next_statetransition_costnew_costnew_paths r0find_min_cost_pathz-DTensorRedistributePlanner.find_min_cost_pathsh  7CYK 8 9 %% .3mmB.?.? +Aq$ t )) '' KK & & &--(-*NK0;/@/@/B/B  + OW,,#C@H#zl2HqLGNN7AA#$&$    ! 4 PI P PY P P   r2r@full_tensor_shapec t|}|jD])}|j}|j}t |dkrt |D]}||kr |j|} t| trR| |||j ||j |\} } nyt| trR| |||j ||j |\} } ntd| | ||<+|S)NrrzUnsupported placement type: )rrr rprirErr#rlocal_shard_size_and_offsetrr_sym_get_coordinaterrj) rGrIr@rnnew_logical_shaper&r rpmdimr[new_sizerss r0get_logical_shapez,DTensorRedistributePlanner.get_logical_shapesm 044E/F/F5 9 9E)JI9~~""$$! 9 98##%06 i// Q"+"G"G)*5(--t-<<(<._try_normalize_specsw4 9F 9=, &{22#+$%KT%K%KLLL(888r2rz@Multiple mesh_dims are different between cur_state and nxt_stater@rArB)rrXrrr#rr>rk itertoolschainrr?rrrmpairwiserr0ziprErurmr?)rGrvrwrnr{rr(rdst_shard_orderr[rIrJr state_pathr2 nxt_stateupdate_mesh_dimr@r1 nxt_placementrBs r0$generate_graph_based_transform_infosz?DTensorRedistributePlanner.generate_graph_based_transform_infoss2 9 9 5C(*4 5 9 9 9 9"+>*=h*G*G'*=*=h*G*G' ( G GI)]33 G7;;IFFF #HH K KI)W-- K155i6IJJJNN>?CC NN>?CC 02,,Y BB $-$6z$B$B   Iy#y';;;"$@I ,i.BCCAA(>%x1B)) (..*)84A=3Q.;r2c Vt|j}|g}g}|jjdkr[|jd|jdkr=|t d|jd|jdf||St|jD]\}}||}t|tr||jjdz kr|j |} | ||j | |j |\} } t|} | | |j <|| ||t|j} t|j}|jdkr,tt!t#| D]}| |}||}t|tr|j }gg}}tt%| |D]d\}\}}||krnU||r||||r||e||krt)}||kr2|t |||f|||| |<tt%| |D]@\}\}}||kr2|t |||f|||| |<A|S)a Generate the transform infos from the source placements to the target placements. To transform from source to target placement it might have multiple steps, i.e. it might decompose Si -> Sj into Si -> R -> Sj. This would detect if there're mis-aligned/nested shardings between src/dst placements. E.g. Suppose the redistribution to perform is (Shard(0), Shard(0)) -> (Replicate(), Shard(0)), in this case Shard(0) -> Shard(0) for mesh dimension 1 actually needs resharding, because in the former is a nested-sharding of a tensor already sharded dimension 0, whereas the latter is the first sharding on tensor dimension 0. rDrr|r)rr<rr-rrmr?r0r#rr_local_shard_size_and_offsetrgrq num_shardsreversedrRriris_shardr)rGrvrwinitial_logical_shapemesh_dims_to_logical_shaperrzrPcurrent_logical_shape mesh_dim_sizelocal_shard_sizersrrrdtarget_placementsr@currenttarget shard_dimcurrent_mesh_shardingtarget_mesh_shardingsrPs r0generate_greedy_transform_infosz:DTensorRedistributePlanner.generate_greedy_transform_infos4s$!%X^ 4 4&;%<"02   A % %"1%)q$A !#u%% It',q000$($4$9$91$9$E$EM*-*J*J-cg6%(<.B$$$\$ .216 G$G$G$!.1G$Y^,G$$d* G$ +/ G$ G$G$G$\G$R$$!$  $$$$.+ ++++6V)S.)V *V 4e; < VVVVt< "< /8< 4 5< < < < |!9!!!c? ! k  !!!!BGGG!c? G n  GGGGR~~~ n  ~~~~~~r2rvrwuse_graph_based_transformc|j}|j}|j}td||fD }tdg|j|jRD}|s|rd}nt t }n|d}|jtt||j}|rD | |||j } n<#t$r| ||} YnwxYw| ||} | S)Nc3>K|]}tj|VdSrx)ris_default_device_order)ryorders r0r|z2_gen_transform_infos_non_cached..sA$$  +E22$$$$$$r2c3@K|]}t|tVdSrx)r#rrOs r0r|z2_gen_transform_infos_non_cached..s@  1m$$r2TF) rrFranyrrrErErrr<r r) rvrwrrr(rhas_non_default_orderhas_strided_sharddrprs r0_gen_transform_infos_non_cachedrsz &K*O*O!$$$%7$$$!!=8&=)<==* 1*$(!! , 8$G!! " *$)!# "  C!R V!FF(HNOO. V V V!AA(HUUOOO V==hQQ sB77 CCc$t|||Srx)r)rvrwrs r0_gen_transform_infosrs +(5  r2)async_opr is_explicit local_tensor current_spec target_specrrc 2|j|jkrtd|jtdt |jt |j|}|j}|s|Strt|||}nt|||}t|||j|j} t} | O| ||j|jt|| |j|j|j|nt!j} | 5| D]} t%| t&r| j} n|} | j}| j\}}t%|t,st%|t,r| |us Jd| |}||kr|}|dkr|}|r|rTt5t6|}||| |}t%| t&r| j || jz }n{|r4t5t>|}| || || j!}n3t%|t,r| |||| j!}ntEd|d |d |rt5t>|}|rUt5t6|}|#|| ||}t%| t&r| j || jz }nV|r-|$|| || %|}n|rJt5t>|}|j&|j&kr#|'|| || j!|j&}nt%|t,rJ| |||| j!}|$|| || %|}nXtd |d ||r|r.t5t6|}|(|| |}ntS|rtEd|d |d ||krtUd |d|d|}nt%|t,r|rYt5t6|}||| |}|+||||%|}n%|r,|+||||%|}n|r^t5t>|}| || || j!}|+||||%|}nst%|t,rI| |||| j!}|+||||%|}ntd |d ||s.t%|tXj-r|.}|} dddn #1swxYwY|S)z This redistribute the local tensor (torch.Tensor) from the current DTensorSpec to the target DTensorSpec, which involves the necessary collective calls to transform the local shard of the DTensor from its current spec to the target spec. z)Cross device mesh comm not supported yet!NzUuse_strided_shard_as_shard_order should be initialized in DTensorSpec.__post_init__())rz:_StridedShard redistribute assumes no flattened transformsrrDzredistribute from z to z not supported yetzUnexpected placement z& for redistribute to target placement z&Redistribution from one partial type (z) to another (z) is unsupported.)/r]rr)rjrrr;r rrrrrecord_redistribute_callsrr9rF contextlib nullcontextr#r\r@rArrrOrNrr _reduce_valuer_rr_to_replicate_tensorrB RuntimeError_reduce_shard_value_replicate_to_shardrqrg_to_new_shard_dim_partition_valuerrE_replicate_to_strided_shardfuncolAsyncCollectiveTensorwait)rrrrrrnew_local_tensorrroptimized_transform_infos debug_moderedistribute_contextrc mesh_to_userzrr num_chunks partial_speccurrent_placementtarget_placement shard_spec replicateds r0redistribute_local_tensorrs K,,,!"MNNN4< c   ","9:::!+"8999##K  4 4 6 6 9 +'@  / +'@   !: !!'((J  ! ,,   #  " & @ @)'(=   $ -  # % %$ b,b,7a ,a ,N.*ABB *,1 ) 'A,?OGV '=11 Z 5V5V "k111P211%))1)55J&  #/ Q$0 ""$$} %%''#'#9#9L'3'A'A$k1(($ #>3JKKW*4@+;n>V+V(%%'' (,UG(<(<%'8'M'M$k1n6R(($$ 77'.'C'C$k1n6R(($$'TWTT&TTT""b #'v#6#6 %%''(#'#9#9L'3'G'G$k16F(($ #>3JKKW*4@+;n>V+V())++'7'K'K$k1k6U6UVW6X6X(($$%%''!%eW!5!5J!~)9)===+5+G+G('*8,0 ,,( 77!(!=!=$k1n6R""J(8'K'K"##77:: (($$%ggg_egg""$$6 ''))4#'#8#8L'3'D'D$k1(($$$G,, 4&TWTT&TTT&((,uWuu\buuu(4$$FM22% %%''##'#9#9L!-!;!;$k1""J(.'I'I"KK4S4STU4V4V(($$))++'-'I'I$k1k6U6UVW6X6X(($$%%''(,UG(<(<%!2!G!G$k1n6R""J(.'I'I"KK4S4STU4V4V(($$ 77 ")!=!=$k1n6R""J(.'I'I"KK4S4STU4V4V(($$%ggg_egg ;  &">!! ;$4#8#8#:#: +LLCa ,b,b,b,b,b,b,b,b,b,b,b,b,b,b,b,F s>W\  \\r grad_outputdtensor.DTensor previous_spec out_dtypeop_dtypec  ||jjkr|j|}t|jj|jjt|j| ||jj }t|j|j|j |j }n|j}|j}g}t|j|jD]s\}} t|s|r6| r"|t#^|| tt|jt%||j |j }t'||||} | j|kr| |} t|jt%|t|j| | j|j } | | fS)a Common function for redistributing a distributed tensor during backward and twice-backward backpropagation steps. Args: grad_output: The output gradient tensor. previous_spec: DTensorSpec prior to redistribution. out_dtype: dtype to cast the returned gradient to. Pinned by autograd to match the dtype of the input of the node above this one. op_dtype: dtype to run the backward collective at. async_op: whether to perform the DTensor redistribute operation asynchronously or not. Default: False Returns: A :class:`torch.Tensor` object. A :class:`DTensorSpec` object. dtyper<striderr]rrEr))rrEr)rrEr)) _local_tensorrtor_specrrrr<rr)rErrrOrNrmrrXr) rrrrrrrnormalized_placementsrroutputrys r0_redistribute_backwardrs82;,222"033(3CC ""."(3"!'"))++ .9->-_    $*$/$0-:-[    #0 "( .0|6 8PQQ11 7 # # 1w';';'='= 16CTCTCVCV 1 ! ( ( 5 5 5 5 ! ( ( 0 0 0 0!.//!-)6)W M' F|y  9%% ! #$$#%%'',   *7)W   D 4<r2c8eZdZUejed<ejed<dS)_BackwardDtypeConfigrrN)rSrTrUr%rrWrZr2r0rrIs.k{r2rcBeZdZUejed<ejed<eed<dS) _DtypeConfigrrbackward_optionsN)rSrTrUr%rrWrrZr2r0rrNs:k{******r2rc beZdZedddedeedfdedef dZ ed d Z d S) Redistributeinputrrr.r dtype_configc ||_|d}|d|_|d|_|d}|d}||jjkrj|j|} t ||jjt|j | ||jj } n|j} |j} | |_ | j|kr,t ||| j} t| | | |d } n| } | } | j|krY| |} t || jt|j | || j } t!j| | |j S) Nrrrrrr)rET)rrr requires_grad)r bwd_op_dtype bwd_out_dtyperrrrrrrr<rr)rrErdtensorDTensorr) ctxrrrrrbwdrrrrrrs r0forwardzRedistribute.forwardUs  -.z? , + - u*0 0 0 .111AAL&  ;1&+ <<>>" 271]   LL!.L ;L'  "j 0 0%Z\5MK/! FF"F&K <9 $ $YYy))F%&&+ <<>># 2=1]   K  -     r2rctt||j|j|j|j}|ddddfSrx)NestedRedistributeapplyrrrr)rroutput_dtensors r0backwardzRedistribute.backwardsL+11    L               r2N)rr) rSrTrUrrrXrrrrrrZr2r0rrTsH !H  H )S.) H  H #H H H \H T   \   r2rc jeZdZdZedddededejdejf dZ ed d Z d S) ra This class is used to make the redistribution of a DTensor twice-differentiable. This is called during the `Redistribute.forward`. Therefore, `NestedRedistribute.forward` is called during the first backward pass, and `NestedRedistribute.backward` is called during the second backward pass. Note: `NestedRedistribute.backward` is not differentiable, and therefore triple backward is not yet supported. rrrrrrc||_|jj|_||_t |||||\}}||_tj|||j S)N)rrrr) rrroriginal_dtyperrrrrr)rrrrrrrrys r0rzNestedRedistribute.forwardsx  (6< -         %3     r2 grad2_outputcz|j}t|||j|j|j}|dddddfSrx)rrrrrr)rrrrs r0rzNestedRedistribute.backwardsU( ,11   L L              r2N)rr) rSrTrUrbrrrr%rrrrZr2r0rrs  '  #    +  ;    \  D   \   r2r)Trarx)`rrr}loggingrr collectionsrcollections.abcr functoolsrtypingrrrr%)torch.distributed._functional_collectivesr_functional_collectivesrtorch.distributed.tensor._apitensor_apirr torch.distributed._mesh_layoutr *torch.distributed.tensor._collective_utilsr &torch.distributed.tensor._dtensor_specr rrrrtorch.distributed.tensor._utilsr$torch.distributed.tensor.device_meshr(torch.distributed.tensor.placement_typesrrrrrr torch.typesrrtorch.utils._debug_moder getLoggerrSrrrrWrr$r1contextmanagerr6r:rr?r\rrrVrurXrrrhrrYrrr ReferenceTyperrrrrTensorrrrrrautogradFunctionrrrZr2r0rs? ######$$$$$$'''''''''' :::::::::////////////EEEEEE666666QQQQQQJIIIII;;;;;;32222222999999  8 $ $ 48#TD[777 6;,d:::MmMMMMM& 9898d98989898x AA$AAAA8d$///       0/ Fd$///n0/4.5".5Y.53S >*.5 .5.5.5.5b !&sCx$>? ?!&sCx?$????0ADE#uS#X";<=EEE99S#X9)S.)9)S.) 9  9  9 9 9999xp.)pp)S.)p)S.) p  .2 23 ppppn '  +Z 78 " %%%"%%%%4 N N N N N N N N h.26666 $d{6 . 6666r.2 $d{ . -1ooo,ooo  o $d{ oo \oooop [[["[[{ [ k [  [[[[|9 +++++9+++ [ [ [ [ [ 5>*[ [ [ |C C C C C 0C C C C C r2