j vUddlZddlZddlZddlmZddlmZddlmZm Z m Z m Z m Z m Z ddlZddlmZddlmZddgZes:ddlZGd d Zd Zeejd _eejd _dSdd lmZddlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)ej*e+Z,e r( ddl-m.Z.n #e/$re,0dYnwxYwGddej1Z2e2Z3e2e4d<dde5fdZ6GddZddde5de e7dfde e e5dfdefdZdS)Nreduce)chain)DictListOptionalTuple TYPE_CHECKINGUnion) is_available)not_noneinit_device_mesh DeviceMeshceZdZdS)_DeviceMeshStubN)__name__ __module__ __qualname__NC:\Projects\Historical\venv\Lib\site-packages\torch/distributed/device_mesh.pyrrs rrcdSNrrrr_init_device_mesh_stubrs rztorch.distributed.device_mesh)Backend) _find_pg_by_ranks_and_tag_get_default_group_get_group_tag get_backendget_process_group_ranksget_rankget_world_sizeinit_process_groupis_initialized new_group ProcessGroup split_group) ArrayLikezCDeviceMesh requires numpy >= 1.21 to be installed for type checkingc xeZdZddZddZdddeedfd eeedfddfd Z dddd e eddfd Z dd Z ddde efdZ ededefdZededefdZddd edefdZ ddedede ejddfdZdeeedffdZddd ededfdZdS)_MeshEnvreturnNcLg|_i|_i|_i|_i|_dSr) mesh_stackchild_to_root_mappingmesh_dim_group_optionsroot_to_flatten_mappingflatten_name_to_root_dimsselfs r__init__z_MeshEnv.__init__Cs=02DOGID &  'UWD (  * * *rrcjt|jdkrtd|jdS)Nrz#No device mesh is currently active!)lenr- RuntimeErrorr2s rget_current_meshz_MeshEnv.get_current_meshOs34?##q(("#HIII?2& &r device_meshsubmesh_dim_names. submesh_dimscpfd|D}j}g}g}d}t||D]\} } t| dkr|| d|z | d|z }|| d|z |t| dz z }||j| jd|| d|z |j| dtt|j } |D]} | | |j g| |Rj dg|R}  }| D] }tj||d}||vr|}!||_|j|<|S)Nc8g|]}tfd|dS)c>|j|zSr)meshsize)xyr:s rz5_MeshEnv.create_sub_mesh...`s[%5%:%:1%=%=!=rr).0mesh_dimr:s r z,_MeshEnv.create_sub_mesh..^sG  ====rrrEr6) start_dimend_dimFmesh_dim_names _init_backend)r@zipr7flattenappendr0_dim_group_infoslistrangendimremovepermutereshaper!r device_typer.)r3r:r;r<slice_dim_size mesh_tensor slice_dim_idxslice_dim_group_infonum_dims_flattenmesh_dim_indices mesh_dim_namemesh_dims_remained_idxidxpg_ranks_by_dimcur_rankmesh_ndsubmesh res_submeshs ` rcreate_sub_meshz_MeshEnv.create_sub_meshTsy !- N&*KM#%  ! 36|EV3W3W  / -'((1,,"-"5"5"21"58H"H 0 47G G#6##K"(()9!).sC$$$+122889PQQ$$$r_cDg|]}tj|Sr)r rL)rFdimrls rrHz0_MeshEnv.create_flatten_mesh..s9!!9::3?rz# already exists for submesh of the . z5The mesh_dim_names of submesh and flattened mesh are z-. Please specify another valid mesh_dim_name.r6rL)_mesh_resources get_root_meshr rLjoinr1 setdefaultrrRkeysr8r0mathprodr@rArSrTrUrVrWr!rrXr.tuple)r3r:r_flatten_dims_in_rootinvalid_dim_namesflattened_mesh_dim_sizeremained_dims_in_rootflatten_dim_in_rootrbrcrdflattened_meshres_flattened_meshrls @rcreate_flatten_meshz_MeshEnv.create_flatten_meshs(55kBBI$$$$/7 8R/S/S$$$ !  ##7!!   * 5 5i D D D %!hy78899!/ :??AA!!!  111"$VVVVVCL]CCCT999!T%A)%LLL3I>}MM&*i 0@0E0E0G0G&H&H #$(y~/B)C)C$D$D !'; B B#%,,-@AAAA4in4&)=gb122 !))++H* 8 8!+)$1#3""" w&&)7&=FD &'9 :TfD ( 3 3Ir B B= QGLMaGbGbD *9 5m D% %rcD|j|d}|s|n|Sr)r.get)r3r:rls rrsz_MeshEnv.get_root_meshs+266{DIII&/>;;Y >rc||}|j}|r=|r;t|dks Jd|d}|||SdS)z Returns the index of the mesh dim in the root mesh. The device_mesh passed in needs to be sliced out from the root mesh or submesh of the root mesh. rEz"The submesh can only be a 1D mesh.rN)rsrLr7get_mesh_dim_by_name)r3r:rlchild_mesh_dim_nameschild_mesh_dim_names rget_root_mesh_dimz_MeshEnv.get_root_mesh_dims~ **;77I#.#=  Q1 Q,--2227322&:1&=#00>>G}GGGa[E_aaK6<<]KKLL Lrrobackend pg_optionsc||f|j|<dSr)r/)r3rorrs r_set_mesh_dim_group_optionsz$_MeshEnv._set_mesh_dim_group_optionss 18/DD ' , , ,rc( |||krtd|j|i|j|}g|j| t fd|Dst d|d dd}g}|D]}}||vr&||}|d}||n0|j|}||f||krt d|dd|d d |}~|S) z Validate whether the mesh_dim_names is valid for slicing the given device_mesh. If valid, return dim indexes of the slice mesh in the device mesh. z'Cannot create a submesh from a submesh.c3 K|]}|vV dSrr)rFr_valid_mesh_dim_namess r z0_MeshEnv._get_slice_mesh_dims..2s=!!55rzInvalid mesh_dim_names z% specified. Valid mesh_dim_names are .r6z specified. z!Found mesh dim indices to slice: rpz.Mesh dim indices should be in ascending order.) rsr8r1rurLallrrPrj) r3r:rLr1curr_idxslice_mesh_dimsr_ mesh_indicesnext_idxrs @r_get_slice_mesh_dimsz_MeshEnv._get_slice_mesh_dimssd00===="#LMMM  * 5 5k2 F F F(,(F{(S %$+$*$ %3 HnHH0DHHHH O!/ $ $  $===#<]#KL ,B/H#**<8888*9?? NNH#**H;777x''"N.NNNOOOOOH $" "rc|||}|jd|d|j|}|}g}|D]I}t |j||fd}||vr|j|gng|_| |J|S)z` Return all the submeshes of a given mesh dimension of the device mesh. r6FrK) rr@swapdimsrWrAr!rrXrQrP) r3r:r_rGrbrc res_submeshesmesh_1dres r_get_all_submeshesz_MeshEnv._get_all_submeshesSs 00mLLH).77HEEMMK$))(33O#++--HM* . .$+$1#3"'  7**!1(;<<( $$W---- rr+Nr+rr)r:rr+r)rrrr4r9r strrintrgrrrsr staticmethodrrr C10dBackendOptionsrrrrrrr*r*Bs^     ' ' ' ' M %M  %S#XM uS#X/ M   M M M M `MQ@ &@ &+@ &>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import DeviceMesh >>> >>> # Initialize device mesh as (2, 4) to represent the topology >>> # of cross-host(dim 0), and within-host (dim 1). >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]]) rXr@.rLNTrKr(rMr+c||_t|tjr"|jjdkrt d|t|tjr2|tj n tj |dtj |_ |rt|nd|_ t|j |_d|_|dkr|r(||t)r*t+dkrt-j|_|j t1k}|ddvsJ|ddkr|dnd|_dSdS) Ncpuz!`mesh` must be a CPU tensor, got dtypedevicerxlathreadedr)rrE)rX isinstancerTensorrtype ValueErrordetachtortensorr@ryrLrOtolist_flatten_mesh_list _thread_id_get_or_create_default_group_init_process_groupsr$r threading get_identr!nonzerorA_coordinate_on_dim)r3rXr@rLrM rank_coordss rr4zDeviceMesh.__init__s +D $ -- M$+2Be2K2K !KT!K!KLLLdEL11G   uy 111\$uEIFFF I >&e%ee2FeeIMIYeee((6J)JKKK%'' 'rc g}t}|jjdkr|jt krt t t }tj r%t|dkrtd|dn|}| t|||jfnt |jjD]}|jd|d|j|}|t$jvrt$j|\}}nd\}}|jrd|j|nd |} d}t+|d dx} %t-|||| }|D]} | } | t| ||| }|| vrrt3||kr(t5d |d| d| tt7|| |jf||_dS)NrEgloozcpu:gloo,cuda:nccl mesh_default)rranks group_descr6)NNmesh_ mesh_dim_bound_device_id) parent_pgr split_ranksr)rrrrzFEach device mesh dimension should get only one process group, but got z in !)rr@rTrr"rRrSrrr rr%rPr group_namerrWrArrr/rLrr'rr!r7r8r rQ) r3dim_group_infos default_groupr dim_grouprorbrrrrdim_meshsubgroup_rankss rrzDeviceMesh._init_process_groupssACO.00My~""ty'8'8N#3#34455z..00 ' $M22f<<  4##1' &&&y11!,!00PPC'+i&8&8S&A&A&I&IDINN3//''O oDDD,B3G#&J/9+ ./: 3C 8:::...!%I+2)+(>(@(@'1 %%% %4)1):): +2(1&4(/+5+5 )))I ==??n<<"?33c99&2%>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import DeviceMesh >>> >>> # Initialize a 2D device mesh as (2, 4) to represent the topology >>> # of cross-host(dim 0), and within-host (dim 1). >>> mesh_2d = init_device_mesh(device_type="cuda", (2,4), mesh_dim_names=("dp", "tp")) >>> tp_mesh = mesh_2d["tp"] >>> dp_mesh = mesh_2d["dp"] >>> >>> # Initialize a 3D mesh. >>> mesh_3d = init_device_mesh(device_type="cuda", (2,2,2), mesh_dim_names=("dp", "pp", "cp")) >>> # The order of the mesh_dim_names provided deteremines the order of dimensions in the submesh. >>> dp_cp_mesh = mesh_3d["dp", "cp"] >>> cp_dp_mesh = mesh_3d["cp", "dp"] z1Cannot slice a DeviceMesh without mesh_dim_names!N) rLr8rrrrrr _subclasses fake_tensorunset_fake_temporarilyrg)r3rLrres r __getitem__zDeviceMesh.__getitem__sZ& X"#VWWW&0%D%DX!!. !444 "1"F"F.##&2IIKK-==noGs<B%%B),B)rGct|dstd|jjdkr!|td|jjddd|jjdkr+|)t t |jd dd St|}tj |d}|rG|| vr1||jd dd }t t |St|trt||n|}t t |j|dd S) a Returns the single ProcessGroup specified by mesh_dim, or, if mesh_dim is not specified and the DeviceMesh is 1-dimensional, returns the only ProcessGroup in the mesh. Args: mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index of the mesh dimension. Default is None. Returns: A :class:`ProcessGroup` object. rQz*DeviceMesh process groups not initialized!rENFound the DeviceMesh have dimensionsJOptional kwarg `mesh_dim` needs to be specified when device_mesh.ndim > 1.zmIf you want to get the list of all the ProcessGroups in the DeviceMesh,please use `get_all_groups()` instead.r)hasattrr8r@rTr rrQrrrsr0rrvrrr)r3rGrlr0rs r get_groupzDeviceMesh.get_groups4!344 Q"#OPPPy~!!h&6"LLLL`=y~""x'7-t/DQ/G/KL(55d;;I&5&M&Q&Q4'' #' 87N7S7S7U7U+U+U"9("C"TUV"WXZYZXZ"[ 9? KLLL"(C00"O88xHHH!  -t/DX/NrPQr/RSrcNfdtjjDS)z Returns a list of ProcessGroups for all mesh dimensions. Returns: A list of :class:`ProcessGroup` object. c:g|]}|Sr)r)rFir3s rrHz-DeviceMesh.get_all_groups..s%EEE!DNN1%%EEEr)rSr@rTr2s`rget_all_groupszDeviceMesh.get_all_groupss,FEEEuTY^/D/DEEE Errqgroupc"t|trt|}t|tjr||ks"|Bt|tjs(||kr"t dt|d|tj|dtj }t|||d}t|||j fg|_ |St|}t|dkrt d |t d t|tjr3|tj d n tj|dtj }|jt|kr5t d |d t|dt|||d}d|D|_ |S)aM Constructs a :class:`DeviceMesh` with ``device_type`` from an existing :class:`ProcessGroup`. The constructed device mesh has number of dimensions equal to the number of groups passed. If more than one group is passed, then the ``mesh`` argument is required. Nz Invalid mesh z for ProcessGroup with ranks rrFrKrz.Expects at least one ProcessGroup to be passedz0Must pass mesh if passing multiple ProcessGroups)rrzEExpects mesh with ndim equal to number of ProcessGroups but got mesh z and z ProcessGroupscVg|]&}t|t||jf'Sr)rr r)rFrs rrHz)DeviceMesh.from_group..JsH,,,  #5))+E22$,,,r)rr&r rrrrrrrrrrrQrRr7rrrT)rrXr@rL group_ranksr:groupss r from_groupzDeviceMesh.from_groups; %.. #5e<< tU\22 7;{{}} 7S7S$&tU\::% ++$]D ]]P[]]|KUYOOO(#1"'  $E**K9IJ0 ,#"%[[F6{{a !QRRR| !STTTdEL11G   uy ???\$uEIFFF  yCKK'' L KKMMLL03F LLL%T.PUK,, $ ,,,K ( rcl||jn|j|Sr)r@rrA)r3rGs rrAzDeviceMesh.sizeTs,(0(849??$$$dinnX>V>V Vrc|jjSr)r@rTr2s rrTzDeviceMesh.ndimWs 9> !rc4t|jjSr)ryr@rr2s rrzDeviceMesh.shape[s)) )rctS)z: Returns the current global rank. )r!r2s rr!zDeviceMesh.get_rank_s:: rc|jdkr |td|jjdd|d}t||}t |t s Jdtt|S)a{ Returns the local rank of the given mesh_dim of the DeviceMesh. Args: mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index of the mesh dimension. Default is None. Returns: An integer denotes the local rank. The following program runs on each process/rank in an SPMD manner. In this example, we have 2 hosts with 4 GPUs each. Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 0, 1, 2, 3 would return 0. Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 4, 5, 6, 7 would return 1. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 0, 4 would return 0. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 1, 5 would return 1. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 2, 6 would return 2. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 3, 7 would return 3. Example:: >>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import DeviceMesh >>> >>> # Initialize device mesh as (2, 4) to represent the topology >>> # of cross-host(dim 0), and within-host (dim 1). >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]]) rENrrrrz1We expect ProcessGroup before calling `get_rank`!)rTr8r@r rrr&r!)r3rGmesh_dim_groups rget_local_rankzDeviceMesh.get_local_rankes8y1}}!1"LLLL`!%dnnX&>&>??N  C CB C CH^4455 5rc"|jr|jndS)z Return the relative indices of this rank relative to all dimensions of the mesh. If this rank is not part of the mesh, return None. N)rr2s rget_coordinatezDeviceMesh.get_coordinates /3.EO4**4 Orr_cd|jstdt||S)a\ Returns a 1D DeviceMesh by flattening the current DeviceMesh. If no mesh_dim_name is provided, the default is a string concatentaing the mesh_dim_names of the given submesh with each mesh_dim_name separated by "_". For example, if we have a 3D mesh DeviceMesh([[[0, 1], [2, 3]], [[4, 5], [6, 7]]], mesh_dim_names=("dp", "cp", "tp")), calling mesh_3d["dp", "cp"]._flatten() will create a 1D submesh DeviceMesh([0, 1, 2, 3], mesh_dim_names=("dp_cp",)) on rank 0, 1, 2, 3 and a 1D submesh DeviceMesh([4, 5, 6, 7], mesh_dim_names=("dp_cp",)) on rank 4, 5, 6, 7. After the flattened dimension is created, to access the flattened dimesnion in mesh_3d, one can use the existing slicing method to obtain the flattened mesh through calling mesh_3d["dp_cp"]. z3Cannot flatten a DeviceMesh without mesh_dim_names!)rLr8rrr)r3r_s r_flattenzDeviceMesh._flattens<& "I#66t]KK Krrrr)%rrr__doc__r__annotations__rrrr r boolr4rrrrrrobjectrrrr&rrrrr rApropertyrTrr!rrrrrrrr{s% % Nl sCx11119="& ( ( ( (  k12(  %U38_5 (   ( ( ( ( ( T ( ( (:r 4r 4r 4h      - - - - $c $ $ $ $     4    G "'U38_(<"=G  G G G G R, , huS#X&?, <, , , , \ FD$6 F F F F @D@  9= @ @ @ tL'99:@ @ 5{!:;<@  %U38_5 @   @ @ @  @ D W W# W# W W W W  "# " " "  "  *5c? * * *  * c     ( 6( 68E#s(O+D( 6PS( 6( 6( 6( 6T PHT#Y$7 P P P P L L(3- L< L L L L L Lrrq mesh_shape.rLr+c |tt|t|krtdd|t|t|kr1tddt|dt|d|r(|std|d d t jd 5t jtj|tj  |}dddn #1swxYwYt||| }|S)a Initializes a `DeviceMesh` based on `device_type`, `mesh_shape`, and `mesh_dim_names` parameters. This creates a DeviceMesh with an n-dimensional array layout, where `n` is the length of `mesh_shape`. If `mesh_dim_names` is provided, each dimension is labeled as `mesh_dim_names[i]`. .. note:: `init_device_mesh` follows SPMD programming model, meaning the same PyTorch Python program runs on all processes/ranks in the cluster. Ensure `mesh_shape` (the dimensions of the nD array describing device layout) is identical across all ranks. Inconsistent `mesh_shape` may lead to hanging. .. note:: If no process group is found, init_device_mesh will initialize distributed process group/groups required for distributed communications behind the scene. Args: device_type (str): The device type of the mesh. Currently supports: "cpu", "cuda/cuda-like". Passing in a device type with a GPU index, such as "cuda:0", is not allowed. mesh_shape (Tuple[int]): A tuple defining the dimensions of the multi-dimensional array describing the layout of devices. mesh_dim_names (Tuple[str], optional): A tuple of mesh dimension names to assign to each dimension of the multi-dimensional array describing the layout of devices. Its length must match the length of `mesh_shape`. Each string in `mesh_dim_names` must be unique. Returns: DeviceMesh: A :class:`DeviceMesh` object representing the device layout. Example:: >>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import init_device_mesh >>> >>> mesh_1d = init_device_mesh("cuda", mesh_shape=(8,)) >>> mesh_2d = init_device_mesh("cuda", mesh_shape=(2, 8), mesh_dim_names=("dp", "tp")) Nz"Each mesh_dim_name must be unique.z/Found repeated mesh_dim_name in mesh_dim_names z6mesh_shape and mesh_dim_names should have same length!zFound len(mesh_dim_names): z and len(mesh_shape):rz0Device type with index is not supported but got rpzUIf you maintained a 'torch.device' object, it's recommended to pass in 'device.type'.rr)rXr@rL) r7setr8isalpharrarangerwrxrviewr)rXrrLr@r:s rrrsR  %3~&&''3~+>+>>>"8VnVV :#n"5"555"Ln#n2E2Enn\_`j\k\knnn  {2244 R;RRRg \%  Y Y< * 5 5UYGGGLLZXXD Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y #)   sAD$$D(+D()r)8loggingrwr functoolsr itertoolsrtypingrrrr r r rtorch.distributedr torch.utils._typing_utilsr __all__sysrrmodulesrrtorch._C._distributed_c10drr"torch.distributed.distributed_c10drrrrr r!r"r#r$r%r&r' getLoggerrlogger numpy.typingr( ImportErrorwarninglocalr*rrrrrrrrrr5sd DDDDDDDDDDDDDDDD ******...... | ,|~~_JJJ           ?NCK/0;0K'BAAAAA                            W x ( (F  . . . . . . .    NNU       l!l!l!l!l!9?l!l!l!\ !) OX***111111mLmLmLmLmLmLmLmLf59 GGGG#s(OG!sCx1 G  GGGGGGs7B>>CC