jgAddlZddlZddlmZmZmZmZmZddlZddl m Z ddl m Z mZddlmZmZgdZejddd gZd Zd Zejd i aet2_t2jr et2_dZdZejejfdZ dZ!dZ"ej#ddfdZ$dZ%dZ& ddZ'dZ(dZ)dZ*dZ+dZ,ede$e,dS)N)castDictListSetTuple)api constants)_group_membership_management_update_group_membership)backend_registeredregister_backendconstruct_rpc_backend_options init_backend BackendValue BackendTyper%construct_rpc_backend_options_handlerinit_backend_handlercd|jzS)Nz BackendType.)name)selfs WC:\Projects\Historical\venv\Lib\site-packages\torch/distributed/rpc/backend_registry.py_backend_type_reprrs DI %%z An enum class of available backends. PyTorch ships with a builtin ``BackendType.TENSORPIPE`` backend. Additional ones can be registered using the :func:`~torch.distributed.rpc.backend_registry.register_backend` function. rvaluenamescB|tjvS)z Checks if backend_name is registered as an RPC backend. Args: backend_name (str): string to identify the RPC backend. Returns: True if the backend has been registered with ``register_backend``, else False. )r __members__keys) backend_names rr r 3s ;27799 99rcFt|rtd|ddtD}t|t ||ifi|}t jd|att_tj rtt_ t|S)aRegisters a new RPC backend. Args: backend_name (str): backend string to identify the handler. construct_rpc_backend_options_handler (function): Handler that is invoked when rpc_backend.construct_rpc_backend_options(**dict) is called. init_backend_handler (function): Handler that is invoked when the `_init_rpc_backend()` function is called with a backend. This returns the agent. z RPC backend z: already registeredc(i|]}|j|jS)rr).0members r z$register_backend..RsNNN&+v|NNNr)rrrr) r RuntimeErrorrdictrenumEnumr__repr____doc___backend_type_doc)r!rrexisting_enum_dictextended_enum_dicts rrr@s,''NL,LLLMMMNN+NNN ,6[%9  )-7IJJJK-K0/  | $$rc *|jj||fi|SN)rr)backend rpc_timeout init_methodkwargss rrres/ ?7= >[  $*  rc&|jj|i|Sr2)rr)r3argsr6s rrrps -7= -t >v > >>rcxtj}tj||||}| Jd|dkr?||kr't d|d||dkr?||kr't d|d||S)Nz*Failed to initialize default ProcessGroup.zrank argument z doesn't match pg rank zworld_size argument z doesn't match pg size ) rpc_constantsDEFAULT_PROCESS_GROUP_TIMEOUTdistProcessGroupGloorankr(size)storer? world_sizeprocess_group_timeoutgroups r_init_process_grouprEts)G  !%z;P Q QE   J     --WDWWWWXXXbzUZZ\\99 T: T Tejjll T T    Lrc .ddlm}||||||S)Nr)TensorPipeRpcBackendOptions)r4r5num_worker_threads _transports _channels)rG)r4r5rHrIrJr6rGs r1_tensorpipe_construct_rpc_backend_options_handlerrLs?.----- & &-    rc:tfd|DS)Nc3nK|]/}|jdkp|jdkod|jcxkokncV0dS)cpucudarN)typeindex)r%d device_counts r z/_tensorpipe_validate_devices..sl  %MAFf,Lag1L1L1L1L 1L1L1L1Lr)all)devicesrTs `r_tensorpipe_validate_devicesrXs;   rc\dt|D}tj|||||f|d|D}d|D}d|D}d|D} t |||| t |||} t ||| }| |fS)Ncg|]}ddigf S)rKrr$)r%_s r zB_tensorpipe_exchange_and_check_all_device_maps..s 666Q"aR666rcg|] \}}}}| Sr$r$)r%rr[s rr\zB_tensorpipe_exchange_and_check_all_device_maps..s 444-$1a444rci|] \}}}}|| Sr$r$)r%rcountr[s rr'zB_tensorpipe_exchange_and_check_all_device_maps..s#GGG):uauGGGrci|] \}}}}|| Sr$r$)r%rr[map_s rr'zB_tensorpipe_exchange_and_check_all_device_maps..s#CCC&6dAtQtTCCCrci|] \}}}}|| Sr$r$)r%rr[rWs rr'zB_tensorpipe_exchange_and_check_all_device_maps..s#EEE%8T1a4EEEr)ranger@r=all_gather_object_validate_device_maps_create_reverse_mapping_create_device_list) my_namemy_device_countmy_device_maps my_devicesrDgathered all_namesall_device_countsall_device_maps all_devicesreverse_device_mapss r._tensorpipe_exchange_and_check_all_device_mapsrrs 76% "5"5666  7O^ZH%548444IGGhGGGCC(CCCOEEHEEEK)%6UUU2'9oVV$ZATUUJ  **rTc |D]}||}tt|t|krtd|d|t|||std|d|d|||D]U}|rjt|||s0td|d||d|||D]\}} tt| t| krtd|d|d| ||r\t| ||s!td|d |d| d ||nIt| ||s!td|d |d| d||||gr]t| ||s!td|d |d| d ||}||vrIt| ||s!td|d |d| d||̐WdS)NzNode z" has duplicated devices devices = z, has devices with invalid indices devices = z device count = z@ has invalid target node names in its device maps device maps = z node names = z5 has duplicated target devices in its device map for z device map = z5 has unexpected source devices in its device map for z devices = z? has source devices with invalid indices in its device map for z5 has unexpected target devices in its device map for z? has target devices with invalid indices in its device map for ) lenset ValueErrorrXr issubsetitemsvaluesget) rmrnrorpis_static_groupnoderW source_node target_noderas rreresG  d# s7||  G , ,NNNWNN ,G5Ft5LMM <<<$<<"3D"9<<  !22  3{'C'H'H'J'J#K#K#T#T $ $  , ,,!0!=!B!B!D!D,, ),,  "1!=!C!C!E!E( (  K3t{{}}%%&&#d))33 +K++-8++$(++ ;' 499;;''00JJ$@ @@1<@@(,@@&1%=@@2 .{; !GKGG-8GG$(GG'8 &DGG {B// 4;;==))22;{3KLL$@ @@1<@@(,@@&1%=@@ 111:V 0=;;1!GKGG-8GG$(GG'8 &DGGG( 22rc|st}|D])}||*|D])}||*|t jdt|}t|d}|S)NrOc|jSr2)rR)rSs rz%_create_device_list..s!'r)key) ruryupdater discardtorchdevicelistsorted)rkrjrq devices_setras rrgrgs '), "))++ , ,D   tyy{{ + + + +'..00 , ,D   tyy{{ + + + +EL//000+&&  (9(9:::J rci}|D]9}|||vr-d|||D||<:|S)Nci|]\}}|| Sr$r$)r%kvs rr'z+_create_reverse_mapping..s+)))A1)))r)rx)rhrmrorqr|s rrfrf sjGI od+ + +))!0!6w!?!E!E!G!G)))  % rcXddlm}t|tj}|}t j}t j r%|j rt j ||j |j fS)NrTensorPipeAgent) rKrrr _get_current_rpc_agent_get_backend_optionsrrPrT is_availablerWinit device_maps)ragentoptsrTs r_get_device_infosrs!!!!!! #"<">"> ? ?E  % % ' 'D:**,,L z  T\  )4< 77rc ddlm}t||}|}|j}|}iiigf\}}}}|D]} | j} | |krt j| t\} } } n@| }tj |j |j} } } | || <| || <| || <|| t!||||dt#|||}|D]H} t%|| || ||| <t j| t&||| |dfIdS)NrrF)r{T)r8)rKrrget_worker_inforget_worker_infosr rpc_syncrrrrPrTrrWappendrerfrgr )rrmy_worker_inforhall_worker_infosrnrorprm worker_info worker_namerT device_maprWrrqs r#_set_devices_and_reverse_device_mapr s!!!!!! % ( (E**,,N!G--//ACRR> Y'&& !& ' ! !03 .11 -L*gg--//D ''))  '.*L *6+&'1 $#* K %%%% 2'9oVV!   #6  $ok&BDW$ $ K   $ +k":>>>>>> eTZ ( (CA%AABBB )+F G G  cNa c c   :**,,L(3ddeOD$E4<<'U    +  '  ( ( $W : " " $ $   JOO              U### &9&EFFFF   *%t < <  $O#E   ' ' ' 4E::::     +                  s*%&G  FG F;;G  GG TENSORPIPE)T)- collectionsr*typingrrrrrrtorch.distributed distributedr=rKr r r;_utilsr r __all__ namedtuplerrr.r+rr,r-r rDEFAULT_RPC_TIMEOUT_SECDEFAULT_INIT_METHODrrrEDEFAULT_NUM_WORKER_THREADSrLrXrrrergrfrrrr$rrrsG ////////////// --------JJJJJJJJ   &{%<>TU &&&dim2666 ) ,+K : : :"%"%"%N51???,%? &+++.QUBBBBJ   8881 1 1 hUUUp5$r