§
    J�’jA/  ã                   ór  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
 d dlmZ d dlmZmZ d dlmZ  ej        d¦  «        Z ej        ¦   «         Z ej        d¦  «        Ze                     e¦  «         e                     e¦  «         e                     ej        ¦  «         d	e_         ed
¦  «        Z ed¦  «        Z G d„ de ¦  «        Z!dde"dz  de#fd„Z$dde"dz  de"fd„Z%dee#         de#fd„Z& G d„ d¦  «        Z'	 dde'dz  dedeeeef         geeedz  f         f         fd„Z(dS )é    N)ÚCallableÚSequence)ÚLock)Údefault_timer)ÚAnyÚTypeVar)Ú	ParamSpecÚstrobelight_function_profilerzB%(name)s, line %(lineno)d, %(asctime)s, %(levelname)s: %(message)sFÚ_PÚ_Rc                   ó   — e Zd ZdZdS )ÚStrobelightCLIProfilerErrorzC
    Raised when an error happens during strobelight profiling
    N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© ó    úVC:\Projects\DSCons\.venv\Lib\site-packages\torch\_strobelight\cli_function_profiler.pyr   r       s   € € € € € ðð ð ð r   r   ÚpidÚreturnc                 ó~   — d}| pt          j        ¦   «         } t          j        |                     | ¦  «        ¦  «        S )zFReturns the link to the process's namespace, example: pid:[4026531836]z/proc/{}/ns/pid)ÚosÚgetpidÚreadlinkÚformat)r   ÚPID_NAMESPACE_PATHs     r   Ú_pid_namespace_linkr   &   s7   € à*ÐØ
Ð
•”‘”€CÝŒ;Ð)×0Ò0°Ñ5Ô5Ñ6Ô6Ð6r   c                 ó¤   — | pt          j        ¦   «         } t          | ¦  «        }t          ||                     d¦  «        dz   d…         ¦  «        S )z"Returns the process's namespace idú[é   éÿÿÿÿ)r   r   r   ÚintÚfind)r   Úlinks     r   Ú_pid_namespacer&   -   sF   € à
Ð
•”‘”€CÝ˜sÑ#Ô#€DÝˆt�D—I’I˜c‘N”N QÑ&¨Ð+Ô,Ñ-Ô-Ð-r   Úcommandc                 ó,   — d                      | ¦  «        S )Nú )Újoin)r'   s    r   Ú_command_to_stringr+   4   s   € Ø�8Š8�GÑÔÐr   c                   ó  — e Zd ZdZ e¦   «         Zddddddddddd	œ
d
ededede	dedede
e	         dz  de
e	         dz  dedefd„Zd#d„Zd$deddfd„Zd#d„Zd#d„Zdeddfd„Zdefd„Zdeeef         d ej        d!ej        dedz  fd"„ZdS )%ÚStrobelightCLIFunctionProfileraž  
    Note: this is a Meta only tool.

    StrobelightCLIFunctionProfiler can be used to profile a python function and
    generate a strobelight link with the results. It works on meta servers but
    does not require an fbcode target.
    When stop_at_error is false(default), error during profiling does not prevent
    the work function from running.

    Check function_profiler_example.py for an example.
    FiX  g    ÐcAzpytorch-strobelight-ondemandé<   Né   )
Ústop_at_errorÚmax_profile_duration_secÚsample_eachÚrun_user_nameÚtimeout_wait_for_running_secÚtimeout_wait_for_finished_secÚrecorded_env_variablesÚsample_tagsÚstack_max_lenÚasync_stack_max_lenr0   r1   r2   r3   r4   r5   r6   r7   r8   r9   c       
         ó„   — || _         || _        || _        || _        || _        || _        d | _        d | _        || _        d S ©N)	r0   r1   r2   r3   r4   r5   Úcurrent_run_idÚprofile_resultr7   )Úselfr0   r1   r2   r3   r4   r5   r6   r7   r8   r9   s              r   Ú__init__z'StrobelightCLIFunctionProfiler.__init__H   sT   € ð +ˆÔØ(@ˆÔ%Ø&ˆÔØ*ˆÔØ,HˆÔ)Ø-JˆÔ*ð +/ˆÔØ04ˆÔØ&ˆÔÐÐr   r   c                 ó  — t          j        ¦   «         }t          |¦  «        }ddddddddt          | j        ¦  «        › d	t          | j        d
z  ¦  «        › d|› d|› �g}| j        rB|                     d¦  «         |                     d                     | j        ¦  «        ¦  «         t           
                    dt          |¦  «        ¦  «         t          j        |d¬¦  «        }|j                             d¦  «        }t           
                    d|¦  «         |j        dk    rt#          d|› �¦  «        ‚t%          j        d|¦  «        x}r)t          |                     d¦  «        ¦  «        | _        d S t#          d|› �¦  «        ‚)NÚstrobeclientÚrunz
--profilerÚpyperfz--eventÚcyclesz--asyncz--sample-intervalz--duration-msiè  z--pidú:z--sample-tagsú,úrunning command: %sT©Úcapture_outputúutf-8úoutput:
{%s}r   z:failed to start strobelight profiling, error in run_async:zINFO Run Id: (-?\d+)r!   z9failed to start strobelight profiling, unexpected result )r   r   r&   r#   r2   r1   r7   Úappendr*   ÚloggerÚdebugr+   Ú
subprocessrB   ÚstderrÚdecodeÚ
returncoder   ÚreÚsearchÚgroupr<   )r>   Ú	processIdÚ	namespacer'   ÚresultÚoutputÚmatchs          r   Ú
_run_asyncz)StrobelightCLIFunctionProfiler._run_asyncb   s•  € Ý”I‘K”Kˆ	Ý" 9Ñ-Ô-ˆ	àØØØØØØØÝ�4Ô#Ñ$Ô$Ð&ØÝ�4Ô0°4Ñ7Ñ8Ô8Ð:ØØÐ&Ð&˜9Ð&Ð&ð
ˆð  Ôð 	7Ø�NŠN˜?Ñ+Ô+Ð+Ø�NŠN˜3Ÿ8š8 DÔ$4Ñ5Ô5Ñ6Ô6Ð6å�ŠÐ*Õ,>¸wÑ,GÔ,GÑHÔHÐHÝ” ¸Ð=Ñ=Ô=ˆØ”×%Ò% gÑ.Ô.ˆÝ�Š�_ fÑ-Ô-Ð-àÔ Ò!Ð!Ý-ØUÈVÐUÐUñô ð õ ”IÐ5°vÑ>Ô>Ð>ˆ5ð 	Ý"% e§k¢k°!¡n¤nÑ"5Ô"5ˆDÔØˆFå)ØPÈÐPÐPñ
ô 
ð 	
r   r   Úcounterc                 ó€  — |dk    rt          d¦  «        ‚ddd| j        › g}t                               dt	          |¦  «        ¦  «         t          j        |d¬¦  «        }|j                             d	¦  «        }t                               d
|¦  «         |j	        dk    rt          d|› �¦  «        ‚t          j        d|¦  «        x}rd|                     d¦  «        }|dk    rd S |dk    r.t          j        d¦  «         |                      |dz   ¦  «         d S t          d|› d�¦  «        ‚t          d|› d�¦  «        ‚)Né   z*wait_for_running called more than 20 timesrA   ÚgetRunStatusú--run-idrG   TrH   rJ   rK   r   zAfailed to start strobelight profiling, error in wait_for_running:zProfile run status: (.*)r!   ÚRUNNINGÚ	PREPARINGé
   zunexpected z phaseúunexpected output
: r)   )r   r<   rM   rN   r+   rO   rB   rP   rQ   rR   rS   rT   rU   ÚtimeÚsleepÚ_wait_for_running)r>   r\   r'   rX   rY   rZ   Úcurrent_statuss          r   rg   z0StrobelightCLIFunctionProfiler._wait_for_running‹   sb  € Ø�RŠ<ˆ<Ý-Ø<ñô ð ð " >°:À$ÔBUÐ?WÐXˆÝ�ŠÐ*Õ,>¸wÑ,GÔ,GÑHÔHÐHÝ” ¸Ð=Ñ=Ô=ˆØ”×%Ò% gÑ.Ô.ˆÝ�Š�_ fÑ-Ô-Ð-àÔ Ò!Ð!Ý-Ø\ÐTZÐ\Ð\ñô ð õ ”IÐ8¸&ÑAÔAÐAˆ5ð 		XØ"Ÿ[š[¨™^œ^ˆNØ Ò*Ð*Ø�Ø ;Ò.Ð.Ý”
˜2‘”�Ø×&Ò& w°¡{Ñ3Ô3Ð3Ø�å1Ð2VÀÐ2VÐ2VÐ2VÑWÔWÐWå)Ð*KÀ&Ð*KÐ*KÐ*KÑLÔLÐLr   c                 ó$  — dddt          | j        ¦  «        g}t                               dt	          |¦  «        ¦  «         t          j        |d¬¦  «        }|j                             d¦  «        }t                               d|¦  «         |j	        d	k    rt          d
|› �¦  «        ‚t          j        d|¦  «        x}r?|                     d¦  «        }|                     d¦  «        rd S t          d|› d�¦  «        ‚t          d|› d�¦  «        ‚)NrA   ÚstopRunr`   rG   TrH   rJ   rK   r   z<failed to stop strobelight profiling, return code is not 0 :úINFO ::1:(.*)r!   zSuccess!z*failed to stop strobelight profiling, got z resultrd   r)   )Ústrr<   rM   rN   r+   rO   rB   rP   rQ   rR   r   rS   rT   rU   Ú__contains__)r>   r'   rX   rY   rZ   rh   s         r   Ú	_stop_runz(StrobelightCLIFunctionProfiler._stop_run©   s"  € Ø! 9¨j½#¸dÔ>QÑ:RÔ:RÐSˆÝ�ŠÐ*Õ,>¸wÑ,GÔ,GÑHÔHÐHÝ” ¸Ð=Ñ=Ô=ˆØ”×%Ò% gÑ.Ô.ˆÝ�Š�_ fÑ-Ô-Ð-àÔ Ò!Ð!Ý-ØWÈvÐWÐWñô ð õ ”I˜o¨vÑ6Ô6Ð6ˆ5ð 	Ø"Ÿ[š[¨™^œ^ˆNØ×*Ò*¨:Ñ6Ô6ð Ø�å1ØXÀÐXÐXÐXñô ð õ *Ð*KÀ&Ð*KÐ*KÐ*KÑLÔLÐLr   c                 ó$  — dddt          | j        ¦  «        g}t                               dt	          |¦  «        ¦  «         t          j        |d¬¦  «        }|j                             d¦  «        }t                               d|¦  «         |j	        d	k    rt          d
|› �¦  «        ‚t          j        d|¦  «        x}r{|                     d¦  «        }|                     d¦  «        r*t          j        d¦  «         |                      ¦   «          d S |                     d¦  «        st          d|› �¦  «        ‚g | _        t          j        d|¦  «        D ]8}| xj        |d	         z  c_        t                               |d	         ¦  «         Œ9d S )NrA   r_   r`   rG   TrH   rJ   rK   r   z<failed to extract profiling results, return code is not 0 : rk   r!   zProfile run status: PROCESSINGrc   z!Profile run finished with SUCCESSz9failed to extract profiling results, unexpected response zF(Total samples(.*)|GraphProfiler(.*)|Icicle view \(python stack\)(.*)))rl   r<   rM   rN   r+   rO   rB   rP   rQ   rR   r   rS   rT   rU   rm   re   rf   Ú_get_resultsr=   ÚfindallÚinfo)r>   r'   rX   rY   rZ   rh   Úitems          r   rp   z+StrobelightCLIFunctionProfiler._get_resultsÀ   s£  € Ø! >°:½sÀ4ÔCVÑ?WÔ?WÐXˆÝ�ŠÐ*Õ,>¸wÑ,GÔ,GÑHÔHÐHÝ” ¸Ð=Ñ=Ô=ˆØ”×%Ò% gÑ.Ô.ˆÝ�Š�_ fÑ-Ô-Ð-àÔ Ò!Ð!Ý-ØWÈvÐWÐWñô ð õ ”I˜o¨vÑ6Ô6Ð6ˆ5ð 		Ø"Ÿ[š[¨™^œ^ˆNØ×*Ò*Ð+KÑLÔLð Ý”
˜2‘”�Ø×!Ò!Ñ#Ô#Ð#Ø�Ø#×0Ò0Ð1TÑUÔUð Ý1ØXÐPVÐXÐXñô ð ð !ˆÔÝ”JØUØñ
ô 
ð 	!ð 	!ˆDð ÐÔ 4¨¤7Ñ*ÐÔÝ�KŠK˜˜QœÑ Ô Ð Ð ð	!ð 	!r   Úcollect_resultsc                 ó"  — 	 |                       ¦   «          t                               d¦  «         t                               d¦  «         |sd S |                      ¦   «          d S # t
          $ r  t                               dd¬¦  «         Y d S w xY w)Nzstrobelight profiling stoppedzcollection stoppedzerror during stop_strobelightT©Úexc_info)rn   rM   rr   rN   rp   Ú	ExceptionÚwarning)r>   rt   s     r   Ú_stop_strobelight_no_throwz9StrobelightCLIFunctionProfiler._stop_strobelight_no_throwß   s¢   € ð	Kà�NŠNÑÔÐÝ�KŠKÐ7Ñ8Ô8Ð8å�LŠLÐ-Ñ.Ô.Ð.à"ð Ø�à×ÒÑÔÐÐÐøÝð 	Kð 	Kð 	KÝ�NŠNÐ:ÀTˆNÑJÔJÐJÐJÐJÐJð	Køøøs   ‚A
A$ ÁA$ Á$&BÂBc                 ó^  — d}	 |                       ¦   «          d}t                               d| j        ¦  «         |                      ¦   «          t                               d¦  «         dS # t
          $ r8 t                               dd¬¦  «         |r|                      d¬¦  «         Y dS w xY w)NFTzstrobelight run id is: %szstrobelight profiling runningzerror during start_strobelight:rv   ©rt   )r[   rM   rr   r<   rg   rx   ry   rz   )r>   Ústrobelight_starteds     r   Ú_start_strobelightz1StrobelightCLIFunctionProfiler._start_strobelightò   s¾   € Ø#Ðð	Ø�OŠOÑÔÐØ"&ÐÝ�KŠKÐ3°TÔ5HÑIÔIÐIØ×"Ò"Ñ$Ô$Ð$Ý�KŠKÐ7Ñ8Ô8Ð8Ø�4øåð 	ð 	ð 	Ý�NŠNÐ<ÀtˆNÑLÔLÐLØ"ð GØ×/Ò/ÀÐ/ÑFÔFÐFØ�5�5ð		øøøs   „A$A* Á*>B,Â+B,Úwork_functionÚargsÚkwargsc                 ó°  — d | _         d | _        t          j                             d¦  «        x}�r¥|s8| j        rt          d¦  «        ‚t                               d¦  «          ||i |¤ŽS |  	                    ¦   «         }|s\| j        r-t          j         
                    ¦   «          t          d¦  «        ‚ ||i |¤Ž}t          j         
                    ¦   «          |S 	 t                               d¦  «         t          ¦   «         } ||i |¤Ž}t          ¦   «         }||z
  }	t                               d|	¦  «         |                      d¬¦  «         t          j         
                    ¦   «          |S # t          $ rW}
t                               dd¬	¦  «         |                      d¬¦  «         t          j         
                    ¦   «          |
‚d }
~
ww xY wd S )
NFzconcurrent runs not supportedz%failed to start strobelight profilingzcollection startedzwork function took %s secondsTr|   zwork function throw exceptionrv   )r<   r=   r-   Ú_lockÚacquirer0   r   rM   ry   r~   ÚreleaserN   Útimerrr   rz   rx   )r>   r   r€   r�   ÚlockedÚstartedrX   ÚstartÚendÚ
total_timeÚerrors              r   Úprofilez&StrobelightCLIFunctionProfiler.profile  sö  € ð #ˆÔØ"ˆÔå3Ô9×AÒAÀ%ÑHÔHÐHˆ6ñ !	Øð 6ØÔ%ð WÝ5Ð6UÑVÔVÐVå—’Ð>Ñ?Ô?Ð?Ø$�} dÐ5¨fÐ5Ð5Ð5à×-Ò-Ñ/Ô/ˆGØð ØÔ%ð Ý2Ô8×@Ò@ÑBÔBÐBÝ5Ø?ñô ð ð '˜¨Ð7°Ð7Ð7�Ý.Ô4×<Ò<Ñ>Ô>Ð>Ø�ðÝ—’Ð1Ñ2Ô2Ð2Ý™œ�Ø&˜¨Ð7°Ð7Ð7�Ý‘g”g�Ø  5™[�
Ý—’Ð;¸ZÑHÔHÐHØ×/Ò/ÀÐ/ÑEÔEÐEÝ.Ô4×<Ò<Ñ>Ô>Ð>Ø�øÝð ð ð Ý—’Ð>È�ÑNÔNÐNØ×/Ò/ÀÐ/ÑFÔFÐFÝ.Ô4×<Ò<Ñ>Ô>Ð>Ø�øøøøð	øøøð
 ˆts   ÃBE2 Å2
GÅ<AGÇG)r   N)r   )r   r   r   r   r   rƒ   Úboolr#   Úfloatrl   Úlistr?   r[   rg   rn   rp   rz   r~   r   r   r   r€   r�   r�   r   r   r   r-   r-   8   sÚ  € € € € € ð
ð 
ð ˆD‰FŒF€Eð
 $Ø(/Ø Ø;Ø,.Ø-/Ø37Ø(,Ø Ø#&ð'ð 'ð 'ð ð'ð #&ð	'ð
 ð'ð ð'ð '*ð'ð (+ð'ð !% S¤	¨DÑ 0ð'ð ˜#”Y Ñ%ð'ð ð'ð !ð'ð 'ð 'ð 'ð4'
ð '
ð '
ð '
ðRMð M¨ð M°Tð Mð Mð Mð Mð<Mð Mð Mð Mð.!ð !ð !ð !ð>KàðKð 
ðKð Kð Kð Kð& Dð ð ð ð ð (Ø% b¨" fÔ-ð(Ø68´gð(ØIKÌð(à	ˆd‰ð(ð (ð (ð (ð (ð (r   r-   Úprofilerr�   c                 óš   ‡ — ‰ st          di |¤ŽŠ dt          t          t          f         dt          t          t          d z  f         fˆ fd„}|S )Nr   r   c                 ó�   •‡ — t          j        ‰ ¦  «        dt          j        dt          j        dt
          d z  fˆˆ fd„¦   «         }|S )Nr€   r�   r   c                  ó&   •—  ‰j         ‰g| ¢R i |¤ŽS r;   )r�   )r€   r�   r‘   r   s     €€r   Úwrapper_functionz@strobelight.<locals>.strobelight_inner.<locals>.wrapper_function;  s(   ø€ ð $�8Ô# MÐC°DÐCÐCÐC¸FÐCÐCÐCr   )Ú	functoolsÚwrapsr   r€   r�   r   )r   r•   r‘   s   ` €r   Ústrobelight_innerz&strobelight.<locals>.strobelight_inner8  sm   øø€ õ 
Œ˜Ñ	'Ô	'ð	D¥B¤Gð 	Dµr´yð 	DÅRÈ$ÁYð 	Dð 	Dð 	Dð 	Dð 	Dð 	Dñ 
(Ô	'ð	Dð  Ðr   r   )r-   r   r   r   )r‘   r�   r˜   s   `  r   Ústrobelightr™   2  sl   ø€ ð ð <Ý1Ð;Ð;°FÐ;Ð;ˆð Ý¥¥B Ô'ð å	•"•b˜4‘i�-Ô	 ð ð  ð  ð  ð  ð  ð Ðr   r;   ))r–   Úloggingr   rS   rO   re   Úcollections.abcr   r   Ú	threadingr   Útimeitr   r†   Útypingr   r   Útyping_extensionsr	   Ú	getLoggerrM   ÚStreamHandlerÚconsole_handlerÚ	FormatterÚ	formatterÚsetFormatterÚ
addHandlerÚsetLevelÚINFOÚ	propagater   r   rx   r   r#   rl   r   r&   r+   r-   r™   r   r   r   ú<module>rª      s„  ðð Ð Ð Ð Ø €€€Ø 	€	€	€	Ø 	€	€	€	Ø Ð Ð Ð Ø €€€Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð Ø )Ð )Ð )Ð )Ð )Ð )Ø Ð Ð Ð Ð Ð Ð Ð Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'ð 
ˆÔ	Ð:Ñ	;Ô	;€à'�'Ô'Ñ)Ô)€ØˆGÔØHñô €	ð × Ò ˜YÑ 'Ô 'Ð 'à × Ò �/Ñ "Ô "Ð "Ø ‡‚�”Ñ Ô Ð Ø€Ô à€Yˆt�_„_€Ø€WˆT�]„]€ðð ð ð ð  )ñ ô ð ð7ð 7˜S 4™Zð 7°3ð 7ð 7ð 7ð 7ð.ð .˜˜d™
ð .¨cð .ð .ð .ð .ð ¨¤ð °#ð ð ð ð ðrð rð rð rð rñ rô rð rðv 7;ðð Ø,¨tÑ3ðØFIðàˆx˜˜B˜ÔÐ  (¨2¨r°D©y¨=Ô"9Ð9Ô:ðð ð ð ð ð r   