XjddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z ddl mZddlmZddlmZmZmZmZmZmZmZddlmZddlZd d lmZd d lmZm Z m!Z!m"Z"m#Z#m$Z$d d l%m&Z&d d l'm(Z(m)Z)m*Z*d dl+m,Z,m-Z-d dl.m/Z/m0Z0d dl1m2Z2e0j3e4Z5dZ6dZ7dZ8dZ9dZ:dZ;dZ<e=e7e8e;edZ?dZ@dZAdZBdZCdZDdZEdZFd ZGd!ZHd"ZId#ZJd$ZKd%ZLd&ZMd'ZNd(ZOd)ePd*ePd+ePfd,ZQd-eReSzePzd+eRfd.ZTe Gd/d0ZUe Gd1d2ZVe Gd3d4ZWe d5ed+eefd6ZXGd7d8ZYd9ePeePzd:eZdzd+e[ePeffd;Z\dZ_d@iown namespacenever.) COMPLETEDERRORDELETEDCANCELEDz/.hf-sbx-serveraset -e d=/tmp/.sbx-server if command -v wget >/dev/null 2>&1; then wget -q -O "$d" "$SBX_SERVER_URL" elif command -v curl >/dev/null 2>&1; then curl -fsSL -o "$d" "$SBX_SERVER_URL" else cp "$SBX_SERVER_MOUNT/sbx-server-$SBX_SERVER_SHA256" "$d"; fi if command -v sha256sum >/dev/null 2>&1; then actual=$(sha256sum "$d" | cut -d' ' -f1) elif command -v openssl >/dev/null 2>&1; then actual=$(openssl dgst -sha256 -r "$d" | cut -d' ' -f1) elif [ "${SBX_ALLOW_UNVERIFIED_SERVER:-}" = 1 ]; then echo "sbx: SBX_ALLOW_UNVERIFIED_SERVER=1: running the sandbox server unverified" >&2 actual=$SBX_SERVER_SHA256 # opted out, so there is nothing left to compare against else echo "sbx: cannot verify the sandbox server digest: this image has neither sha256sum nor" >&2 echo "sbx: openssl, so refusing to run it. Use an image with either hash tool installed." >&2 echo "sbx: Dedicated jobs may opt out with env={'SBX_ALLOW_UNVERIFIED_SERVER': '1'};" >&2 echo "sbx: pool hosts require setting it in the image environment instead." >&2 exit 1 fi if [ "$actual" != "$SBX_SERVER_SHA256" ]; then echo "sbx: sandbox server digest mismatch: got $actual," >&2 echo "sbx: expected $SBX_SERVER_SHA256." >&2 echo "sbx: refusing to run it. Upgrade huggingface_hub if this client is pinned to a" >&2 echo "sbx: digest that is no longer published." >&2 exit 1 fi chmod +x "$d" unset SBX_SERVER_URL SBX_SERVER_MOUNT SBX_SERVER_SHA256 SBX_ALLOW_UNVERIFIED_SERVER exec "$d" hf_tokennoncereturnctj|d|tjS)aODerive the sandbox auth token from the user's HF token and the job's nonce. Stateless: any machine holding the same HF token can recompute it from the nonce stored in the job's labels, so `Sandbox.connect(job_id)` needs no local state. Only the derived token is passed to the sandbox server as a job secret; the HF token itself is not. Scope: one nonce is minted per *job*, not per sandbox. For a dedicated sandbox those are the same thing. In a pool this derives the *host* credential, which manages the pool and can recover per-sandbox tokens -- a leak there is host-wide. Each pooled sandbox also has its own random capability token, minted by the host server, which is what per-sandbox operations and `proxy_headers` use. This is not a hardened boundary in either direction. An untrusted image can own the sandbox port, so don't treat it as a guarantee that credentials stay out of the sandbox; and the token is derived for whichever job carries the matching nonce label, so don't treat holding it as proof of which job you are talking to. z hf-sandbox:)hmacnewencodehashlibsha256 hexdigest)r1r2s IC:\Projects\Historical\venv\Lib\site-packages\huggingface_hub/_sandbox.py_derive_sandbox_tokenr<sE& 8HOO%%'s r;_duration_to_secsrCs2(S%L))8}} ( # ##r=ceZdZUdZedzed<eed<eed<dZedzed<dZe ed<d Z eed <e d e fd Z d efd Z dS)SandboxCommandResultz?Result of a command executed in a sandbox with [`Sandbox.run`].N exit_codestdoutstderrsignalF timed_outr duration_msr3c|jdkSNr)rFselfs r;okzSandboxCommandResult.oks~""r=ct|jdkr|jn|jdddz}d|jd|d|jdS)NPMz...zSandboxCommandResult(exit_code=z , stdout=z, duration_ms=))lenrGrFrK)rOouts r;__repr__zSandboxCommandResult.__repr__sX --33dkkSbS9IE9Qrrr#rr_c_orrrrr=)__name__ __module__ __qualname____doc__rA__annotations__strrIrJboolrKpropertyrPrWr=r;rErEsIITz KKK KKKFC$JItK #D###X#s#ssssssr=rEceZdZUdZedzed<eed<eeezed<eddZ ded <dZ edzed <dZ edzed <d Z e ed <dZedzed<de fdZdS)SandboxProcessaA background process started in a sandbox with [`Sandbox.run`]`(..., background=True)`. List a sandbox's processes with [`Sandbox.processes`] and stop one with [`SandboxProcess.kill`]. Recently completed processes stay in the listing (the server keeps a bounded number of them), so `running` and `exit_code` tell whether a process is still alive or already exited (as of when it was listed). NidpidcmdF)reprcompareSandbox_sandboxtag started_at_msTrunningrFr3c|jtd|jdd|j}t |ddS)aTerminate the background process. Idempotent. Returns whether this call is what stopped it: `False` means it had already exited or been terminated, which is not an error. Note that a descendant which detaches with `setsid()` leaves the signalled process group and outlives this call. Delete the sandbox to be certain everything it started is gone. NzThis process cannot be stopped individually: its sandbox runs a sandbox server that predates opaque process ids, so the server never issued one. Recycle the pool's hosts, or delete the sandbox to stop everything it started.DELETEz /processes/killedF)rcrri_requestr^jsonget)rOresponses r;killzSandboxProcess.killsm 7?U  =))(4K$'4K4KLLHMMOO''%88999r=)rXrYrZr[r]r\rAr rrirjrkrlr^rFrtr`r=r;rbrbs d NNN HHH tCy %UE:::Hi:::Ct $M3:$$$GT IsTz   :d::::::r=rbcjeZdZUdZeed<eed<eded<eed<dZedzed<d Z eed <dS) FileEntryz%A file or directory inside a sandbox.namepath)filedirsymlinktypesizeNmtime_msmode) rXrYrZr[r]r\rrAr~rr`r=r;rvrvsd// III III * ++++ IIIHcDjD#NNNNNr=rv local_pathc#K|jd|jdtddz }tjtjztjzttddzttddz}tj||d} tj |d5}|Vd d d n #1swxYwYtj ||d S#t$r| d wxYw) a Yield a writer whose bytes only appear at `local_path` once the download completes. Writing straight to the destination has three problems: an interrupted transfer leaves a truncated file at the final name; `open(path, "wb")` follows a symlink sitting there, so the sandbox's output can be redirected into any file this process can write; and a predictable temp name is a race another local process can win. So the bytes go to a temp file in the destination directory, created `O_EXCL` (nobody else's file) and `O_NOFOLLOW` (not a symlink) with mode `0600` and an unguessable name, then `os.replace`d into place. `rename` does not follow symlinks either, so a symlinked destination is *replaced* rather than written through, and a failure leaves nothing behind. r+r0z.part O_NOFOLLOWrO_BINARYiwbNT) missing_ok) parentrwrosO_WRONLYO_CREATO_EXCLgetattropenfdopenreplace BaseExceptionunlink)rtmpflagsfdfs r;_open_download_targetr s;  G*/GGIaLLGGG GC K"* $ry 072|Q3O3O ORYZ\^hjkRlRl lE eU # #B Yr4  AGGG                3 #####  d ### s0C&B7+ C7B;;C>B;?C"C;c 6eZdZdZdZdZdZdZd,d Zd e de fd Z d-d e de de fdZ d.d e de e ze zde d zdd fdZd.de ezd e de d zdd fdZd e de ezdd fdZdedeeeeffdZdeedeegefdeefdZd e dedede fdZd e dedeeee ffdZd e de de d zdd fdZd e d e dede d zdd f d!Zd e ded"e de d zd#edd f d$Zd e deefd%Z d e defd&Z!d e de"fd'Z#d/d e d)e"dd fd*Z$d e dd fd+Z%d S)0 SandboxFilesu$Filesystem operations inside a sandbox, available as [`Sandbox.files`]. In shared (pool) mode, paths are rooted at the sandbox's private home — the only place its code can write — so a leading `/` is taken relative to that home. In dedicated mode, paths are absolute on the container filesystem. Paths are normalized lexically, then resolved by the host server relative to an open home-directory descriptor. The file API refuses symlinks on every component, including links whose targets are inside the same home. i i sandboxrhr3Nc||_dSN)rirOrs r;__init__zSandboxFiles.__init__?s  r=rxcl||j}||jkrt|d|d|jd||jkrQt |}|||D]\}}||||t|z<t|S| |d|S)zRead a file from the sandbox and return its content as bytes. Raises [`SandboxError`] above `MAX_READ_BYTES`; use [`download`] for anything that large, which streams to disk instead of buffering. z is z bytes, over the zc-byte limit for reading into memory. Use `files.download(path, local_path)`, which streams to disk.r) statr}MAX_READ_BYTESrPARALLEL_THRESHOLD bytearray _read_rangesrUbytes _read_range)rOrxr}bufferoffsetparts r;readzSandboxFiles.readBs yy# $% % %YYTYYD4GYYY  $) ) )t__F $ 1 1$ = = ; ; 6:vT 2233== a...r=utf-8encodingcR|||S)z@Read a file from the sandbox and return its content as a string.)rdecode)rOrxrs r; read_textzSandboxFiles.read_textWs yy%%h///r=datarc`t|tr|}n)t|ts|}t ||jkr||||dSd|i}|||d<|j dd||dS)zHWrite content to a file in the sandbox (parent directories are created).NrxrPUT /files/writeparamscontent) r@r]r7rrrUr _write_rangesrirp)rOrxrrrs r;writezSandboxFiles.write[s dC  ;;==DDD%(( 99;;D t99t. . .   tT4 0 0 0 F$  !F6N unVTRRRRRr=rc&t|d5}tj|j}||jkr|||||n||||ddddS#1swxYwYdS)a Upload a local file to the sandbox. The file is opened once and both sized (`fstat`) and read through that same descriptor, so what is uploaded is the file that was measured -- rather than whatever the path resolves to a moment later. rb)rN)rrfstatfilenost_sizer_write_ranges_from_filer)rOrrxrrr}s r;uploadzSandboxFiles.uploadis*d # # /q8AHHJJ''/Dd---,,T1dDAAAA 4 ... / / / / / / / / / / / / / / / / / /sA(BB  B c.||j}tt|5}||jkrT|||D]/\}}||||0 ddddS|j ddd|i5}| dD]}|| dddn #1swxYwYddddS#1swxYwYdS)a!Download a file from the sandbox. The bytes land in a private temp file next to the destination and are renamed over it at the end, so an interrupted download leaves no truncated file and a symlink sitting at `local_path` is replaced rather than followed. NGET /files/readrxrr chunk_size) rr}rrrrseekrri_stream iter_bytes) rOrxrr}rrrrschunks r;downloadzSandboxFiles.downloadzsyy# "4 #3#3 4 4 #d---%)$5$5dD$A$A""LFDFF6NNNGGDMMMM # # # # # # # #&&umVTN&SS #W_%00K0HH##EGGENNNN# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #s<AD D 6/C2& D 2C6 6D 9C6 :D  DDr}cP|jfdtdDS)Nc:g|]}|t|z fSr`)min).0rrr}s r; z(SandboxFiles._ranges..s,XXXUD6M223XXXr=r)PARALLEL_CHUNK_SIZErange)rOr}rs `@r;_rangeszSandboxFiles._rangess6(XXXXX%4QVBWBWXXXXr=itemsfnct|jt|}t|5}t |||cdddS#1swxYwYdS)zRun `fn(item)` over items concurrently. All workers share the sandbox's `httpx2.Client`, which is thread-safe and pools connections, so parallel transfers fan out over several streams at once. N)rPARALLEL_MAX_WORKERSrUrlistmap)rOrrworkersexecutors r; _parallelzSandboxFiles._parallels d/U<<  ( ( 1H R//00 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s#A""A&)A&rlengthct}|jdd|||d5}|dD]I}t |t |z|krt d||J dddn #1swxYwYt ||krt dt|S) Nrr)rxrrrrrz+File response exceeded the requested range.z6File changed size during transfer; retry the download.)rrirrrUrextendr)rOrxrrrrsrs r;rzSandboxFiles._read_ranges {{ ] " " =$&TZ)[)[#   # !,, ,DD # #t99s5zz)F22&'TUUU E"""" # # # # # # # # # # # # # # # # t99  WXX XT{{sA!BB"%B"c #Kdtttfdtttfffd }|}t jt |}t|5}tdt ||D])}| |||||zEd{V* ddddS#1swxYwYdS)aFetch every range in parallel, yielding `(offset, bytes)` as they land. Yielding rather than returning a list is the point: the caller places each chunk and drops it, so peak memory is a few chunks instead of the whole file plus a copy. rngr3c@|\}}|||fSr)r)rrrrxrOs r;fetchz(SandboxFiles._read_ranges..fetchs) NFF4++D&&AAA Ar=rN) tuplerArrrrrUrrr)rOrxr}rrangesrrstarts`` r;rzSandboxFiles._read_rangess` BuS#X B5e+< B B B B B B Bd##d/V==  ( ( PHq#f++w77 P P#<<veego6M/NOOOOOOOOOO P P P P P P P P P P P P P P P P P P PsA CC"Ccdtttfddffd }t |dS)Nrr3c v|\}}||||ztdS)Ntotal) _put_rangerU)rrrrrrxrOs r;pushz(SandboxFiles._write_ranges..pushsF NFF OOD&$v/G*H$VYZ^V_V_O ` ` ` ` `r=)rrArrrU)rOrxrrrs```` r;rzSandboxFiles._write_rangess aeCHo a$ a a a a a a a a a t||CII..55555r=sourcectjdtttfddffd }|dS)zUpload `source` in parallel ranges, reading each range on demand. Serialize reads from the shared descriptor; network writes still overlap. rr3Nc|\}}5 | |}dddn #1swxYwYt||krtd||dS)Nz+Upload source changed size during transfer.r)rrrUrr) rrrrrrx read_lockrOr}rs r;rz2SandboxFiles._write_ranges_from_file..pushs NFF , , F### F++ , , , , , , , , , , , , , , ,5zzV##"#PQQQ OOD&%TO B B B B Bs+AAA) threadingLockrrArr)rOrxrr}rrrs````` @r;rz$SandboxFiles._write_ranges_from_files N$$  CeCHo C$ C C C C C C C C C C C t||D))400000r=rrc||d}|||d<|t|z|kr||d<|jdd||dS)N)rxrr truncate_torrr)rUrirp)rOrxrrrrrs r;rzSandboxFiles._put_rangese*.&!A!A  !F6N CJJ % ' '$)F= ! unVUSSSSSr=cg}d} d|i}|||d<|jdd|}|d|d D|d }|s|Sy) zList a directory in the sandbox. The server paginates, so this follows the cursor and returns everything -- a caller sees one list regardless of how the directory is chunked on the wire. NTrxafterrz /files/listrc3.K|]}tdi|VdS)Nr`)rv)rentrys r; z$SandboxFiles.list..s0NN%9--u--NNNNNNr=entriesnext)rirprqrrr)rOrxrrrpayloads r;rzSandboxFiles.lists$&  &,d^F "'wm,,UM&,QQVVXXG NNNN79;MNNN N N NKK''E  r=c||jddd|i}tdi|S)z3Get metadata of a file or directory in the sandbox.rz /files/statrxrr`)rirprvrq)rOrxrss r;rzSandboxFiles.stats?=))%PT~)VV++8==??+++r=cz ||dS#t$r}|jdkrYd}~dSd}~wwxYw)z+Check whether a path exists in the sandbox.TNF)rr status_code)rOrxes r;existszSandboxFiles.existssW  IIdOOO4   }##uuuuu  s : 55:F recursivecVd|i}|rd|d<|jdd|dS)z*Delete a file or directory in the sandbox.rx1rrnz /files/deleterNrirp)rOrxrrs r;deletezSandboxFiles.deletes@$  &"%F;  xHHHHHr=cD|jddd|idS)z0Create a directory (and parents) in the sandbox.POSTz /files/mkdirrxrNr)rOrxs r;mkdirzSandboxFiles.mkdirs) v~vtnMMMMMr=)rrhr3N)rr)F)&rXrYrZr[rrrrrr]rrrr rrrrrAr rrr r rrr rrrrrvrrr^rrr r`r=r;rr%s  ))'N    //////*00c0S0s0000 S S# SS5[8%; S3: SY] S S S S//t/3/cDj/TX////"#S#cDj#T####(YCYDsCx$9YYYY1tCy1hucz.B1tCy1111  S # %    PPCPHU3:=N4OPPPP$6#6U6#*666661C111TWZ^T^1cg1111$ Ts TC T TS4Z T[^ Tcg T T T Ti&,,,,,, 34II3I4IDIIIIN#N$NNNNNNr=rreshellc|dur$t|tstd|dur$t|trtdd|i}|||d<|S)zQBuild the `cmd`/`shell` part of an `/exec` payload, validating their consistency.TzCshell=True requires `cmd` to be a shell command string, not a list.FzRshell=False requires `cmd` to be an argv list (e.g. ['echo', 'hi']), not a string.reNr )r@r] ValueError)rer rs r; _exec_payloadr sl }}ZS11}^___ ~~*S#..~mnnn$clG   Nr=rsc#K|D]6}|stj|}|ddkr|V7dS)zJIterate NDJSON events from a streaming response, skipping keepalive pings.eventpingN) iter_linesrqloadsrr)rsliners r; _iter_eventsrsf##%%   4   99W   ' 'KKK r=c| |d}n#t$r|jdd}YnwxYwt d|jd||j)zQRead the error body and raise a SandboxError (works for streaming responses too).errorNizSandbox API error (z): )r)rrq Exceptiontextrr)rsmessages r;_raise_for_statusr#s MMOOO&--//'* &&&-%& OX-AOOgOO]e]q r r rrs1A  A job_idc . |d}n#t$rd}YnwxYwt|tr"t|t s |t krdS||nd}td|d|dt d)aRefuse a server whose wire contract this client cannot drive. Permissive in one direction only. A *newer* server is accepted: it declares its own backwards compatibility by keeping the protocol integer it still supports. An *older* one is refused, because the mismatches are things like the per-sandbox capability token, which surfaces as a 403 on an unrelated route several calls later instead of here. `version` is no use for this: it moves for a doc fix as readily as for a protocol break. protocolNz1 (declares no protocol) Sandbox job z runs sbx-server protocol z5, but this version of huggingface_hub needs protocol z or newer. A pool host keeps the server binary it downloaded at boot for up to 24h, so recycle this pool's hosts (kill them, or let them idle-time-out) to pick up the current server.)rqrrrr@rAr^SANDBOX_SERVER_PROTOCOLr)rsrrreporteds r;_check_server_protocolr"-s==??&&z22 (C  Hd)C)CTkHkHk#/xx5OH  Dv D D D D*A D D D  s '* 99c(eZdZdZdeddfdZdZdS) _SandboxAuthaAttach the *current* HF bearer to every request. The bearer used to be captured once when the transport was built. A sandbox host lives up to 24h, so an OIDC/OAuth credential can expire or rotate inside that window, after which every request to the Jobs proxy failed even though the handle was still perfectly good. Reading it per request fixes that. The `X-Sandbox-Token` is deliberately *not* re-derived: the server holds the value derived from the bearer that created the job, so re-deriving from a rotated bearer would produce a token the server has never seen. The two credentials answer different gates -- the proxy wants a live bearer, the server wants the original capability -- and only the first needs refreshing. apir3Nc||_dSr)_api)rOr%s r;rz_SandboxAuth.__init__Vs  r=c#PKdt|j|jd<|VdS)NBearer Authorization)_effective_tokenr'headers)rOrequests r; auth_flowz_SandboxAuth.auth_flowYs2+R5Edi5P5P+R+R( r=)rXrYrZr[rrr.r`r=r;r$r$GsO  Edr=r$tokencRt|tr|std|S)Nz\The pool host returned no valid sandbox-scoped token; recycle it with an updated sbx-server.)r@r]rr/s r;_require_scoped_tokenr2^s7 eS ! !   j    Lr=cpeZdZdZddddedededzd ed ed ed ed ededdfdZedddde d ed ed ed ededdfdZ e dedzfdZ de d edzde fdZdddeded edzdejfdZedddeded edzdeejfdZdedefdZd!dZd!dZdeddfd ZdS)"_SandboxServeruFHTTP transport to one `sbx-server` instance — a dedicated job or a shared host. Owns the `httpx2.Client`, the base URL and the auth headers. In dedicated mode a server is paired 1:1 with its [`Sandbox` In pool mode one server (one host job) is shared by many sandboxes, and `live`/`capacity` track packing. r)max_connectionscapacityrownerimageNbase_urlr2 sandbox_tokenr%r6r7r3c  ||_||_||_||_||_||_||_| |_d|_d|_ d|_ tj d|it|tj||d|_dS)NrTFX-Sandbox-Token)r6max_keepalive_connections)r,authlimitsfollow_redirects)rr8_imager:r2r'_sandbox_tokenr7liveverifiedownedhttpx2Clientr$Limits_client) rOrr8r9r:r2r;r%r6r7s r;rz_SandboxServer.__init__ns       +     }& 6c""=\klll #     r=jobc z||j|jj|jp|jt |||||| S)zIBuild a server from a freshly fetched job (reads its exposed server URL). rr8r9r:r2r;r%r6r7)rcr8rw docker_imagespace_id_find_server_url)clsrKr2r;r%r6r7s r;from_jobz_SandboxServer.from_jobsPs6)."2cl%c**'+    r=c|jSr)rBrNs r;r9z_SandboxServer.images {r=kwargscH|i|did|i|d<|S)aBOverride `X-Sandbox-Token` for one request. The client-level header carries the *host* credential, which manages the pool. A pooled sandbox's own operations should present that sandbox's capability token instead, so nothing wider than the sandbox is offered on a per-sandbox call. Nr,r=)rr)rOrTr;s r; _with_tokenz_SandboxServer._with_tokens8  $ _6::i#<#< _>OQ^ _ _F9  r=r;methodrxc |dtjdd}|||}|jj||j|zfd|i|}|jdkrt||S)z@Request to the in-job server. Raises SandboxError on API errors.timeoutgN@$@connect) poprGTimeoutrVrJr-r:rrrOrXrxr;rTrZrss r;r-z_SandboxServer.requests**YtT(J(J(JKK!!&-88'4<' 0D``g`Y_``  3 & & h ' ' 'r=c+4K|dtjdd}|||}|jj||j|zfd|i|5}|jdkrt||VddddS#1swxYwYdS)zJStreaming request to the in-job server. Raises SandboxError on API errors.rZgQ@r[r\r^N) r_rGr`rVrJstreamr:rrras r;rcz_SandboxServer.streams **YtT(J(J(JKK!!&-88 T\ )= Y Yw YRX Y Y ]e#s**!(+++NNN                  s!B  BBlocal_idc|dd|d}t|dS)zKRecover a pooled sandbox's capability using the host management credential.r/v1/sandboxes/z/tokenr/)r-rqr2rr)rOrdrs r;r;z_SandboxServer.sandbox_tokensG||E#DH#D#D#DEEJJLL$TXXg%6%6777r=c8|jdSr)rJcloserNs r;rhz_SandboxServer.closes r=cR|j|j|jdS)Nrr))r' cancel_jobrr8rNs r;rkz_SandboxServer.cancel_jobs' DK4:FFFFFr= start_timeoutc Ftj|z}d}tj|krV |j|jdzt jd}|jdkrt||jdSn#tj $rYnwxYwtj|z dkrtj}|j |j|j }|j jtvrUt!|j |j|j }t#d |jd |j jd |j jd |tjdtj|kVt#d |jd|dd)zMPoll /health until the server answers; fail fast (with logs) if the job dies.gz/healthg@rZNg@rjr)rz$ terminated during startup (status: z , message: ).g333333?z did not become ready within z.0fzs.)timerJrrr:rGr`rr"r RequestErrorr' inspect_jobr8statusstage_TERMINAL_STAGES_tail_job_logsrrsleep)rOrldeadlinelast_job_checkrsrKlogss r; wait_readyz_SandboxServer.wait_readys9;;.ikkH$$ <++DMI,Ev~^aObOb+cc'3..*8T[AAAF/&    y{{^+c11!%i++4;$*+UU:#'777)$)T[DJWWWD&^t{^^$'J$4^^ADAS^^W[^^ Jt   #ikkH$$$i$+iiTaiiiijjjsAB BBr3N)rXrYrZr[r]rrAr classmethodrrRr_r9dictrVrGResponser-rr rcr;rhrkrBr}r`r=r;r4r4fs" "- - - -  - Tz -  - - - - - -  - - - - ^ "                [ .sTzX $ sTz d    NRcd `f`oEI    !$ 8;d   &/ "   ^ 8c8c8888 GGGGkk$kkkkkkr=r4c eZdZdZedddededfdZedddedegdffd Zdd dededfd ZdS) _KillMethodz>Lets `kill` work both as a classmethod and an instance method.instanceNr8r3).NcdSrr`rOrr8s r;__get__z_KillMethod.__get__sKN3r=rhcdSrr`rs r;rz_KillMethod.__get__sORsr=Sandbox | Nonec h||jSddddtdtdzdtdzddffd}|S)Nr)r/ sandbox_idr)r/r3c\|||dS)Nr)r]rt)rr)r/r8s r;rtz!_KillMethod.__get__..kill s- MM* M G G L L N N N N Nr=)_killr])rOrr8rts ` r;rz_KillMethod.__get__sx  > !=AW[ O O OS Od  O#PT* O`d O O O O O O r=)rXrYrZr[rr|r rr`r=r;rrsHH NNTNhy6INNNXN R R$R8BH;MRRRXR 0(9BUr=rceZdZdZdddedededzdeded edzd dfd Zee fd e ddddd dddd dedede e zezdzde eefdzde eefdzdeedzdedzdede eefdzde dedzd dfdZeddddededzdedzd dfdZeZdDdZdDd ZdEd!ZdDd"Zed#d#d#d#d#d#d#d#d#d#d$ d%eeezd&edzde eefdzd'edzd(e dzd)edzd*eegdfdzd+eegdfdzd,ed-ed.ed d efd/Zed#d#d#d0d%eeezd&edzde eefdzd'edzd.ed1d ef d2Zdddddddd1d1d d$ d%eeezd&edzde eefdzd'edzd(e dzd)edzd*eegdfdzd+eegdfdzd,ed-ed.ed eezfd3Zd eefd4Ze d edzfd5Z!e d edzfd6Z"dFd8d9d:e ezd;edZ$d efd?Z%d@edAed e&j'fdBZ(e)d@edAed e*e&j'fdCZ+dS)GrhaQAn isolated cloud machine running on Hugging Face Jobs. > [!NOTE] > The Sandbox API is experimental. Its API and behavior may change without notice. Create a dedicated one with [`Sandbox.create`] (one job per sandbox), or get many cheap shared ones from a [`SandboxPool`]. Reattach to a running sandbox from anywhere with [`Sandbox.connect`]. Use as a context manager to terminate it on exit: ```python >>> from huggingface_hub import Sandbox >>> with Sandbox.create(image="python:3.12") as sbx: ... print(sbx.run("python --version").stdout) ``` NrWrcserverrd owns_sandbox owns_serverr;r3c||_||_||jnt||_||_|dnd||_||_||_d|_d|_ t||_ dS)Nz/v1rfF) rc_serverrCr2 _local_id _base_path _owns_sandbox _owns_server_on_kill_killedrfiles)rOrcrrdrrr;s r;rzSandbox.__init__s 8@7Gf33MbcpMqMq!$,#3%%9T(9T9T*(<@  !$'' r= cpu-basicF^@) flavor idle_timeoutenvsecretsvolumesr)forward_hf_tokenlabelsrlr/r9rrrrrr)rrrlr/c h| t| ts$tdt| j| D]Z\} } t| t rt| t std| tvrtd| d[t| }t|}td}t||}t||||||||\}}}}| |||||ti| pitd t t"t$|i|pdt&g| }d} t(||||t,jd z }|| n#t2$r| ||j|jj n<#t2$r/}t<d|jd|Yd}~nd}~wwxYw|| wxYw||j|dddS)u Create a dedicated sandbox (one HF Job) and block until it is ready (~7s on cpu-basic). Each sandbox is a full isolated VM, so this is the right choice for GPU workloads or untrusted code. To fan out many cheap CPU sandboxes instead, use [`SandboxPool`]. The job runs with a fixed 24h maximum lifetime; `idle_timeout` is the real keeper — an idle sandbox shuts itself down well before that. Args: image (`str`, *optional*, defaults to `"python:3.12"`): Any Docker image with `/bin/sh` (Docker Hub or `hf.co/spaces/...`). flavor (`str`, *optional*, defaults to `"cpu-basic"`): Hardware flavor, e.g. `"cpu-basic"`, `"a10g-small"`. See `hf jobs hardware`. idle_timeout (`int` or `float` or `str`, *optional*, defaults to `600`): Auto-shutdown after this much inactivity (no API calls, no running processes). Defaults to 10 minutes; pass `None` to disable. Note that a *foreground* command is not currently counted as activity, so a single `run()` that takes longer than this without other API traffic can have its sandbox shut down under it — raise the timeout (or pass `None`) for long single commands. env (`dict[str, Any]`, *optional*): Environment variables available in the sandbox. secrets (`dict[str, Any]`, *optional*): Secret environment variables (encrypted server-side). volumes (`List[Volume]`, *optional*): HF repos/buckets to mount, see [`Volume`]. namespace (`str`, *optional*): User or org namespace to run under (defaults to current user). forward_hf_token (`bool`, *optional*, defaults to `False`): If True, your HF token is injected as `HF_TOKEN` (opt-in). labels (`dict[str, str]`, *optional*): Labels to attach to the underlying HF Job. start_timeout (`float`, *optional*, defaults to `120.0`): Max seconds to wait for the sandbox to become ready. token (`str`, *optional*): HF token override. The image only needs `/bin/sh`. The sandbox server is downloaded at startup with `wget`/`curl` if available, otherwise read off an always-mounted server bucket (which adds ~2-3s to cold start, so shipping `wget`/`curl` keeps it fast). Nz'`labels` must be a dict[str, str], got z(`labels` keys and values must be stringszLabel 'zB' is reserved by huggingface_hub Sandbox and cannot be overridden.r1rrrrrrr;r r9commandrrrrZrrexposer)rKr2r;r%r6rjzFailed to cancel sandbox job  after startup failure: Trcrrdrr)!r@rr r|rXrr]RESERVED_SANDBOX_LABELSrr+rr<_bootstrap_job_specrun_jobSANDBOX_MAX_LIFETIME SANDBOX_LABEL MODE_LABELMODE_DEDICATED NONCE_LABELSANDBOX_SERVER_PORTr4rRrrr}rrkrcr8rwloggerwarningrh)rQr9rrrrrr)rrrlr/keyvaluer%r1r2r;rjob_env job_secrets job_volumesrKrrs r;createzSandbox.createCst  fd++ d !b4PV<> 5H  %-' 6 6 6 2+{kk(iv|imS*nVachii'4'(   +/ #,,+ , AA E -F   m , , , ,    dcf GGGG d d dbsvbb_`bbcccccccc d!   scfVd[_````s7AF H!&GH H%G<7H<HHrrcjt|}t||\}}t|vr|td\}}t |||} d|ddD}||vrtd|d|d ||} n##t$r| wxYw||||d d | S| || } | j pi} | t} | tdks| td|dtd| t t"krtd|dtd| jjdkr td|d| jjdt)t+|| } t,| | | |t0jdz}|| j|dd d S)z?Reattach to a running sandbox from anywhere, using only its id.r1rrpch|] }|d S)rcr`)ritems r; z"Sandbox.connect..saaa4DJaaar=r /v1/sandboxeszSandbox z no longer exists on host r+FTrcrrdrrr;rjrNJob z is not a sandbox (missing 'z ' label).z` is a sandbox host, not a single sandbox. Connect to one of its sandboxes with id 'z '.RUNNING is not running (status: rqrrr)r_split_sandbox_id SHARED_ID_SEPsplit _connect_hostr-rqrr;rrhrtrrrrrr MODE_POOLrurvr<r+r4rRrrrc) rQrr)r/r% host_job_idrdrexistingr;rKrr2s r;r]zSandbox.connects%    1*i H H I J & &$.$4$4]A$F$F !K"3 yIIIF aa6>>%3Y3Y3^3^3`3`aaa8++&'f*'f'fXc'f'f'fggg!' 4 4X > >      3!" +  ooZ9oEE!r ;'' ::m $ $ + +u}fjffmfffgg g ::j ! !Y . .OzOO3@OOO  : y ( (c*ccszO_cccdd d-.>s.C.CUKK (('(=A )  scfVd\`aaaas A!C C c|jrdS |j|jn*|jdd|j|jn=#t $r0}td|j d|Yd}~dSd}~wwxYwd|_|j | ||j r|j dSdS)zTerminate the sandbox. Idempotent. Dedicated sandboxes cancel their underlying job; shared sandboxes are removed from their host (freeing a slot) while the host keeps running. NrnrfrWzFailed to kill sandbox : T) rrrrkr-rCrrrrcrrrhrOrs r;rz Sandbox._kills <  F ~% '')))) $$X/P/P/P`d`s$ttt    NNCTWCCCC D D D FFFFF  = $ MM$      ! L    ! !sA A B!%B  BcJ|jr|jdSdS)zRelease the local HTTP client without terminating the sandbox. Idempotent. No-op for pool sandboxes (the client belongs to the pool's host). N)rrrhrNs r;rhz Sandbox.closes3   ! L    ! !r=c|Srr`rNs r; __enter__zSandbox.__enter__  r=ch|jr|dS|dSr)rrtrhrOexc_infos r;__exit__zSandbox.__exit__s/    IIKKKKK JJLLLLLr=.) r rcwdrZstdin on_stdout on_stderrcheckcapture_output backgroundrer rrZrrrrrrc dSrr`) rOrer rrrZrrrrrrs r;runz Sandbox.runs  #sr=)r rrTcdSrr`)rOrer rrrs r;rz Sandbox.run(s r=c Z t||} |r|| d<|r|| d<| rj|dd| } t| d| d|| d| S||| d <||| d <g}g}d d }dt t dt dd f fd }|dd| 5}t|D]}|ddkr&|||d|||d4|ddkr&|||d|||df|ddkr~t|dd |d ||d|dd|dd }nd d d n #1swxYwY|td| r#|j d ks|j rt|||S) u Run a command in the sandbox and wait for it, streaming output live. With `background=True` the command is started detached and `run` returns a [`SandboxProcess`] immediately, without waiting for it to finish — handy for servers and other long-running processes. List them later with [`Sandbox.processes`] and stop one with [`SandboxProcess.kill`]. The streaming/wait-only options (`timeout`, `stdin`, `on_stdout`, `on_stderr`, `check`) don't apply in that mode. Args: cmd (`str` or `List[str]`): A shell command string (run with `/bin/sh -c`) or an argv list (exec'd directly). shell (`bool`, *optional*): Force the execution mode instead of inferring it from the type of `cmd`. `True` runs through `/bin/sh -c` and requires `cmd` to be a string; `False` exec's `cmd` directly and requires it to be an argv list. `None` (default) infers from the type. Set it explicitly to avoid the type-driven footgun (e.g. `["echo hi"]` being exec'd as a single program named `"echo hi"`). env (`dict[str, Any]`, *optional*): Extra environment variables for this command. cwd (`str`, *optional*): Working directory. timeout (`float`, *optional*): Kill the command (whole process group) after this many seconds. stdin (`str`, *optional*): Data to write to the command's stdin. on_stdout (`Callable[[str], None]`, *optional*): Callback invoked with stdout chunks as they arrive. on_stderr (`Callable[[str], None]`, *optional*): Callback invoked with stderr chunks as they arrive. check (`bool`, *optional*, defaults to `True`): If True, raise [`SandboxCommandError`] on non-zero exit. capture_output (`bool`, *optional*, defaults to `True`): If True, accumulate stdout/stderr into the returned result. Pass `False` when you only want `on_stdout`/`on_stderr`: output is then handed to the callbacks and dropped, so a command producing gigabytes does not have to fit in memory. `result.stdout`/`result.stderr` are empty in that case. background (`bool`, *optional*, defaults to `False`): If True, start the command detached and return a [`SandboxProcess`] right away instead of waiting for it and returning a [`SandboxCommandResult`]. Returns: a [`SandboxCommandResult`] (with `exit_code`, `stdout`, `stderr`, `duration_ms`), or a [`SandboxProcess`] when `background=True`. rrr  /processesrqrcrdrj)rcrdrerjriNrZrrpartsrr3csdSt|ztkrtdtdt|z ||dS)Nzcommand produced more than z characters of output. Pass `capture_output=False` with `on_stdout`/`on_stderr` to stream it instead, or redirect it to a file in the sandbox and download that.)rUMAX_CAPTURED_OUTPUT_CHARSrappend)rrrcaptureds r;keepzSandbox.run..keeps~! #d))#&???"Q2KQQQ D !H LL     r=z/execrrGrHexitrFrrIrJFrK)rFrGrHrIrJrKz%connection lost while running command)reresult)rrprqrbrrrr]rrrEjoinrrFrJr)rOrer rrrZrrrrrrrr stdout_parts stderr_partsrrrsrrs ` @r;rz Sandbox.run3st U++  ! GEN  ! GEN  ==G=DDIIKKD!88D>>KHHUOO   !(GI   $GG  #% "$ .2 S         \\&'\ 8 8 H%h//  >X--DuV}555 ,! %-0007^x//DuV}555 ,! %-0007^v--1"' "4!ww|44!ww|44$yy22"'))K"?"?$)IImQ$?$? FE.               * >FGG G  >f&!++v/?+%#f=== = sDG**G.1G.cpdd}fd|DS)aList the background processes of this sandbox. Returns the processes started with [`Sandbox.run`]`(..., background=True)`; stop one with [`SandboxProcess.kill`]. Recently completed processes stay listed (with `running=False` and their `exit_code`); the server keeps a bounded number of them, so a sandbox that has run thousands of short commands will not list them all. rrc g|]w}t|d|d|d|d|d|d|dxS)rcrdrerjrkrlrF)rcrdrerjrkrlrFri)rbrr)rprOs r;rz%Sandbox.processes..s     55;;eHeHEE%LLeeO44) %% ,,       r=)rprq)rOrs` r; processeszSandbox.processessQ}}UL116688         r=c|jjSr)rr9rNs r;r9z Sandbox.images |!!r=c,|j |jjndS)zIFor a shared/pool sandbox, the job id of the host running it (else None).N)rrrrNs r;host_idzSandbox.host_ids'+n&@t|""dJr=/zhttps://)schemeportrxrc|jjddd}|dr|nd|z}|||jd||S)a4Public URL that proxies through to a server running *inside* this sandbox. Requests to the returned URL are forwarded by the in-job sandbox server to a server you started in the sandbox on `port`, including WebSocket (`ws(s)://`) upgrades and streamed responses. Pair it with [`proxy_headers`] for auth. How the sandbox must listen on `port`: - **Pool / shared sandbox**: it cannot bind a TCP port (Landlock), so bind a **unix socket** at `$SBX_PROXY_DIR/.sock` (the `SBX_PROXY_DIR` env var is set in every sandbox). E.g. `uvicorn app:app --uds $SBX_PROXY_DIR/8000.sock`. - **Dedicated sandbox**: bind a normal TCP port on `127.0.0.1:`. (You can also expose the port directly via the job proxy without going through here.) Args: port (`int` or `str`): The port (pool: the `` of the unix socket) the inner server listens on. path (`str`, *optional*, defaults to `"/"`): Path on the inner server to point at, e.g. `"/ws"`. scheme (`str`, *optional*, defaults to `"https://"`): URL scheme to build the link with. Defaults to `"https://"`; pass `"wss://"` for a WebSocket client (the proxy is protocol-agnostic, so only the client-side scheme changes). Returns: `str`: a URL like `https://--49983.hf.jobs/v1/.../proxy/8000/ws` (or `wss://...` with `scheme="wss://"`). Example: ```python >>> url = sandbox.proxy_url_for(8000, "/ws", scheme="wss://") >>> import websockets >>> async with websockets.connect(url, additional_headers=sandbox.proxy_headers) as ws: ... await ws.send("hello") ``` z://rrz/proxy/)rr:r startswithr)rOrrxr host_and_rests r; proxy_url_forzSandbox.proxy_url_forsgL -33E1==bA s++;tttM-MMMMtMMMr=cJdt|jj|jdS)aAuth headers to send with [`proxy_url_for`] requests (HF token + sandbox token). Read these at the moment you use them: the HF token is resolved on access, so a long-lived handle hands out a current bearer rather than one captured when the sandbox was created. For a pooled sandbox this is the sandbox's own capability token, not the pool host's: these headers typically end up in a browser or WebSocket client, so they should confer access to this sandbox and nothing else. r))r*r=)r+rr'rCrNs r; proxy_headerszSandbox.proxy_headerss3M'7 8I'J'JLL#2   r=c(d|jd|jdS)Nz Sandbox(id=z, image=rT)rcr9rNs r;rWzSandbox.__repr__s?TW?? ????r=rXresourcec H|jj||j|zfd|ji|SNr;)rr-rrC)rOrXrrTs r;rpzSandbox._requests2#t|#FDOh,FttVZVitmstttr=c+K|jj||j|zfd|ji|5}|VddddS#1swxYwYdSr)rrcrrC)rOrXrrTrss r;rzSandbox._streams T\ DOh.  >B>Q U[    NNN                  s 8<<r~)r3rh)r),rXrYrZr[r]r4r^rr DEFAULT_IMAGEDEFAULT_IDLE_TIMEOUTrArBrr r rrr]rrtrrhrrrr rrErrbrr_r9rrrrWrGrrprr rr`r=r;rhrhs  .%) ( ( (  ( ( * (  ( (Tz (  ( ( ( (H#ra"1E%))-'+ $!&(,$ rararara ra EkC'$. ra #s(^d " rac3h$&raf$ra:raraS#X%raraTzra rarara[rahAE[_.b.b.b.bC$J.bcTXj.bdm.b.b.b[.bd ;==D!!!!.!!!! !%( #2525"%(### 49_#d{ # #s(^d " # 4Z ##Tz#SE4K(4/#SE4K(4/###EN# ###X#  !%(  49_d{  #s(^d "  4Z DM X"%) $ 2626# }}} 49_}d{ } #s(^d " } 4Z }}Tz}SE4K(4/}SE4K(4/}}}}  .}}}}~ 4/    2"sTz"""X"KtKKKXK (Nz(N(N(N#)(N3(N(N^a(N(N(N(NT tCH~   X @#@@@@ usucuuuuucSx?X^r=rhceZdZdZefdedddeddeddd ded ed e d e d e dzd edzde e zezdzdedzde dededzde ddfdZ e dedddededzdededzddf dZdFde deefdZdedddeeefdzde e zezdzde ddfdZdeeefdzde e zezdzde ddfd Zede fd!Zede fd"Zedeefd#ZdGd$ZdHd%ZdGd&ZdGd'Zdedzfd(Zd)ede fd*Zd)ede fd+Z dId-Z!d.ed/dd,fd0Z"de fd1Z#dGd2Z$de fd3Z%dd4d5e d6e dee&fd7Z'de&fd8Z(d9d/deeefd:e dzdd;fd<Z)d=e*ddfd>Z+d9e&ddfd?Z,dGd@Z-dAe.de fdBZ/dJdKdEZ0dS)L SandboxPoolu A fleet of shared "host" jobs, each packing many landlock-isolated sandboxes. > [!WARNING] > The Sandbox API is experimental. Its API and behavior may change without notice. > > **Pooled sandboxes are for workloads inside one trust boundary.** A pooled sandbox is a uid plus a Landlock > ruleset inside a shared VM — not a VM of its own — and every sandbox on a host shares that host's auth token > and its privileged control plane. Use a pool to fan out *your own* code cheaply. For mutually distrusting > workloads, use [`Sandbox.create`], which gives each one its own VM. The specific gaps are listed under > "Known limitations" in the sandbox conceptual guide. One host is one billed HF Job (a VM); it runs the sandbox server and multiplexes up to `sandboxes_per_host` lightweight sandboxes, isolated from each other by uid + the Landlock LSM. This makes large fan-outs cheap (the VM cost is shared across all its sandboxes) and fast (creating a sandbox is ~one proxy round-trip once a host is warm). Best for many parallel CPU sandboxes such as RL rollouts; for GPU or strong VM-level isolation between mutually-distrusting workloads, use [`Sandbox.create`] instead. The constructor pre-provisions `warm_up` hosts (default 1) and blocks until they are ready; further hosts are then provisioned on demand as sandboxes are requested, and all are torn down on `close()` (or when idle, via `idle_timeout`). The user never manages jobs: ```python >>> from huggingface_hub import SandboxPool >>> with SandboxPool(image="python:3.12", flavor="cpu-basic", warm_up=2) as pool: ... boxes = [pool.create() for _ in range(100)] # packed across the warm hosts ... print(boxes[0].run("echo hi").stdout) hi ``` `create()` makes **one** sandbox at a time: it reuses a host that still has free capacity before booting a new one, so you grow on demand as work arrives. To avoid a cold start on the first few calls, pre-provision hosts with `warm_up` (or [`warm`]). Warm hosts are discovered via job labels, so reuse works **across processes** too (a fresh pool with the same `image`/`flavor`/`name` attaches to hosts an earlier run left behind): ```python >>> pool = SandboxPool(image="python:3.12") >>> sbx = pool.create() # finds a warm host (here or in another process), else boots one ``` rrNrF) rsandboxes_per_hostwarm_up max_hostsrwrr)rl adopt_hostsr/ _connect_moder9rr r rrwrr)rlrr/rr3c | tttfvr+tdtdtdtd| d |dkrtd|dkrtdt | |_||_||_||_||_ ||_ ||nd td |_ ||_ ||_t|j||_| |_| |_d |_d |_g|_t+j|_t+j|_t+j|_d |_d |_| |_| |_d |_t+j|j|_ tC|_"| s|#d Sd S)uConfigure a pool and pre-provision `warm_up` hosts (blocks until they are ready). Env/secrets are *not* set here: they belong to each sandbox and are passed to `create(env=...)`, so sandboxes in the same pool can have different environments. Args: image (`str`, *optional*, defaults to `"python:3.12"`): Docker image for the hosts (needs `/bin/sh`). All sandboxes in the pool share this image. flavor (`str`, *optional*, defaults to `"cpu-basic"`): Hardware flavor for the host jobs (e.g. `"cpu-basic"`). sandboxes_per_host (`int`, *optional*, defaults to `50`): How many sandboxes to pack per host (per VM density). warm_up (`int`, *optional*, defaults to `1`): How many hosts to pre-provision in the constructor (which blocks until they are ready), so an initial burst of `create()` calls doesn't pay a host cold start each. Existing warm hosts (from the cache / other processes) count towards it, so only the shortfall is booted; capped by `max_hosts`. Defaults to 1 (a single host). max_hosts (`int`, *optional*): Optional cap on the number of host jobs (a cost ceiling). When reached and all hosts are full, `create()` raises. The cap is enforced per-process against the hosts this handle knows about, so two processes using the same pool concurrently can exceed it. (Per-host density, `sandboxes_per_host`, *is* enforced by the host itself.) name (`str`, *optional*): Pool name, used as the `hf-sandbox-pool` job label so the pool is discoverable (e.g. `hf sandbox pool ls`, `connect()`). `create()` reuses running hosts carrying this label (including from other processes) before booting new ones, so distinct names keep separate pools from sharing hosts. A random name is generated when omitted. idle_timeout (`int` or `float` or `str`, *optional*, defaults to `600`): Host idle timeout — a host shuts down once it has had no sandboxes for this long (a billing backstop). Each sandbox also has its own idle timeout, set at `create()`. Pass `None` to disable. namespace (`str`, *optional*): User or org namespace to run hosts under. start_timeout (`float`, *optional*, defaults to `120.0`): Max seconds to wait for a host to become ready. adopt_hosts (`str`, *optional*, defaults to `"own"`): Which already-running hosts `create()` may pack onto. Hosts are found by filtering Jobs on labels, and labels are set by whoever creates the Job — so a label match is a claim, not proof. This decides how much more than the claim is required: - `"own"` (default): only Jobs this principal started, per the Jobs API's `initiator`. Safe in a shared namespace. - `"namespace"`: any Job in the namespace carrying the pool's labels, provided its image, flavor, command and exposed URL all match. Restores cross-user host sharing; only use it in a namespace whose members you trust, since any of them can publish a Job your client will then send a token to. - `"never"`: no adoption at all — this handle only uses hosts it booted itself. token (`str`, *optional*): HF token override. zadopt_hosts must be one of z, z; got r+rz sandboxes_per_host must be >= 1.zwarm_up must be >= 1.r1Nzpool-Fr)$ ADOPT_OWNADOPT_NAMESPACE ADOPT_NEVERr rr'r9rr _warm_uprrrw _idle_timeout _namespace_cache_context_start_timeout _adopt_hosts _principal_principal_resolved_hostsrr_lock _warmup_lock _boot_lock_closed _warmed_up_require_live_host _owns_hosts _in_flight Condition_drainedset_dead_host_ids_ensure_warmed_up) rOr9rr r rrwrr)rlrr/rs r;rzSandboxPool.__init__NsN y/;G G Gwiww_wwR]wwgrwww   ! !?@@ @ Q;;455 5&&&   "4 " ,DD2H)A,,2H2H )#-TY BB+''+#( ,. ^%% &N,, $.** #0 -,!+DJ77 (+  %  " " $ $ $ $ $ % %r=)r)rr/pool_idc rt|}t|t||}|.||j|j|j|j||j|||d St||||}t|||}| d} | d} ||j p |j pt|jt|jnd t| d t | t| nd|| t| nd|||d S) uhReattach to a running pool by id, from any machine — no local state needed. Finds a running host labelled with `pool_id` and rebuilds the pool's config (image/flavor/density/host-idle) from that host job's spec and env vars, returning a [`SandboxPool`] ready to `create()` more sandboxes — packing onto the running hosts, or booting a duplicate (same config) when they are full. Raises [`SandboxError`] if no running host is found (a pool stops existing once all of its hosts are gone — idle-timed-out or killed). Args: pool_id (`str`): The id returned when the pool was first created. namespace (`str`, *optional*): Namespace to search for the pool's hosts (defaults to yours). adopt_hosts (`str`, *optional*, defaults to `"own"`): Which hosts may be attached to. See [`SandboxPool`]. Reattaching to a pool whose hosts another member of the namespace started needs `"namespace"`. token (`str`, *optional*): HF token override. r1NT) r9rr rrwrr)rr/rr)policyrpSBX_IDLE_TIMEOUT SBX_MAX_HOSTSr SBX_CAPACITY)rrrr9rr rr_find_pool_host_job _host_envrrrNrOrr]rADEFAULT_SANDBOXES_PER_HOST) rQr,r)rr/r%cacherKridle_raw max_hosts_raws r;r]zSandboxPool.connectsX<%    Y)G)GHH  3k|#(#;/"/#'"    "#w)KXXXSI66677-..00 s"CclCm&)j&<3sz???+"377>;U#V#VWW,9,Ec-(((4*2*>XD#    r= num_hostsc|jrtd||j5|t |jz }dddn #1swxYwY|dkrP||d}|j5|j|dddn #1swxYwY|j5d|_ dddn #1swxYwY| |j S)uEnsure `num_hosts` empty host(s) are running and leave them running. Returns the pool's host job ids. Used to "create" a pool up front: the hosts carry the pool label and config (in their env vars), so a later `SandboxPool.connect(pool_id)` (even from another machine) finds them and spawns sandboxes without a cold start. The hosts keep billing until killed or idle. Adopts hosts already running for this pool (found via job labels) before booting, so a `warm()` after `connect()` — or a repeated `warm()` — tops up to `num_hosts` instead of duplicating live hosts and blowing past `max_hosts`. This SandboxPool is closed.NrT allow_partial) r"r_discover_hostsrrUr_provision_hostsrr r# _save_cachehost_ids)rOr9 shortfallbooteds r;warmzSandboxPool.warms < ><== =  Z 5 5!C $4$44I 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 q==**9D*IIF + + ""6*** + + + + + + + + + + + + + + +   # #"DO # # # # # # # # # # # # # # # }s5AAAB,,B03B0>CCC)rrrrrrhc|j5|jrtd|xjdz c_dddn #1swxYwY |||||j5|xjdzc_|jdddS#1swxYwYS#|j5|xjdzc_|jdddw#1swxYwYwxYw)uCreate one sandbox, provisioning a host if needed. Reuses a host with free capacity (this pool's, or a warm host found via job labels / the local cache) before booting a new one, so a `create()` against a warm host costs ~one round-trip. Call it repeatedly to fan out; use `warm_up` (or [`warm`]) to pre-provision hosts and avoid a cold start on the first calls. If a host fills up under us (another process packed it) or a cached host is gone, the sandbox is re-placed on another host (or a fresh one). Args: env (`dict[str, Any]`, *optional*): Environment variables for this sandbox (each sandbox gets its own). idle_timeout (`int` or `float` or `str`, *optional*, defaults to `600`): Per-sandbox idle timeout — a sandbox is evicted from its host after this much inactivity (no API calls, no running process). Distinct from the host idle timeout. Pass `None` to disable. forward_hf_token (`bool`, *optional*, defaults to `False`): If True, inject your HF token as `HF_TOKEN` in the sandbox (opt-in). Unlike a dedicated sandbox's `secrets`, a pooled sandbox's env is delivered to the host server at creation (never stored in the host job), so it doesn't appear in any job's metadata. r;rN)rr"rr&_create_lockedr( notify_all)rOrrrs r;rzSandboxPool.create7s:Z ! !| B"#@AAA OOq OO  ! ! ! ! ! ! ! ! ! ! ! ! ! ! !  +&&sL:JKK + +1$ ((*** + + + + + + + + + + + + + + + + + +1$ ((*** + + + + + + + + + + + + + + + +sO';??B&$*BB !B &C0.*C$ C0$C( (C0+C( ,C0c t|pi}|rt|j|d<|t|nd}g d}d} | |}|,|s*d}||}|W|jr;|r9|js2t|j |j td|j d| }| ||||} nK#t$r>|j5t#d|jdz |_dddn #1swxYwYwxYw| || S| vr |d}|dz }|t*krtdU#t$r|j5 fd |jD|_dddn #1swxYwY D]U} |n#t$rYnwxYw|=#|wxYwwxYw) NHF_TOKENFrT No running host found for pool 'X'. The pool has stopped (all its hosts were killed or idle-timed-out); create a new one.rzZCould not place sandbox: hosts kept reporting full. Raise max_hosts or sandboxes_per_host.cg|]}|v| Sr`r`)rh new_hostss r;rz.SandboxPool._create_locked..s#LLLQ)9K9Kq9K9K9Kr=)rr+r'rCr+ _reserve_oner>r$rrrwrr_boot_one_host _create_onerrmaxrDr@remove_MAX_PACK_ROUNDSrkrh) rOrrr sandbox_env idle_secs discoveredroundshostrrNs @r;rFzSandboxPool._create_lockedas{ 39"oo  B&6ty&A&AK #8D7O%l333UY *,  <   " " $ $ $, ((**< >(+Aty1}(=(=DI>>>>>>>>>>>>>>>  *((***& y((!((..." ! ,,,&tU, Z    M MLLLL$+LLL  M M M M M M M M M M M M M M M! ! !!OO%%%% DJJLLLLDJJLLLL  sB2F09DF0E"E  E E EE EF04.s$99Tty999999r=N)rsumrrNs r; num_sandboxeszSandboxPool.num_sandboxessZ : :99T[99999 : : : : : : : : : : : : : : : : : :s 377cf|j5d|jDcdddS#1swxYwYdS)z!Job ids of the provisioned hosts.cg|] }|j Sr`rr]s r;rz(SandboxPool.host_ids..s888DDK888r=N)rrrNs r;rAzSandboxPool.host_idssZ 9 988DK888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9s &**c j5jr ddddSd_jtz}jrMjfd|s.tdj djd|dj }g_ dddn #1swxYwYg}g}|D]} |j r| nq#t$rd}||||jd |d td |jd |Yd}~nd}~wwxYw|#|wxYw|r| n!jrt'j j|r.t+dj dd|ddS)uRelease the pool. Idempotent. For a pool we created, this terminates all host jobs (and therefore all their sandboxes). For a `connect()`'d handle it only releases the local HTTP clients: the shared hosts may be serving other clients, so — like [`Sandbox.connect`] — leaving a `with` block must not tear them down. Terminate a connected pool's hosts explicitly with `hf sandbox pool delete `. Only hosts *this handle started* are cancelled. A host discovered via labels may be serving another process' sandboxes, so it is released rather than terminated. Raises [`SandboxError`] if a host job could not be cancelled, naming the jobs that are still running — they keep billing, and their cache entries are kept so they stay discoverable. When `close()` is reached through `__exit__` with an exception already in flight, the failure is logged instead, so it cannot mask the original error. NTcjdkSrM)r&rNsr;z#SandboxPool.close..sdoYZFZr=rnPool '': z$ operation(s) still in flight after z(s; any host they boot will keep running. (rTFailed to cancel sandbox host r)hostsz#' could not terminate host job(s): ; zM. They are still running and billing; cancel them with `hf jobs cancel `.)rr"r_CLOSE_DRAIN_TIMEOUTr&r(wait_forrrrwrrFrkrrrrhr@r%rrrr)rO drain_timeoutrj survivorsfailuresrYrs` r;rhzSandboxPool.closes"Z  |          DL !/2FFM t}'='=>Z>Z>Z>Zdq'='r'r OTYOO4?OO$OOOKEDK               "+-    D :&OO%%% T T T  &&&4; 6 6! 6 6 6777R RRqRRSSSSSSSS  T   >   9  - - - -   > di)< = = =  ^^^tyyQYGZGZ^^^   sH B#A8B##B'*B'7CE EAD<7E<EEE/c|Srr`rNs r;rzSandboxPool.__enter__rr=c |dS#t$r2|dtd|jddYdSwxYw)Nrrfz/' teardown failed while handling another error.T)r)rhrrrrwrs r;rzSandboxPool.__exit__su o JJLLLLL o o o{" NN^DI^^^imN n n n n n n  os8AAc|j5|jr ddddS||j|jnt |j|j}|j5|t|jz }dddn #1swxYwY|dkr| |j s|j5|t|jz }dddn #1swxYwY|dkrP| |d}|j5|j |dddn #1swxYwYd|_dddn #1swxYwY|jr| dSdS)uOne-time: seed from the cache and pre-provision up to `warm_up` hosts. Cheap by default: seeding is local (no HTTP), and if the cache already gives at least `warm_up` hosts we trust them and skip discovery/booting (dead ones are pruned lazily by `create()`). Only when short do we list_jobs and boot the shortfall in parallel, capped by `max_hosts`. Runs at most once per pool, under `_warmup_lock`: concurrent first `create()` calls block here until the warm-up completes — and see its hosts — rather than racing past a half-set flag and each booting their own. `_warmed_up` is only set once the work succeeds, so a failed warm-up is retried (seeding/discovery dedupe by job id). Warm-up hosts are pool-level (like [`warm`]): they are not torn down if the triggering `create()` later fails — `close()` (or the `with` block) reclaims them. NrTr<)r r#_seed_hosts_from_cacherrrrrUrr>r$r?rr@)rOtargetrBrCs r;r+zSandboxPool._ensure_warmed_ups   # #  # # # # # # # #  ' ' ) ) )&*n&>$*S-=-=$= >>>>>>>>>>>>>>> 1}}!%!6!6yPT!6!U!U!Z77 K..v666777777777777777"DO' # # # # # # # # # # # # # # #, ;           s EAE!B9 EB E B +E8C EC E#C $'E D2& E2D6 6E9D6 : EEEc |jsw |jdd|_|jdu|_n4#t $r'}t d|Yd}~nd}~wwxYw|jS)zGId of the principal this pool authenticates as, for the `"own"` policy.Tr6rcN;Could not resolve the current user to validate pool hosts: )rr'whoamirrrrrrrs r; _principal_idzSandboxPool._principal_id8s' b b"&)"2"2"2">">"B"B4"H"H+/?$+F(( b b b`]^``aaaaaaaa bsAA A<A77A<rKc t||j|jtkr|nd|j|j|j}|0td|j d|j d|ddSdS) z9Whether `job` may be adopted as one of this pool's hosts.Nr/ principal_idr)r9rzNot adopting job  as a host for pool 'rgr+FT _host_rejectionrrrzrr9rrdebugrcrwrOrKreasons r; _adoptablezSandboxPool._adoptableCs $151Bi1O1O++---UYo*;       LLaSVaa$)aaX^aaa b b b5tr=c t||j|jtkr|nd|j|j|jd}|0td|j d|j d|ddSdS) zFLike `_adoptable`, minus the checks a not-yet-running job cannot pass.NF)r/r}r)r9r check_urlzNot waiting for job r~rgr+Trrs r;_adoptable_pendingzSandboxPool._adoptable_pendingRs $151Bi1O1O++---UYo*;      LLdddTYdd[addd e e e5tr=_SandboxServer | Nonec|j5|jD]5}|j|jz dkr |xjdz c_|ccdddS6 dddn #1swxYwYdS)zNReserve one slot on the first host with free capacity (under lock), else None.rrN)rrr7rDrOrYs r;rOzSandboxPool._reserve_onebs Z     =49,q00IINIIKK         0                ts0AAAArNr4c|j5|}||cdddS|r$|}||cdddS|d}|||j5|j|dddn #1swxYwY|cdddS#1swxYwYdS)uhProvision (or adopt) one host and reserve a slot on it. Returns None to retry. Held under `_boot_lock` so that, within a process, only one host is booted at a time: a burst of create() calls that all found every host full queue here, and each new host frees `sandboxes_per_host` slots for the threads still waiting — so they reuse it instead of each booting their own. Before booting, we reuse a slot freed by a concurrent boot and, failing that, adopt a host already SCHEDULING for this pool (here or in another process) rather than piling on a duplicate. Nr)r!rO_adopt_pending_hostr?rrr)rOrNrYrCs r;rPzSandboxPool._boot_one_hostks_ ' '$$&&D  ' ' ' ' ' ' ' ''')) ((**# ' ' ' ' ' ' ' '**1--F   V $ $ $ + + ""6*** + + + + + + + + + + + + + + +$$&&! ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's@C(+C(%1C(B=1 C(=C C(C C((C,/C,c djDtfdjdtt t jijDddSt djdj d tj j z}tj |krjj j jj}|d vrdS|d kr6t%fd jDrdStjdtj |kdS)aWait for and adopt a host already SCHEDULING for this pool, if any. Avoids over-provisioning when a host is already on its way up for this pool (started by another process, or an earlier create() in this one): rather than booting a duplicate, wait for it to reach RUNNING and adopt it via discovery. Returns True if a pending host was found (whether or not it eventually came up), False if none is scheduling. ch|] }|j Sr`rbr]s r;rz2SandboxPool._adopt_pending_host..555555r=c3XK|]$}|jv | |V%dSr)rcrrrKknownrOs r;rz2SandboxPool._adopt_pending_host..sS  6&&4+B+B3+G+G&'&&&  r= SCHEDULINGrurr)NFrfz' host z: is already SCHEDULING; waiting for it instead of booting.rj)rrrc38K|]}|jjkVdSrrrc)rrYpendings r;rz2SandboxPool._adopt_pending_host..s,IITt{gj0IIIIIIr=Tg?)rrr' list_jobsrr POOL_LABELrwrrrrcrrrrtrurvr>anyry)rOrzrvrrs` @@r;rzSandboxPool._adopt_pending_hosts65555     9..'& :tyI"o/       ?5 vdivv vvvwww9;;!44ikkH$$I))t)WW^dE555u !!$$&&&IIIIT[IIIII 4 JsOOOikkH$$ur=cdjDfdjdttt jijD}|D]~d} tjj j}tjj}t| dj |_t|dd |_ne#t&t(jt,f$rF}t.d j d |||Yd}~d}~wwxYwj5t7fd jDr|nj|dddn #1swxYwYdS) a<Attach to running host jobs carrying this pool's labels. Lets `create()` reuse a host warmed by an earlier call or another process instead of booting a new one. Hosts are found via job labels; each adopted host's free capacity is read from the server, so packing stays accurate. Labels select candidates; creator, job specification and proxy URL admission checks decide which candidates can receive credentials. Adopted hosts are released by close(), not cancelled, because another handle may still be using them. ch|] }|j Sr`rbr]s r;rz.SandboxPool._discover_hosts..rr=cPg|]"}|jv | |#Sr`)rcrrs r;rz/SandboxPool._discover_hosts..sE    vU""ts';';"  #""r=rrNrpr2rrzSkipping host z during discovery: c38K|]}|jjkVdSrr)rrYrKs r;rz.SandboxPool._discover_hosts..s,EEt{cf,EEEEEEr=)rr'rrrrrwrrrcr4rArrr r7rUr-rqrDrrG HTTPErrorrrrrhrrr)rOmatchesrrrrKrs` @@r;r>zSandboxPool._discover_hostss465555     y** "Iz49E/+    / /CF &ty#&DOTTT  3$/JJJ"%cggnd>U&V&V"W"W!&.."H"H"M"M"O"OPP  &"2NC    LcfLLLLMMM%LLNNN   / /EEEEEEEEE/LLNNNNK&&v...  / / / / / / / / / / / / / / /! / /s,$B$D  E+%;E&&E+5AGG G c|j5d|jD}dddn #1swxYwY |jdtt t |ji|jD]}| |j n<#t$r/}t d|jd|Yd}~nd}~wwxYwt|S)aHow many hosts are running for this pool, not just the ones we track. `max_hosts` is documented as a cost ceiling, but it was compared against `len(self._hosts)` -- an in-process count. Two processes using the same pool each counted only their own hosts, so the number of billed host jobs could quietly exceed the cap by a factor of however many processes were involved. Counting via labels narrows that considerably. It does not close it: two processes can still list simultaneously and both decide there is room. A hard cap has to be enforced where the jobs are created, not by the clients racing to count them. ch|] }|j Sr`rbr]s r;rz/SandboxPool._live_host_count..s;;;tt{;;;r=Nrrz%Could not count live hosts for pool 'rg)rrr'rrrrrwraddrcrrrrU)rOtrackedrKrs r;_live_host_countzSandboxPool._live_host_counts=Z < <;;t{;;;G < < < < < < < < < < < < < < < Ty** "Iz49E/+ $ $  CF####  $  T T T LLRRRqRR S S S S S S S S T7||s$&**AB C%B>>Cr<num_newr=c  j}|r&t|tdj|z }||zjkr9j|z }t d|djdtd|d|d |dkrgSt t|d 5 fd t |D}dddn #1swxYwYg}d}|D]C} ||+#t$r } | }Yd} ~ . s'PPPAxt77PPPr=) rrrrRrrrrrrrkrh) rOrr=currentallowedfuturesrCrfuturerrrs ` @r;r?zSandboxPool._provision_hostssR > %++--G Igs1dnw.F'G'GHH 4>11.72">'>>t~>>#&q'??>>;B>>> a<<I C,<,< = = = QPPPPPwPPPG Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q')"&  F  fmmoo....        # ##%%'''' DLLNNNNFLLNNNNK sT3CC #C 0'D D."D))D.8E  E2 EE2EE22Fc t|j}td}t||}t |j|ddd|jd|\}}}}d|d<t |j|d<|jt |j|d<tdttt|j t|i}|j|j||||jt$||pdt&g|j } d} t*| |||jt/|jd zd |j } | |jn#t4$r |j| j| jj n<#t4$r/} t<d| jd| Yd} ~ nd} ~ wwxYw| | wxYw|j!| _"| S)z6Start one host job and wait until its server is ready.rNFrr SBX_HOST_MODEr2r1rr0)rKr2r;r%r6r7rjrir)#r+r'rr<rrr]r rrrrrrwrrr9rrrrr4rRrr}rrrkrcr8rrrhr%rF) rOr1r2r;rrrrrrKrrs r;rzSandboxPool._boot_hosts9#DI.." -h>> 5H I +"' 6 6 6 2+{$' #&d&=">"> > %'*4>':':GO $j)ZT_afgi*;('4'(o   +/ $,,+I #D$;a$? E E0 -F   d1 2 2 2 2    e $$CFcin$MMMM e e eccc`accdddddddd e!   '  s7?AE G%%+FG% G %GG%G  G%rYrVrc ddi}|||d<|r||d< |dd|}nf#ttjf$rM}|jrt d |jd || |Yd}~dSd}~wwxYwd |_| d pg}t| d ds|s-|j 5|j |_dddn #1swxYwYdS|d}t|jt |d||dd d| d} |j| _| S)uCreate one sandbox on a reserved `host`. Returns the sandbox, or None to retry. None means either the host filled up between our reservation and the create (server-authoritative capacity — another client packed it, so we mark it full and place the sandbox elsewhere) or a host rebuilt from the cache is gone/unreachable (dropped and re-placed via discovery / a fresh boot). countrNidle_timeout_secsrr rrz!Dropping unreachable cached host rT sandboxesrejectedrrcFr/r)r-rqrrGrrErrr _drop_hostrrrArr7rDrhr_on_sandbox_killedr) rOrYrrVbodyrrrrrs r;rQzSandboxPool._create_one`s!(|  (1D$ %  DK <<d<CCHHJJDDf./   }  LLOT[OOAOO P P P OOD ! ! !44444    HH[))/R txx A&& ' ' y  * * M  * * * * * * * * * * * * * * *4|+:}:d4j::$Z((7++     2s(*?B"ABB"/ DD D rc|j5td|jjdz |j_ddddS#1swxYwYdS)z:Free the packing slot of a shared sandbox that was killed.rrN)rrRrrDrs r;rzSandboxPool._on_sandbox_killeds Z D D#&q'/*>*B#C#CGO  D D D D D D D D D D D D D D D D D Ds(=AAc|j5fd|jD|_|jjdddn #1swxYwYdS)zFForget a host found dead this session (and mark it for cache pruning).cg|]}|u| Sr`r`)rrMrYs r;rz*SandboxPool._drop_host..sCCCQd]]1]]]r=N)rrr*rrrhrs `r;rzSandboxPool._drop_hosts Z 1 1CCCCdkCCCDK   # #DK 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 s8A  AAcVt|j|j}|dSt|j}t j}|jD]ي|j5d|jD}dddn #1swxYwYj |vsj j krMt}|mtdj d|jd|d|j5|jj dddn #1swxYwY|jz t$kr|st)j j|jjjt3|j|jt5|jdzdj }j |_ d |_|j5t;fd |jDr|n|j|dddn #1swxYwYdS) uAdopt the pool's cached hosts without any HTTP (rebuilt from cached URL + nonce). The first thing that happens to a seeded host is an authenticated `POST`, so what the file says about *where* it lives is not taken on trust: the URL must be one only this job can have (`_cached_host_rejection`), and an entry older than `HOST_TRUST_TTL` is re-checked against the Jobs API first (`_confirm_cached_host`). A fresh entry — the `pool create` → `create --pool` case the cache exists for — is used as is. Liveness stays best-effort: a seeded host is confirmed by its first successful request and dropped on the first failure (see `_create_one`). Hosts the cache believes are full are skipped — label discovery re-checks them with fresh counts if the seeded ones don't satisfy the request, so a stale-full entry never blocks a create. Nch|] }|j Sr`rbr]s r;rz5SandboxPool._seed_hosts_from_cache..s======r=zDropping cached host of pool 'z': it r+r0rrMFc38K|]}|jjkVdSrrb)rrYchs r;rz5SandboxPool._seed_hosts_from_cache..s,HHDt{bi/HHHHHHr=) rrwrr+r'rrrjrrrrDr7_cached_host_rejectionrrr*r updated_atr_confirm_cached_hostr4r8r9r:r2r<rr rErrhr)rOr6r1nowrrrrs @r;rtz"SandboxPool._seed_hosts_from_caches  4+>?? = F#DI..ikk+ / /B > >===== > > > > > > > > > > > > > > >yE!!RW %;%;+B//F! dRYdd$)dd[adddeeeZ77'++BI666777777777777777R]"^33D.s`   ;*!]*!]#y{{   r=)contextr9rr rrrj dead_host_ids) rrr)r*rrwrr9rr rrrC)rOrjrrdeads r;r@zSandboxPool._save_cachesZ , ,$)MT[[uF  #   Gt*++D , , , , , , , , , , , , , , ,  I'*;#6nBFBTB`*4+=>>>fj s,AAA)rr~)r3r )r3rr)rjrr3N)1rXrYrZr[rr5r rr]rArBr^rrr]r rDrr rrFr_r9r_rArhrrr+rzrrrrOrPrr>rr4r?rrQrhrrrtrrr@r`r=r;r r !s**\#D%""< $1E $$$ #D%D%D%D% D%  D%  D%:D%DjD%EkC'$.D%:D%D%D%TzD%D% D%D%D%D%P !%$ C C C C : C  C Tz C  C C C [C Jc$s)<&*1E!& (+(+(+#s(^d "(+EkC'$. (+  (+  (+(+(+(+TL #s(^d "LEkC'$.L L  LLLL\$3$$$X$ :s:::X: 9$s)999X9 ====~oooo''''R sTz     g $    g$ '-=(>'CZ''''8'T''''R*/*/*/*/X#8GL$$$$t$PTUcPd$$$$L@N@@@@D% 0%tCH~%RUX\R\%aq%%%%ND'DdDDDD ~$1/1/1/1/f(z(d((((T        r=r r9c|dS|}dD].}||r|t|d}/|S)zCompare Docker images leniently: registries and tags get normalized server-side, and a mis-parse here must not stop a pool from working.N)z docker.io/zindex.docker.io/zlibrary/)striplowerrrU)r9prefixs r;_normalize_imager sf }t KKMM   ! !E@))   F # # )#f++--(E Lr=urlct|}|jdkr d|jdS|jpd|dtdsd|dtd Sd S) aCWhy `url` is not a URL we should send a sandbox job's credentials to. The hostname must be derived from the job id we already hold, so nothing but the client's own knowledge of which job it is talking to can decide where the credentials go. The domain is not hard-coded, so staging endpoints keep working. httpsz uses scheme z , not httpsr--r+z is not job z's port-z URLN)rrhostnamerrr)rrparseds r;_server_host_rejectionr! sc]]F }:fm:::: O !r - -.Q.Q;N.Q.Q.Q.W.W.Y.Y Z ZGFVFF-@FFFF 4r=rKc|jt|jjpgng}t|dkrdt|dtSt |d|j}|d|dd|ndS)zDWhy this Job's exposed URL is not one we should send credentials to.Nrzexposes z' URL(s), expected exactly one for port rrb, which )rur expose_urlsrUrrrc)rKurlsrs r;_server_url_rejectionr1 s141G4 &," - - -RD 4yyA~~a#d))aaL_aaa #DGCF ; ; ;F5;5G 1d1g 1 1 1 1 1TQr=rYcZt|j|j}| d|jd|ndS)agWhy a cached host entry must not be rebuilt into a credentialed transport. Same reasoning as `_server_url_rejection`, for a URL that comes off the local disk instead of the Jobs API. Without this, the cache file gets to choose where the HF bearer and the derived host token are sent -- `http://127.0.0.1:1234` included -- and it gets to choose it *before* any API call could have contradicted it. Deriving the expected hostname from the entry's own `job_id` leaves it able to name only a job this client can independently identify, which is the same bar a discovered host has to clear. rbNznames r)rr:r)rYrs r;rr: s>$DM$+ F F FF9?9K 5DM 5 5V 5 5 5QUUr=r%cTtj|jt||S)zOThe local pool cache's security context for this endpoint/credential/namespace.)endpointr/r))rfor_credentialrr+)r%r)s r;rrH s'  & DTUXDYDYen o o oor=T)rr/r}rrcF|tkrdS|tkr3|j}||jdS|dS|j|krd|jp|jdS|)|j"|jj|krd|jjd|S|jp|j}|(t|t|krd |d|S|2t|j |krd t|j d|S|j $t|j tkrd S|rt|ndS) aWhy this Job must not be adopted as one of our pool hosts, or `None`. Hosts are found by filtering Jobs on labels, and labels are set by whoever creates the Job -- so a label match says "this Job claims to be one of our hosts", not "this Job is one of our hosts". Anyone who can create a Job in the namespace can make that claim, copy the public nonce from a real host's labels, and receive the token derived for that host. Everything checked here is asserted by the backend rather than by the Job's creator: the initiator, the owner, the image, the flavor, the command, and the exposed URL. `initiator` is the load-bearing one -- there is no client-side way to set it -- and it is what makes `policy="own"` meaningful. z/host adoption is disabled (adopt_hosts='never')NzJthe Jobs API did not say who started it, so it cannot be confirmed as ourszHcould not resolve the current principal to compare against its initiatorz"started by a different principal (rTzruns under namespace z, not z runs image z runs flavor z*does not run the sandbox bootstrap command)rr initiatorrcrwr8rNrOrr]rrr_bootstrap_commandr) rKr/r}r)r9rrr job_images r;rrM se.@@ M   4__  ]] << ' 'Y 8V),YYY Y!639>Y;V;VLsy~LLyLLL 0CLI -i88Q:R:RSSG"Qgm"Qk="Q"QKw}"%%K&)*;L*I*I&J&J"#+"* J :"799M#&< s s):Y s sdq s sG #8G  #5G HY%DQcosttt   +{ BBr=cddtgS)z>The Job command every sandbox host and dedicated sandbox runs.z/bin/shz-c)_BOOTSTRAP_DOWNLOADr`r=r;rr s t0 11r=ct|jtr|jni}d|vr<||j|jpi}t|tr|ni}|S)z6Return a host job's env vars (where pool config lives)r2rj)r@ environmentrrtrc)r%rKr)rs r;r4r4 sf'>> F#//BCS  ooSVyoAAMSQST**2cc Jr=r.r,c ,d}|tkr_ |dd}n4#t$r'}td|Yd}~nd}~wwxYwg}|dttt|i|D]=}t||||dd}||cS| |j d |d >|r)td |d d |dtd|d)aReturn a running host job belonging to `pool_id`, found via the pool label. This one feeds `SandboxPool.connect`'s cold path, which rebuilds the pool's whole configuration (image, flavor, density, caps) from the job it returns -- so an unvalidated match would let a Job in the namespace choose the image that the pool's *next* hosts boot. Image and flavor are unknown here (they are what we are about to learn), so this checks ownership and shape. NTrwrcrxrrr|rhrTz%Found host job(s) labelled for pool 'z' but none usable: rkzp. Pass adopt_hosts='namespace' to SandboxPool if you intend to share hosts with other members of this namespace.rJrK)rryrrrrrrrrrrrrcrr) r%r,r)r/ principalrrrKrs r;r3r3 sI  ^  ..22488II ^ ^ ^ NN\YZ\\ ] ] ] ] ] ] ] ] ^H}}*iW!MYb00! Y)SW`d    >JJJ36..V...////  )G ) )PTPYPYZbPcPc ) ) )    K7 K K K  s)9 A*A%%A*rprc 8|||}|jpi}|t}|#|tt krt d|d|jjdkr t d|d|jjdt|td|dd }|t d|d |d t ||tt|||tjd z S)z?Reattach to a running host job and return its server transport.rjNrz is not a sandbox host.rz Sandbox host rrqr|z+ does not look like a usable sandbox host: r+rr)rtrrrrrrrrurvrrr4rRr<r+rr)r%rr)rKrr2rs r;rr s9 // / B BC Z 2F JJ{ # #E } :..);;F+FFFGGG z9$$e;eeQTQ[Qaeeefff O$)SW`dFc+ccZ`cccddd  " " +,. s(&E&Et{D{{&E&E&E&E&E&Er=)rfetch_job_logsrr)r%rr)rliness r;rxrx sS''v'KKLLeVWWU rrIN V?TYY&E&Eu&E&E&EEE E ETVVs -0 >>)yr8r5rqrrrrconcurrent.futuresr contextlibr dataclassesrrpathlibrrrtypingr r r r r rr urllib.parserrGrr_sandbox_cacherrrrrr _space_apirerrorsrrrhf_apirrutilsrr utils._parsingr! get_loggerrXrrrrrrrr frozensetrrSANDBOX_SERVER_VERSIONrr r rr5rlrrrrrrwrTrrr]r<rArBrCrErbrvrrr^rrrrrr"Authr$r2r4rrhr rrrrrrr+rrrrr4r3rrrPrxr`r=r;rs   111111%%%%%%((((((((MMMMMMMMMMMMMMMMMM!!!!!! EEEEEEEEEE""""""""%%%%%%%%******  H % %     ! #)]J K$XYY !Z   -    A'BcCcccccc,$e c 1$c$$$$ sssssss s& ):):):):):):): ):X  dx/A2eNeNeNeNeNeNeNeNP sT#Y td{ tCH~    6?x~ssDssssV_cd46;.UkUkUkUkUkUkUkUkp"OOOOOOOOdq q q q q q q q h C$J 3:       d     RwR3:RRRR V Vd V V V Vpp#*ppppp3=3=3= 3= 3=* 3= Tz 3= : 3= $J3=3= 4Z3=3=3=3=l%C(C (C(C c3h$  (C #s(^d " (C &\D (C+#d*(C(C(C 49d38nd38nd6l BC(C(C(C(CV2DI2222 5wcDjT#s(^NRaj"""U"S"d "[^"ov""""JLPu3cDjTb4 *# *#* *sCRVJAW * * * *g'gcggggHLZ\WWWWsW#*WTWWadWWWWWWr=