AxjUdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z e Z dZ dZdZdZd Zd Zd ZGd d ZdZgZdZdZdZejedS)aDirectory-mirror warm cache between local directories and a network volume. ``VolumeCache`` keeps a browsable mirror of one or more local directories on a mounted network volume, and reconciles the two directions on demand: - ``hydrate()``: copy files that are missing or newer on the volume mirror into the container (used on cold start, before the cache is populated). - ``sync()``: copy files that are missing or newer in the container onto the volume mirror (used after the cache has been populated/updated). Transport adapts to the tree's shape: files below 256 KiB are packed into a single ``small.tar`` on the volume (collapsing per-file metadata round-trips), larger files are copied unpacked into ``big/`` across a thread pool, and a versioned ``manifest.json`` (written last) records the layout. Large-file transfers stay incremental (size/mtime diff); the small-file archive is re-packed whole when any small file changes. Packing uses the ``tar`` binary when present (else stdlib ``tarfile``); extraction always uses ``tarfile``, validating every member against the configured dirs. Best-effort: any failure degrades to a cold worker and never raises unless ``best_effort=False``. N)ThreadPoolExecutor) RunPodLoggerg@g>@iz manifest.jsonz small.tarbigceZdZdZejdzejzejdzejzfZddddddZed Z ed Z ed Z ed Z ed Z dZdZdZdZdZdZdZedZdZdZdZedZdZdZdZdZedZdZ d Z!dd!d"Z"d#Z#d$Z$d%Z%dS)& VolumeCacheaoWarm-cache local directories across serverless workers via a network volume. Maintains a mirror of ``dirs`` at ``{volume_path}/.cache/{namespace}``. ``hydrate()`` reconciles volume -> container; ``sync()`` reconciles container -> volume. Used as a context manager, the object is itself the "warm cache" closure: hydrate on enter, sync (in the background by default) on exit. Requires a network volume mounted at ``volume_path`` (default ``/runpod-volume``) and a non-empty ``namespace`` (default ``RUNPOD_ENDPOINT_ID``); otherwise ``available`` is False and all operations are no-ops. Args: dirs: Local directories to cache (e.g. a model cache like ``HF_HOME``). namespace: Isolation key for the on-volume mirror. Must be a single safe path component. Defaults to ``RUNPOD_ENDPOINT_ID``. volume_path: Network-volume mount point. Defaults to ``/runpod-volume``. best_effort: When True (default), swallow and log errors instead of raising. max_workers: Thread count for parallel copy of large files. Defaults to ``min(32, (os.cpu_count() or 4) * 4)`` (I/O-bound, so oversubscribing the CPU count is fine). Example: >>> with VolumeCache(dirs=["/root/.cache/huggingface"]): ... model = load_model() # downloads land in the cached dir refsz .no_existNz/runpod-volumeT) namespace volume_path best_effort max_workersct|tttjfr|g}d|D|_|p tjdpd|_|jritj |js.tj |jvsd|jvsd|jvs |jdvrtd|jtj tj||_||_||dkrtd ||p%t#d tjpd d z|_dS) Nc g|]E}tjtjtj|FS)ospathrealpathfsdecodefspath).0ds XC:\Projects\Historical\venv\Lib\site-packages\runpod/serverless/utils/rp_volume_cache.py z(VolumeCache.__init__..Vs:PPPabg&&r{29Q<<'@'@AAPPPRUNPOD_ENDPOINT_ID/\).z..z4namespace must be a single safe path component, got rz,max_workers must be a positive integer, got ) isinstancestrbytesrPathLike_dirsenvironget _namespacerisabssep ValueErrorrr _volume_path _best_effortmin cpu_count _max_workers)selfdirsr r r r s r__init__zVolumeCache.__init__LsQ dS%5 6 6 6DQP4PPP #Qrz~~6J'K'KQr ?  GMM$/ * * v((do%%t&&+--ZtZZ K +(>(>??'  "{Q[K[[\\ \'M3rBLNN4Ga13L+M+MrcXtj|jd|jS)Nz.cache)rrjoinr-r)r2s r _mirror_rootzVolumeCache._mirror_rootosw||D-xIIIrcVtj|jtSN)rrr6r8_MANIFEST_NAMEr7s r_manifest_pathzVolumeCache._manifest_pathssw||D-~>>>rcVtj|jtSr:)rrr6r8_SMALL_ARCHIVE_NAMEr7s r_small_archive_pathzVolumeCache._small_archive_pathwsw||D-/BCCCrcVtj|jtSr:)rrr6r8 _BIG_SUBDIRr7s r _big_rootzVolumeCache._big_root{sw||D-{;;;rcrt|jo#tj|jS)zATrue iff volume_path is a mounted dir AND namespace is non-empty.)boolr)rrisdirr-r7s r availablezVolumeCache.availables*DO$$It7H)I)IIrc#Ktj|D]\}}}|D]}tj|||ds|drMt fd|jDrn tjrn#t$rYwxYwVdS)Nz.lock .rpvc.tmpc3 K|]}|vV dSr:r)rsubrs r z*VolumeCache._iter_files..s'GGssd{GGGGGGr) rwalkrr6endswithany_EXCLUDE_SUBSTRINGSislinkOSError)r2rootdirpathr&filesnamers @r _iter_fileszVolumeCache._iter_filess%'WT]]   !GUE  w||GT22==))T]];-G-GGGGGd.FGGGGGw~~d++! !H    sB00 B=<B=cgg}}|D]O} tj|j}n#t$rY)wxYw|tkr|n||P||fSr:)rstatst_sizerQ_SMALL_FILE_THRESHOLDappend)r2pathssmallrpsizes r _partitionzVolumeCache._partitionss G GA wqzz)    222UU C CA F F F Fczs $ 11cp tj|}n#t$rYdSwxYw||j|jdS)Nrr_mtime)rrXrQrYst_mtime)r2rsts r _file_metazVolumeCache._file_metasL BB   44 bj2;GGGs  %%cptj|jdtt||d}|jdtjdtjd}t|d5}tj ||dddn #1swxYwYtj ||jdS)NTexist_ok)version thresholdr]rrrHw) rmakedirsr8_FORMAT_VERSIONrZr<getpid threading get_identopenjsondumpreplace)r2 small_metabig_metapayloadtmpfhs r_write_manifestzVolumeCache._write_manifests D%5555&.    $UUry{{UUY5H5J5JUUU #s^^ #r Igr " " " # # # # # # # # # # # # # # # 3+,,,,,s/BBBc$ t|j5}tj|}dddn #1swxYwYn#tt f$rYdSwxYwt |tr|dtkrdS|S)Nrj) rrr<rsloadrQr,r"dictr(rn)r2rzobjs r_read_manifestzVolumeCache._read_manifests d)** $bimm $ $ $ $ $ $ $ $ $ $ $ $ $ $ $$   44 #t$$  (:(:o(M(M4 s,A7 A;A;AAAc tj|d}n#t$rYdSwxYw|j|dko|j|dt z kS)NrFr_rcrrXrQrYrd_MTIME_TOLERANCE)r2metares r_meta_satisfied_by_localz$VolumeCache._meta_satisfied_by_localse f&&BB   55 zT&\)]bkT']M]=].]]s  ++cd|r||gngD}g}|D]`}||d}|,|d|dks|d|dtzkr||a|S)Nc i|] }|d| Srrres r z4VolumeCache._changed_vs_manifest..sSSS!6ASSSrrr_rc)r(rr[)r2metasmanifestkeypriorchangedmr^s r_changed_vs_manifestz VolumeCache._changed_vs_manifestsSS(Q S"(=(=(=rSSS " "A !F)$$AyAfI622aj1W:P`C`6`6`q!!!rc tj|}n#t$rYdSwxYw tj|}n#t$rYdSwxYw|j|jkp|j|jt zkS)NFTr)src_pathdst_pathsrs r _needs_copyzVolumeCache._needs_copys !!AA   55  !!AA   44 yAI%SajCS6S)SSs %%> A  A ctj|tfd|jDS)Nc3hK|],}|kp!|tjzV-dSr:) startswithrr+)rrtargets rrKz,VolumeCache._is_safe_dest..s@TTA6Q;?&"3"3AJ"?"?TTTTTTr)rrrrNr&)r2dst_absrs @r _is_safe_destzVolumeCache._is_safe_dests=!!'**TTTTTTTTTTrc |dtjdtjd} tjtj|dtj||tj ||dS#t$rs}t d|d|d| tj |rtj|n#t$rYnwxYwYd}~dSd}~wwxYw) NrrHTrhzVolumeCache: skip z -> : F)rrorprqrmrdirnameshutilcopy2rurQlogdebugexistsremove)r2srcdstryexcs r _copy_filezVolumeCache._copy_files"EEry{{EEY%8%:%:EEE  K,,t < < < < Lc " " " JsC 4    II@3@@C@@3@@ A A A 7>>#&&#IcNNN    55555 s<AB D #D<3C0/D0 C=:D<C==DD c>fd|D}d|Dd|Dz }|st|fStj5}|fd|}dt ||D}dddn #1swxYwY||fS)aCopy every (src, dst) that needs it, in parallel. Returns ``(copied, current)`` sets of source paths: - ``copied`` -- sources transferred successfully this call. - ``current`` -- sources skipped because the destination was already up to date. Their union is the set of sources the destination now holds a current copy of; a source in neither failed to copy. Callers that record a "mirror complete" manifest MUST include only sources in that union, or a failed copy would be marked present and later served stale. copy2 + atomic os.replace means a source in ``copied`` matches its destination at that instant, so no re-stat is needed to confirm it. cJg|]\}}||||f Sr)r)rrrr2s rrz.VolumeCache._copy_parallel..s6CCC41aD,<,z-VolumeCache._copy_parallel..s'''A1'''rch|]\}}|Srrrs rrz-VolumeCache._copy_parallel..s*>*>*>A1*>*>*>r)r cH|d|dS)Nrr)r)sdr2s rz,VolumeCache._copy_parallel.. s$//"Q%A*G*Grc"h|] \\}}}| | Srr)rrroks rrz-VolumeCache._copy_parallel.. s%CCCJFQBCaCCCrN)setrr1mapzip)r2pairstodocurrentpoolresultscopieds` r_copy_parallelzVolumeCache._copy_parallelsDCCC5CCC'''''*>*>*>*>*>> "55'> ! D,= > > > D$hhGGGGNNGCC#dG*<*<CCCF  D D D D D D D D D D D D D D D ws4BBBctj|}tj|}||kp!||tjzS)z?True iff ``path`` resolves inside ``base`` (symlinks resolved).)rrrrr+)baser base_real path_reals r_withinzVolumeCache._within sSG$$T** G$$T** I%Q)=)=i"&>P)Q)QQrc |S#t$r1}|jstd||cYd}~Sd}~wwxYw)NzVolumeCache operation failed: ) Exceptionr.rwarn)r2fndefaultrs r_guardzVolumeCache._guardsi 244K   $  HH;c;; < < <NNNNNN  s A&AAAc*tjdS)Ntar)rwhichr7s r _tar_binaryzVolumeCache._tar_binarys|E"""rc tj|jd|jdtjdt jd}d|D} |r|d} t|d5}| d |dddn #1swxYwYtj d d d d d d|d|g dtj tj itjddit|np#t|wxYwt!j|d5}t#||D]\}}|||d dddn #1swxYwYtj||jdS#t(tjt jf$r7} t.d| t|Yd} ~ dSd} ~ wwxYw)NTrhrrHcNg|]"}tj|d#Sr)rrrelpath)rfs rrz+VolumeCache._pack_small..#s(777A3''777rz.listrl rz-Crz-cz--no-recursionz-fz-TCOPYFILE_DISABLE1)checkstdoutstderrenvF)arcname recursivez'VolumeCache: pack small bucket failed: )rrmr8r?rorprqrrrwriter6 subprocessrunDEVNULLr'_silent_removetarfileraddrurQSubprocessErrorTarErrorrr) r2rTryrelslistingrztfrrelrs r _pack_smallzVolumeCache._pack_small s D%5555)ZZBIKKZZ):M:O:OZZZ77777 !! @ --- ,gs++2r4111222222222222222Nc41A4dT[\")1)1CrzC+=sCC #7++++N7++++\#s++@r"%eT"2"2@@3q#????@@@@@@@@@@@@@@@@ JsD4 5 5 543W5EF    IIEEE F F F 3   55555 syF'5D)B:. D:B>>DB>A D F'D,,F'/F4 F'FF'FF''!G?,G::G?c tj|j}n#ttjf$rYdSwxYwd} |}n^#ttjf$rE}t d|jd||cYd}~|Sd}~wwxYw|D]}| stj d|j }||sR||j|jd}||rx ||||dz }#ttjf$r/}t d|j d|Yd}~d}~wwxYw |n#|wxYw|rt d|d |S) NrzVolumeCache: corrupt archive rrrbrzVolumeCache: skip extract zVolumeCache: extracted z small file(s))rrrr?rQr getmembersrrcloseisfilerrr6rUrr_rcr_extract_memberinfo)r2r extractedmembersrmemberrrs r_extract_smallzVolumeCache._extract_small?s2 d677BB)*   11    !--//W-. ! ! ! [$:R[[VY[[\\\      ( HHJJJJ- !" Q Q}}gll3 44))#.. #V[6<PP0066Q((V444NII!12QQQIIO6;OO#OOPPPPPPPPQ! Q& HHJJJJBHHJJJJ  J HHHyHHH I I Isl66AF%B-(&B(B-F%(B--A=F%+EF%F %FF%F  F%%F;cttdr||dddS||ddS)N data_filterrdata)filter)hasattrrextract)rrs rrzVolumeCache._extract_membercsN 7M * * $ JJvs6J 2 2 2 2 2 JJvs # # # # #rcJ|jsdS||jdS)zReconcile volume mirror -> container. Copy files missing or newer in the container. Returns the number of files copied. No-op (0) if unavailable. r)rFr _do_hydrater7s rhydratezVolumeCache.hydrateps* ~ 1{{4+Q///rc|}|dSd}|dr||z }g}|dgD]}tj|dd}tjd|}tj|j|}||r2| |j|r| ||f| |\}} |t|z }|r%td|d|j|S)Nrr]rrrzVolumeCache: hydrated z file(s) from )rr(rrrrr6rBrrr[rlenrrr8) r2rrestored big_pairsentryrrrr_currents rrzVolumeCache._do_hydratezsX&&((  1 <<  . ++-- -H \\%,, - -E'//%-55C',,sC((C',,t~s33C!!#&& -4<<+L+L -  #s,,,..y99CKK  [ HHYhYYdFWYY Z Z Zr backgroundcjsdS|r>tjfdd}t||dSjddS)aOReconcile container -> volume mirror. Copy files missing or newer on the volume. When ``background=True`` (default), run on a daemon thread and return immediately; a process-exit hook joins outstanding syncs so short-lived processes still complete. When False, run inline and return when done. Nc:jdS)Nr)r_do_syncr7srrz"VolumeCache.sync..s DM10M0MrT)rdaemonr)rFrpThread_register_pendingstartrr )r2rts` rsynczVolumeCache.syncsw~  F  * (M(M(M(MVZ[[[A a GGIIIII KK q ) ) ) ) )rcg}jD]I}tj|r(||J|\}}}dfd|DD}d}|r||d}d|r| dgngD} t| d|Dz } |s| r@ d|Drt|}n|d|Dz}g}|stjfd |D} | \} } | | zfd fd |DD}t| }||||z}|r%t"d |d j|S)Ncg|]}||Srrrrs rrz(VolumeCache._do_sync..sPPPAaPaPPPrc3BK|]}|VdSr:rfrrr2s rrKz'VolumeCache._do_sync..s/!J!J$//!"4"4!J!J!J!J!J!Jrrr]ch|] }|d Srrrs rrz'VolumeCache._do_sync..s^^^1V9^^^rch|] }|d Srrrs rrz'VolumeCache._do_sync..s)H)H)H!F))H)H)Hrcg|] }|d Srrrs rrz(VolumeCache._do_sync..s$C$C$C1QvY$C$C$Crcg|] }|d Srrrs rrz(VolumeCache._do_sync..s,K,K,K1QvY,K,K,Krc g|]G}|tjjtj|dfHSr)rrr6rBrrs rrz(VolumeCache._do_sync..sAcccTUadnbgooa6M6MNNOcccrc,g|]}|r |dv|Srr)rrmirroreds rrz(VolumeCache._do_sync..s7   ! HI& U]H]H]AH]H]H]rc3BK|]}|VdSr:rrs rrKz'VolumeCache._do_sync..s/>>q**>>>>>>rzVolumeCache: synced z file(s) to )r&rrrEextendrVr`rrr(rDrrrr?rr{rrr8)r2 all_filesrR small_files big_filesrrvsmall_transferredr prior_smalldeletedrrrrw big_copiedtotalrs` @rr zVolumeCache._do_syncso J 9 9Dw}}T"" 9  !1!1$!7!7888!%!;!; Y&&((PP!J!J!J!Jk!J!J!JPPP   $// HgNNG^^H/\x||GR/H/H/HZ\^^^K;)H)HZ)H)H)HHIIG $' $ ##$C$C $C$C$CDD$(+G %%!*,K,K ,K,K,K KI!#J 5 43 4 4 4ccccYbccc --i88 G#    >>>>I>>>   [[  Z222..  T HHRERRt?PRR S S S rc.||Sr:)rr7s r __enter__zVolumeCache.__enter__s  rc2|ddS)NTr)r)r2rs r__exit__zVolumeCache.__exit__s T """tr)&__name__ __module__ __qualname____doc__rr+rOr4propertyr8r<r?rBrFrVr`rfr{rrr staticmethodrrrrrrrrrrrrrr r(r*rrrrr,s:6F?RV3RVk5IBF5RS"&3CQUcg!N!N!N!N!NFJJXJ??X?DDXD<<X<JJXJ   HHH - - -   ^^^ T T\ TUUU"4RR\R ###>"""H$$\$000."&*****"222prrcR tj|dS#t$rYdSwxYwr:)rrrQrs rrrs:  $      s  &&cpdtDtdd<t|dS)Nc:g|]}||Sr)is_alive)rrs rrz%_register_pending..s%CCCqajjllCCCCr)_pending_syncsr[)threads rr r s8CCNCCCN111&!!!!!rc.ttD]f} |t|r$t dtddW#t$rYcwxYwtdS)N)timeoutz3VolumeCache: background sync did not finish within z.0fz"s at exit; cache may be incomplete) listr5r6_JOIN_TIMEOUT_SECONDSr4rrrclear)rs r_join_pending_syncsr<s . ! !    FF0F 1 1 1zz|| U,TUUU    D sAA,, A98A9c8tdSr:)r5r;rrr_reset_pending_for_testr> sr)r.atexitrsrrrrrpconcurrent.futuresr#runpod.serverless.modules.rp_loggerrrrr:rZrnr;r>rArrr5r r<r>registerrrrrCs;* 111111<<<<<<lnn" ! ttttttttx    """   #$$$$$r