§
    AxÅj)  ã                   ó  — d Z ddlZddlZddlZddlmZ ddlmZmZm	Z	 ddl
mZ ddlZddlmZ ddlmZmZmZ dd	iZd
edefd„Zdefd„Zdede	eef         fd„Zdedeeee         f         dee         fd„Zdedefd„ZdS )zÊ
PodWorker | modules | download.py

Called when inputs are images or zip files.
Downloads them into a temporary directory called "input_objects".
This directory is cleaned up after the job is complete.
é    N)ÚThreadPoolExecutor)ÚListÚUnionÚDict)Úurlparse)ÚRequestException)Úiter_content_cappedÚmax_download_bytesÚsafe_getz
User-Agentz:runpod-python/0.0.0 (https://runpod.io; support@runpod.io)Ú	file_sizeÚreturnc                 ó&   — | dk    rdS | dk    rdS dS )z;
    Calculates the chunk size based on the file size.
    i   i   i   @i    © )r   s    úTC:\Projects\Historical\venv\Lib\site-packages\runpod/serverless/utils/rp_download.pyÚcalculate_chunk_sizer      s+   € ð �KÒÐØˆtØÐ&Ò&Ð&Øˆ{àÐó    c                 óz   — 	 t          |                      dd¦  «        ¦  «        S # t          t          f$ r Y dS w xY w)a=  
    Reads Content-Length for chunk sizing, tolerating a missing or malformed value.

    The header is remote-controlled, so a non-integer value must not abort an
    otherwise valid download; 0 simply yields the smallest chunk size. The real
    size limit is enforced while streaming by iter_content_capped().
    zContent-Lengthr   )ÚintÚgetÚ	TypeErrorÚ
ValueError)Úheaderss    r   Úparse_content_lengthr   (   sK   € ðÝ�7—;’;Ð/°Ñ3Ô3Ñ4Ô4Ð4øÝ•zÐ"ð ð ð Øˆqˆqðøøøs   ‚"% ¥:¹:Úcontent_dispositionc                 ó\   — d„ |                       d¦  «        D ¦   «         }d„ |D ¦   «         }|S )Nc              3   ó>   K  — | ]}|                      ¦   «         V — Œd S )N)Ústrip)Ú.0Úps     r   ú	<genexpr>z-extract_disposition_params.<locals>.<genexpr>7   s*   è è € Ð?Ð?˜1ˆQ�WŠW‰YŒYÐ?Ð?Ð?Ð?Ð?Ð?r   ú;c                 óà   — i | ]k}d |v ¯|                      d d¦  «        \  }}|                     ¦   «                              ¦   «         |                     ¦   «                              d¦  «        “ŒlS )ú=é   ú")Úsplitr   Úlower)r   ÚpartÚkeyÚvalues       r   ú
<dictcomp>z.extract_disposition_params.<locals>.<dictcomp>9   sl   € ð ð ð àØ�$ˆ;ˆ;ØŸ:š: c¨1Ñ-Ô-‰JˆC�ð 	�	Š	‰Œ×ÒÑÔ˜UŸ[š[™]œ]×0Ò0°Ñ5Ô5àˆ;ˆ;r   )r&   )r   ÚpartsÚparamss      r   Úextract_disposition_paramsr.   6   sI   € Ø?Ð?Ð 3× 9Ò 9¸#Ñ >Ô >Ð?Ñ?Ô?€Eðð àðñ ô €Fð €Mr   Újob_idÚurlsc                 ó(  ‡‡— t           j                             t           j                             d| d¦  «        ¦  «        Št          j        ‰d¬¦  «         t          j        t
          j        t          d¬¦  «        dt          dt          d	t          fd
„¦   «         Šdt          d	t          fˆˆfd„}t          |t          ¦  «        r|g}t          ¦   «         5 }t          |                     ||¦  «        ¦  «        }ddd¦  «         n# 1 swxY w Y   |S )zÔ
    Accepts a single URL or a list of URLs and downloads the files.
    Returns the list of downloaded file absolute paths.
    Saves the files in a directory called "downloaded_files" in the job directory.
    ÚjobsÚdownloaded_filesT©Úexist_oké   )Ú	max_triesÚurlÚpath_to_saver   c                 óÔ  — t          | ddt          ¬¦  «        5 }|                     ¦   «          |j                             d¦  «        }d}|rHt          |¦  «        }t          j                             |                     dd¦  «        ¦  «        d         }|s7t          j                             t          | ¦  «        j        ¦  «        d         }t          t          |j        ¦  «        ¦  «        }t          ||z   d¦  «        5 }t          ||t          ¦   «         ¦  «        D ]}|                     |¦  «         Œ	 d d d ¦  «         n# 1 swxY w Y   |cd d d ¦  «         S # 1 swxY w Y   d S )	NTé   ©ÚstreamÚtimeoutr   úContent-DispositionÚ Úfilenamer$   Úwb)r   ÚHEADERSÚraise_for_statusr   r   r.   ÚosÚpathÚsplitextr   r   r   Úopenr	   r
   Úwrite)	r8   r9   Úresponser   Úfile_extensionr-   Ú
chunk_sizeÚ	file_pathÚchunks	            r   Údownload_filez/download_files_from_urls.<locals>.download_fileL   sÙ  € å�c $°½7ÐCÑCÔCð 	"ÀxØ×%Ò%Ñ'Ô'Ð'Ø"*Ô"2×"6Ò"6Ð7LÑ"MÔ"MÐØˆNØ"ð QÝ3Ð4GÑHÔH�Ý!#¤×!1Ò!1°&·*²*¸ZÈÑ2LÔ2LÑ!MÔ!MÈaÔ!P�ð "ð IÝ!#¤×!1Ò!1µ(¸3±-´-Ô2DÑ!EÔ!EÀaÔ!H�å-Õ.BÀ8ÔCSÑ.TÔ.TÑUÔUˆJõ �l ^Ñ3°TÑ:Ô:ð +¸iÝ0°¸:ÕGYÑG[ÔG[Ñ\Ô\ð +ð +�EØ—O’O EÑ*Ô*Ð*Ð*ð+ð+ð +ð +ñ +ô +ð +ð +ð +ð +ð +ð +øøøð +ð +ð +ð +ð "ð'	"ð 	"ð 	"ð 	"ñ 	"ô 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"øøøð 	"ð 	"ð 	"ð 	"ð 	"ð 	"s6   ™C(EÄ6EÄ8EÅE	ÅEÅE	ÅEÅE!Å$E!c                 ó0  •— | €d S t          j        ¦   «         › }t          j                             ‰|¦  «        }	  ‰| |¦  «        }n-# t
          $ r }t          d| › d|› �¦  «         Y d }~d S d }~ww xY wt          j                             |› |› �¦  «        S )NzFailed to download z: )ÚuuidÚuuid4rE   rF   Újoinr   ÚprintÚabspath)r8   Ú	file_nameÚoutput_file_pathrK   ÚerrÚdownload_directoryrO   s        €€r   Údownload_file_to_pathz7download_files_from_urls.<locals>.download_file_to_pathc   s¸   ø€ Øˆ;Ø�4å”z‘|”|Ð%ˆ	Ýœ7Ÿ<š<Ð(:¸IÑFÔFÐð	Ø*˜]¨3Ð0@ÑAÔAˆNˆNøÝð 	ð 	ð 	ÝÐ4¨Ð4Ð4¨sÐ4Ð4Ñ5Ô5Ð5Ø�4�4�4�4�4øøøøð	øøøõ Œw�ŠÐ"2ÐD°NÐDÐDÑEÔEÐEs   »A Á
A2ÁA-Á-A2N)rE   rF   rU   rS   ÚmakedirsÚbackoffÚon_exceptionÚexpor   ÚstrÚ
isinstancer   ÚlistÚmap)r/   r0   rZ   Úexecutorr3   rY   rO   s        @@r   Údownload_files_from_urlsrd   C   su  øø€ õ œŸš­¬¯ª°f¸fÐFXÑ)YÔ)YÑZÔZÐÝ„KÐ"¨TÐ2Ñ2Ô2Ð2åÔ�'œ,Õ(8ÀAÐFÑFÔFð"�3ð "­cð "µcð "ð "ð "ñ GÔFð"ð,F¥3ð F­3ð Fð Fð Fð Fð Fð Fð Fõ �$�ÑÔð Øˆvˆå	Ñ	Ô	ð K Ý §¢Ð-BÀDÑ IÔ IÑJÔJÐðKð Kð Kñ Kô Kð Kð Kð Kð Kð Kð Køøøð Kð Kð Kð Kð Ðs   Ã$DÄDÄDÚfile_urlc                 ó  — t          j        dd¬¦  «         t          | ddt          ¬¦  «        5 }|                     ¦   «          |j                             d¦  «        }d}|r%t          |¦  «        }|                     dd¦  «        }|s3t          | ¦  «        j	        }t           j	         
                    |¦  «        }t           j	                             |¦  «        d	                              d
d¦  «        }t          j        ¦   «         › }t           j	                             d|› d
|› �¦  «        }t!          t#          |j        ¦  «        ¦  «        }	t%          |d¦  «        5 }
t'          ||	t)          ¦   «         ¦  «        D ]}|
                     |¦  «         Œ	 ddd¦  «         n# 1 swxY w Y   ddd¦  «         n# 1 swxY w Y   |dk    r˜t           j	                             d|¦  «        }t          j        |d¬¦  «         t-          j        |d¦  «        5 }|                     |¦  «         ddd¦  «         n# 1 swxY w Y   t           j	                             |¦  «        }nd}t           j	                             |¦  «        |||dœS )aƒ  
    Downloads a single file from a given URL, file is given a random name.
    First checks if the content-disposition header is set, if so, uses the file name from there.
    If the file is a zip file, it is extracted into a directory with the same name.

    Returns an object that contains:
    - The absolute path to the downloaded file
    - File type
    - Original file name
    Ú	job_filesTr4   é   r<   r?   r@   rA   r$   ú.rB   NÚzipÚr)rM   ÚtypeÚoriginal_nameÚextracted_path)rE   r[   r   rC   rD   r   r   r.   r   rF   ÚbasenamerG   ÚreplacerQ   rR   rS   r   r   rH   r	   r
   rI   ÚzipfileÚZipFileÚ
extractallrU   )re   Údownload_responser   Úoriginal_file_namer-   Údownload_pathÚ	file_typerV   rW   rL   Úoutput_filerN   Úunzipped_directoryÚzip_refs                 r   Úfiler{   {   s&  € õ „K� dÐ+Ñ+Ô+Ð+å	�( 4°½WÐ	EÑ	EÔ	Eð )ÐIZð 	×*Ò*Ñ,Ô,Ð,à/Ô7×;Ò;Ð<QÑRÔRÐàÐØð 	<Ý/Ð0CÑDÔDˆFà!'§¢¨J¸Ñ!;Ô!;Ðà!ð 	AÝ$ XÑ.Ô.Ô3ˆMÝ!#¤×!1Ò!1°-Ñ!@Ô!@Ðå”G×$Ò$Ð%7Ñ8Ô8¸Ô;×CÒCÀCÈÑLÔLˆ	å”z‘|”|Ð%ˆ	åœ7Ÿ<š<¨¸	Ð5OÐ5OÀIÐ5OÐ5OÑPÔPÐõ *Õ*>Ð?PÔ?XÑ*YÔ*YÑZÔZˆ
ÝÐ" DÑ)Ô)ð 	)¨[Ý,Ð->À
ÕL^ÑL`ÔL`ÑaÔað )ð )�Ø×!Ò! %Ñ(Ô(Ð(Ð(ð)ð	)ð 	)ð 	)ñ 	)ô 	)ð 	)ð 	)ð 	)ð 	)ð 	)ð 	)øøøð 	)ð 	)ð 	)ð 	)ð5)ð )ð )ñ )ô )ð )ð )ð )ð )ð )ð )øøøð )ð )ð )ð )ð< �EÒÐÝœWŸ\š\¨+°yÑAÔAÐÝ
ŒÐ&°Ð6Ñ6Ô6Ð6ÝŒ_Ð-¨sÑ3Ô3ð 	3°wØ×ÒÐ1Ñ2Ô2Ð2ð	3ð 	3ð 	3ñ 	3ô 	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3øøøð 	3ð 	3ð 	3ð 	3åœWŸ_š_Ð-?Ñ@Ô@ÐÐà!Ðõ ”W—_’_Ð%5Ñ6Ô6ØØ+Ø,ð	ð ð sH   ¯D0F9Å6F"ÆF9Æ"F&	Æ&F9Æ)F&	Æ*F9Æ9F=Ç F=ÈH7È7H;È>H;)Ú__doc__rE   rQ   rq   Úconcurrent.futuresr   Útypingr   r   r   Úurllib.parser   r\   Úrequestsr   Úrunpod.serverless.utils.rp_ssrfr	   r
   r   rC   r   r   r   r_   r.   rd   Údictr{   r   r   r   ú<module>rƒ      s–  ððð ð 
€	€	€	Ø €€€Ø €€€Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø $Ð $Ð $Ð $Ð $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø %Ð %Ð %Ð %Ð %Ð %ðð ð ð ð ð ð ð ð ð ð ÐUÐ
V€ð	 Cð 	¨Cð 	ð 	ð 	ð 	ð Sð ð ð ð ð
°Cð 
¸DÀÀcÀ¼Nð 
ð 
ð 
ð 
ð5 Sð 5°°c¸4À¼9°nÔ0Eð 5È$ÈsÌ)ð 5ð 5ð 5ð 5ðp9�3ð 9˜4ð 9ð 9ð 9ð 9ð 9ð 9r   