jBdZddlmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl mZmZmZmZddlmZddlmZddlmZddlmZmZdd lmZdd lmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9m:Z:m;Z;e j<e=Z>ee?e?fZ@dMdZAGddeBZCdNd!ZDGd"d#eBZEdOd&ZF dPdQd*ZGdRd-ZHGd.d/ZIGd0d1eZJdSd3ZKeKdTd7ZLed89Gd:d5ZMGd;d$.F.FMMM 4c eZdZdfd ZxZS)_NotAPIContent content_typer$ request_descr%Nonecht||||_||_dSN)super__init__r2r3)selfr2r3 __class__s r-r8z_NotAPIContent.__init__@s3 |444((r/)r2r$r3r$r%r4)__name__ __module__ __qualname__r8 __classcell__r:s@r-r1r1?s=))))))))))r/r1responserr4c|jdd}|}|drdSt ||jj)z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr)r*r1requestmethod)r@r2content_type_ls r-_ensure_api_headerrJFsg#'' BBL!''))N     x'7'> ? ??r/ceZdZdS)_NotHTTPN)r;r<r=r/r-rLrL\sDr/rLsessionrctj|\}}}}}|dvrt||d}t |t |dS)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httphttpsT)allow_redirectsN)urllibparseurlsplitrLheadrrJ)r#rNr,netlocpathqueryfragmentresps r-_ensure_api_responser\`sq-3L,A,A#,F,F)FFD% &&&jj <<T< 2 2DTtr/Fforce_revalidateboolctt|jrt||tdt |ddgdi}|rtdd|d<||| }t|t|td t ||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. )rNzGetting page %sAcceptz, )rDz*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z"Refreshing package index response.z max-age=0z Cache-Control)rEzFetched page %s as %srBrC) rrfilenamer\loggerdebugrjoinrFrrJrE)r#rNr]rEr[s r-_get_simple_responserers"tCyy)**3S'2222 LL"$8$=$=>>> $))     G/  9:::#. ;;sG; , ,DTt LLS!! 33 Kr/rEResponseHeadersc|rSd|vrOtj}|d|d<|d}|rt |SdS)z=Determine if we have any encoding information in our headers.rBz content-typecharsetN)emailmessageMessage get_paramr$)rEmrhs r-_get_encoding_from_headersrns` >W,, M ! ! # ##N3.++i((  w<<  4r/c&eZdZd dZdd Zdd Zd S)CacheablePageContentpage IndexContentr%r4c&|jsJ||_dSr6)cache_link_parsingrqr9rqs r-r8zCacheablePageContent.__init__s&&&& r/otherobjectr^cpt|t|o|jj|jjkSr6) isinstancetyperqr#)r9rvs r-__eq__zCacheablePageContent.__eq__s*%d,,P%*.1PPr/intc4t|jjSr6)hashrqr#r9s r-__hash__zCacheablePageContent.__hash__sDIM"""r/N)rqrrr%r4)rvrwr%r^)r%r|)r;r<r=r8r{rrMr/r-rprpsTQQQQ######r/rpceZdZddZdS) ParseLinksrqrrr%Iterable[Link]cdSr6rMrus r-__call__zParseLinks.__call__sr/Nrqrrr%r)r;r<r=rrMr/r-rrsAAAAAAr/rfnc|tjd fd tjd fd }|S) z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. cacheable_pagerpr% list[Link]c>t|jSr6)listrq)rrs r-wrapperz*with_cached_index_content..wrappersBB~*++,,,r/rqrrcr|jrt|St|Sr6)rtrpr)rqrrs r-wrapper_wrapperz2with_cached_index_content..wrapper_wrappers;  " 77/5566 6BBtHH~~r/)rrpr%r)rqrrr%r) functoolscachewraps)rrrs` @r-with_cached_index_contentrsr_-----_-_R r/rqrrrc#K|j}|drUtj|j}|dgD]#}tj||j }||V$dSt|j }|j pd}| |j ||j }|jp|}|jD] } tj| ||}||V!dS)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. rDfilesNzutf-8)page_urlbase_url)r2r)r*jsonloadscontentrFr from_jsonr#HTMLLinkParserencodingfeeddecoderanchors from_element) rqrIdatafilelinkparserrr#ranchors r- parse_linksrs! &,,..N  !FGGz$,''HHWb))  D>$11D|JJJJ DH % %F}'H KK ##H--... (C%#H. #III <  r/T)frozencReZdZUdZded<ded<ded<ded<d Zd ed <dd ZdS)rraRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. bytesrr$r2r&rr#Tr^rtr%c*t|jSr6)rr#rs r-__str__zIndexContent.__str__ s#DH---r/N)r%r$)r;r<r=__doc____annotations__rtrrMr/r-rrrrsrNNN HHH#####......r/c4eZdZdZd fd Zdd Zdd ZxZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. r#r$r%r4cvtd||_d|_g|_dS)NT)convert_charrefs)r7r8r#rr)r9r#r:s r-r8zHTMLLinkParser.__init__s7 $///$( 46 r/tagattrslist[tuple[str, str | None]]c|dkr)|j"||}| ||_dSdS|dkr)|jt |dSdS)Nbasea)rget_hrefrappenddict)r9rrhrefs r-handle_starttagzHTMLLinkParser.handle_starttagsn &==T]2==''D $   CZZ L  U , , , , ,Zr/r&c*|D]\}}|dkr|cSdS)NrrM)r9rnamevalues r-rzHTMLLinkParser.get_href#s1   KD%v~~ tr/)r#r$r%r4)rr$rrr%r4)rrr%r&)r;r<r=rr8rrr>r?s@r-rrso 777777----r/rrrreasonstr | ExceptionmethCallable[..., None] | Nonec<| tj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rbrc)rrrs r-_handle_get_simple_failr*s+  ||D 0$?????r/rtc|t|j}t|j|jd||j|S)NrB)rr#rt)rnrErrrr#)r@rtrs r-_make_index_contentr4sG*(*:;;H ( L-    r/)r]IndexContent | Nonec|jddd}t|}|rtd||dS|drt jt|rU| ds|dz }tj |d}td| t||| }t!||j S#t$$rtd |Yn-t&$r1}td ||j|jYd}~nd}~wt,t.f$r}t1||Yd}~nd}~wt2t4f$r0}d |j}t1||tjYd}~nd}~wt:$r"}t1|d|jYd}~nfd}~wt<$r"}t1|d|jYd}~n$r,}t1|tA|jYd}~nd}~wwxYwdS)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.zfile:/z index.htmlz# file: URL is directory, getting %srNr])rtz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: zproxy connection error: )!r#splitr.rbwarningr*osrXisdirrendswithrSrTurljoinrcrerrtrLr1r3r2rrrrrcontextinforrrr$)rrNr]r# vcs_schemer[excrs r-_get_index_contentrAs (..a  #C#3''J W     t ~~g A27==S1A1A#B#B A||C    3JC l""3 55 :C@@@U# 3C   <#4Dz'LinkCollector.create..s+IIs-c22IIIIIIr/)rrno_index)rNr) index_urlextra_index_urlsrrbrcrdrrcreater)clsrNrrrrrlink_collectors r-rzLinkCollector.creates'(7+CC   $5  LL&IIjIIIII   J'-2 ")!!%   '%   r/ list[str]c|jjSr6)rrrs r-rzLinkCollector.find_linkss ++r/Nlocationr package_namer&rcz|jj}d|vp|duot||v}t||j|S)z> Fetch an HTML page containing package links. z:all:Nr)rNrefresh_packager r)r9rrr]should_force_revalidates r-fetch_responsezLinkCollector.fetch_responsese <7")-="=#  $ D!,//3CC " L4    r/ project_namer$candidates_from_pager rcztjfdjD}tjfdjD}t tj redtj ||D}t|ddg|z}t d|tt!|t!|S)Nc 3VK|]#}t|jjddV$dS)Frpage_validator expand_dirrtrNr"rNis_secure_originrlocrrr9s r-rz0LinkCollector.collect_sources..s_ 4 4  %9#|< #()     4 4 4 4 4 4 r/c 3VK|]#}t|jjddV$dS)TrNrrs r-rz0LinkCollector.collect_sources..s_ 5 5  %9#|<#')     5 5 5 5 5 5 r/c4g|]}||j d|jS)Nz* )r)rss r- z1LinkCollector.collect_sources..s6=QV%7QV %7%7%7r/z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrrb isEnabledForloggingDEBUG itertoolschainr+rcrdrr)r9rrindex_url_sourcesfind_links_sourcesliness``` r-collect_sourceszLinkCollector.collect_sourcess (3 4 4 4 4 4 4 (AA,OO 4 4 4    &(( )4 5 5 5 5 5 5  5 5 5    &((    w} - - +");=NOOE u::33#/333E LL5)) * * *.//-..    r/)rNrrrr%r4F)rNrrr rr^r%r)r%rr6)rrrr&r%r)rr$rr r%r) r;r<r=rr8 classmethodrpropertyrrrrMr/r-rrs #( [B,,,X,:>     (, , , , , , r/r)r#r$r%r&)r@rr%r4)r#r$rNrr%r4r)r#r$rNrr]r^r%r)rErfr%r&)rrr%rrr6)rrrrrrr%r4)T)r@rrtr^r%rr)rrrNrr]r^r%r)Tr __future__rr email.messagerirr rr r urllib.parserScollections.abcrrrr dataclassesr html.parserr optparser typingr r pip._vendor.packaging.utilsr pip._vendor.requestsrpip._vendor.requests.exceptionsrpip._internal.exceptionsrrrrrrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.utils.urlsrpip._internal.vcsrsourcesr r!r" getLoggerr;rbr$rfr. Exceptionr1rJrLr\rernrprrrrrrrrrrrrMr/r-r-s#"""""  HHHHHHHHHHHH!!!!!!"""""" :99999))))))666666+*****999999444444888888999999999999000000!!!!!!AAAAAAAAAA  8 $ $ c*)))))Y)))@@@@,     y   *#88888v # # # # # # # #BBBBBBBB(8 $........(Z>(,@@@@@48     "# ??????D,,,,,z,,, v v v v v v v v v v r/