§
    »2«j 3  ã                  óÎ   — d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZmZ d dlZd dlmZ d dlmZmZmZ  ej        d¦  «        Zd	Zd
Z G d„ de¦  «        ZdS )é    )ÚannotationsN)ÚPath)ÚAny)ÚurljoinÚunquote)Úasync_playwright)ÚISxdCrawlerPortÚSxdAnnouncementÚSxdAttachmentInfoz*dscons.infrastructure.adapters.sxd_crawlerzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36aé  
// 1. Mask navigator.webdriver
Object.defineProperty(navigator, 'webdriver', {
    get: () => undefined,
});

// 2. Inject realistic window.chrome runtime
window.chrome = {
    runtime: {},
    app: {},
    loadTimes: function() {},
    csi: function() {}
};

// 3. Emulate languages and realistic plugins
Object.defineProperty(navigator, 'languages', {
    get: () => ['vi-VN', 'vi', 'en-US', 'en'],
});

Object.defineProperty(navigator, 'plugins', {
    get: () => [1, 2, 3, 4, 5],
});
c                  óH   — e Zd ZdZ	 	 d"d#d
„Zd$d%d„Z	 	 d&d'd„Zd(d„Zd)d „Zd!S )*ÚSxdStealthCrawlerAdapteruB  Secondary Infrastructure Adapter for crawling Department of Construction (Sá»Ÿ XÃ¢y Dá»±ng) data.

    Implements AIHawk-inspired browser stealth and anti-bot bypass techniques:
    1. Removes Chromium automation flags (--disable-blink-features=AutomationControlled).
    2. Deep prototype masking of navigator.webdriver and window.chrome runtime.
    3. Handles self-signed/government CA SSL certificate chains (ignore_https_errors).
    4. Human-like behavioral simulation (jitter delay, natural scrolling).
    5. Resilient streaming file downloader for government CDNs.
    ú!https://soxaydung.haiphong.gov.vné¨a  Úbase_domainÚstrÚ
timeout_msÚintÚreturnÚNonec                óH   — |                      d¦  «        | _        || _        d S )Nú/)Úrstripr   r   )Úselfr   r   s      ú_C:\Projects\DSCons\app\modules\inventory\infrastructure\adapters\sxd_stealth_crawler_adapter.pyÚ__init__z!SxdStealthCrawlerAdapter.__init__A   s%   € ð
 '×-Ò-¨cÑ2Ô2ˆÔØ$ˆŒˆˆó    é�  é°  Úmin_msÚmax_msc              ƒ  ón   K  — t          j        ||¦  «        dz  }t          j        |¦  «        ƒ d{V —† dS )z4Simulate natural human hesitation/interaction delay.g     @�@N)ÚrandomÚuniformÚasyncioÚsleep)r   r   r    Údelays       r   Ú_human_jitterz&SxdStealthCrawlerAdapter._human_jitterI   sD   è è € å”˜v vÑ.Ô.°Ñ7ˆÝŒm˜EÑ"Ô"Ð"Ð"Ð"Ð"Ð"Ð"Ð"Ð"Ð"r   ú6https://soxaydung.haiphong.gov.vn/thong-bao-chi-so-giaé   Úbase_urlÚ	max_itemsúlist[SxdAnnouncement]c              ƒ  óÈ  K  — g }t          ¦   «         4 ƒd{V —†}t                               d¦  «         |j                             dg d¢¬¦  «        ƒ d{V —†}|                     t          dddœd	d
ddddddœ¬¦  «        ƒ d{V —†}|                     t          ¦  «        ƒ d{V —† | 	                    ¦   «         ƒ d{V —†}	 t                               d|¦  «         | 
                    || j        d¬¦  «        ƒ d{V —†}|                      ¦   «         ƒ d{V —† |                     d¦  «        ƒ d{V —† |                      dd¦  «        ƒ d{V —† |                     d¦  «        ƒ d{V —†}	t          ¦   «         }
g }|	D ]8}|d         }||
vr*|
                     |¦  «         |                     |¦  «         Œ9t                               dt#          |¦  «        ¦  «         |d|…         D �]h}|d         }|d         }t%          | j        |¦  «        }t)          j        d|t(          j        ¦  «        }|r|                     d¦  «        nd}t)          j        d|t(          j        ¦  «        }|r|                     d¦  «        nd}t)          j        d|t(          j        ¦  «        }|ret1          |                     d¦  «        ¦  «        }|                     d ¦  «        r"t1          |                     d ¦  «        ¦  «        nd!}|d"›d#|d$›�}nd%}|                      ||¦  «        ƒ d{V —†}|                     t5          |||||d&|¬'¦  «        ¦  «         �Œjn0# t6          $ r#}t                               d(|d¬)¦  «         ‚ d}~ww xY w	 |                     ¦   «         ƒ d{V —† n# |                     ¦   «         ƒ d{V —† w xY w	 ddd¦  «        ƒd{V —† n# 1 ƒd{V —†swxY w Y   |S )*zTCrawl the latest material price and price index announcements using Stealth Browser.Nz<Launching AIHawk Stealth Chromium instance for SXD portal...T)z---disable-blink-features=AutomationControlledz--disable-infobarsz--no-sandboxz--disable-dev-shm-usagez--disable-web-securityz--ignore-certificate-errors)ÚheadlessÚargsi€  i8  )ÚwidthÚheightzvi-VNzAsia/Ho_Chi_Minhz#vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7zA"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"z?0z	"Windows")zAccept-Languagez	Sec-Ch-UazSec-Ch-Ua-MobilezSec-Ch-Ua-Platform)Ú
user_agentÚviewportÚlocaleÚtimezone_idÚignore_https_errorsÚextra_http_headerszNavigating to %s...Údomcontentloaded©ÚtimeoutÚ
wait_untilzwindow.scrollBy(0, 400);i,  i¼  u  () => {
                    const links = Array.from(document.querySelectorAll("a"));
                    const items = [];
                    for (const a of links) {
                        const href = a.getAttribute("href") || "";
                        const title = (a.innerText || "").trim();
                        if (!href || !title) continue;
                        
                        const titleLower = title.toLowerCase();
                        if (
                            (titleLower.includes("giÃ¡") || titleLower.includes("váº­t liá»‡u") || titleLower.includes("chá»‰ sá»‘")) &&
                            (titleLower.includes("thÃ´ng bÃ¡o") || titleLower.includes("cÃ´ng bá»‘") || titleLower.includes("quyáº¿t Ä‘á»‹nh"))
                        ) {
                            items.push({
                                title: title,
                                href: href
                            });
                        }
                    }
                    return items;
                }Úhrefz9Found %d candidate price announcement links on SXD portalÚtitleu   (\d+[\w\-]*/(?:TB|QÄ�|QD)-SXD)é   Ú u-   ngÃ y\s*(\d{1,2}[\/\-\.]\d{1,2}[\/\-\.]\d{4})u3   thÃ¡ng\s*(\d{1,2})(?:[\s\/\-]*(?:nÄƒm)?\s*(\d{4}))?é   iê  Ú04dú-Ú02dz2026-08Ú	HAI_PHONG)r=   Úannouncement_numberÚpublish_dateÚperiodÚ
source_urlÚprovinceÚattachmentsz%Error during SXD stealth crawling: %s)Úexc_info)r   ÚloggerÚinfoÚchromiumÚlaunchÚnew_contextÚDEFAULT_DESKTOP_UAÚadd_init_scriptÚAIHAWK_STEALTH_JSÚnew_pageÚgotor   r'   ÚevaluateÚsetÚaddÚappendÚlenr   r   ÚreÚsearchÚ
IGNORECASEÚgroupr   Ú_extract_detail_attachmentsr
   Ú	ExceptionÚerrorÚclose)r   r*   r+   ÚannouncementsÚpÚbrowserÚcontextÚpageÚresponseÚraw_articlesÚ
seen_hrefsÚcandidate_itemsÚitemÚhÚ	candidateÚ	raw_titleÚrel_urlÚfull_urlÚdoc_num_matchÚdoc_numÚ
date_matchrF   Úperiod_matchÚmonthÚyearrG   rJ   Úexcs                               r   Úcrawl_latest_announcementsz3SxdStealthCrawlerAdapter.crawl_latest_announcementsN   sñ  è è € ð 02ˆå#Ñ%Ô%ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&¨Ý�KŠKÐVÑWÔWÐWØœJ×-Ò-Øðð ð ð .ñ 
ô 
ð 
ð 
ð 
ð 
ð 
ð 
ˆGð $×/Ò/Ý-Ø#'°4Ð8Ð8ØØ.Ø$(à'LØ!dØ(,Ø*5ð	$ð $ð 0ñ ô ð ð ð ð ð ð ˆGð ×)Ò)Õ*;Ñ<Ô<Ð<Ð<Ð<Ð<Ð<Ð<Ð<à ×)Ò)Ñ+Ô+Ð+Ð+Ð+Ð+Ð+Ð+ˆDðg&Ý—’Ð1°8Ñ<Ô<Ð<Ø!%§¢Ø d¤oÐBTð "+ñ "ô "ð ð ð ð ð ð �ð ×(Ò(Ñ*Ô*Ð*Ð*Ð*Ð*Ð*Ð*Ð*ð —m’mÐ$>Ñ?Ô?Ð?Ð?Ð?Ð?Ð?Ð?Ð?Ø×(Ò(¨¨cÑ2Ô2Ð2Ð2Ð2Ð2Ð2Ð2Ð2ð &*§]¢]ðñ&ô &ð  ð  ð  ð  ð  ð  �õ2 !™UœU�
Ø"$�Ø(ð 5ð 5�DØ˜Vœ�AØ 
Ð*Ð*Ø"Ÿš qÑ)Ô)Ð)Ø'×.Ò.¨tÑ4Ô4Ð4øå—’ØOÝ˜Ñ(Ô(ñô ð ð "1°°)°Ô!<ð .ñ .�IØ )¨'Ô 2�IØ'¨Ô/�GÝ& tÔ'7¸ÑAÔA�Hõ %'¤IØ9¸9ÅbÄmñ%ô %�Mð 9FÐM˜m×1Ò1°!Ñ4Ô4Ð4È2�Gå!#¤ØHØ!Ýœñ"ô "�Jð
 ;EÐ#L :×#3Ò#3°AÑ#6Ô#6Ð#6È"�Lå#%¤9ØNØ!Ýœñ$ô $�Lð
 $ð +Ý # L×$6Ò$6°qÑ$9Ô$9Ñ :Ô :˜Ø=I×=OÒ=OÐPQÑ=RÔ=RÐ\�s <×#5Ò#5°aÑ#8Ô#8Ñ9Ô9Ð9ÐX\˜Ø$(Ð!:Ð!:Ð!:¨uÐ!:Ð!:Ð!:˜˜à!*˜ð )-×(HÒ(HØ ñ)ô )ð #ð #ð #ð #ð #ð #�Kð "×(Ò(Ý'Ø"+Ø07Ø)5Ø#)Ø'/Ø%0Ø(3ðñ ô ñ
ô 
ð 
ñ 
ðI.øõ` ð ð ð Ý—’ÐDÀcÐTX�ÑYÔYÐYØøøøøðøøøða.ðh —m’m‘o”oÐ%Ð%Ð%Ð%Ð%Ð%Ð%Ð%ø�g—m’m‘o”oÐ%Ð%Ð%Ð%Ð%Ð%Ð%Ð%øøøÐ%ðQH	&ð H	&ð H	&ñ H	&ô H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&ð H	&øøøð H	&ð H	&ð H	&ð H	&ðT ÐsI   ™B'OÃJMÍNÍ
N ÍM;Í;N Î NÎOÎN;Î;OÏ
OÏOrf   r   Ú
detail_urlúlist[SxdAttachmentInfo]c           	   ƒ  ó|  K  — |                      ¦   «         ƒ d{V —†}g }	 |                     || j        d¬¦  «        ƒ d{V —† |                      dd¦  «        ƒ d{V —† |                     d¦  «        ƒ d{V —†}|D ]Ñ}|d         }t          | j        |¦  «        }t          |¦  «                             ¦   «         }	d}
dD ]}||	v r| 	                    d	¦  «        }
 nŒ|
rmt          |                     d
¦  «        d         ¦  «        j        }t          |¦  «        }|rd	|vrd|
› �}|                     t          |||
d¬¦  «        ¦  «         ŒÒn3# t          $ r&}t                                d||¦  «         Y d}~nd}~ww xY w|                     ¦   «         ƒ d{V —† n# |                     ¦   «         ƒ d{V —† w xY w|S )zONavigate to detail announcement page and extract all attachment download links.Nr8   r9   éÈ   iX  aP  () => {
                const links = Array.from(document.querySelectorAll("a[href]"));
                return links.map(a => ({
                    href: a.getAttribute("href") || "",
                    text: (a.innerText || "").trim(),
                    download: a.getAttribute("download") || ""
                }));
            }r<   )z.pdfú.xlsxz.xlsz.docxz.docú.Ú?r   zattachment.)ÚfilenameÚdownload_urlÚ	file_typeÚ
size_bytesz0Could not extract detail attachments from %s: %s)rT   rU   r   r'   rV   r   r   r   ÚlowerÚlstripr   ÚsplitÚnamerY   r   r`   rL   Úwarningrb   )r   rf   rz   rg   rJ   Ú	raw_linksÚlinkr<   Ú	full_linkÚunquotedÚextÚcandidate_extÚraw_filenameÚclean_filenamerx   s                  r   r_   z4SxdStealthCrawlerAdapter._extract_detail_attachmentsâ   s•  è è € ð ×%Ò%Ñ'Ô'Ð'Ð'Ð'Ð'Ð'Ð'ˆØ/1ˆð2	Ø—)’)Ø D¤OÐ@Rð ñ ô ð ð ð ð ð ð ð ð ×$Ò$ S¨#Ñ.Ô.Ð.Ð.Ð.Ð.Ð.Ð.Ð.ð #Ÿmšmðñ	ô 	ð 	ð 	ð 	ð 	ð 	ð 	ˆIð "ð ð �Ø˜F”|�Ý# DÔ$4°dÑ;Ô;�	Ý" 9Ñ-Ô-×3Ò3Ñ5Ô5�à�Ø%Oð ð �MØ$¨Ð0Ð0Ø+×2Ò2°3Ñ7Ô7˜Ø˜ð 1ð ð å#'¨¯
ª
°3©¬¸Ô(:Ñ#;Ô#;Ô#@�LÝ%,¨\Ñ%:Ô%:�NØ)ð =¨S¸Ð-FÐ-FØ)<°sÐ)<Ð)<˜à×&Ò&Ý)Ø%3Ø)2Ø&)Ø'(ð	ñ ô ñô ð øð%øõ6 ð 	ð 	ð 	Ý�NŠNØBÀJÐPSñô ð ð ð ð ð ð øøøøð	øøøð
 —*’*‘,”,ÐÐÐÐÐÐÐÐø�$—*’*‘,”,ÐÐÐÐÐÐÐÐøøøàÐs0    D.E ÅF Å
E?ÅE:Å5F Å:E?Å?F ÆF9Úannouncementr
   Údestination_dirr   údict[str, Path]c              ƒ  ó¼  K  — |                      dd¬¦  «         i }t          | j        dz   ddœ}t          j        d|dd¬¦  «        4 ƒd	{V —†	 }|j        D �]`}||j        z  }t                               d
|j        |j	        ¦  «         	 | 
                    |j	        ¦  «        ƒ d	{V —†}|                     ¦   «          t          |d¦  «        5 }	|	                     |j        ¦  «         d	d	d	¦  «         n# 1 swxY w Y   |j                             ¦   «         }
d|
v sd|
v r||d<   n;d|
v sd|
v s|
                     d¦  «        r||d<   nd|
v sd|
v r||d<   n
|||j        <   t                               d|¦  «         �Œ)# t"          $ r,}t                               d|j	        |¦  «         Y d	}~�ŒZd	}~ww xY w	 d	d	d	¦  «        ƒd	{V —† n# 1 ƒd	{V —†swxY w Y   |S )zGDownload announcement attachments with SSL bypass and anti-bot Referer.T)ÚparentsÚexist_okr   z*/*)z
User-AgentÚRefererÚAcceptFg      N@)ÚverifyÚheadersr:   Úfollow_redirectsNz"Downloading SXD file %s from %s...Úwbz	phu-luc-1Ú	phu_luc_1Úpl1_pdfz	phu-luc-2Ú	phu_luc_2r~   Ú	pl2_excelztb-z	thong-baoÚmain_doc_pdfz'Successfully downloaded and saved to %sz$Failed to download attachment %s: %s)ÚmkdirrQ   r   ÚhttpxÚAsyncClientrJ   r�   rL   rM   r‚   ÚgetÚraise_for_statusÚopenÚwriteÚcontentr…   Úendswithr`   ra   )r   r’   r“   Úsaved_filesr›   ÚclientÚattÚ	dest_pathrh   ÚfÚfn_lowerrx   s               r   Údownload_announcement_filesz4SxdStealthCrawlerAdapter.download_announcement_files!  s(  è è € ð 	×Ò d°TÐÑ:Ô:Ð:Ø')ˆõ -ØÔ'¨#Ñ-Øð
ð 
ˆõ Ô$Ø '°4È$ð
ñ 
ô 
ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	àØ#Ô/ð ñ �Ø+¨c¬lÑ:�	Ý—’Ø8¸#¼,ÈÔHXñô ð ðØ%+§Z¢Z°Ô0@Ñ%AÔ%AÐAÐAÐAÐAÐAÐA�HØ×-Ò-Ñ/Ô/Ð/å˜i¨Ñ.Ô.ð 2°!ØŸš Ô 0Ñ1Ô1Ð1ð2ð 2ð 2ñ 2ô 2ð 2ð 2ð 2ð 2ð 2ð 2øøøð 2ð 2ð 2ð 2ð  #œ|×1Ò1Ñ3Ô3�HØ" hÐ.Ð.°+ÀÐ2IÐ2IØ1:˜ IÑ.Ð.Ø$¨Ð0Ð0°KÀ8Ð4KÐ4KÈx×O`ÒO`ÐahÑOiÔOiÐ4KØ3<˜ KÑ0Ð0Ø (Ð*Ð*¨k¸XÐ.EÐ.EØ6?˜ NÑ3Ð3à4=˜ C¤LÑ1å—K’KÐ IÈ9ÑUÔUÐUÑUøÝ ð ð ð Ý—L’LØ>ÀÔ@PÐRUñô ð ð ð ð ð ñ øøøøðøøøð3ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ðB Ðsb   Á;GÂ
AE?ÃC5Ã)E?Ã5C9Ã9E?Ã<C9Ã=B E?Å=GÅ?
F5Æ	!F0Æ*GÆ0F5Æ5GÇ
GÇGN)r   r   )r   r   r   r   r   r   )r   r   )r   r   r    r   r   r   )r(   r)   )r*   r   r+   r   r   r,   )rf   r   rz   r   r   r{   )r’   r
   r“   r   r   r”   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r'   ry   r_   r²   © r   r   r   r   6   s©   € € € € € ðð ð ?Øð%ð %ð %ð %ð %ð#ð #ð #ð #ð #ð QØðRð Rð Rð Rð Rðh=ð =ð =ð =ð~1ð 1ð 1ð 1ð 1ð 1r   r   )Ú
__future__r   r$   Úloggingr"   r[   ÚsslÚpathlibr   Útypingr   Úurllib.parser   r   r¤   Úplaywright.async_apir   Ú3app.modules.inventory.domain.ports.sxd_crawler_portr	   r
   r   Ú	getLoggerrL   rQ   rS   r   r·   r   r   ú<module>rÁ      s2  ðØ "Ð "Ð "Ð "Ð "Ð "à €€€Ø €€€Ø €€€Ø 	€	€	€	Ø 
€
€
€
Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø )Ð )Ð )Ð )Ð )Ð )Ð )Ð )à €€€Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1ðð ð ð ð ð ð ð ð ð ð 
ˆÔ	ÐGÑ	HÔ	H€ð9ð ð
Ð ð2\ð \ð \ð \ð \˜ñ \ô \ð \ð \ð \r   