§
    y�’j4  ã                  ó²  — d dl mZ d dlZd dlmZmZ d dlmZ d dlmZ d dl	m
Z
 ddlmZmZmZmZmZmZ dd	lmZ dd
lmZmZmZ ddlmZmZ d,d„Zd-d„Zd.d„Zd/d„Zd0d „Ze d!k    rpd dl!Z!d dl"Z"e"j#         $                    d"¬#¦  «          e!j%         e¦   «         ¦  «        Z& e'd$e&d%         › d&e&d'         › d(�¦  «          e'd)e&d*         › d+�¦  «         dS dS )1é    )ÚannotationsN)ÚUTCÚdatetime)ÚPath)ÚAny)Úget_orchestratoré   )ÚEXPECTED_DOCUMENT_TYPESÚINGEST_BATCHÚMANIFEST_PATHÚPROJECT_CANDIDATESÚSOURCE_CODEÚSOURCE_NAME)Úextract_project_root)Ú
chunk_textÚ	load_textÚslugify)ÚProjectCandidateÚProjectDocumentÚdocumentr   Úreturnúlist[dict[str, Any]]c                óú  — t          | j        ¦  «        }t          |¦  «        }|sg S t          | j        j        ¦  «        }g }t          |d¬¦  «        D �]¥\  }}i d| j        “d| j        “d| j        “d| j	        “d| j
        “d| j        “d	| j        “d
t          | j        ¦  «        “d| j        “d| j        “d| j        “d| j        “d| j        “dt(          “dt*          “dt-          | j        ¦  «        “d| j        “i d| j        “d| j        “d| j        “d| j        “d|“dt9          |¦  «        “d| j        “d| j        “d| j        “d| j         “dtB          “dt          | j"        ¦  «        “d t          | j#        ¦  «        “d!t          | j$        ¦  «        “d"| j%        “d#d$| j        v “d%d&| j        v “¥d'| j        v pd(| j        v d)| j        › d*| j        › d*|› d+|› �d,œ¥}| &                    ||d-œ¦  «         �Œ§|S ).Nr	   ©ÚstartÚproject_codeÚproject_nameÚ	task_typeÚ	doc_stageÚ
doc_familyÚknowledge_typeÚdocument_typeÚrequired_forÚissuerÚsignerÚdecision_noÚ
issue_dateÚeffective_dateÚsource_codeÚsource_repositoryÚsource_fileÚrelative_pathÚsource_filenameÚsource_formatÚsection_titleÚfolder_scopeÚchunk_indexÚchunk_countÚclassification_sourceÚmatched_ruleÚclassification_confidenceÚproject_rootÚingest_batchÚtagsÚrelated_docsÚcompliance_refsÚreadiness_weightÚrequired_for_paymentÚpaymentÚrequired_for_finalizationÚfinalizationÚ	pre_awardÚcontractingÚunknownz::z::chunk-)Úrequired_for_commencementÚdraft_statusÚid)ÚtextÚmetadata)'r   Úabsolute_pathr   r   ÚstemÚ	enumerater   r   r   r   r    r!   r"   Úlistr#   r$   r%   r&   r'   r(   r   r   Ústrr,   r-   r.   r/   r0   Úlenr3   r4   Ú
confidencer6   r   r8   r9   r:   r;   Úappend)r   rF   ÚchunksÚ	file_slugÚpayloadÚindexÚchunkrG   s           úBC:\Projects\DSCons\tools\hd2026_ingest_support\knowledge_ingest.pyÚbuild_chunk_payloadrV      s  € Ý�XÔ+Ñ,Ô,€DÝ˜ÑÔ€FØð Øˆ	å˜Ô.Ô3Ñ4Ô4€IØ$&€GÝ! &°Ð2Ñ2Ô2ð )>ñ )>‰ˆˆuð'
Ø˜HÔ1ð'
à˜HÔ1ð'
ð ˜Ô+ð'
ð ˜Ô+ð	'
ð
 ˜(Ô-ð'
ð ˜hÔ5ð'
ð ˜XÔ3ð'
ð �D Ô!6Ñ7Ô7ð'
ð �h”oð'
ð �h”oð'
ð ˜8Ô/ð'
ð ˜(Ô-ð'
ð ˜hÔ5ð'
ð �;ð'
ð  ¥ð'
ð  �3˜xÔ5Ñ6Ô6ð!'
ð" ˜XÔ3ð#'
ð '
ð$ ˜xÔ7ð%'
ð& ˜XÔ3ð''
ð( ˜XÔ3ð)'
ð* ˜HÔ1ð+'
ð, ˜5ð-'
ð. �3˜v™;œ;ð/'
ð0 $ XÔ%Cð1'
ð2 ˜HÔ1ð3'
ð4 (¨Ô)<ð5'
ð6 ˜HÔ1ð7'
ð8 �Lð9'
ð: •D˜œÑ'Ô'ð;'
ð< �D Ô!6Ñ7Ô7ð='
ð> �t HÔ$<Ñ=Ô=ð?'
ð@  Ô 9ðA'
ðB # I°Ô1FÐ$FðC'
ðD (¨¸8Ô;PÐ)PðE'
ð '
ðF *5¸Ô8MÐ)Mð *6Ø Ô 5Ð5Ø%ØÔ*ÐbÐb¨hÔ.DÐbÐbÈ	ÐbÐbÐ[`ÐbÐbðM'
ð '
ð '
ˆðP 	�Š °8Ð<Ð<Ñ=Ô=Ð=Ñ=Ø€Nó    Ú	candidater   Ú	documentsúlist[ProjectDocument]Úingest_summaryúdict[str, Any]r6   úPath | Nonec                óN  ‡
— i }g }d}d}|D ]k}|                      |j        d¦  «        dz   ||j        <   |j        dk    r3|dz  }t          |¦  «        dk     r|                     |j        ¦  «         Œf|dz  }Œld„ |D ¦   «         Š
t          ˆ
fd„t          D ¦   «         ¦  «        }	i d| j        “d| j        “d	|rt          |¦  «        nd “d
t          |o|                     ¦   «         ¦  «        “dt          |¦  «        “dt          |¦  «        “d|“d|“d|                      dd¦  «        “d|                      dd¦  «        “d|“d|	“d|“d|                      dg ¦  «        “d|                      dg ¦  «        “d|                      dg ¦  «        “d|                      dg ¦  «        “S )Nr   r	   rB   é
   c                ó   — h | ]
}|d k    ¯|’ŒS )rB   © )Ú.0Údoc_types     rU   ú	<setcomp>z2build_project_coverage_manifest.<locals>.<setcomp>b   s)   € ð ð ð Ø¸ÀYÒ9NÐ9NˆÐ9NÐ9NÐ9NrW   c              3  ó$   •K  — | ]
}|‰v¯|V — Œd S ©Nra   )rb   rc   Údiscovered_doc_typess     €rU   ú	<genexpr>z2build_project_coverage_manifest.<locals>.<genexpr>e   s;   øè è € ð $ð $àØÐ/Ð/Ð/ð 	à/Ð/Ð/Ð/ð$ð $rW   r   r   r6   Úproject_root_existsÚtotal_filesÚextractable_filesÚclassified_filesÚunknown_filesÚknowledge_files_ingestedÚfiles_ingestedÚknowledge_chunks_ingestedrP   Údocument_type_countsÚmissing_document_typesÚsample_unknown_filesÚsample_skipped_filesÚskipped_filesÚmissing_filesÚingested_document_typesÚdocument_typesÚstages)Úgetr"   rM   rO   r,   Úsortedr
   r   r   rL   ÚboolÚexists)rX   rY   r[   r6   rq   rs   rl   rm   r   rr   rg   s             @rU   Úbuild_project_coverage_manifestr~   L   ss  ø€ ð ,.ÐØ&(ÐØÐØ€Màð 	"ð 	"ˆà ×$Ò$ XÔ%;¸QÑ?Ô?À!ÑCð 	˜XÔ3Ñ4ð Ô! YÒ.Ð.Ø˜QÑˆMÝÐ'Ñ(Ô(¨2Ò-Ð-Ø$×+Ò+¨HÔ,BÑCÔCÐCøà Ñ!ÐÐðð Ø!5ðñ ô Ðõ $ð $ð $ð $ð $å/ð$ñ $ô $ñ ô ÐðØ˜	Ô.ðà˜	Ô.ðð 	¨\ÐC�˜LÑ)Ô)Ð)¸tðð 	�t LÐ$J°\×5HÒ5HÑ5JÔ5JÑKÔKð	ð
 	•s˜9‘~”~ðð 	�S ™^œ^ðð 	Ð,ðð 	˜ðð 	# N×$6Ò$6Ð7GÈÑ$KÔ$Kðð 	$ ^×%7Ò%7¸À!Ñ%DÔ%Dðð 	Ð 4ðð 	!Ð"8ðð 	Ð 4ðð 	 × 2Ò 2°?ÀBÑ GÔ Gðð 	˜×+Ò+¨O¸RÑ@Ô@ðð  	" >×#5Ò#5Ð6FÈÑ#KÔ#Kð!ð" 	�.×$Ò$ X¨rÑ2Ô2ð#ð rW   ÚmanifestÚNonec                óž   — t           j                             dd¬¦  «         t          j        t	          j        | dd¬¦  «        d¬¦  «         d S )NT)ÚparentsÚexist_okFé   )Úensure_asciiÚindentúutf-8©Úencoding)r   ÚparentÚmkdirÚ
write_textÚjsonÚdumps)r   s    rU   Úwrite_manifestr�   €   sX   € åÔ×Ò t°dÐÑ;Ô;Ð;ÝÔÝŒ
�8¨%¸Ð:Ñ:Ô:ÀWðñ ô ð ð ð rW   Úproject_documents_by_projectú dict[str, list[ProjectDocument]]c              ƒ  óÀ  ‡ ‡K  — t          ¦   «         }d}d}g }i }ddddddddddd	ddddd
œ}t          t          d¬¦  «        D �]×\  }}‰                      |j        g ¦  «        }	|                     |j        |j        t          |	¦  «        ddt          ¦   «         t          ¦   «         g g dœ¦  «        }
i }|	D �]´}|j	        dk    rK|
d          
                    |j        ¦  «         | 
                    |j        |j        dd|j	        dœ¦  «         ŒY|j                             ¦   «         sK|
d          
                    |j        ¦  «         | 
                    |j        |j        dd|j	        dœ¦  «         Œ½t          |¦  «        }|r™|                     |j        d¦  «        }|                     |g ¦  «                             |¦  «         |
dxx         dz  cc<   |
d                              |j	        ¦  «         |
d                              |j        ¦  «         |dz  }| 
                    |j        |j        |rdndt          |¦  «        |j	        |j        |j        |j        dœ¦  «         �Œ¶d}|                     ¦   «         D ]B\  }}|r;|                     ||t,          t.          t0          dœ¬¦  «        ƒ d {V —†}||z  }||z  }ŒC||
d<   t3          d|d›dt          t          ¦  «        d›d |j        › d!t          |	¦  «        › d"|
d         › d#|› d$�d%¬&¦  «         �ŒÙd'„ t5          |                     ¦   «         ¦  «        D ¦   «         Šˆˆ fd(„t          D ¦   «         }t,          t.          d)t0          t7          j        t:          ¦  «                             ¦   «         t          t          ¦  «        t?          d*„ ‰                       ¦   «         D ¦   «         ¦  «        ||t?          d+„ ‰                       ¦   «         D ¦   «         ¦  «        d,œ|d-œ}tC          |¦  «         t0          t?          d.„ ‰                       ¦   «         D ¦   «         ¦  «        ||tE          tF          ¦  «        ‰|d/œS )0Nr   Úproject_planningÚprocurement_tenderÚcontract_managementÚtechnical_deliveryÚaudit_inspectionÚpayment_settlementÚdossier_reviewÚlegal_compliance)Úproject_approvalr”   r•   r–   Úcost_managementÚsite_handoverÚsite_executionÚquality_managementÚpayment_supportÚcloseout_documentationrš   r—   r™   r“   r˜   r	   r   ©r   Ú
files_seenro   rP   rx   ry   rv   ru   rB   ru   Úunclassified)Úfiler   ÚstatusrP   r"   rv   Úmissingro   rx   ry   ÚingestedÚempty)r¥   r   r¦   rP   r"   r   r   r3   )r)   r*   r7   )r   rP   Údefault_metadatarP   ú[Ú02dú/z] z: u    tá»‡p (u    há»£p lá»‡) -> u    chunks náº¡p Qdrant.T)Úflushc                óÀ   — i | ][\  }}||d          |d         |d         |d         t          |d         ¦  «        t          |d         ¦  «        |d         |d         dœ“Œ\S )	r   r£   ro   rP   rx   ry   rv   ru   r¢   )r{   )rb   r   Úsummarys      rU   ú
<dictcomp>z+ingest_qdrant_documents.<locals>.<dictcomp>ý   sŠ   € ð ð ð ñ "ˆL˜'ð 	Ø# NÔ3Ø! ,Ô/Ø%Ð&6Ô7Ø˜hÔ'Ý$ WÐ-=Ô%>Ñ?Ô?Ý˜W XÔ.Ñ/Ô/Ø$ _Ô5Ø$ _Ô5ð	
ð 	
ðð ð rW   c           
     ó²   •— g | ]S}t          |‰                     |j        g ¦  «        ‰                     |j        i ¦  «        t          |¦  «        ¦  «        ‘ŒTS ra   )r~   rz   r   r   )rb   rX   Únormalized_projectsr�   s     €€rU   ú
<listcomp>z+ingest_qdrant_documents.<locals>.<listcomp>  so   ø€ ð ð ð ð õ 	(ØØ(×,Ò,¨YÔ-CÀRÑHÔHØ×#Ò# IÔ$:¸BÑ?Ô?Ý  Ñ+Ô+ñ		
ô 	
ðð ð rW   u   HÄ�-2026c              3  ó4   K  — | ]}t          |¦  «        V — Œd S rf   ©rM   ©rb   Úitemss     rU   rh   z*ingest_qdrant_documents.<locals>.<genexpr>  s9   è è € ð (ð (Ø$•�E‘
”
ð(ð (ð (ð (ð (ð (rW   c              3  ó   K  — | ]}|¯d V — Œ	dS )r	   Nra   r·   s     rU   rh   z*ingest_qdrant_documents.<locals>.<genexpr>!  s;   è è € ð +ð +ØÈð+Øð+ð +ð +ð +ð +ð +rW   )Údocuments_discoveredÚdocuments_ingestedÚchunks_ingestedÚprojects_with_documents)r)   Úsource_nameÚsource_pathr7   Úgenerated_atÚprojects_processedÚtotalsÚprojectsc              3  ó4   K  — | ]}t          |¦  «        V — Œd S rf   r¶   r·   s     rU   rh   z*ingest_qdrant_documents.<locals>.<genexpr>+  s9   è è € ð $
ð $
Ø �C�‰JŒJð$
ð $
ð $
ð $
ð $
ð $
rW   )r7   Údocuments_configuredr»   Útotal_chunksÚmanifest_pathrÃ   Úfiles)$r   rJ   r   rz   r   Ú
setdefaultr   rM   Úsetr"   rO   r,   rH   r}   rV   r   ÚextendÚaddr   r3   r¸   Úingest_knowledger   r   r   Úprintr{   r   Únowr   Ú	isoformatÚsumÚvaluesr�   rL   r   )r�   ÚorchestratorrÆ   Útotal_documents_ingestedÚper_fileÚper_projectÚTASK_TYPE_MAPÚp_idxrX   rY   Úproject_summaryÚproject_chunks_by_taskr   rR   Úmapped_task_typeÚproj_chunk_countÚmapped_taskÚchunks_batchÚinsertedÚcoverage_projectsr   r³   s   `                    @rU   Úingest_qdrant_documentsrá   ˆ   sê  øøè è € õ $Ñ%Ô%€LØ€LØ ÐØ%'€HØ-/€Kð /Ø2Ø4Ø2Ø/Ø-Ø.Ø0Ø/Ø"2Ø.Ø.Ø*Ø.Ø2ðð €Mõ$ &Õ&8ÀÐBÑBÔBð X
ñ X
ÑˆˆyØ0×4Ò4°YÔ5KÈRÑPÔPˆ	Ø%×0Ò0ØÔ"à )Ô 6Ý! )™nœnØ"#ØÝ"%¡%¤%Ý™%œ%Ø!#Ø!#ð	ð 	ñ
ô 
ˆð CEÐà!ð 0	ñ 0	ˆHØÔ%¨Ò2Ð2Ø Ô0×7Ò7¸Ô8NÑOÔOÐOØ—’à (Ô 6Ø(0Ô(=Ø"0Ø"#Ø)1Ô)?ðð ñô ð ð ØÔ)×0Ò0Ñ2Ô2ð Ø Ô0×7Ò7¸Ô8NÑOÔOÐOØ—’à (Ô 6Ø(0Ô(=Ø"+Ø"#Ø)1Ô)?ðð ñô ð ð å)¨(Ñ3Ô3ˆGØð .Ø#0×#4Ò#4ØÔ&Ð(8ñ$ô $Ð ð '×1Ò1Ð2BÀBÑGÔG×NÒNÈwÑWÔWÐWØÐ 0Ð1Ð1Ô1°QÑ6Ð1Ð1Ñ1ØÐ 0Ô1×5Ò5°hÔ6LÑMÔMÐMØ Ô)×-Ò-¨hÔ.@ÑAÔAÐAØ(¨AÑ-Ð(à�OŠOà$Ô2Ø$,Ô$9Ø,3Ð@˜j˜j¸Ý! '™lœlØ%-Ô%;Ø!)Ô!3Ø!)Ô!3Ø-5Ô-Kð	ð 	ñô ð ñ ð ÐØ)?×)EÒ)EÑ)GÔ)Gð 	)ð 	)Ñ%ˆK˜Øð )Ø!-×!>Ò!>Ø)Ø'å'2Ý-8Ý(4ð&ð &ð "?ñ "ô "ð ð ð ð ð ð �ð ! HÑ,Ð Ø Ñ(�øà$4ˆ˜Ñ!Ýð H�ð  Hð  Hð  H�CÕ 2Ñ3Ô3ð  Hð  Hð  H¸9Ô;Qð  Hð  HÕUXÐYbÑUcÔUcð  Hð  HÐm|ð  ~Nô  nOð  Hð  Hð  aqð  Hð  Hð  HØð	
ñ 	
ô 	
ð 	
ñ 	
ð
ð õ &,¨K×,=Ò,=Ñ,?Ô,?Ñ%@Ô%@ðñ ô Ððð ð ð ð õ ,ðñ ô Ðõ #Ý"Ø!Ý$Ý œ¥SÑ)Ô)×3Ò3Ñ5Ô5Ý!Õ"4Ñ5Ô5å$'ð (ð (Ø(D×(KÒ(KÑ(MÔ(Mð(ñ (ô (ñ %ô %ð #;Ø+Ý'*ð +ð +Ø;×BÒBÑDÔDð+ñ +ô +ñ (ô (ð	
ð 	
ð &ð#ð €Hõ& �8ÑÔÐõ %Ý #ð $
ð $
Ø$@×$GÒ$GÑ$IÔ$Ið$
ñ $
ô $
ñ !
ô !
ð 7Ø$Ý�]Ñ+Ô+Ø'Øð
ð 
ð 
rW   c               ƒ  óš   K  — ddl m} m} ddlm}  |¦   «         }i }| D ]} |||¦  «        }|||j        <   Œt          |¦  «        ƒ d {V —†S )Nr	   )r   Úbuild_document_type_rules)Údiscover_project_documents)Ú	constantsr   rã   Ú	discoveryrä   r   rá   )r   rã   rä   ÚrulesÚdocs_by_projectrX   Údocss          rU   Úrun_full_hd2026_ingestrê   6  s”   è è € ØHÐHÐHÐHÐHÐHÐHÐHØ5Ð5Ð5Ð5Ð5Ð5à%Ð%Ñ'Ô'€EØ8:€OØ'ð 7ð 7ˆ	Ø)Ð)¨)°UÑ;Ô;ˆØ26ˆ˜	Ô.Ñ/Ð/Ý(¨Ñ9Ô9Ð9Ð9Ð9Ð9Ð9Ð9Ð9rW   Ú__main__r‡   rˆ   z=== DA NAP rÆ   z CHUNKS TU r»   z TAI LIEU VAO QDRANT ===z=== MANIFEST: rÇ   z ===)r   r   r   r   )
rX   r   rY   rZ   r[   r\   r6   r]   r   r\   )r   r\   r   r€   )r�   r‘   r   r\   )r   r\   )(Ú
__future__r   r�   r   r   Úpathlibr   Útypingr   Úapp.api.dependenciesr   rå   r
   r   r   r   r   r   ræ   r   Úhelpersr   r   r   Úmodelsr   r   rV   r~   r�   rá   rê   Ú__name__ÚasyncioÚsysÚstdoutÚreconfigureÚrunÚresultrÎ   ra   rW   rU   ú<module>rù      s,  ðØ "Ð "Ð "Ð "Ð "Ð "à €€€Ø "Ð "Ð "Ð "Ð "Ð "Ð "Ð "Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à 1Ð 1Ð 1Ð 1Ð 1Ð 1ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ,Ð +Ð +Ð +Ð +Ð +Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5ð2ð 2ð 2ð 2ðj1ð 1ð 1ð 1ðhð ð ð ðkð kð kð kð\	:ð 	:ð 	:ð 	:ð ˆzÒÐØ€N€N€NØ€J€J€Jà„J×Ò GÐÑ,Ô,Ð,ØˆWŒ[Ð/Ð/Ñ1Ô1Ñ2Ô2€FØ	€EØo�f˜^Ô,ÐoÐo¸Ð@TÔ9UÐoÐoÐoñô ð ð 
€EÐ
8˜6 /Ô2Ð
8Ð
8Ð
8Ñ9Ô9Ð9Ð9Ð9ð ÐrW   