§
    ÔŸ’j±  ã                  óˆ   — d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ dd„Zdd„Z G d„ d¦  «        ZdS )zwHybrid Search Service combining BM25 Lexical Search and Dense Vector Search (Qdrant) with Reciprocal Rank Fusion (RRF).é    )ÚannotationsN)ÚCounter)ÚAny)ÚEmbeddingService)ÚQdrantStoreÚtextÚstrÚreturnc                óf   — d                      d„ t          j        d| ¦  «        D ¦   «         ¦  «        S )u@   Loáº¡i bá»� dáº¥u tiáº¿ng Viá»‡t Ä‘á»ƒ Ä‘á»‘i sÃ¡nh linh hoáº¡t.Ú c              3  óB   K  — | ]}t          j        |¦  «        °|V — Œd S )N)ÚunicodedataÚ	combining)Ú.0Úchs     ú8C:\Projects\DSCons\app\services\hybrid_search_service.pyú	<genexpr>z strip_accents.<locals>.<genexpr>   sG   è è € ð ð àÝÔ$ RÑ(Ô(ðØ
ðð ð ð ð ð ó    ÚNFKD)Újoinr   Ú	normalize)r   s    r   Ústrip_accentsr      sA   € à�7Š7ð ð åÔ'¨°Ñ5Ô5ðñ ô ñ ô ð r   ú	list[str]c                ó~   — | sg S |                       ¦   «         }t          j        d|¦  «        }d„ |D ¦   «         }||z   S )uˆ   PhÃ¢n tÃ¡ch tá»« chuáº©n hÃ³a cho BM25 (há»— trá»£ tiáº¿ng Viá»‡t vÃ  kÃ½ tá»± Ä‘áº·c biá»‡t nhÆ° sá»‘ hiá»‡u vÄƒn báº£n/mÃ£ váº­t tÆ°).uÁ   [a-z0-9Ã Ã¡áº£Ã£áº¡Äƒáº±áº¯áº³áºµáº·Ã¢áº§áº¥áº©áº«áº­Ã¨Ã©áº»áº½áº¹Ãªá»�áº¿á»ƒá»…á»‡Ã¬Ã­á»‰Ä©á»‹Ã²Ã³á»�Ãµá»�Ã´á»“á»‘á»•á»—á»™Æ¡á»�á»›á»Ÿá»¡á»£Ã¹Ãºá»§Å©á»¥Æ°á»«á»©á»­á»¯á»±á»³Ã½á»·á»¹á»µÄ‘/_.\-]+c                óR   — g | ]$}t          |¦  «        |k    ¯t          |¦  «        ‘Œ%S © )r   )r   Úts     r   ú
<listcomp>z!bm25_tokenize.<locals>.<listcomp>#   s3   € ÐSÐSÐS¨a½]È1Ñ=MÔ=MÐQRÒ=RÐ=R� qÑ)Ô)Ð=RÐ=RÐ=Rr   )ÚlowerÚreÚfindall)r   Ú
normalizedÚtokensÚunaccented_tokenss       r   Úbm25_tokenizer%      sY   € àð Øˆ	Ø—’‘”€JåŒZð 	MØñô €Fð
 TÐS°6ÐSÑSÔSÐØÐ%Ñ%Ð%r   c                  ó4   — e Zd ZdZd d„Zd!d„Z	 	 	 	 d"d#d„ZdS )$ÚHybridSearchServiceuZ   Dá»‹ch vá»¥ tÃ¬m kiáº¿m há»—n há»£p (BM25 Lexical + Dense Vectors Qdrant + RRF Reranking).Úqdrant_storer   Úembedding_servicer   r
   ÚNonec                ó>   — || _         || _        d| _        d| _        d S )Ng      ø?g      è?)Ú_qdrant_storeÚ_embedding_serviceÚ_k1Ú_b)Úselfr(   r)   s      r   Ú__init__zHybridSearchService.__init__*   s&   € ð *ˆÔØ"3ˆÔØˆŒØˆŒˆˆr   Úquery_tokensr   Ú
doc_tokensÚavg_dlÚfloatÚdoc_lenÚintÚterm_idfúdict[str, float]c           
     ó2  — |r|sdS t          |¦  «        }d}|D ]|}||vrŒ||         }	|                     |d¦  «        }
|	| j        dz   z  }|	| j        d| j        z
  | j        |t	          d|¦  «        z  z  z   z  z   }||
|t	          d|¦  «        z  z  z  }Œ}|S )uF   TÃ­nh Ä‘iá»ƒm BM25 cho má»™t tÃ i liá»‡u Ä‘á»‘i vá»›i cÃ¢u truy váº¥n.ç        ç      ð?g�íµ ÷Æ°>)r   Úgetr.   r/   Úmax)r0   r2   r3   r4   r6   r8   Údoc_counterÚscoreÚq_termÚtfÚidfÚ	numeratorÚdenominators                r   Ú_compute_bm25_scorez'HybridSearchService._compute_bm25_score2   sÔ   € ð ð 	 ð 	Ø�3å˜jÑ)Ô)ˆØˆà"ð 		@ð 		@ˆFØ˜[Ð(Ð(ØØ˜VÔ$ˆBØ—,’,˜v sÑ+Ô+ˆCØ˜dœh¨™nÑ-ˆIØ˜tœxØ�d”g‘ ¤¨7µS¸¸fÑ5EÔ5EÑ+EÑ FÑFñ ñ ˆKð �S˜I­¨D°+Ñ(>Ô(>Ñ>Ñ?Ñ?ˆEˆEàˆr   é   NÚhybridÚqueryr	   Útop_kÚ	task_typeú
str | NoneÚretrieval_filtersúdict[str, Any] | NoneÚmodeúlist[dict[str, Any]]c           	   ƒ  ó$  ‡&‡'K  — |r|                      ¦   «         sg S t          |dz  d¦  «        }| j                             |¦  «        }g Š&	 | j                             |||¬¦  «        Š&n# t          $ r g Š&Y nw xY w|dk    r
‰&d|…         S ‰&sg S t          |¦  «        }d„ ‰&D ¦   «         }	d„ |	D ¦   «         }
t          |
¦  «        t          dt          |
¦  «        ¦  «        z  }t          ‰&¦  «        }i }t          |¦  «        D ]CŠ't          ˆ'fd	„|	D ¦   «         ¦  «        }t          j        d
||z
  dz   |dz   z  z   ¦  «        |‰'<   ŒDg }t          t          |	|
¦  «        ¦  «        D ]8\  }\  }}|                      |||||¦  «        }|                     ||f¦  «         Œ9|dk    rr|                     d„ d¬¦  «         g }t          |d|…         d¬¦  «        D ]<\  }\  }}t%          ‰&|         ¦  «        }||d<   ||d<   |                     |¦  «         Œ=|S d„ t          ‰&¦  «        D ¦   «         }t'          |d„ d¬¦  «        }ˆ&fd„t          |¦  «        D ¦   «         }ˆ&fd„|D ¦   «         }d}d}d}g }‰&D ]÷}|                     dd¦  «        } |                     | t          ‰&¦  «        ¦  «        }!|                     | d¦  «        }"|||!z   z  }#|"dk    r)|||                     | t          ‰&¦  «        ¦  «        z   z  nd}$|#|$z   }%t%          |¦  «        }t+          |%d¦  «        |d<   |                     d d¦  «        |d!<   t+          |"d¦  «        |d<   d"|d#<   |                     |¦  «         Œø|                     d$„ d¬¦  «         |d|…         S )%u4   TÃ¬m kiáº¿m ngá»¯ nghÄ©a vÃ  tá»« khÃ³a káº¿t há»£p.é   é   )Úquery_vectorÚlimitÚfiltersÚdenseNc                ó¢   — g | ]L}t          |                     d d¦  «        dz   t          |                     di ¦  «        ¦  «        z   ¦  «        ‘ŒMS )r   r   ú Úmetadata)r%   r=   r	   )r   Úress     r   r   z.HybridSearchService.search.<locals>.<listcomp>p   s^   € ð 
ð 
ð 
àõ ˜#Ÿ'š' &¨"Ñ-Ô-°Ñ3µc¸#¿'º'À*ÈbÑ:QÔ:QÑ6RÔ6RÑRÑSÔSð
ð 
ð 
r   c                ó,   — g | ]}t          |¦  «        ‘ŒS r   )Úlen)r   r#   s     r   r   z.HybridSearchService.search.<locals>.<listcomp>t   s   € Ð@Ð@Ð@ v•s˜6‘{”{Ð@Ð@Ð@r   é   c              3  ó$   •K  — | ]
}‰|v ¯d V — ŒdS )r^   Nr   )r   r#   rA   s     €r   r   z-HybridSearchService.search.<locals>.<genexpr>{   s-   øè è € ÐRÐR VÀÈ6ÐAQÐAQ˜qÐAQÐAQÐAQÐAQÐRÐRr   r<   g      à?Úbm25c                ó   — | d         S ©Nr^   r   ©Úitems    r   ú<lambda>z,HybridSearchService.search.<locals>.<lambda>‰   s
   € °D¸´G€ r   T)ÚkeyÚreverse)ÚstartÚ
bm25_scoreÚrankc                ób   — i | ],\  }}|                      d t          |¦  «        ¦  «        |dz   “Œ-S ©Úidr^   ©r=   r	   )r   Úir[   s      r   ú
<dictcomp>z.HybridSearchService.search.<locals>.<dictcomp>”   sC   € ð 
ð 
ð 
Ù-3¨Q°ˆC�GŠG�D�#˜a™&œ&Ñ!Ô! 1 q¡5ð
ð 
ð 
r   c                ó   — | d         S rb   r   rc   s    r   re   z,HybridSearchService.search.<locals>.<lambda>š   s
   € °°Q´€ r   c                óv   •— i | ]5\  }\  }}‰|                               d t          |¦  «        ¦  «        |dz   “Œ6S rl   rn   )r   rj   ÚidxÚ_Údense_resultss       €r   rp   z.HybridSearchService.search.<locals>.<dictcomp>œ   sR   ø€ ð 
ð 
ð 
á�‘h�s˜Að ˜#Ô×"Ò" 4­¨S©¬Ñ2Ô2°D¸1±Hð
ð 
ð 
r   c                ój   •— i | ]/\  }}‰|                               d t          |¦  «        ¦  «        |“Œ0S )rm   rn   )r   rs   r@   ru   s      €r   rp   z.HybridSearchService.search.<locals>.<dictcomp>    sJ   ø€ ð 
ð 
ð 
á��Uð ˜#Ô×"Ò" 4­¨S©¬Ñ2Ô2°Eð
ð 
ð 
r   g      N@g333333ã?gš™™™™™Ù?rm   r   r;   r   é   Ú	rrf_scorer@   Údense_scoreÚ
hybrid_rrfÚsearch_modec                ó.   — |                       dd¦  «        S )Nrx   r;   )r=   )Úxs    r   re   z,HybridSearchService.search.<locals>.<lambda>¿   s   €  q§u¢u¨[¸#Ñ'>Ô'>€ r   )Ústripr>   r-   Úembed_queryr,   ÚsearchÚ	Exceptionr%   Úsumr]   ÚsetÚmathÚlogÚ	enumerateÚziprF   ÚappendÚsortÚdictÚsortedr=   Úround)(r0   rI   rJ   rK   rM   rO   Úcandidate_limitrT   r2   Údoc_token_listÚdoc_lengthsr4   Ún_docsr8   Ún_containingÚbm25_scored_itemsrs   r3   r6   r@   Úresultsrj   rd   Údense_ranksÚsorted_by_bm25Ú
bm25_ranksÚbm25_score_mapÚkÚw_denseÚw_bm25Úfused_itemsr[   Údoc_idÚd_rankÚb_scoreÚdense_componentÚbm25_componentrx   ru   rA   s(                                         @@r   r€   zHybridSearchService.searchN   s·  øøè è € ð ð 	˜EŸKšK™MœMð 	ØˆIõ ˜e a™i¨Ñ,Ô,ˆØÔ.×:Ò:¸5ÑAÔAˆà.0ˆð	Ø Ô.×5Ò5Ø)Ø%Ø)ð 6ñ ô ˆMˆMøõ
 ð 	ð 	ð 	ØˆMˆMˆMð	øøøð �7Š?ˆ?Ø   % Ô(Ð(àð 	ØˆIõ % UÑ+Ô+ˆð
ð 
à$ð
ñ 
ô 
ˆð AÐ@°Ð@Ñ@Ô@ˆÝ�[Ñ!Ô!¥C¨­3¨{Ñ+;Ô+;Ñ$<Ô$<Ñ<ˆÝ�]Ñ#Ô#ˆð &(ˆÝ˜,Ñ'Ô'ð 	ð 	ˆFÝÐRÐRÐRÐR¨~ÐRÑRÔRÑRÔRˆLÝ#œxØ�v Ñ,¨sÑ2°|ÀcÑ7IÑJÑJñ ô  ˆH�VÑÐð
 68ÐÝ*3µC¸ÈÑ4TÔ4TÑ*UÔ*Uð 	3ð 	3Ñ&ˆCÑ&�*˜gØ×,Ò,Ø˜j¨&°'¸8ñô ˆEð ×$Ò$ c¨5 \Ñ2Ô2Ð2Ð2à�6Š>ˆ>Ø×"Ò"Ð';Ð';ÀTÐ"ÑJÔJÐJØˆGÝ&/Ð0AÀ&À5À&Ô0IÐQRÐ&SÑ&SÔ&Sð %ð %Ñ"�‘l�s˜EÝ˜M¨#Ô.Ñ/Ô/�Ø%*��\Ñ"Ø#��V‘Ø—’˜tÑ$Ô$Ð$Ð$ØˆNð
ð 
Ý7@ÀÑ7OÔ7Oð
ñ 
ô 
ˆõ
  ØÐ#7Ð#7Àð
ñ 
ô 
ˆð
ð 
ð 
ð 
å"+¨NÑ";Ô";ð
ñ 
ô 
ˆ
ð
ð 
ð 
ð 
à/ð
ñ 
ô 
ˆð
 ˆØˆØˆà,.ˆØ ð 	%ð 	%ˆCØ—W’W˜T 2Ñ&Ô&ˆFØ —_’_ V­S°Ñ-?Ô-?Ñ@Ô@ˆFØ$×(Ò(¨°Ñ5Ô5ˆGà%¨¨V©Ñ4ˆOð ˜Q’;�;ð ˜1˜zŸ~š~¨fµc¸-Ñ6HÔ6HÑIÔIÑIÑJÐJàð ð (¨.Ñ8ˆIå˜‘9”9ˆDÝ % i°Ñ 3Ô 3ˆD�ÑØ"%§'¢'¨'°3Ñ"7Ô"7ˆD�ÑÝ!& w°Ñ!2Ô!2ˆD�ÑØ".ˆD�ÑØ×Ò˜tÑ$Ô$Ð$Ð$à×ÒÐ>Ð>ÈÐÑMÔMÐMØ˜6˜E˜6Ô"Ð"s   ÁA+ Á+A:Á9A:)r(   r   r)   r   r
   r*   )r2   r   r3   r   r4   r5   r6   r7   r8   r9   r
   r5   )rG   NNrH   )rI   r	   rJ   r7   rK   rL   rM   rN   rO   r	   r
   rP   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r1   rF   r€   r   r   r   r'   r'   '   sp   € € € € € ØdÐdðð ð ð ðð ð ð ð> Ø $Ø37Øðr#ð r#ð r#ð r#ð r#ð r#ð r#r   r'   )r   r	   r
   r	   )r   r	   r
   r   )r¤   Ú
__future__r   r„   r    r   Úcollectionsr   Útypingr   Úapp.services.embeddingsr   Úapp.services.qdrant_storer   r   r%   r'   r   r   r   ú<module>rª      sç   ðØ }Ð }à "Ð "Ð "Ð "Ð "Ð "à €€€Ø 	€	€	€	Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1ðð ð ð ð&ð &ð &ð &ðY#ð Y#ð Y#ð Y#ð Y#ñ Y#ô Y#ð Y#ð Y#ð Y#r   