§
    ÔŸ’j÷  ã                  óŽ   — d Z ddlmZ ddlZddlZddlmZ ddlmZ 	 ddl	m
Z
 n# e$ r dZ
Y nw xY wddlmZ  G d„ d	¦  «        ZdS )
z2Embedding services for DSCons knowledge retrieval.é    )ÚannotationsN)ÚSequence)ÚPath)ÚSentenceTransformer)ÚSettingsc                  ój   — e Zd ZdZdd„Zdd„Zedd	„¦   «         Zedd„¦   «         Zdd„Z	dd„Z
dd„ZdS )ÚEmbeddingServicez?PhoBERT-oriented embedding wrapper with safe fallback behavior.Úsettingsr   ÚreturnÚNonec                óœ  — || _         |j        pd| _        d| _        d| _        t          |j        ¦  «        | _        | j                             dd¬¦  «         t          j
                             dt          | j        ¦  «        ¦  «         t          j
                             dt          | j        ¦  «        ¦  «         t          j
                             dt          | j        ¦  «        ¦  «         t          j        d	d
¦  «                             ¦   «         dv | _        t           �$| j        s|j        r|                      ¦   «          dS dS dS dS )z.Initialize the embedding backend if available.zvinai/phobert-base-v2i   NT)ÚparentsÚexist_okÚHF_HOMEÚTRANSFORMERS_CACHEÚSENTENCE_TRANSFORMERS_HOMEÚ#DSCONS_DISABLE_EMBEDDING_MODEL_LOADÚ0>   Ú1ÚonÚyesÚtrue)Ú	_settingsÚembedding_model_nameÚ_model_nameÚ_fallback_dimensionÚ_modelr   Úembedding_cache_dirÚ
_cache_dirÚmkdirÚosÚenvironÚ
setdefaultÚstrÚgetenvÚlowerÚ_disable_model_loadr   Úembedding_eager_loadÚ_load_model)Úselfr
   s     ú-C:\Projects\DSCons\app\services\embeddings.pyÚ__init__zEmbeddingService.__init__   s6  € à!ˆŒØ#Ô8ÐSÐ<SˆÔØ#&ˆÔ ØˆŒÝ˜xÔ;Ñ<Ô<ˆŒØŒ×Ò d°TÐÑ:Ô:Ð:Ý
Œ
×Ò˜i­¨T¬_Ñ)=Ô)=Ñ>Ô>Ð>Ý
Œ
×ÒÐ2µC¸¼Ñ4HÔ4HÑIÔIÐIÝ
Œ
×ÒÐ:½CÀÄÑ<PÔ<PÑQÔQÐQÝ#%¤9Ø1°3ñ$
ô $
ç
Š%‰'Œ'ð 
ð$
ˆÔ õ  Ð+ØÔ,ð ,àÔ-ð ,ð ×ÒÑÔÐÐÐð	 ,Ð+Ð+Ð+Ð+Ð+ó    c                ój  — | j         €t          €dS 	 t          | j        t          | j        ¦  «        | j        j        ¬¦  «        | _         n¢# t          $ r• 	 ddlm	}m
}m}  || j        t          | j        ¦  «        ¬¦  «        }t          |dd¦  «        p|j        } | |¦   «         ¦  «        }t          || |¦   «         g¬¦  «        | _         n# t          $ r
 d| _         Y nw xY wY nw xY w| j         �=t          | j         d¦  «        r*t          | j         j        pd	d
¦  «        | j         _        dS dS dS )z;Load the embedding model once and reuse it across requests.N)Úcache_folderÚlocal_files_onlyr   )Ú	NormalizeÚPoolingÚTransformer)Ú	cache_dirÚget_word_embedding_dimension)ÚmodulesÚmax_seq_lengthi   i    )r   r   r   r$   r   r   Úembedding_local_files_onlyÚ	ExceptionÚsentence_transformers.modelsr1   r2   r3   ÚgetattrÚget_embedding_dimensionÚhasattrÚminr7   )r*   r1   r2   r3   Ú
word_modelÚdimÚpooling_models          r+   r)   zEmbeddingService._load_model0   s�  € àŒ;Ð"Õ&9Ð&AØˆFð	#å-ØÔ Ý  ¤Ñ1Ô1Ø!%¤Ô!Jðñ ô ˆDŒKˆKøõ
 ð 	#ð 	#ð 	#ð#ØXÐXÐXÐXÐXÐXÐXÐXÐXÐXà(˜[ØÔ$µ°D´OÑ0DÔ0Dðñ ô �
õ ˜JÐ(FÈÑMÔMð :Ø!Ô9ð ð !( ¨¨©¬¡¤�Ý1Ø'¨¸	¸	¹¼ÐDðñ ô �”�øõ ð #ð #ð #Ø"�”��ð#øøøøøð!	#øøøð& Œ;Ð"¥w¨t¬{Ð<LÑ'MÔ'MÐ"Ý),¨T¬[Ô-GÐ-OÈ4ÐQUÑ)VÔ)VˆDŒKÔ&Ð&Ð&ð #Ð"Ð"Ð"s6   ’8A Á
C*ÁA9CÃC*ÃC$Ã!C*Ã#C$Ã$C*Ã)C*r$   c                ó   — | j         S )z+Return the configured embedding model name.)r   )r*   s    r+   Ú
model_namezEmbeddingService.model_nameR   s   € ð ÔÐr-   Úintc                óÎ   — | j         €| j        s|                      ¦   «          | j         �6| j                              ¦   «         }t	          |t
          ¦  «        r|dk    r|S | j        S )z+Return the active embedding dimensionality.Nr   )r   r'   r)   Ú get_sentence_embedding_dimensionÚ
isinstancerD   r   )r*   Ú	dimensions     r+   Úvector_sizezEmbeddingService.vector_sizeW   sj   € ð Œ;Ð tÔ'?ÐØ×ÒÑÔÐàŒ;Ð"Øœ×DÒDÑFÔFˆIÝ˜)¥SÑ)Ô)ð !¨i¸!ªm¨mØ Ð ØÔ'Ð'r-   ÚtextsúSequence[str]úlist[list[float]]c                óä   ‡ — d„ |D ¦   «         }|sg S ‰ j         €‰ j        s‰                      ¦   «          ‰ j         �)‰ j                              |dd¬¦  «        }d„ |D ¦   «         S ˆ fd„|D ¦   «         S )zEmbed a batch of texts.c                óZ   — g | ](}t          |t          ¦  «        r|nt          |¦  «        ‘Œ)S © )rG   r$   )Ú.0Útexts     r+   ú
<listcomp>z0EmbeddingService.embed_texts.<locals>.<listcomp>e   s>   € ð 
ð 
ð 
Ø=A•J˜t¥SÑ)Ô)Ð8ˆDˆD­s°4©y¬yð
ð 
ð 
r-   NFT)Úconvert_to_numpyÚnormalize_embeddingsc                óR   — g | ]$}t          t          t          |¦  «        ¦  «        ‘Œ%S rO   )ÚlistÚmapÚfloat)rP   Úvectors     r+   rR   z0EmbeddingService.embed_texts.<locals>.<listcomp>t   s*   € ÐCÐCÐC°•D��U FÑ+Ô+Ñ,Ô,ÐCÐCÐCr-   c                ó:   •— g | ]}‰                      |¦  «        ‘ŒS rO   )Ú_fallback_embed)rP   rQ   r*   s     €r+   rR   z0EmbeddingService.embed_texts.<locals>.<listcomp>v   s'   ø€ ÐHÐHÐH¨t�×$Ò$ TÑ*Ô*ÐHÐHÐHr-   )r   r'   r)   Úencode)r*   rJ   Únormalized_textsÚvectorss   `   r+   Úembed_textszEmbeddingService.embed_textsc   s²   ø€ ð
ð 
ØEJð
ñ 
ô 
Ðð  ð 	ØˆIàŒ;Ð tÔ'?ÐØ×ÒÑÔÐàŒ;Ð"Ø”k×(Ò(Ø Ø!&Ø%)ð )ñ ô ˆGð
 DÐC¸7ÐCÑCÔCÐCàHÐHÐHÐHÐ7GÐHÑHÔHÐHr-   rQ   úlist[float]c                óX   — |                       |g¦  «        }|r|d         n
dg| j        z  S )zEmbed a single search query.r   ç        )r_   rI   )r*   rQ   r^   s      r+   Úembed_queryzEmbeddingService.embed_queryx   s4   € à×"Ò" D 6Ñ*Ô*ˆØ$ÐBˆw�qŒzˆz¨3¨%°$Ô2BÑ*BÐBr-   c                óJ  ‡— dg| j         z  }|                     d¦  «        }|s|S d}d}| j         dz  }t          |¦  «        |k     rX|t          j        ||                     dd¦  «        z   ¦  «                             ¦   «         z  }|dz  }t          |¦  «        |k     °Xt          | j         ¦  «        D ]?}|dz  }|||dz   …         }	t           	                    |	dd¬	¦  «        }
|
d
z  dz  dz
  ||<   Œ@t          d„ |D ¦   «         ¦  «        dz  Š‰dk    r|S ˆfd„|D ¦   «         S )z>Generate a deterministic fallback embedding from text content.rb   zutf-8r-   r   é   Úlittleé   F)Úsignedg  àÿÿÿïAg       @g      ð?c              3  ó    K  — | ]	}||z  V — Œ
d S )NrO   )rP   Ú	components     r+   ú	<genexpr>z3EmbeddingService._fallback_embed.<locals>.<genexpr>”   s'   è è € ÐAÐA¨Y�9˜yÑ(ÐAÐAÐAÐAÐAÐAr-   g      à?c                ó   •— g | ]}|‰z  ‘ŒS rO   rO   )rP   rj   Únorms     €r+   rR   z4EmbeddingService._fallback_embed.<locals>.<listcomp>—   s   ø€ Ð9Ð9Ð9 Y�	˜DÑ Ð9Ð9Ð9r-   )r   r\   ÚlenÚhashlibÚsha256Úto_bytesÚdigestÚrangerD   Ú
from_bytesÚsum)r*   rQ   rY   ÚencodedÚdigest_streamÚcounterÚrequired_bytesÚindexÚstartÚchunkÚvaluerm   s              @r+   r[   z EmbeddingService._fallback_embed}   se  ø€ à�˜Ô1Ñ1ˆØ—+’+˜gÑ&Ô&ˆØð 	ØˆMàˆØˆØÔ1°AÑ5ˆå�-Ñ Ô  >Ò1Ð1Ø�Wœ^Ø˜'×*Ò*¨1¨hÑ7Ô7Ñ7ñô çŠf‰hŒhñˆMð �q‰LˆGõ	 �-Ñ Ô  >Ò1Ð1õ ˜4Ô3Ñ4Ô4ð 	?ð 	?ˆEØ˜A‘IˆEØ! %¨%°!©)Ð"3Ô4ˆEÝ—N’N 5¨(¸5�NÑAÔAˆEØ" \Ñ1°SÑ8¸3Ñ>ˆF�5‰MˆMåÐAÐA¸&ÐAÑAÔAÑAÔAÀSÑHˆØ�1Š9ˆ9ØˆMØ9Ð9Ð9Ð9°&Ð9Ñ9Ô9Ð9r-   N)r
   r   r   r   )r   r   )r   r$   )r   rD   )rJ   rK   r   rL   )rQ   r$   r   r`   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r,   r)   ÚpropertyrC   rI   r_   rc   r[   rO   r-   r+   r	   r	      sË   € € € € € ØIÐIðð ð ð ð6 Wð  Wð  Wð  WðD ð ð  ð  ñ „Xð ð ð	(ð 	(ð 	(ñ „Xð	(ðIð Ið Ið Ið*Cð Cð Cð Cð
:ð :ð :ð :ð :ð :r-   r	   )r�   Ú
__future__r   ro   r!   Úcollections.abcr   Úpathlibr   Úsentence_transformersr   ÚImportErrorÚapp.core.settingsr   r	   rO   r-   r+   ú<module>r‰      sä   ðØ 8Ð 8à "Ð "Ð "Ð "Ð "Ð "à €€€Ø 	€	€	€	Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð ðØ9Ð9Ð9Ð9Ð9Ð9Ð9øØð ð ð ØÐÐÐðøøøð 'Ð &Ð &Ð &Ð &Ð &ðE:ð E:ð E:ð E:ð E:ñ E:ô E:ð E:ð E:ð E:s   ž% ¥/®/