§
    +�’jÛ˜  ã                   óÚ  — d dl Z d dlZd dlZd dlZd dlZddlmZ d„ Zda	da
d„ Zdadad„ Zd„ Zd	„ Zed
ededddddddddiedededddddddddidœZededddddiededdddd ied!ed"dd#ddied$ed%dd&dd id'œZej        Zej        Zej        Zej        Zej        ZeZeZeZ G d(„ d)¦  «        Z G d*„ d+¦  «        Z G d,„ d-¦  «        Z  G d.„ d/¦  «        Z!d0„ Z"ej#        e_$        ej%        e_&         e"e¦  «          e"e ¦  «          e"e!¦  «         dd1l'm(Z( 	  ee j)         *                     eej+         ,                    d2¦  «        ¦  «        d3¦  «        ¦  «         dS # e-$ r Y dS w xY w)4é    Né   )Ú_sentencepiecec                 ó\   — t          | d¦  «        ot          | t          t          f¦  «         S )NÚ__len__)ÚhasattrÚ
isinstanceÚstrÚbytes)Úxs    úDC:\Projects\DSCons\.venv\Lib\site-packages\sentencepiece\__init__.pyÚ_is_sequencer      s(   € Ý�1�iÑ Ô ÐD­°A½½U°|Ñ)DÔ)DÐ%DÐDó    Fc                  ót   — t           s0	 ddlm}  | an"# t          $ r}t	          d¦  «        |‚d }~ww xY wda d S d S )Nr   )Úsentencepiece_pb2z_protobuf is required when using out_type='proto'. Please install protobuf: pip install protobufT)Ú_protobuf_loadedÚsentencepiecer   Ú_sentencepiece_pb2ÚImportError)r   Úes     r   Ú_load_protobufr      s~   € åð 	 ð	Ø7Ð7Ð7Ð7Ð7Ð7Ø!2ÐÐøÝð 	ð 	ð 	Ýð@ñô ð ðøøøøð	øøøð
  ÐÐÐð	 ð 	 s   ‰ ’
1œ,¬1c                  óv   — t           s,	 dd l} | an"# t          $ r}t          d¦  «        |‚d }~ww xY wda t          S )Nr   zSnumpy is required when return_type='numpy'. Please install numpy: pip install numpyT)Ú_numpy_loadedÚnumpyÚ_numpyr   )Únpr   s     r   Ú_load_numpyr      si   € åð 	ð	ØÐÐÐØˆFˆFøÝð 	ð 	ð 	Ýð:ñô ð ðøøøøð	øøøð
 ˆÝ€Ms   ‰ �
/š*ª/c                 ó~   — t          ¦   «          t                               ¦   «         }|                     | ¦  «         |S ©N)r   r   ÚSentencePieceTextÚParseFromString)Ú
serializedÚspts     r   Ú_to_sptr#   ,   s8   € ÝÑÔÐÝ
×
.Ò
.Ñ
0Ô
0€CØ×Ò˜
Ñ#Ô#Ð#Ø€Jr   c                 ó~   — t          ¦   «          t                               ¦   «         }|                     | ¦  «         |S r   )r   r   ÚNBestSentencePieceTextr    )r!   Ú	nbest_spts     r   Ú_to_nbest_sptr'   2   s9   € ÝÑÔÐÝ"×9Ò9Ñ;Ô;€IØ×Ò˜jÑ)Ô)Ð)ØÐr   Ú_EncodeAsIdsÚ_EncodeAsPiecesÚserialized_protoÚ_EncodeAsSerializedProtoÚprotoÚoffset_mappingÚ_EncodeAsOffsetMappingr   Ú_EncodeAsBufferÚ_EncodeAsIdsBatchÚ_EncodeAsPiecesBatchÚ_EncodeAsSerializedProtoBatchÚ_EncodeAsOffsetMappingBatchÚ_EncodeAsBufferBatch©FTÚ
_DecodeIdsÚ_DecodeIdsAsBytesÚ_DecodeIdsAsSerializedProtoÚ_DecodeAsOffsetMappingÚ_DecodeIdsBatchÚ_DecodeIdsAsBytesBatchÚ _DecodeIdsAsSerializedProtoBatchÚ_DecodeAsOffsetMappingBatchÚ_DecodePiecesÚ_DecodePiecesAsBytesÚ_DecodePiecesAsSerializedProtoÚ_DecodePiecesBatchÚ_DecodePiecesAsBytesBatchÚ#_DecodePiecesAsSerializedProtoBatch))FFr5   )TF)TTc                   ó’  — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 dOd„Zd„ ZdPd„Zed„ ¦   «         Zed	„ ¦   «         ZdQd
„Z	d„ Z
d„ Z	 	 	 	 	 	 	 	 	 	 	 	 dRd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ ZdQd„ZdQd„ZdQd„ZdQd„ZdQd„ZdQd„Z	 	 	 	 	 	 	 dSd„ZdPd„ZdPd„ZdPd„ZdPd„ZdPd „ZdPd!„Z 	 	 	 	 	 	 	 	 	 	 dTd"„Z!dQd#„Z"dQd$„Z#dQd%„Z$dQd&„Z%dQd'„Z&	 	 	 	 	 	 	 	 	 dUd(„Z'd)„ Z(d*„ Z)d+„ Z*d,„ Z+d-„ Z,d.„ Z-dVd/„Z.dWd0„Z/e0fd1„Z1e0fd2„Z2dXd4„Z3dXd5„Z4d6„ Z5d7„ Z6d8„ Z7d9„ Z8d:„ Z9dPd;„Z:d<„ Z;d=„ Z<d>„ Z=d?„ Z>d@„ Z?dA„ Z@dB„ ZAdC„ ZBdD„ ZCdE„ ZDdF„ ZEdG„ ZFdH„ ZGdI„ ZHe<ZIe<ZJdJ„ ZKdK„ ZLdL„ ZMdM„ ZNdN„ ZOdS )YÚSentencePieceProcessorNFéÿÿÿÿçš™™™™™¹?c                 ó$  — t          j        ¦   «         | _        |                      ||t          ¬¦  «        | _        || _        || _        || _        || _	        || _
        |	| _        |
| _        || _        |s|r|                      ||¬¦  «         d S d S )N©Údefault)Ú
model_fileÚmodel_proto)r   rE   Ú
_processorÚ_resolve_return_typeÚintÚ_return_typeÚ_add_bosÚ_add_eosÚ_reverseÚ_emit_unk_pieceÚ_enable_samplingÚ_nbest_sizeÚ_alphaÚ_num_threadsÚLoad)ÚselfrK   rL   Úout_typeÚadd_bosÚadd_eosÚreverseÚemit_unk_pieceÚenable_samplingÚ
nbest_sizeÚalphaÚnum_threadsÚreturn_types                r   Ú__init__zSentencePieceProcessor.__init__‰   s©   € õ )Ô?ÑAÔAˆŒð !×5Ò5°kÀ8ÕUXÐ5ÑYÔYˆÔØˆŒØˆŒØˆŒØ-ˆÔØ /ˆÔØ%ˆÔØˆŒØ'ˆÔØð 	F˜ð 	FØ�IŠI ¸ˆIÑEÔEÐEÐEÐEð	Fð 	Fr   c                 óÀ   — |€| j         }|dk    rt          d¦  «        ‚t          t          t          ddddf}||vr#t          d                     ||¦  «        ¦  «        ‚|S )NÚimmutable_protoúWimmutable_proto is deprecated and no longer supported. Use return_type='proto' instead.r,   r*   r   r-   z*Invalid return_type: {}. Must be one of {})rP   Ú
ValueErrorrO   r	   r
   Úformat)rZ   rd   Úvalid_typess      r   Ú!_resolve_and_validate_return_typez8SentencePieceProcessor._resolve_and_validate_return_type¥   ss   € ØÐØÔ+ˆKØÐ+Ò+Ð+ÝÐvÑwÔwÐwå�C¥¨Ð1CÀWÐN^Ð_ˆØ˜kÐ)Ð)ÝÐI×PÒPÐQ\Ð^iÑjÔjÑkÔkÐkØÐr   c                 óp   — |�|�t          d¦  «        ‚|}|€|�|n| j        }|                      |¦  «        S )NzFCannot specify both out_type and return_type. Use return_type instead.)ri   rP   rl   )rZ   rd   r[   rJ   s       r   rN   z+SentencePieceProcessor._resolve_return_type°   sM   € ØÐØÐ&Ý Ð!iÑjÔjÐjØ"ˆKØÐØ%,Ð%8˜'˜'¸dÔ>OˆKØ×5Ò5°kÑBÔBÐBr   c                 ón   — d|v sd|v rt          d¦  «        ‚ | di |¤Ž}|                     |¦  «         |S )zCCreates a SentencePieceProcessor by loading a model file from disk.rK   rL   zXmodel_file and model_proto cannot be passed via kwargs when using from_file classmethod.© )ri   ÚLoadFromFile)ÚclsrK   ÚkwargsÚ	processors       r   Ú	from_filez SentencePieceProcessor.from_file¹   sS   € ð ˜6Ð!Ð! ]°fÐ%<Ð%<ÝÐwÑxÔxÐxØ�C�M�M˜&�M�Mˆ	Ø×Ò˜zÑ*Ô*Ð*ØÐr   c                 ón   — d|v sd|v rt          d¦  «        ‚ | di |¤Ž}|                     |¦  «         |S )zQCreates a SentencePieceProcessor by loading a serialized model proto from memory.rK   rL   zYmodel_file and model_proto cannot be passed via kwargs when using from_proto classmethod.ro   )ri   ÚLoadFromSerializedProto)rq   rL   rr   rs   s       r   Ú
from_protoz!SentencePieceProcessor.from_protoÂ   sS   € ð ˜6Ð!Ð! ]°fÐ%<Ð%<ÝÐxÑyÔyÐyØ�C�M�M˜&�M�Mˆ	Ø×)Ò)¨+Ñ6Ô6Ð6ØÐr   c                 ó¢   — |r|rt          d¦  «        ‚|r|                      |¦  «        S |r|                      |¦  «        S t          d¦  «        ‚)Nz-model_file and model_proto must be exclusive.z3Either model_file or model_proto must be specified.)ri   rv   rp   )rZ   rK   rL   s      r   rY   zSentencePieceProcessor.LoadË   si   € Øð 	N˜+ð 	NÝÐLÑMÔMÐMØð 	=Ø×/Ò/°Ñ<Ô<Ð<Øð 	1Ø×$Ò$ ZÑ0Ô0Ð0ÝÐNÑOÔOÐOr   c                 ó6   — | j                              |¦  «        S r   )rM   rp   )rZ   Úfilenames     r   rp   z#SentencePieceProcessor.LoadFromFileÔ   s   € ØŒ×+Ò+¨HÑ5Ô5Ð5r   c                 ó~   — t          |d¦  «        r|                     ¦   «         }| j                             |¦  «        S )NÚSerializeToString)r   r|   rM   rv   )rZ   r!   s     r   rv   z.SentencePieceProcessor.LoadFromSerializedProto×   s<   € Ý�:Ð2Ñ3Ô3ð 	8Ø#×5Ò5Ñ7Ô7ˆJØŒ×6Ò6°zÑBÔBÐBr   c           	      óÔ  ‡— |                       ||¦  «        }|�|dk    rt          d¦  «        ‚|€| j        }|€| j        }|€| j        }|€| j        }|€| j        }|€| j        }|€| j        }|	€| j	        }	|du r|�|dk    s|dk    s|€t          d¦  «        ‚|	�t          |	t          ¦  «        st          d¦  «        ‚t          |t          ¦  «        }t                               |¦  «        }|r||vr"t          d                     |¦  «        ¦  «        ‚||         }t#          | j        |¦  «        }|t&          t(          fv }|p|d	v }|r||	|
||||||g	}n	|||||||g}|r|                     |¦  «         |r|t(          u }|                     |¦  «         ng|dk    ra|€Ht          |t(          ¦  «        p1t          |t          ¦  «        o|ot          |d         t(          ¦  «        }n|}|                     |¦  «          ||Ž }|d
k    r:t-          ¦   «         Š|rˆfd„|D ¦   «         S ‰                     |‰j        ¬¦  «        S |dk    r|rd„ |D ¦   «         S t3          |¦  «        S |S )Nr-   ú@return_bytes is only supported when return_type='offset_mapping'Tr   r   a  When enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. when "nbest_size = -1" , this method samples from all candidates on the lattice instead of nbest segmentations.únum_threads must be intúunknown return_type={})r*   r,   r-   r   c                 óH   •— g | ]}‰                      |‰j        ¬ ¦  «        ‘ŒS ©©Údtype©Ú
frombufferÚint32©Ú.0Úbufr   s     €r   ú
<listcomp>z1SentencePieceProcessor.Encode.<locals>.<listcomp>4  s+   ø€ ÐNÐNÐN¸s˜Ÿš c°´˜Ñ:Ô:ÐNÐNÐNr   rƒ   r,   c                 ó,   — g | ]}t          |¦  «        ‘ŒS ro   ©r#   ©r‰   Úss     r   r‹   z1SentencePieceProcessor.Encode.<locals>.<listcomp>9  ó   € Ð4Ð4Ð4 q� ™
œ
Ð4Ð4Ð4r   )rN   ri   rQ   rR   rS   rT   rU   rV   rW   rX   r   rO   Ú	TypeErrorÚlistÚ_ENCODE_DISPATCHÚgetrj   ÚgetattrrM   r	   r
   Úappendr   r†   r‡   r#   )rZ   Úinputr\   r]   r^   r_   r`   ra   rb   rc   Úthread_poolrd   r[   Úreturn_bytesÚis_batchÚmethodsÚmethod_nameÚmethodÚis_pieces_methodÚhas_unkÚargsÚreturn_bytes_valÚraw_resr   s                          @r   ÚEncodezSentencePieceProcessor.EncodeÜ   s,  ø€ ð ×/Ò/°¸XÑFÔFˆØÐ#¨Ð7GÒ(GÐ(GÝÐ_Ñ`Ô`Ð`Øˆ?Ø”mˆGØˆ?Ø”mˆGØˆ?Ø”mˆGØÐ!Ø!Ô1ˆNØÐ"Ø"Ô3ˆOØÐØÔ)ˆJØˆ=Ø”KˆEØÐØÔ+ˆKà˜dÐ"Ð"¨
Ð(:¸jÈAºo¸oØ)3°qª¨¸E¸MÝð2ñô ð ð Ð¥j°½cÑ&BÔ&BÐÝÐ5Ñ6Ô6Ð6å˜e¥TÑ*Ô*ˆõ #×&Ò& xÑ0Ô0ˆØð 	K˜+¨WÐ4Ð4ÝÐ5×<Ò<¸[ÑIÔIÑJÔJÐJà˜kÔ*ˆÝ˜œ¨+Ñ6Ô6ˆð
 '­3µ¨,Ð6ÐØ"Ðd kÐ5dÐ&dˆØð 	ZØ˜;¨°_ÀjÐRWÐY`ÐbiÐkrÐsˆDˆDà˜?¨J¸¸wÈÐQXÐYˆDØð 	(Ø�KŠK˜Ñ'Ô'Ð'Øð 		*Ø +­uÐ 4ÐØ�KŠKÐ(Ñ)Ô)Ð)Ð)ØÐ,Ò,Ð,ØÐ#å#-¨eµUÑ#;Ô#;ð  $CÅ
È5ÕRVÑ@WÔ@Wð  ABÐ\að  ABÕfpÐqvÐwxÔqyõ  |Añ  gBô  gBÐ Ð à#/Ð Ø�KŠKÐ(Ñ)Ô)Ð)à�&˜$�-ˆð ˜'Ò!Ð!å‘”ˆBØð OØNÐNÐNÐNÀgÐNÑNÔNÐNØ—=’= °´�=Ñ9Ô9Ð9Ø˜GÒ#Ð#àð 5Ø4Ð4¨GÐ4Ñ4Ô4Ð4Ý˜7Ñ#Ô#Ð#àˆr   c                 ó,   —  | j         d|t          dœ|¤ŽS ©N©r—   rd   ro   ©r£   r	   ©rZ   r—   rr   s      r   ÚEncodeAsPiecesz%SentencePieceProcessor.EncodeAsPieces>  ó!   € ØˆtŒ{ÐB µCÐBÐB¸6ÐBÐBÐBr   c                 ó,   —  | j         d|t          dœ|¤ŽS r¥   ©r£   rO   r¨   s      r   ÚEncodeAsIdsz"SentencePieceProcessor.EncodeAsIdsA  rª   r   c                 ó"   —  | j         d|ddœ|¤ŽS ©Nr   r¦   ro   ©r£   r¨   s      r   ÚEncodeAsNumpyz$SentencePieceProcessor.EncodeAsNumpyD  ó!   € ØˆtŒ{ÐF °GÐFÐF¸vÐFÐFÐFr   c                 ó"   —  | j         d|ddœ|¤ŽS ©Nr*   r¦   ro   r°   r¨   s      r   ÚEncodeAsSerializedProtoz.SentencePieceProcessor.EncodeAsSerializedProtoG  s"   € ØˆtŒ{ÐQ Ð4FÐQÐQÈ&ÐQÐQÐQr   c                 ó"   —  | j         d|ddœ|¤ŽS ©Nr,   r¦   ro   r°   r¨   s      r   ÚEncodeAsProtoz$SentencePieceProcessor.EncodeAsProtoJ  r²   r   c                 ó"   —  | j         d|ddœ|¤ŽS )Nr-   r¦   ro   r°   r¨   s      r   ÚEncodeAsOffsetMappingz,SentencePieceProcessor.EncodeAsOffsetMappingM  s"   € ØˆtŒ{ÐO Ð4DÐOÐOÈÐOÐOÐOr   c                 ó    — t          d¦  «        ‚©Nrh   ©ri   r¨   s      r   ÚEncodeAsImmutableProtoz-SentencePieceProcessor.EncodeAsImmutableProtoP  ó   € ÝÐrÑsÔsÐsr   c           	      ó2   —  | j         d|||t          ddœ|¤ŽS ©NT©r—   ra   rb   rd   r`   ro   r§   ©rZ   r—   ra   rb   rr   s        r   ÚSampleEncodeAsPiecesz+SentencePieceProcessor.SampleEncodeAsPiecesS  ó;   € ØˆtŒ{ð L °:ÀUÝ'*¸DðLð LØDJðLð Lð 	Lr   c           	      ó2   —  | j         d|||t          ddœ|¤ŽS rÁ   r¬   rÃ   s        r   ÚSampleEncodeAsIdsz(SentencePieceProcessor.SampleEncodeAsIdsW  rÅ   r   c           	      ó(   —  | j         d|||dddœ|¤ŽS )Nr   TrÂ   ro   r°   rÃ   s        r   ÚSampleEncodeAsNumpyz*SentencePieceProcessor.SampleEncodeAsNumpy[  ó;   € ØˆtŒ{ð P °:ÀUØ'.ÀðPð PØHNðPð Pð 	Pr   c           	      ó(   —  | j         d|||dddœ|¤ŽS )Nr*   TrÂ   ro   r°   rÃ   s        r   ÚSampleEncodeAsSerializedProtoz4SentencePieceProcessor.SampleEncodeAsSerializedProto_  s;   € ØˆtŒ{ð [ °:ÀUØ'9È4ð[ð [ØSYð[ð [ð 	[r   c           	      ó(   —  | j         d|||dddœ|¤ŽS )Nr,   TrÂ   ro   r°   rÃ   s        r   ÚSampleEncodeAsProtoz*SentencePieceProcessor.SampleEncodeAsProtoc  rÊ   r   c                 ó    — t          d¦  «        ‚r¼   r½   rÃ   s        r   ÚSampleEncodeAsImmutableProtoz3SentencePieceProcessor.SampleEncodeAsImmutableProtog  r¿   r   c	                 ó¶  ‡ ‡‡‡‡‡‡‡	‡
‡— ‰                       ‰|¦  «        Š‰€‰ j        Š‰€‰ j        Š‰€‰ j        Š‰€‰ j        Š‰€‰ j        Š‰dk    rdŠ‰dk    rFt          ¦   «         Šˆˆˆˆˆˆ fd„Š
t          |t          ¦  «        rˆ
fd„|D ¦   «         S  ‰
|¦  «        S ˆˆˆˆˆˆˆ fd„Š	t          |t          ¦  «        rˆ	fd„|D ¦   «         S  ‰	|¦  «        S )Nr   r   r   c                 ó\   •— ‰j                              | ‰‰‰‰¦  «        }ˆfd„|D ¦   «         S )Nc                 óH   •— g | ]}‰                      |‰j        ¬ ¦  «        ‘ŒS r‚   r…   rˆ   s     €r   r‹   zJSentencePieceProcessor.NBestEncode.<locals>._encode_np.<locals>.<listcomp>ˆ  s+   ø€ ÐRÐRÐR¸s˜Ÿš c°´˜Ñ:Ô:ÐRÐRÐRr   )rM   Ú_NBestEncodeAsBuffer)ÚtextÚraw_buffersr\   r]   ra   r   r^   rZ   s     €€€€€€r   Ú
_encode_npz6SentencePieceProcessor.NBestEncode.<locals>._encode_np…  sA   ø€ Ø"œo×BÒBØ˜* g¨w¸ñAô A�àRÐRÐRÐRÀkÐRÑRÔRÐRr   c                 ó&   •— g | ]} ‰|¦  «        ‘ŒS ro   ro   ©r‰   Únr×   s     €r   r‹   z6SentencePieceProcessor.NBestEncode.<locals>.<listcomp>‹  ó!   ø€ Ð5Ð5Ð5¨!˜
˜
 1™œÐ5Ð5Ð5r   c           	      ó¾  •— ‰t           u r‰	j                             | ‰‰‰‰¦  «        S ‰t          u s	‰t          u r)‰t          u }‰	j                             | ‰‰‰‰‰|¦  «        S ‰dk    r‰	j                             | ‰‰‰‰‰¦  «        S ‰dk    r.‰	j                             | ‰‰‰‰‰¦  «        }t          |¦  «        S t          d 	                    ‰¦  «        ¦  «        ‚©Nr*   r,   r€   )
rO   rM   Ú_NBestEncodeAsIdsr	   r
   Ú_NBestEncodeAsPiecesÚ_NBestEncodeAsSerializedProtor'   ri   rj   )
rÕ   r™   r!   r\   r]   r_   ra   rd   r^   rZ   s
      €€€€€€€r   Ú_encodez3SentencePieceProcessor.NBestEncode.<locals>._encodeŽ  s  ø€ Ø�cÐ!Ð!Ø”×8Ò8Ø˜* g¨w¸ñAô Að Aà�cÐ!Ð! [µEÐ%9Ð%9Ø +­uÐ 4�Ø”×;Ò;Ø˜* g¨w¸ÀÐQ]ñ_ô _ð _àÐ0Ò0Ð0Ø”×DÒDØ˜* g¨w¸ÀñQô Qð Qà˜gÒ%Ð%Ø!œ_×JÒJØ˜* g¨w¸ÀñQô Q�
å$ ZÑ0Ô0Ð0ÝÐ5×<Ò<¸[ÑIÔIÑJÔJÐJr   c                 ó&   •— g | ]} ‰|¦  «        ‘ŒS ro   ro   ©r‰   rÚ   rá   s     €r   r‹   z6SentencePieceProcessor.NBestEncode.<locals>.<listcomp>   ó!   ø€ Ð.Ð.Ð. 1�G�G˜A‘J”JÐ.Ð.Ð.r   )	rN   rQ   rR   rS   rT   rV   r   r   r’   )rZ   r—   r\   r]   r^   r_   ra   rd   r[   rá   r×   r   s   ` `````` @@@r   ÚNBestEncodez"SentencePieceProcessor.NBestEncodej  s}  øøøøøøøøøø€ ð ×/Ò/°¸XÑFÔFˆØˆ?Ø”mˆGØˆ?Ø”mˆGØˆ?Ø”mˆGØÐ!Ø!Ô1ˆNØÐØÔ)ˆJà˜Š?ˆ?ØˆJà˜'Ò!Ð!Ý‘”ˆBðSð Sð Sð Sð Sð Sð Sð Sð Sð Sõ
 ˜%¥Ñ&Ô&ð 6Ø5Ð5Ð5Ð5¨uÐ5Ñ5Ô5Ð5Ø�:˜eÑ$Ô$Ð$ð	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kõ" �e�TÑ"Ô"ð 	/Ø.Ð.Ð.Ð.¨Ð.Ñ.Ô.Ð.Øˆw�u‰~Œ~Ðr   c                 ó.   —  | j         d||t          dœ|¤ŽS ©N©r—   ra   rd   ro   )rå   r	   ©rZ   r—   ra   rr   s       r   ÚNBestEncodeAsPiecesz*SentencePieceProcessor.NBestEncodeAsPieces£  ó&   € ØˆtÔÐ^ e¸
ÕPSÐ^Ð^ÐW]Ð^Ð^Ð^r   c                 ó.   —  | j         d||t          dœ|¤ŽS rç   )rå   rO   ré   s       r   ÚNBestEncodeAsIdsz'SentencePieceProcessor.NBestEncodeAsIds¦  rë   r   c                 ó$   —  | j         d||ddœ|¤ŽS )Nr   rè   ro   ©rå   ré   s       r   ÚNBestEncodeAsNumpyz)SentencePieceProcessor.NBestEncodeAsNumpy©  ó&   € ØˆtÔÐb e¸
ÐPWÐbÐbÐ[aÐbÐbÐbr   c                 ó$   —  | j         d||ddœ|¤ŽS )Nr*   rè   ro   rï   ré   s       r   ÚNBestEncodeAsSerializedProtoz3SentencePieceProcessor.NBestEncodeAsSerializedProto¬  s&   € ØˆtÔÐm e¸
ÐPbÐmÐmÐflÐmÐmÐmr   c                 ó$   —  | j         d||ddœ|¤ŽS )Nr,   rè   ro   rï   ré   s       r   ÚNBestEncodeAsProtoz)SentencePieceProcessor.NBestEncodeAsProto¯  rñ   r   c                 ó    — t          d¦  «        ‚r¼   r½   ré   s       r   ÚNBestEncodeAsImmutableProtoz2SentencePieceProcessor.NBestEncodeAsImmutableProto²  r¿   r   c           
      óž  ‡ ‡‡‡‡‡‡‡‡	‡
‡— ‰                       ‰
|¦  «        Š
‰€‰ j        Š‰€‰ j        Š‰€‰ j        Š‰€‰ j        Š‰€dŠ‰€dŠ‰€dŠ‰	€dŠ	‰dk    rt          d¦  «        ‚‰	r‰st          d¦  «        ‚‰
dk    rt          d¦  «        ‚ˆˆˆˆˆ	ˆˆ
ˆˆ ˆf
d	„Št          |t          ¦  «        rˆfd
„|D ¦   «         S  ‰|¦  «        S )Nr   g      ð?Fr   znum_samples must be positivez8When include_best is True, We must specify "wor = True".r   z:numpy return_type is not supported in SampleEncodeAndScorec                 óÖ  •
— ‰	t           u r!‰j                             | ‰‰‰‰‰‰‰
¦  «        S ‰	t          u s	‰	t          u r,‰	t          u }‰j                             | ‰‰‰‰‰‰‰
‰|¦
  «
        S ‰	dk    r"‰j                             | ‰‰‰‰‰‰‰
‰¦	  «	        S ‰	dk    r1‰j                             | ‰‰‰‰‰‰‰
‰¦	  «	        }t          |¦  «        S t          d 	                    ‰	¦  «        ¦  «        ‚rÝ   )
rO   rM   Ú_SampleEncodeAndScoreAsIdsr	   r
   Ú_SampleEncodeAndScoreAsPiecesÚ&_SampleEncodeAndScoreAsSerializedProtor'   ri   rj   )rÕ   r™   r!   r\   r]   rb   r_   Úinclude_bestÚnum_samplesrd   r^   rZ   Úwors      €€€€€€€€€€r   rá   z<SentencePieceProcessor.SampleEncodeAndScore.<locals>._encodeÛ  s3  ø€ Ø�cÐ!Ð!Ø”×AÒAØ˜+ u¨c°<ÀÈ'ÐSZñ\ô \ð \à�cÐ!Ð! [µEÐ%9Ð%9Ø +­uÐ 4�Ø”×DÒDØ˜+ u¨c°<ÀÈ'ÐSZÐ\jÐlxñzô zð zàÐ0Ò0Ð0Ø”×MÒMØ˜+ u¨c°<ÀÈ'ÐSZÐ\jñlô lð là˜gÒ%Ð%Ø!œ_×SÒSØ˜+ u¨c°<ÀÈ'ÐSZÐ\jñlô l�
å$ ZÑ0Ô0Ð0ÝÐ5×<Ò<¸[ÑIÔIÑJÔJÐJr   c                 ó&   •— g | ]} ‰|¦  «        ‘ŒS ro   ro   rã   s     €r   r‹   z?SentencePieceProcessor.SampleEncodeAndScore.<locals>.<listcomp>í  rä   r   )rN   rQ   rR   rS   rT   ri   r   r’   )rZ   r—   r\   r]   r^   r_   rþ   rb   rÿ   rý   rd   r[   rá   s   ` ````````` @r   ÚSampleEncodeAndScorez+SentencePieceProcessor.SampleEncodeAndScoreµ  sd  øøøøøøøøøøø€ ð ×/Ò/°¸XÑFÔFˆØˆ?Ø”mˆGØˆ?Ø”mˆGØˆ?Ø”mˆGØÐ!Ø!Ô1ˆNØÐØˆKØˆ=ØˆEØˆ;ØˆCØÐØ ˆLà˜!ÒÐÝÐ;Ñ<Ô<Ð<Øð 	Y ð 	YÝÐWÑXÔXÐXà˜'Ò!Ð!ÝÐYÑZÔZÐZð	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kõ" �e�TÑ"Ô"ð 	/Ø.Ð.Ð.Ð.¨Ð.Ñ.Ô.Ð.Øˆw�u‰~Œ~Ðr   c                 ó0   —  | j         d|||t          dœ|¤ŽS ©N©r—   rþ   rb   rd   ro   )r  r	   ©rZ   r—   rþ   rb   rr   s        r   ÚSampleEncodeAndScoreAsPiecesz3SentencePieceProcessor.SampleEncodeAndScoreAsPiecesð  ó)   € Ø(ˆtÔ(Ðv¨uÀ+ÐUZÕhkÐvÐvÐouÐvÐvÐvr   c                 ó0   —  | j         d|||t          dœ|¤ŽS r  )r  rO   r  s        r   ÚSampleEncodeAndScoreAsIdsz0SentencePieceProcessor.SampleEncodeAndScoreAsIdsó  r  r   c                 ó&   —  | j         d|||ddœ|¤ŽS )Nr*   r  ro   ©r  r  s        r   Ú%SampleEncodeAndScoreAsSerializedProtoz<SentencePieceProcessor.SampleEncodeAndScoreAsSerializedProtoö  s?   € Ø(ˆtÔ(ð  F¨uÀ+ÐUZÐhzð  Fð  Fð  Eð  Fð  Fð  	Fr   c                 ó&   —  | j         d|||ddœ|¤ŽS )Nr,   r  ro   r  r  s        r   ÚSampleEncodeAndScoreAsProtoz2SentencePieceProcessor.SampleEncodeAndScoreAsProtoù  s)   € Ø(ˆtÔ(Ðz¨uÀ+ÐUZÐhoÐzÐzÐsyÐzÐzÐzr   c                 ó    — t          d¦  «        ‚r¼   r½   r  s        r   Ú$SampleEncodeAndScoreAsImmutableProtoz;SentencePieceProcessor.SampleEncodeAndScoreAsImmutableProtoü  r¿   r   c           	      óV  ‡ ‡‡‡‡‡‡‡‡	‡‡‡— ‰                       ‰	|
¦  «        Š	‰€‰ j        Š‰€‰ j        Š‰€‰ j        Š‰€‰ j        Š‰€‰ j        Š‰�t          ‰t          ¦  «        st          d¦  «        ‚‰�t          ‰t          ¦  «        r‰dk    rt          d¦  «        ‚‰	dk    rHt          ¦   «         Šˆˆˆˆˆˆˆ ˆfd„Št          |t          ¦  «        rˆfd„|D ¦   «         S  ‰|¦  «        S ˆˆˆˆˆˆ	ˆˆ ˆf	d„Št          |t          ¦  «        rˆfd„|D ¦   «         S  ‰|¦  «        S )	Nr   r   zchunk_len must be positive.r   c           	      ó|   •— ‰j                              | ‰‰‰	‰‰‰¦  «        }‰                     |‰j        ¬¦  «        S )Nrƒ   )rM   Ú_ParallelEncodeAsBufferr†   r‡   )
rÕ   rŠ   r\   r]   Ú	chunk_lenr   rc   r^   rZ   r˜   s
     €€€€€€€€r   r×   z9SentencePieceProcessor.ParallelEncode.<locals>._encode_np  sD   ø€ Ø”o×=Ò=Ø˜) [°+¸wÈÐQXñZô Z�à—}’} S°´�}Ñ9Ô9Ð9r   c                 ó&   •— g | ]} ‰|¦  «        ‘ŒS ro   ro   rÙ   s     €r   r‹   z9SentencePieceProcessor.ParallelEncode.<locals>.<listcomp>$  rÛ   r   c                 óÎ  •	— ‰t           u r ‰
j                             | ‰‰‰‰‰‰	¦  «        S ‰t          u s	‰t          u r+‰t          u }‰
j                             | ‰‰‰‰‰‰	‰|¦	  «	        S ‰dk    r!‰
j                             | ‰‰‰‰‰‰	‰¦  «        S ‰dk    r0‰
j                             | ‰‰‰‰‰‰	‰¦  «        }t          |¦  «        S t          d 	                    ‰¦  «        ¦  «        ‚rÝ   )
rO   rM   Ú_ParallelEncodeAsIdsr	   r
   Ú_ParallelEncodeAsPiecesÚ _ParallelEncodeAsSerializedProtor#   ri   rj   )rÕ   r™   r!   r\   r]   r  r_   rc   rd   r^   rZ   r˜   s      €€€€€€€€€r   rá   z6SentencePieceProcessor.ParallelEncode.<locals>._encode'  s+  ø€ Ø�cÐ!Ð!Ø”×;Ò;Ø˜) [°+¸wÈÐQXñZô Zð Zà�cÐ!Ð! [µEÐ%9Ð%9Ø +­uÐ 4�Ø”×>Ò>Ø˜) [°+¸wÈÐQXÐZhÐjvñxô xð xàÐ0Ò0Ð0Ø”×GÒGØ˜) [°+¸wÈÐQXÐZhñjô jð jà˜gÒ%Ð%Ø!œ_×MÒMØ˜) [°+¸wÈÐQXÐZhñjô j�
å˜zÑ*Ô*Ð*ÝÐ5×<Ò<¸[ÑIÔIÑJÔJÐJr   c                 ó&   •— g | ]} ‰|¦  «        ‘ŒS ro   ro   rã   s     €r   r‹   z9SentencePieceProcessor.ParallelEncode.<locals>.<listcomp>9  rä   r   )rN   rQ   rR   rS   rT   rX   r   rO   r‘   ri   r   r’   )rZ   r—   r\   r]   r^   r_   r  rc   r˜   rd   r[   rá   r×   r   s   ` ```````` @@@r   ÚParallelEncodez%SentencePieceProcessor.ParallelEncodeÿ  sÍ  øøøøøøøøøøøø€ ð ×/Ò/°¸XÑFÔFˆØˆ?Ø”mˆGØˆ?Ø”mˆGØˆ?Ø”mˆGØÐ!Ø!Ô1ˆNØÐØÔ+ˆKàÐ¥j°½cÑ&BÔ&BÐÝÐ5Ñ6Ô6Ð6ØÐ¥J¨y½#Ñ$>Ô$>ÐÀ)ÈqÂ.À.ÝÐ:Ñ;Ô;Ð;à˜'Ò!Ð!Ý‘”ˆBð:ð :ð :ð :ð :ð :ð :ð :ð :ð :ð :ð :õ
 ˜%¥Ñ&Ô&ð 6Ø5Ð5Ð5Ð5¨uÐ5Ñ5Ô5Ð5Ø�:˜eÑ$Ô$Ð$ð	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kð 	Kõ" �e�TÑ"Ô"ð 	/Ø.Ð.Ð.Ð.¨Ð.Ñ.Ô.Ð.Øˆw�u‰~Œ~Ðr   c                 ó,   —  | j         d|t          dœ|¤ŽS r¥   )r  r	   r¨   s      r   ÚParallelEncodeAsPiecesz-SentencePieceProcessor.ParallelEncodeAsPieces<  ó"   € Ø"ˆtÔ"ÐJ¨½CÐJÐJÀ6ÐJÐJÐJr   c                 ó,   —  | j         d|t          dœ|¤ŽS r¥   )r  rO   r¨   s      r   ÚParallelEncodeAsIdsz*SentencePieceProcessor.ParallelEncodeAsIds?  r  r   c                 ó"   —  | j         d|ddœ|¤ŽS r¯   ©r  r¨   s      r   ÚParallelEncodeAsNumpyz,SentencePieceProcessor.ParallelEncodeAsNumpyB  ó"   € Ø"ˆtÔ"ÐN¨¸GÐNÐNÀvÐNÐNÐNr   c                 ó"   —  | j         d|ddœ|¤ŽS r´   r"  r¨   s      r   ÚParallelEncodeAsSerializedProtoz6SentencePieceProcessor.ParallelEncodeAsSerializedProtoE  s$   € Ø"ˆtÔ"ÐY¨Ð<NÐYÐYÐRXÐYÐYÐYr   c                 ó"   —  | j         d|ddœ|¤ŽS r·   r"  r¨   s      r   ÚParallelEncodeAsProtoz,SentencePieceProcessor.ParallelEncodeAsProtoH  r$  r   c                 ó    — t          d¦  «        ‚r¼   r½   r¨   s      r   ÚParallelEncodeAsImmutableProtoz5SentencePieceProcessor.ParallelEncodeAsImmutableProtoK  r¿   r   c                 ó®  — |                       ||t          ¬¦  «        }|�|dk    rt          d¦  «        ‚|€| j        }|�t	          |t
          ¦  «        st          d¦  «        ‚|dk    rt          |¦  «        nd}|dk    rdn|}|                      |||||¬¦  «        }	|dk    r0t	          |	t          ¦  «        rd	„ |	D ¦   «         S t          |	¦  «        S |	S )
NrI   r-   r~   r   Fr,   r*   )r™   c                 ó,   — g | ]}t          |¦  «        ‘ŒS ro   r�   rŽ   s     r   r‹   z1SentencePieceProcessor.Decode.<locals>.<listcomp>]  r�   r   )rN   r	   ri   rX   r   rO   r‘   ÚboolÚ_decode_rawr’   r#   )
rZ   r—   rd   r[   rc   r˜   r™   r¡   Úcpp_return_typeÚraw_vals
             r   ÚDecodezSentencePieceProcessor.DecodeN  s  € Ø×/Ò/°¸XÍsÐ/ÑSÔSˆØÐ#¨Ð7GÒ(GÐ(GÝÐ_Ñ`Ô`Ð`ØÐØÔ+ˆKØÐ¥j°½cÑ&BÔ&BÐÝÐ5Ñ6Ô6Ð6à1<Ð@PÒ1PÐ1P�4 Ñ-Ô-Ð-ÐV[ÐØ0;¸wÒ0FÐ0FÐ,Ð,ÈKˆØ×"Ò" 5¨/¸;ÈÐbrÐ"ÑsÔsˆà˜'Ò!Ð!Ý˜'¥4Ñ(Ô(ð 5Ø4Ð4¨GÐ4Ñ4Ô4Ð4Ý˜7Ñ#Ô#Ð#Øˆr   c                 óÀ  — |€dS t          |¦  «        rt          |¦  «        dk    rdS n1t          |t          t          f¦  «        rt          |¦  «        dk    rdS d„ }t          |¦  «         }|r|g}t          |d¦  «        r|j        dk    }nt          |d         ¦  «        }|r ||d         ¦  «        }	n ||¦  «        }	t                               |	|f¦  «        }
|
r||
vr"t          d 
                    |¦  «        ¦  «        ‚|
|         }t          | j        |¦  «        }|dk    r|r |||||¦  «        S  |||¦  «        S |r ||||¦  «        S  ||¦  «        S )NÚ r   c                 ó”   — t          | ¦  «        dk    rdS t          | d¦  «        rdS t          | d         t          t          f¦  «        S )Nr   Fr„   )Úlenr   r   r	   r
   )Úseqs    r   Ú
_is_piecesz6SentencePieceProcessor._decode_raw.<locals>._is_piecesk  sE   € Ý�3‰xŒx˜1Š}ˆ}Ø�uÝ�s˜GÑ$Ô$ð Ø�uÝ˜c !œf¥s­E lÑ3Ô3Ð3r   Úndimr   z$unknown return_type={} or input typer-   )r   r5  r   r	   r
   r   r8  Ú_DECODE_DISPATCHr”   ri   rj   r•   rM   )rZ   r—   rd   rc   r˜   r™   r7  Ú	is_singlerš   Úinput_is_piecesr›   rœ   r�   s                r   r.  z"SentencePieceProcessor._decode_rawa  s¹  € Øˆ=Ø�2Ý˜ÑÔð 	Ý�5‰zŒz˜QŠˆØ�rð å˜¥¥U˜|Ñ,Ô,ð 	µ°U±´¸q²°Ø�2ð	4ð 	4ð 	4õ % UÑ+Ô+Ð+ˆ	Øð 	Ø�GˆEõ �5˜&Ñ!Ô!ð 	.Ø”z A’~ˆHˆHå# E¨!¤HÑ-Ô-ˆHð ð 	0Ø(˜j¨¨q¬Ñ2Ô2ˆOˆOà(˜j¨Ñ/Ô/ˆOõ #×&Ò&¨¸Ð'BÑCÔCˆØð 	Y˜+¨WÐ4Ð4ÝÐC×JÒJÈ;ÑWÔWÑXÔXÐXà˜kÔ*ˆÝ˜œ¨+Ñ6Ô6ˆàÐ*Ò*Ð*Øð 3Ø�v˜e [°+¸|ÑLÔLÐLà�v˜e \Ñ2Ô2Ð2àð %Ø�v˜e [°+Ñ>Ô>Ð>à�v˜e‘}”}Ð$r   c                 ó"   —  | j         d||dœ|¤ŽS r¥   ©r1  ©rZ   r—   rd   rr   s       r   ÚDecodePiecesz#SentencePieceProcessor.DecodePieces–  ó!   € ØˆtŒ{ÐJ °KÐJÐJÀ6ÐJÐJÐJr   c                 ó"   —  | j         d||dœ|¤ŽS r¥   r=  r>  s       r   Ú	DecodeIdsz SentencePieceProcessor.DecodeIds™  r@  r   r*   c                 ó"   —  | j         d||dœ|¤ŽS r¥   r=  r>  s       r   ÚDecodePiecesAsSerializedProtoz4SentencePieceProcessor.DecodePiecesAsSerializedProtoœ  r@  r   c                 ó"   —  | j         d||dœ|¤ŽS r¥   r=  r>  s       r   ÚDecodeIdsAsSerializedProtoz1SentencePieceProcessor.DecodeIdsAsSerializedProtoŸ  r@  r   c                 ó"   —  | j         d|ddœ|¤ŽS r·   r=  r¨   s      r   ÚDecodePiecesAsProtoz*SentencePieceProcessor.DecodePiecesAsProto¢  r²   r   c                 ó"   —  | j         d|ddœ|¤ŽS r·   r=  r¨   s      r   ÚDecodeIdsAsProtoz'SentencePieceProcessor.DecodeIdsAsProto¥  r²   r   c                 ó    — t          d¦  «        ‚r¼   r½   r¨   s      r   ÚDecodePiecesAsImmutableProtoz3SentencePieceProcessor.DecodePiecesAsImmutableProto¨  r¿   r   c                 ó    — t          d¦  «        ‚r¼   r½   r¨   s      r   ÚDecodeIdsAsImmutableProtoz0SentencePieceProcessor.DecodeIdsAsImmutableProto«  r¿   r   c                 ó„   ‡ ‡— t          |t          ¦  «        rˆˆ fd„|D ¦   «         S ‰ j                             |‰¦  «        S )Nc                 óF   •— g | ]}‰j                              |‰¦  «        ‘ŒS ro   )rM   Ú_CalculateEntropy)r‰   r   rb   rZ   s     €€r   r‹   z;SentencePieceProcessor.CalculateEntropy.<locals>.<listcomp>°  s+   ø€ ÐOÐOÐOÀA�D”O×5Ò5°a¸Ñ?Ô?ÐOÐOÐOr   )r   r’   rM   rQ  )rZ   r—   rb   s   ` `r   ÚCalculateEntropyz'SentencePieceProcessor.CalculateEntropy®  sL   øø€ Ý�e�TÑ"Ô"ð 	PØOÐOÐOÐOÐOÈÐOÑOÔOÐOØŒ×0Ò0°¸Ñ>Ô>Ð>r   c                 óÖ   ‡ — t          |t          ¦  «        r|rˆ fd„|D ¦   «         S ˆ fd„|D ¦   «         S |r‰ j                             |¦  «        S ‰ j                             |¦  «        S )Nc                 óD   •— g | ]}‰j                              |¦  «        ‘ŒS ro   )rM   Ú_NormalizeWithOffsets©r‰   r   rZ   s     €r   r‹   z4SentencePieceProcessor.Normalize.<locals>.<listcomp>¶  s)   ø€ ÐPÐPÐPÀQ˜œ×=Ò=¸aÑ@Ô@ÐPÐPÐPr   c                 óD   •— g | ]}‰j                              |¦  «        ‘ŒS ro   )rM   Ú
_NormalizerV  s     €r   r‹   z4SentencePieceProcessor.Normalize.<locals>.<listcomp>·  s)   ø€ ÐAÐAÐA°a�D”O×.Ò.¨qÑ1Ô1ÐAÐAÐAr   )r   r’   rM   rU  rX  ©rZ   r—   Úwith_offsetss   `  r   Ú	Normalizez SentencePieceProcessor.Normalize³  sˆ   ø€ Ý�e�TÑ"Ô"ð 	BØð QØPÐPÐPÐPÈ%ÐPÑPÔPÐPØAÐAÐAÐA¸5ÐAÑAÔAÐAØð 	@Ø”?×8Ò8¸Ñ?Ô?Ð?ØŒ×)Ò)¨%Ñ0Ô0Ð0r   c                 ór   — d„ |                      ¦   «         D ¦   «         }| j                             |¦  «        S )Nc                 ó4   — i | ]\  }}|t          |¦  «        “ŒS ro   ©r	   )r‰   ÚkeyÚvalues      r   ú
<dictcomp>zASentencePieceProcessor.OverrideNormalizerSpec.<locals>.<dictcomp>½  s$   € ÐGÐGÐG©*¨#¨u�c�3˜u™:œ:ÐGÐGÐGr   )ÚitemsrM   Ú_OverrideNormalizerSpec)rZ   rr   Ú
new_kwargss      r   ÚOverrideNormalizerSpecz-SentencePieceProcessor.OverrideNormalizerSpec¼  s3   € ØGÐG¸¿º¹¼ÐGÑGÔGˆ
ØŒ×6Ò6°zÑBÔBÐBr   c                 ó4   — | j                              ¦   «         S r   )rM   ÚGetPieceSize©rZ   s    r   rg  z#SentencePieceProcessor.GetPieceSizeÀ  s   € ØŒ×+Ò+Ñ-Ô-Ð-r   c                 ó6   — | j                              |¦  «        S r   )rM   Ú	PieceToId©rZ   Úpieces     r   rj  z SentencePieceProcessor.PieceToIdÃ  s   € ØŒ×(Ò(¨Ñ/Ô/Ð/r   c                 ó6   — | j                              |¦  «        S r   )rM   Ú	IdToPiece©rZ   Úids     r   rn  z SentencePieceProcessor.IdToPieceÆ  ó   € ØŒ×(Ò(¨Ñ,Ô,Ð,r   c                 ó6   — | j                              |¦  «        S r   )rM   ÚGetScorero  s     r   rs  zSentencePieceProcessor.GetScoreÉ  ó   € ØŒ×'Ò'¨Ñ+Ô+Ð+r   c                 ó6   — | j                              |¦  «        S r   )rM   Ú	IsUnknownro  s     r   rv  z SentencePieceProcessor.IsUnknownÌ  rq  r   c                 ó6   — | j                              |¦  «        S r   )rM   Ú	IsControlro  s     r   rx  z SentencePieceProcessor.IsControlÏ  rq  r   c                 ó6   — | j                              |¦  «        S r   )rM   ÚIsUnusedro  s     r   rz  zSentencePieceProcessor.IsUnusedÒ  rt  r   c                 ó6   — | j                              |¦  «        S r   )rM   ÚIsBytero  s     r   r|  zSentencePieceProcessor.IsByteÕ  s   € ØŒ×%Ò% bÑ)Ô)Ð)r   c                 ó4   — | j                              ¦   «         S r   )rM   Úunk_idrh  s    r   r~  zSentencePieceProcessor.unk_idØ  ó   € ØŒ×%Ò%Ñ'Ô'Ð'r   c                 ó4   — | j                              ¦   «         S r   )rM   Úbos_idrh  s    r   r�  zSentencePieceProcessor.bos_idÛ  r  r   c                 ó4   — | j                              ¦   «         S r   )rM   Úeos_idrh  s    r   rƒ  zSentencePieceProcessor.eos_idÞ  r  r   c                 ó4   — | j                              ¦   «         S r   )rM   Úpad_idrh  s    r   r…  zSentencePieceProcessor.pad_idá  r  r   c                 ó4   — | j                              ¦   «         S r   )rM   Úserialized_model_protorh  s    r   r‡  z-SentencePieceProcessor.serialized_model_protoä  s   € ØŒ×5Ò5Ñ7Ô7Ð7r   c                 ó*   — |                       ¦   «         S r   ©r‡  rh  s    r   Ú__getstate__z#SentencePieceProcessor.__getstate__ê  ó   € Ø×*Ò*Ñ,Ô,Ð,r   c                 ó*   — |                       ¦   «         S r   r‰  rh  s    r   Ú__getstate_for_pickle__z.SentencePieceProcessor.__getstate_for_pickle__í  r‹  r   c                 óX   — |                       ¦   «          |                      |¦  «         d S r   )re   rv   ©rZ   r‡  s     r   Ú__setstate__z#SentencePieceProcessor.__setstate__ð  s*   € Ø�Š‰ŒˆØ×$Ò$Ð%;Ñ<Ô<Ð<Ð<Ð<r   c                 ó*   — |                       ¦   «         S r   )rg  rh  s    r   r   zSentencePieceProcessor.__len__ô  s   € Ø× Ò Ñ"Ô"Ð"r   c                 ó,   — |                       |¦  «        S r   )rj  rk  s     r   Ú__getitem__z"SentencePieceProcessor.__getitem__÷  s   € Ø�~Š~˜eÑ$Ô$Ð$r   )NNNFFFFFrF   rG   rF   Nr   ©NN)NNNNNNNNNNNN)NNNNNNN)
NNNNNNNNNN)	NNNNNNNNN)NNNNN)F)r*   )PÚ__name__Ú
__module__Ú__qualname__re   rl   rN   Úclassmethodrt   rw   rY   rp   rv   r£   r©   r­   r±   rµ   r¸   rº   r¾   rÄ   rÇ   rÉ   rÌ   rÎ   rÐ   rå   rê   rí   rð   ró   rõ   r÷   r  r  r	  r  r  r  r  r  r   r#  r&  r(  r*  r1  r.  r	   r?  rB  rD  rF  rH  rJ  rL  rN  rR  r[  re  rg  rj  rn  rs  rv  rx  rz  r|  r~  r�  rƒ  r…  r‡  Ú
piece_sizeÚ
vocab_sizerŠ  r�  r�  r   r“  ro   r   r   rE   rE   ˆ   sd  € € € € € à Ø!ØØØØØ %Ø!&ØØØØ!ðFð Fð Fð Fð8	ð 	ð 	ðCð Cð Cð Cð ðð ñ „[ðð ðð ñ „[ððPð Pð Pð Pð6ð 6ð 6ðCð Cð Cð ØØØ"Ø#ØØØØØØØ ð`ð `ð `ð `ðDCð Cð CðCð Cð CðGð Gð GðRð Rð RðGð Gð GðPð Pð Pðtð tð tðLð Lð Lð LðLð Lð Lð LðPð Pð Pð Pð[ð [ð [ð [ðPð Pð Pð Pðtð tð tð tð
 !Ø Ø Ø#'Ø#Ø $Ø!ð7ð 7ð 7ð 7ðr_ð _ð _ð _ð_ð _ð _ð _ðcð cð cð cðnð nð nð nðcð cð cð cðtð tð tð tð
 &*Ø%)Ø%)Ø,0Ø)-Ø#'Ø!%Ø*.Ø)-Ø&*ð9ð 9ð 9ð 9ðvwð wð wð wðwð wð wð wðFð Fð Fð Fð{ð {ð {ð {ðtð tð tð tð
  $Ø#Ø#Ø&*Ø!%Ø#'Ø#'Ø#'Ø $ð;ð ;ð ;ð ;ðzKð Kð KðKð Kð KðOð Oð OðZð Zð ZðOð Oð Oðtð tð tðð ð ð ð&3%ð 3%ð 3%ð 3%ðj /2ð Kð Kð Kð Kð ,/ð Kð Kð Kð KðKð Kð Kð KðKð Kð Kð KðGð Gð GðGð Gð Gðtð tð tðtð tð tð?ð ?ð ?ð
1ð 1ð 1ð 1ðCð Cð Cð.ð .ð .ð0ð 0ð 0ð-ð -ð -ð,ð ,ð ,ð-ð -ð -ð-ð -ð -ð,ð ,ð ,ð*ð *ð *ð(ð (ð (ð(ð (ð (ð(ð (ð (ð(ð (ð (ð8ð 8ð 8ð €JØ€Jð-ð -ð -ð-ð -ð -ð=ð =ð =ð#ð #ð #ð%ð %ð %ð %ð %r   rE   c                   ó"   — e Zd Zdd„Zd„ Zd„ ZdS )Ú
_LogStreamNc                 ól   — || _         | j         �%t          j                             ¦   «         | _        d S d S r   )ÚostreamÚsysÚstderrÚfilenoÚorig_stream_fileno)rZ   rž  s     r   re   z_LogStream.__init__ü  s5   € ØˆŒØŒ<Ð#Ý&)¤j×&7Ò&7Ñ&9Ô&9ˆDÔ#Ð#Ð#ð $Ð#r   c                 ó¶   — | j         �Qt          j        | j        ¦  «        | _        t          j        | j                              ¦   «         | j        ¦  «         d S d S r   )rž  ÚosÚdupr¢  Úorig_stream_dupÚdup2r¡  rh  s    r   Ú	__enter__z_LogStream.__enter__  sP   € ØŒ<Ð#Ý#%¤6¨$Ô*AÑ#BÔ#BˆDÔ ÝŒG�D”L×'Ò'Ñ)Ô)¨4Ô+BÑCÔCÐCÐCÐCð $Ð#r   c                 óì   — | j         �lt          j        | j        ¦  «         t          j        | j        | j        ¦  «         t          j        | j        ¦  «         | j                              ¦   «          d S d S r   )rž  r¤  Úcloser¢  r§  r¦  )rZ   Útyper`  Ú	tracebacks       r   Ú__exit__z_LogStream.__exit__  sj   € ØŒ<Ð#ÝŒH�TÔ,Ñ-Ô-Ð-ÝŒG�DÔ(¨$Ô*AÑBÔBÐBÝŒH�TÔ)Ñ*Ô*Ð*ØŒL×ÒÑ Ô Ð Ð Ð ð	 $Ð#r   r   )r•  r–  r—  re   r¨  r­  ro   r   r   rœ  rœ  û  sI   € € € € € ð:ð :ð :ð :ð
Dð Dð Dð
!ð !ð !ð !ð !r   rœ  c                   ó>   — e Zd Zedd„¦   «         Zedd„¦   «         ZdS )ÚSentencePieceTrainerNc                 ój  — | �4t          | t          ¦  «        rt          j                             | ¦  «        S d„ }d }d }d }i }|                     ¦   «         D ](\  }}|dv r|}Œ|dv r|}Œ|dv r|}Œ ||¦  «        ||<   Œ)|r|                     ¦   «         |d<   |rX|r!t          j                             ||¦  «        }	nt          j                             |¦  «        }	| 	                    |	¦  «         nA|r t          j         
                    ||¦  «        S t          j                             |¦  «        S d S )Nc                 ó  — t          | t          ¦  «        r\t          j        ¦   «         }t	          j        |d¬¦  «        }|                     d„ | D ¦   «         ¦  «         |                     ¦   «         S t          | ¦  «        S )Nr3  )Úlineterminatorc                 ó,   — g | ]}t          |¦  «        ‘ŒS ro   r^  )r‰   Úvs     r   r‹   z@SentencePieceTrainer._Train.<locals>._encode.<locals>.<listcomp>  s   € Ð 7Ð 7Ð 7¨A¥ Q¡¤Ð 7Ð 7Ð 7r   )	r   r’   ÚioÚStringIOÚcsvÚwriterÚwriterowÚgetvaluer	   )r`  Úfr¸  s      r   rá   z,SentencePieceTrainer._Train.<locals>._encode  sn   € Ý˜%¥Ñ&Ô&ð "Ý”K‘M”M�Ýœ A°bÐ9Ñ9Ô9�Ø—’Ð 7Ð 7°Ð 7Ñ 7Ô 7Ñ8Ô8Ð8Ø—z’z‘|”|Ð#å˜5‘z”zÐ!r   )Úsentence_iteratorÚsentence_reader)Úmodel_writer)Ú
normalizerÚ_serialized_normalizer_spec)r   r	   r   r¯  Ú_TrainFromStringrb  Úserialized_normalizer_specÚ_TrainFromMap4Ú_TrainFromMap3ÚwriteÚ_TrainFromMap2Ú_TrainFromMap)
Úargrr   rá   r¼  r¾  r¿  rd  r_  r`  rL   s
             r   Ú_TrainzSentencePieceTrainer._Train  su  € àˆ?�z¨#­sÑ3Ô3ˆ?Ý!Ô6×GÒGÈÑLÔLÐLð	"ð 	"ð 	"ð !ÐØˆØˆ
Øˆ
Ø Ÿ,š,™.œ.ð 	1ð 	1‰JˆC�ØÐ>Ð>Ð>Ø$)Ð!Ð!ØÐ(Ð(Ð(Ø$��Ø˜Ð&Ð&Ø"�
�
à") '¨%¡.¤.�
˜3‘�àð 	`Ø8B×8]Ò8]Ñ8_Ô8_ˆJÐ4Ñ5àð 
	UØ ð ]Ý,ÔA×PÒPÐQ[Ð]nÑoÔo��å,ÔA×PÒPÐQ[Ñ\Ô\�Ø×Ò˜{Ñ+Ô+Ð+Ð+à ð UÝ%Ô:×IÒIÈ*ÐVgÑhÔhÐhå%Ô:×HÒHÈÑTÔTÐTØˆtr   c                 ó|   — t          |¬¦  «        5  t          j        dd| i|¤Žcd d d ¦  «         S # 1 swxY w Y   d S )N)rž  rÈ  ro   )rœ  r¯  rÉ  )rÈ  Ú	logstreamrr   s      r   ÚTrainzSentencePieceTrainer.Train;  s¤   € å 	Ð*Ñ*Ô*ð 	Bð 	BÝ'Ô.ÐAÐA°3ÐA¸&ÐAÐAð	Bð 	Bð 	Bð 	Bñ 	Bô 	Bð 	Bð 	Bð 	Bð 	Bð 	Bð 	Bøøøð 	Bð 	Bð 	Bð 	Bð 	Bð 	Bs   ‘1±5¸5r   r”  )r•  r–  r—  ÚstaticmethodrÉ  rÌ  ro   r   r   r¯  r¯    sY   € € € € € Øð)ð )ð )ñ „\ð)ðV ðBð Bð Bñ „\ðBð Bð Br   r¯  c                   óN   — e Zd Z	 	 	 	 	 	 	 	 	 d
d„Zd„ Zd„ Zd„ Zdd„Zd„ Zd	„ Z	dS )ÚSentencePieceNormalizerNFc
                 óâ  — t          j        ¦   «         | _        |r| j                             |¦  «         nè|r?t	          |d¦  «        r|                     ¦   «         }| j                             |¦  «         n§|r?t	          |d¦  «        r|                     ¦   «         }| j                             |¦  «         nf|r| j                             |¦  «         nI|r| j         	                    |¦  «         n,|r| j         
                    |¦  «         nt          d¦  «        ‚| j                             d|¦  «         | j                             d|¦  «         | j                             d|	¦  «         d S )Nr|   zno model is specifiedÚadd_dummy_prefixÚescape_whitespacesÚremove_extra_whitespaces)r   rÏ  Ú_normalizerrp   r   r|   rv   Ú LoadFromSerializedNormalizerSpecÚLoadFromRuleTSVÚLoadFromRuleNameÚLoadFromMapri   Ú_SetProtoField)
rZ   rK   rL   Únormalizer_specÚrule_tsvÚ	rule_nameÚnorm_maprÑ  rÒ  rÓ  s
             r   re   z SentencePieceNormalizer.__init__B  s�  € õ *ÔAÑCÔCˆÔàð 	6ØÔ×)Ò)¨*Ñ5Ô5Ð5Ð5Øð 	6Ý�{Ð$7Ñ8Ô8ð >Ø)×;Ò;Ñ=Ô=�ØÔ×4Ò4°[ÑAÔAÐAÐAØð 	6Ý�Ð(;Ñ<Ô<ð FØ"1×"CÒ"CÑ"EÔ"E�ØÔ×=Ò=¸oÑNÔNÐNÐNØð 	6ØÔ×,Ò,¨XÑ6Ô6Ð6Ð6Øð 	6ØÔ×-Ò-¨iÑ8Ô8Ð8Ð8Øð 	6ØÔ×(Ò(¨Ñ2Ô2Ð2Ð2åÐ4Ñ5Ô5Ð5àÔ×'Ò'Ð(:Ð<LÑMÔMÐMØÔ×'Ò'Ð(<Ð>PÑQÔQÐQØÔ×'Ò'Ð(BÐD\Ñ]Ô]Ð]Ð]Ð]r   c                 ó4   — | j                              ¦   «         S r   )rÔ  Ú	Decompilerh  s    r   rß  z!SentencePieceNormalizer.Decompilee  s   € ØÔ×)Ò)Ñ+Ô+Ð+r   c                 ó4   — | j                              ¦   «         S r   ©rÔ  r‡  rh  s    r   r‡  z.SentencePieceNormalizer.serialized_model_protoh  ó   € ØÔ×6Ò6Ñ8Ô8Ð8r   c                 ó4   — | j                              ¦   «         S r   )rÔ  rÂ  rh  s    r   rÂ  z2SentencePieceNormalizer.serialized_normalizer_speck  s   € ØÔ×:Ò:Ñ<Ô<Ð<r   c                 óÖ   ‡ — t          |t          ¦  «        r|rˆ fd„|D ¦   «         S ˆ fd„|D ¦   «         S |r‰ j                             |¦  «        S ‰ j                             |¦  «        S )Nc                 óD   •— g | ]}‰j                              |¦  «        ‘ŒS ro   )rÔ  rU  rV  s     €r   r‹   z5SentencePieceNormalizer.Normalize.<locals>.<listcomp>q  s*   ø€ ÐQÐQÐQÀa˜Ô(×>Ò>¸qÑAÔAÐQÐQÐQr   c                 óD   •— g | ]}‰j                              |¦  «        ‘ŒS ro   )rÔ  rX  rV  s     €r   r‹   z5SentencePieceNormalizer.Normalize.<locals>.<listcomp>r  s*   ø€ ÐBÐBÐB°q�DÔ$×/Ò/°Ñ2Ô2ÐBÐBÐBr   )r   r’   rÔ  rU  rX  rY  s   `  r   r[  z!SentencePieceNormalizer.Normalizen  sŠ   ø€ Ý�e�TÑ"Ô"ð 	CØð RØQÐQÐQÐQÈ5ÐQÑQÔQÐQØBÐBÐBÐB¸EÐBÑBÔBÐBØð 	AØÔ#×9Ò9¸%Ñ@Ô@Ð@ØÔ×*Ò*¨5Ñ1Ô1Ð1r   c                 ó4   — | j                              ¦   «         S r   rá  rh  s    r   rŠ  z$SentencePieceNormalizer.__getstate__w  râ  r   c                 ój   — t          j        ¦   «         | _        | j                             |¦  «         d S r   )r   rÏ  rÔ  rv   r�  s     r   r�  z$SentencePieceNormalizer.__setstate__z  s1   € Ý)ÔAÑCÔCˆÔØÔ×0Ò0Ð1GÑHÔHÐHÐHÐHr   )	NNNNNNFFFr   )
r•  r–  r—  re   rß  r‡  rÂ  r[  rŠ  r�  ro   r   r   rÏ  rÏ  A  s¬   € € € € € à Ø!Ø!%ØØØØ"'Ø$)Ø*/ð!^ð !^ð !^ð !^ðF,ð ,ð ,ð9ð 9ð 9ð=ð =ð =ð2ð 2ð 2ð 2ð9ð 9ð 9ðIð Ið Ið Ið Ir   rÏ  c                 óV  — dd l }i }| j                             ¦   «         D ]]\  }}|                     d|¦  «        rB|                     dd|¦  «                             ¦   «                              dd¦  «        }|||<   Œ^|                     ¦   «         D ]\  }}t          | ||¦  «         Œd S )Nr   z^[A-Z]+z(?<!^)(?=[A-Z])Ú_Ún_bestÚnbest)ÚreÚ__dict__rb  ÚmatchÚsubÚlowerÚreplaceÚsetattr)Ú	classnamerí  Ú	snake_mapÚkr´  Úsnakes         r   Ú_add_snake_caserø  €  sÂ   € Ø€I€I€IØ€IØÔ"×(Ò(Ñ*Ô*ð !ð !‰ˆˆ1Ø�8Š8�J Ñ"Ô"ð 	!Ø—F’FÐ-¨s°AÑ6Ô6×<Ò<Ñ>Ô>×FÒFÀxÐQXÑYÔYˆEØ ˆI�eÑøØ—’Ñ!Ô!ð !ð !‰ˆˆ1Ý�	˜1˜aÑ Ô Ð Ð ð!ð !r   )Ú__version__r   Úpackage_data).r¤  rŸ  r·  rµ  Úimportlib.resourcesÚ	importlibr3  r   r   r   r   r   r   r   r   r#   r'   rO   r	   r
   r“   r9  Ú
ThreadPoolÚSetRandomGeneratorSeedÚSetMinLogLevelÚSetNBestTimeoutÚ
SetDataDirÚset_random_generator_seedÚset_min_log_levelÚset_nbest_timeoutrE   rœ  r¯  rÏ  rø  r£   ÚTokenizer1  Ú
DetokenizeÚ_versionrù  ÚpathÚjoinÚ	resourcesÚfilesÚ	Exceptionro   r   r   ú<module>r     sx  ðØ 	€	€	€	Ø 
€
€
€
Ø 
€
€
€
Ø 	€	€	€	Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð ðEð Eð Eð Ð ØÐ ð ð  ð  ð €Ø	€ðð ð ðð ð ðð ð ð  	ˆ^ØÐØÐ ØÐ6ØÐ+ØÐ2ØÐ"ðð 	Ð ØÐ#ØÐ%ØÐ;ØÐ0ØÐ7ØÐ'ððð Ð ðB 	ˆ\ØÐ"ØÐ9ØÐ2ð	ð 	ÐØÐ'ØÐ>ØÐ7ð	ð 	ˆ_ØÐ%ØÐ<ØÐ2ð	ð 	Ð!ØÐ*ØÐAØÐ7ð	ð/ð Ð ð@ Ô&€
à'Ô>Ð ØÔ.€Ø Ô0€ØÔ&€
à2Ð Ø"Ð Ø#Ð ðp	%ð p	%ð p	%ð p	%ð p	%ñ p	%ô p	%ð p	%ðf!ð !ð !ð !ð !ñ !ô !ð !ð&0Bð 0Bð 0Bð 0Bð 0Bñ 0Bô 0Bð 0Bðf;Ið ;Ið ;Ið ;Ið ;Iñ ;Iô ;Ið ;Ið~!ð !ð !ð #9Ô"?Ð Ô Ø$:Ô$AÐ Ô !ð €Ð&Ñ 'Ô 'Ð 'Ø €Ð$Ñ %Ô %Ð %Ø €Ð'Ñ (Ô (Ð (ð "Ð !Ð !Ð !Ð !Ð !ð	Ø€JˆrŒw�|Š|˜C˜C 	Ô 3× 9Ò 9¸/Ñ JÔ JÑKÔKÈ^Ñ\Ô\Ñ]Ô]Ð]Ð]Ð]øØð 	ð 	ð 	Ø€D€Dð	øøøs   ÄAE! Å!E*Å)E*