mjdZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl m ZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZmZdd lmZm Z m!Z!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8m9Z9e9j:e;ZdZ?dZ@dZAe.dz Z.ee e!e"dZBe=e@dZCe7e.Gdde1ZDeDZEdS) z Tokenization classes for fast tokenizers (provided by HuggingFace's tokenizers library). For slow (python) tokenizers see tokenization_utils.py N) defaultdict)Iterable)copyfile)Any)is_offline_mode) AddedToken processors)Encoding) Tokenizer)Decoder)BPEUnigram) BpeTrainerUnigramTrainerWordLevelTrainerWordPieceTrainer) cached_file) SpmConverter)convert_gguf_tokenizer)is_tekken_vocab_filename)load_gguf_checkpoint)INIT_TOKENIZER_DOCSTRING BatchEncodingPreTokenizedInputPreTrainedTokenizerBase TextInputTruncationStrategygenerate_merges)PaddingStrategyadd_end_docstringshf_apiloggingztokenizer.jsonzspecial_tokens_map.jsonztokenizer_config.jsonztokenizer.modelzadded_tokens.jsonu tokenizer_object ([`tokenizers.Tokenizer`]): A [`tokenizers.Tokenizer`] object from 🤗 tokenizers to instantiate from. See [Using tokenizers from 🤗 tokenizers](../fast_tokenizers) for more information. tokenizer_file ([`str`]): A path to a local JSON file representing a previously serialized [`tokenizers.Tokenizer`] object from 🤗 tokenizers. )r r WordLevel WordPiece)tokenizer_file vocab_filec) eZdZdZeZdZdZedOdZ fdZ e de fdZ e de fdZdPd ed edzdeefd Z dQd eejzd e dzd edzd e dedzdeedff fd ZdZe dZe dZejdZejdZdZe defdZdeeeffdZe deeeffdZe deeeffdZe deee ffdZ!eZ"e!Z#deeeffdZ$de fdZ%defdZ&e de'fdZ(e de)fd Z* dRd"e+d#e dzd$e dzd%e d&e d'e d(e d)e deeee,fe-e+ffd*Z.d+edefd,Z/d-ededzfd.Z0dOd/e-ee zdefd0Z1dOd1e defd2Z2dOd3ee-ezd4e dee-ezfd5Z3dSd6ed1edzd7e de-efd8Z4d9e5d:e6d;ededzf d?Z7dd!e5j8e6j9dd@ddddddddddd!dfd6e:e;ze-e:ze-e;zdAe:e;ze-e:ze-e;zdzd7e d9e5d:e6d;edzdedzdCe dzd#e dzd$e dzd%e d&e d'e d(e d)e dDe dzde dTdHee-ezd4e dIe dzdefdJZ? dUd eejzdKeedfd e dzd edzdeedff dLZ@ dVdMZAe dWdNZBxZCS)XTokenizersBackendaQ Base class for all fast tokenizers (wrapping HuggingFace tokenizers library). Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`]. Handles all the shared methods for tokenization and special tokens, as well as methods for downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary. This class also contains the added tokens in a unified way on top of all tokenizers so we don't have to handle the specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...). NFc $t|}|dd}|Ltj|r-|t us d|jvs|rtj||d<|S|ntj|rNt|d5}tj |}dddn #1swxYwY| di d}|d vret|}t|d} i| d <|d krg| d <| |d<g|d <tj tj|} ntj|} | j|d<| j|d<| j|d<| j | j|d<| j | j|d<| d} | r| dddkr | d} nt%| t&s| g} | D]C} | ddkr(d| vr$ddl} | | d|d<nD| di d d}|j8t%|t&r"t't/t0|}n|jjdkrFt%|t&r0|r.t%|dt&t0fr d|D}nx|jjdkrdt5|D}nN|jjd ks|jjdkr.t%|t&rdt5|D}||d <t7|dd}d | divr,|r*|jd kr|dd }d |D}||d <|S| d!}| d"}| d }| d }t%|t8rXtj|r9t;|r*d#d$lm}||}| |d<|St%|t8rtj|r|!d%r d#d&l"m#}||}|j$|jfi|} d#d'l"m%}| |j}|tM|d(r |j'dAi|}n=#tP$r0}tR*d)|jd*|d+Yd}~nd}~wwxYwtM|d,r |j+dAi|}d|vr|t us d|jvr|d d}|d d}| d-pi}||rd.|,D}|,D]a\}}t[|}t9|}| |}|r'||kr!||vr||||<|||<bt]j/|j0||/}|||d<|j0j1}|j2dkr|3d0|j4pd1|j5dkr|3d2|j6pd3|j7dkr|3d4|j8pd5nu#tP$rh}tR*d6|d7|d8d#d9l"m9} | || d:;}| |d<Yd}~nd}~wwxYw|S|At%|t8r,tj|r ||d <|d }|At%|t8r,tj|r ||d <|d }||j|jjd krt%|trd $gd?}!ty}"|!D]+}#|#|vr%|"=$||#g,t}||"@}||d <|S)Bz Build a `tokenizers.Tokenizer` backend from the available serialization files (tokenizer.json, sentencepiece models, tekken.json, vocab/merges). r&N__init__tokenizer_objectutf-8encodingmodeltype)Nrvocabr merges added_tokenspost_processortokenizer_paddingtokenizer_truncation_json_truncation _json_padding normalizerSequence normalizers Precompiledprecompiled_charsmapr_spm_precompiled_charsmaprc,g|]}t|S)tuple).0items XC:\Projects\DSCons\.venv\Lib\site-packages\transformers\tokenization_utils_tokenizers.py z>TokenizersBackend.convert_to_native_format..s;;;TU4[[;;;r$ci|]\}}|| SrArArCitokens rE z>TokenizersBackend.convert_to_native_format..sCCChaCCCrGr%cTi|]%\}}t|tr|dn||&Sr) isinstancelistrIs rErLz>TokenizersBackend.convert_to_native_format..s8pppS[STV[E4)@)@KU1XXeQppprGcg|]H}t|tr"t|dnt|IS) )rOstrrBsplit)rCmerges rErFz>TokenizersBackend.convert_to_native_format..sKrrrbgZs5K5K]% C 0 0111QVW\Q]Q]rrrrGr' merges_filer)MistralConverter.model)SentencePieceExtractor)SLOW_TO_FAST_CONVERTERSconvert_from_spmz,Could not reorder vocab using converter for z due to z/. Falling back to raw SentencePiece extraction.convert_from_spm_modeladded_tokens_decoderci|]\}}|| SrArA)rCrKtoken_ids rErLz>TokenizersBackend.convert_to_native_format..s&\&\&\?5(x&\&\&\rG)protor2r3 bos_token eos_token unk_tokenzz+Could not extract SentencePiece model from z$ using sentencepiece library due to z%. Falling back to TikToken extractor.)TikTokenConverterextra_special_tokens)r'rgvaluesreturncg}|D]b}|t|ttfr||@|t |c|SN)rOrPrBextendappendrS)rh collectedval_iter_special_tokenss rErpzHTokenizersBackend.convert_to_native_format.._iter_special_tokens)s{') !33C{ !#e}553!(()=)=c)B)BCCCC!((S2222  rG) pad_tokenrerarc sep_token cls_token mask_tokenadditional_special_tokensrg) skip_tokensrA)?dictpopospathisfiler)__dict__ TokenizerFast from_fileopenjsonloadgetfrom_strdumpsr5padding truncationrOrPbase64 b64decoder0maprB__name__ enumerategetattrrSrintegrations.mistral.tokenizerrW convertedendswithconvert_slow_tokenizerrYextractrZhasattrr[ Exceptionloggerwarningr\itemsintrbuild_tokenizer_from_spm_protor` trainer_specbos_id setdefault bos_pieceeos_id eos_pieceunk_id unk_piecerfrrsetupdater)%clstrust_remote_codekwargs local_kwargsfast_tokenizer_filetokenizer_handletokenizer_json model_typeminimal_tokenizer_json minimal_model tok_from_filenormalizer_configr:rr2r3r'rVrW converterrY extractorrZconverter_classer] id_to_tokenr_ new_token current_tokenr, proto_specrfspecial_tokens_keysrvkeyrps% @rEconvert_to_native_formatz*TokenizersBackend.convert_to_native_formatfs F|| *../?FF +233 ,)))Zs|-K-KO`-K/>CY]gCgCg% DJDTDT&'=>EE %@A"&&w3377FFEy eT**4 UE!2!233E#y00eT**--gr:::: :zObfkOkOk'0:rrkqrrr)/ X& !%%l33 "&&}55   ))!!(++ j# & & 27>>*+E+E JbcmJnJn H H H H H H((44I/8/B/B/D/DL+ ,  j# & &H 27>>*+E+EH *J]J]^fJgJgH F IJJJJJJ32:>> 0y0KKlKK  OOOOOO&=&A&A#,&O&OO&2wPb7c7c2'G'G'W'W,'W'W  NNPs|PP]^PPP3 899N#=3#=#M#M #M#ML &\99,,, #,0N0N(,,Wd;;E)--h==F,8+;+;jc::>rw~~k?Z?Z>%0L "!(+F >ci3 8Je8S8SXbchjnXoXo8S !Xc] !tCy ! ! ! ! ! ! # # # %(EEK* R R,&&&&';';\#=NNr8r9r?r, gguf_filer&r]add_prefix_spaceFr'r2r3 name_or_pathconfigr tokenizertokenizer_configrc i|] \}\}}|| SrArA)rCrJw_s rErLz.TokenizersBackend.__init__..qs%CkCkCkYQPVQRTUAqCkCkCkrGT)r2r3fuse_unkdropoutr)r2ra9Couldn't instantiate the backend tokenizer from one of: (1) a `tokenizers` library serialization file, (2) a slow tokenizer instance to convert or (3) an equivalent slow tokenizer class to instantiate and convert. You need to have sentencepiece or tiktoken installed to convert a slow tokenizer to a fast one.rarbrcrdz3The backend tokenizer is not correctly initialized.r7 max_lengthtruncation_side directionstridetruncation_strategystrategyr6rqpad_token_type_id pad_type_id padding_sidelengthpad_to_multiple_ofbackend tokenizers add_bos_token add_eos_tokenr5cFh|]}tt|SrAhashreprrCrKs rE z-TokenizersBackend.__init__..s&$^$^$^5T$u++%6%6$^$^$^rGcVg|]%\}}tt|v#|&SrAr)rCindexrKadded_tokens_decoder_hashs rErFz.TokenizersBackend.__init__..sA   uDKK  (AAA AAArGc|dSNrrA)xs rEz,TokenizersBackend.__init__..s STUVSWrGrc,g|]}t|SrArSrs rErFz.TokenizersBackend.__init__..s;b;b;b5CJJ;b;b;brGc0g|]}|t|SrAr)rCts rErFz.TokenizersBackend.__init__..s$WWW1UVWAWWWrG)speciali pre_tokenizerfix_mistral_regex) init_kwargsrrA):rxrcopydeepcopyryrzr{r}r~rrrrlen _tokenizerrOrwrr rPrBr ValueErrorrrenable_truncation no_truncationrenable_padding_add_bos_token_add_eos_tokenr5_should_update_post_processorsuperr+r'rsplit_special_tokensencode_special_tokensr]sortedradded_tokens_encoderkeys_special_tokens_maprhrSrm_extra_special_tokensrr add_tokensget_vocab_sizeNotImplementedErrorr_patch_mistral_regexrupdate_post_processor)"selfargsrr8r9r,rrr]rr'r2r3fast_tokenizer gguf_path gguf_param architecturetokenizer_dictradditional_kwargs vocab_dict _truncation_paddingexplicit_bos_eos_in_kwargsr5 tokens_to_addencoderspecial_token_valuerKtokensall_named_tokens vocab_sizer __class__s" @rEr+zTokenizersBackend.__init__Hs"::&8$?? ?D99   .555!::&8$??JJ{D11 $jj)94@@%zz*@"EE!::&8%@@ZZ --  7##H%%  '!]+;<>p$/B\p`p  " -DO - < < < < <   lK ,E F F F   / J J J   h H(= > > >   3[5L M M M M O ) ) + + +::1488dDO ? ? ?   2H=Q4R S S S F " " ,F9 %4%>%[/U[B["$jj%@@$jj%@@#ZZ(8$?? ?> <-;DO *-G-q4?KimqKq*""6"""  !(DO 0040I-$^$^DD]$^$^$^!    &';'A'A'C'C X X X   t0557788;b;bTa;b;b;bb$(#;#B#B#D#D : : "*&''w66;NVc;c;c$$%8999/ , ,E5zz((U--G-G$$U+++ }   ! !FWW0H0O0O0Q0QWWW & % %eS))-&ud;;;EEz22- =-SZZ;K-K-K(,  e$$$$ (''' 7799JJ"   JJJ    74?OT#R#R#^ JJ{D ) ) )7d7 $$^T::!,"(**-@$"G"G   DO  . X$/2PTX2X *  - )  & & ( ( ( ( ( ) )s] ]/.]/ricdS)NTrArs rEis_fastzTokenizersBackend.is_faststrGcd|jvr]|jddr=t|dr+|jr$tj|jSdSdS)z `bool`: Whether or not the slow tokenizer can be saved. For a sentencepiece based slow tokenizer, this can only be `True` if the original `"sentencepiece.model"` was not deleted. r'rXFT)vocab_files_namesrrr'ryrzr{rs rEcan_save_slow_tokenizerz)TokenizersBackend.can_save_slow_tokenizersi 41 1 1d6L\6Z6c6cdl6m6m 1t\** 7t 7w~~do66654rGsave_directoryfilename_prefixctj|s td|ddStj||r|dzndt dz}tj|jtj|krt|j||fS)NzVocabulary path (z) should be a directory-rr') ryrzisdirrerrorjoinVOCAB_FILES_NAMESabspathr'r)rrrout_vocab_files rEsave_vocabularyz!TokenizersBackend.save_vocabularysw}}^,,  LLT^TTT U U U F oM_s222QbcoQpp   7??4? + +rw~/N/N N N T_n 5 5 5  rG legacy_format push_to_hub save_format.c t|dvrtd|d|dkrddlm}i}|r4tj|r||}|||} |r|dd} |d t| tjj d } tj | fd d i|j } ||| || |d | St!j|f|||d|S)a Save the full tokenizer state. Extends [`~tokenization_utils_base.PreTrainedTokenizerBase.save_pretrained`] with `save_format` support, for tokenizers converted from a native vocabulary file (e.g. Mistral's `tekken.json`). See the base method for `legacy_format`, `filename_prefix`, and `push_to_hub`. Args: save_format (`str`, *optional*): `"mistral"` to save as a native `tekken.json` by copying the original file (requires the original tekken vocabulary file to be available). The name is normalized to `tekken.json` on save, and the instantiated tokenizer must still match it exactly. `"hf"` or `None` for the default HuggingFace format. kwargs (`dict[str, Any]`, *optional*): Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. Returns: A tuple of `str`: The files saved. )NhfmistralzUnknown save_format=z$. Supported values: 'hf', 'mistral'.r-r)save_tekken_formatcommit_messageNrepo_idexist_okTrK) working_dirr0files_timestampsr/rK)r(rr))rrr.ryrzr!_get_files_timestampsrxrSrTsepr" create_repor0_upload_modified_filesrrsave_pretrained) rrr(rr)r*rr.r4 save_filesr/r0rs rEr9z!TokenizersBackend.save_pretrained s> 5 5 5gKggghh h ) # # J J J J J J "  Nrw}}^<< N#'#=#=n#M#M ++D.AAJ !',s rErz'TokenizersBackend.update_post_processorNs]n( ;4-;!&D n( ;4-;!&D %)%7?S5[[Rww[_[mEucCiRVFVFVsuwwD0BJ39t++DDgkgyRBRUX[R[^bRbRb@BDD   7  ! !3 "5 6 6 6   7  ! !3 "5 6 6 6)3)F^* * * &&&rGc$t|ddS)NrFrrs rErzTokenizersBackend.add_eos_tokenht-u555rGc$t|ddS)NrFrErs rErzTokenizersBackend.add_bos_tokenlrFrGcft|d||dS)Nrobject __setattr__rrvalues rErzTokenizersBackend.add_eos_tokenp34!15999 ""$$$$$rGcft|d||dS)NrrIrLs rErzTokenizersBackend.add_bos_tokenurNrGcg}|jD]j}|t|tr||0t|t r%|t|ddk|jD]g}t|tr||-t|t r%|t|ddh|r||dt|dds |j j | dSdS)a[ Post-initialization hook that runs after the tokenizer is fully set up. This is called by from_pretrained() after loading the tokenizer, which allows us to add any special tokens that may have been passed as AddedToken objects. Child classes should call super()._post_init() if they override this method. NTF)r normalized)r>r) rrhrOrrmrSrrrrr5r)rr token_valuerKs rE _post_initzTokenizersBackend._post_initzsl 3::<< ^ ^K"+z22 ^$$[1111K-- ^$$Z TV[%\%\%\]]]/ X XE%,, X$$U++++E3'' X$$ZtPU%V%V%VWWW  @ OOM$O ? ? ? 48$ ? ? )4?CaCi  & & ( ( ( ( (DjCirGc8|jdS)zP `int`: Size of the base vocabulary (without the added tokens). Fwith_added_tokensrrrs rErzTokenizersBackend.vocab_sizes ---FFFrGc8|jdS)NTrU)r get_vocabrs rErYzTokenizersBackend.get_vocabs((4(@@@rGc*|Srk)rYrs rEr2zTokenizersBackend.vocabs~~rGchdt|jdDS)z Returns the sorted mapping from string to index. The added tokens encoder is cached for performance optimisation in `self._added_tokens_encoder` for the slow tokenizers. c$i|] \}}|j|SrAcontentrCvks rErLz:TokenizersBackend.added_tokens_encoder.. mmmA 1mmmrGc|dSrrArDs rErz8TokenizersBackend.added_tokens_encoder.. dhijdkrGrrr]rrs rErz&TokenizersBackend.added_tokens_encoders9 nm0I0O0O0Q0QWkWk)l)l)lmmmmrGc4|jS)z Returns the added tokens in the vocabulary as a dictionary of index to AddedToken. Returns: `dict[str, int]`: The added tokens. )rget_added_tokens_decoderrs rEr]z&TokenizersBackend.added_tokens_decoders77999rGchdt|jdDS)z Returns the added tokens in the vocabulary as a dictionary of token to index. Returns: `dict[str, int]`: The added tokens. c$i|] \}}|j|SrAr]r_s rErLz5TokenizersBackend.get_added_vocab..rbrGc|dSrrArds rErz3TokenizersBackend.get_added_vocab..rerGrrfrs rEget_added_vocabz!TokenizersBackend.get_added_vocabs9nm0I0O0O0Q0QWkWk)l)l)lmmmmrGcdS)zN Returns True, to avoid expensive `assert tokenizer` gotchas. TrArs rE__bool__zTokenizersBackend.__bool__s trGc8|jdS)zD Size of the full vocabulary with the added tokens. TrUrWrs rE__len__zTokenizersBackend.__len__s---EEErGc|jS)zc `tokenizers.implementations.BaseTokenizer`: The Rust tokenizer used as a backend. )rrs rEbackend_tokenizerz#TokenizersBackend.backend_tokenizers rGc|jjS)zU `tokenizers.decoders.Decoder`: The Rust decoder for this tokenizer. )rdecoderrs rErtzTokenizersBackend.decoders &&rGTr/return_token_type_idsreturn_attention_maskreturn_overflowing_tokensreturn_special_tokens_maskreturn_offsets_mapping return_lengthverbosec F| d|jv}| d|jv}|r|j |g|jz} n|g} tt} | D]} | d| j|r | d| j|r | d| j|r | d| j|r | d| j |r-| dt| j| | fS)a Convert the encoding representation (from low-level HuggingFace tokenizer output) to a python Dict and a list of encodings, take care of building a batch from overflowing tokens. Overflowing tokens are converted to additional examples (like batches) so the output values of the dict are lists (overflows) of lists (tokens). Output shape: (overflows, sequence length) Ntoken_type_idsattention_mask input_idsspecial_tokens_maskoffset_mappingr) model_input_names overflowingrrPrmidstype_idsr~roffsetsr) rr/rurvrwrxryrzr{ encodings encoding_dictrs rE_convert_encodingz#TokenizersBackend._convert_encodingsV( ! ($48N$N ! ($48N$N ! $ #)=)I! X%99II! I#D))  ; ;A + & - -ae 4 4 4$ C./66qzBBB$ I./66q7GHHH) S34;;As rE _add_tokenszTokenizersBackend._add_tokenss7  B?55jAA A))*555rGr=c6|j|S)aG Returns the number of added tokens when encoding a sequence with special tokens. This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put this inside your training loop. Args: pair (`bool`, *optional*, defaults to `False`): Whether the number of added tokens should be computed in the case of a sequence pair or a single sequence. Returns: `int`: Number of special tokens added to sequences. )rnum_special_tokens_to_add)rr=s rErz+TokenizersBackend.num_special_tokens_to_add s&88>>>rGrskip_special_tokensc<t|tr|j|Sg}|rt |jn t }|D]C}t|}||vr||j|D|S)a Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and added tokens. Args: ids (`int` or `list[int]`): The token id (or token ids) to convert to tokens. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. Returns: `str` or `list[str]`: The decoded token(s). )rOrrrrall_special_idsrm)rrrr ids_to_skiprs rEconvert_ids_to_tokensz'TokenizersBackend.convert_ids_to_tokens5s c3   4?..s33 33FQc$.///CEE  > >EJJE ## MM$/55e<< = = = = rGtextrc H|jd|||d|S)N)r text_pairrrA) _encode_plusr)rrr=rrs rEtokenizezTokenizersBackend.tokenizeOs2 t lddOallekllssuuurGpadding_strategyrrrrrc |jj |jj}|tjkr |jn<|||j|jd} d} n fd|D} | |kr|jjdi||tj kr||j dSdS|tj kr|nd} | ||n|j |j|j|j|d}||kr|jjdi|dSdS)a Define the truncation and the padding strategies for fast tokenizers (provided by HuggingFace tokenizers library) and restore the tokenizer settings afterwards. The provided tokenizer has no padding / truncation strategy before the managed section. If your tokenizer set a padding / truncation strategy before, then it will be reset to no padding / truncation when exiting the managed section. Args: padding_strategy ([`~utils.PaddingStrategy`]): The kind of padding that will be applied to the input truncation_strategy ([`~tokenization_utils_base.TruncationStrategy`]): The kind of truncation that will be applied to the input max_length (`int`): The maximum size of a sequence. stride (`int`): The stride to use when handling overflow. pad_to_multiple_of (`int`, *optional*): If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta). padding_side (`str`, *optional*): The side on which the model should have padding applied. Should be selected between ['right', 'left']. Default value is picked from the class attribute of the same name. N)rrrrc>i|]}||dSrkr)rCrar s rErLz@TokenizersBackend.set_truncation_and_padding..s)GGG11kooa66GGGrG)rrpad_idrqrrrA)rrrrDO_NOT_TRUNCATErrMrrr DO_NOT_PAD no_padding MAX_LENGTHr pad_token_idrqrr) rrrrrrrrtargetcurrentrr s @rEset_truncation_and_paddingz,TokenizersBackend.set_truncation_and_paddingRsiBo0 ?* "4"D D D&--///) /5!1 F"GGGGGGG&  11;;F;;; 9 9 9#**,,,,,$#$47Q#Q#QZZW[F -9-E\\4K\+!^#5&8 F6!!..8888888"!rGrris_split_into_wordsreturn_tensorsrc  #d}||std|||std|rAt|ttfo#|o!t|dttf}nt|ttf}|rt|trt d|Pt |t |kr0tdt |dt |d|tt||n|}n |r||fgn|g}t|ttfs t dt|d  ||||| | |j }j j |kr |j _ j ||| }  fd |D}i}|ddD]##fd |D}||#<d|D}r;g}t|D]$\}\}} ||gt |dzz }%||d<|dD]}!|!|t!||| }"|s5| 3s1t!d|"D|"j}"|"S)Ncrt|trdSt|ttfrt |dkrdSt|dtrdSt|dttfrt |ddks!t|ddtrdSt|ddttfrFt |dddkp&t|dddtSdSdSdS)NTrF)rOrSrPrBr)rs rE_is_valid_text_inputz._is_valid_text_inputs!S!! tAe}-- q66Q;;4!c** !4!tUm44!1Q4yyA~~AaDGS)A)A~#t#AaDGdE];;%"1Q47||q0OJqtAwqz34O4OO$u 5urGztext input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs).rzdwhen tokenizing batches of text, `text_pair` must be a list or tuple with the same length as `text`.zbatch length of `text`: z- does not match batch length of `text_pair`: .z:batch_text_or_text_pairs has to be a list or a tuple (got ))rrrrrr)ris_pretokenizedc Jg|]}|  S))r/rurvrwrxryrzr{)r) rCr/rvrzryrwrxrurr{s rErFz2TokenizersBackend._encode_plus.. sX   " "!&;&;*C+E'=+ #   rGc0g|]\}}|D]}|SrArA)rCrDrrrs rErFz2TokenizersBackend._encode_plus..s.NNN74DINNqQNNNNrGc"g|] \}}|D]}| SrArA)rCrrDrs rErFz2TokenizersBackend._encode_plus..s)SSSWQdSSqSSSSrGroverflow_to_sample_mapping) tensor_typeci|]>\}}|t|dkr#t|dtr|dn|?SrN)rrOrP)rCrrMs rErLz2TokenizersBackend._encode_plus...sW"Uc%jj1nnE!Hd9S9Sn%((Y^rG)rrOrPrBrS TypeErrorrzipr1rrrr encode_batchr&_eventual_warn_about_too_long_sequencerrr)$rrrrrrrrrrrrrurvrwrxryrzr{rrr is_batchedbatch_text_or_text_pairsrtokens_and_encodingssanitized_tokensstacksanitized_encodingsrrJtoksrrbatched_outputrs$` ``````` @rErzTokenizersBackend._encode_pluss0   ($#D)) W   )=)=i)H)H W   9#D4-88hThjQUVWQX[_afZgFhFhJJ#D4-88J  T)S)) $Tc)nn)D)D *s4yy**I***FOEZtCi,@,@'A'A'A`d $ $?H'Sy(9':':dV $2UDMBB nTRjMkMknnn  ''- 3!1% (    '#'#< ? 04H H H4HDO 1O00 $1/1   & '*1- * *CNNNN&:NNNE$) S ! !SS0DSSS % X)+ & )*> ? ? K K 9D!*qcC[8I4J4J.JJ**=W 9 :)+6 X XI  7 7 :w W W W W&'79LZhiii n4=V4*&4&:&:&<&<( NrGrc|jj|jj|nd|S)NrR)rrrtdecoder#)rrs rEconvert_tokens_to_stringz*TokenizersBackend.convert_tokens_to_string7s@%-9  " * 1 1& 9 9 9&!! rG token_idsclean_up_tokenization_spacesc |ddt|tr|g}t|tr|d}|j||}||n|j}|rgt|jj j dkr0|j s)t d|jj dn||}|S)Nuse_source_tokenizerr)rr z=Ignoring clean_up_tokenization_spaces=True for BPE tokenizer aE. The clean_up_tokenization post-processing step is designed for WordPiece tokenizers and is destructive for BPE (it strips spaces before punctuation). Set clean_up_tokenization_spaces=False to suppress this warning, or set clean_up_tokenization_spaces_for_bpe_even_though_it_will_corrupt_output=True to force cleanup anyway.)rxrOrrwrrrr1rrr0rGclean_up_tokenization_spaces_for_bpe_even_though_it_will_corrupt_outputr warning_oncerclean_up_tokenization)rrrrrrs rE_decodezTokenizersBackend._decode>s   )4000 i % % $" I i & & /!+.I%%iEX%YY,7 ) (2 % ( 8 T+122;uDDdE##-/---11$77 rG file_namesct|}tj||r|dzndtz}|j|||fz}|S)Nr r)rSryrzr#TOKENIZER_FILErrsave)rrrr(rr&s rE_save_pretrainedz"TokenizersBackend._save_pretrainedisl^,, oM_s222Q__   ##N333>"33 rGc  tj|j}|d}|d} d} |dddkri|dd<g|dd<n|ddd kr^|dd O|dd } |dd| d }  | vr| } d |dd <| d gg|dd<n;|ddd vr i|dd<nt d|ddd7d|dvr-|ddvr|dd|dd<t jtj|g} |D]} | dd}| dd}|ddd kr|sC| dvr| d| d<| td*i| || ||dddkr#d|vr|dd|dd|d<|dddkr#d|vr|dd|dd|d<|ddd kr| | |d<|dt|dddksA|dddkrPd|dvrFtd|ddDr!tj|d<t |dd}|d*|| d|}|||| ,tj}d| vr| dD]}| d|d}fd |D}|| d|d<|D](}|}|t d!)fd"|D| d|d#<d$D]L}|| vrF| |\}} |vr|}|}|t d!||g| |<M| |d<t jtj||j}t*jD]}t/||t/||} |vr|}|j|d}t5|tr-t||j|j|j|jd%&||<|||<|jr|jng}|| |tA|d kr||d'<|d(< |j!d*i|S#tD$rH}d)tG|vr1|d(d|j!d*i|}|_|cYd}~Sd}~wwxYw)+uf Trains a tokenizer on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline) as the current one. Args: text_iterator (generator of `list[str]`): The training corpus. Should be a generator of batches of texts, for instance a list of lists of texts if you have everything in memory. vocab_size (`int`): The size of the vocabulary you want for your tokenizer. length (`int`, *optional*): The total number of sequences in the iterator. This is used to provide meaningful progress tracking new_special_tokens (list of `str` or `AddedToken`, *optional*): A list of new special tokens to add to the tokenizer you are training. special_tokens_map (`dict[str, str]`, *optional*): If you want to rename some of the special tokens this tokenizer uses, pass along a mapping old special token name to new special token name in this argument. kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the trainer from the 🤗 Tokenizers library. Returns: [`PreTrainedTokenizerFast`]: A new tokenizer of the same type as the original one, trained on `text_iterator`. r4r5Nr0r1r r2r3rrrg)r$r%z;This method does not support this type of tokenizer (found z-) only BPE, Unigram, WordLevel and WordPiece.reridr^continuing_subword_prefixend_of_word_suffixr ByteLevelr; pretokenizersc3.K|]}|ddkVdS)r1rNrA)rC pretokenizers rE z.s@$!(K7rGinitial_alphabet)rr>)rtrainerr>rc<g|]}||SrAr)rCrKspecial_tokens_maps rErFz=TokenizersBackend.train_new_from_iterator..s*![![![5"4"8"8"F"F![![![rGzQAttempted to set a token in the post processor that does not exist in the mappingc:g|]}|SrA)r)rCrKrs rErFz=TokenizersBackend.train_new_from_iterator..s)CuCuCuejIDYDYZ_D`D`CuCuCurGr)rr6T) single_wordlstriprstriprQrrgr,z7multiple values for keyword argument 'tokenizer_object'rA)$rloadsrto_strrxrr}rrrmrrlanypre_tokenizers_fastralphabetMODEL_TO_TRAINER_MAPPINGtrain_from_iteratorrrrrSPECIAL_TOKENS_ATTRIBUTESrrrrOrrrrQrgrrrrS)r text_iteratorrrnew_special_tokensrrrr4r5rerr> added_tokenrr trainer_classrtrained_tokenizer_jsonrrrKr_ special_tokenspecial_token_fullrgr new_tokenizerrs ` @rEtrain_new_from_iteratorz)TokenizersBackend.train_new_from_iteratorzsDDO$:$:$<$<==%)).99 '++,<== ' "6 *e 3 3/1N7 #G ,02N7 #H - - G $V , 9 9g&x0<'0:*73G4D3Ew'0 G $V ,0J J J/1N7 #G , ,>n]dNeflNm>>>   *~g666w' 48JJJ3EnU\F]^iFj3kN7 #K 0!*4:n+E+EFF ' = =K!ooi66Gd++Ag&v.);;G;!-+i2HL^2^2^);K 11)*:; v vC+,<=cB8LF)5![![![![TZ![![![FLN#34S9(C!'#,#8#8#?#?#+", s##, DvCuCuCuntCuCuCuN#34S9%@@!/ F F  N22-m++F++ +   HCPQFFRR -t444 . 8 8 8 8 +4 ($$$$$$$ s$! V.. X8> Model type one of the mistral models (on older versions) 2) Remote models on the hub from official mistral models >> Tags including `base_model:.*mistralai` rN) lru_cache)version)rr")maxsizemodel_idric |}n#t$rYdSwxYw|j0dd|jrdSdS)NFzbase_model:.*mistralairT) model_infortagssearchr#)rr0r"res rEis_base_mistralz?TokenizersBackend._patch_mistral_regex..is_base_mistral[s~ ++H55   uu z%995rwwuz7J7JKK 45s ! //Tz config.jsonF) cache_dirrKlocal_files_only%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errors _commit_hashr-r.transformers_versionrz5.0.0)r-mistral3voxtral ministralpixtralrz$The tokenizer you are loading from 'a' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.z[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+isolated)patternbehavior)r use_regex)r functoolsr packagingrtransformers.utils.hubrr"rSboolrrrrrparsesetattrrrrrpre_tokenizersSplitRegexrrOr; Metaspacer)rrpretrained_model_name_or_pathrKrrr is_localrrrrrrr _config_filemistral_config_detectedf_configr transformers_model_typersplit_pretokenizercurrent_pretokenizerr"rs @@rErz&TokenizersBackend._patch_mistral_regex?s*  ''''''%%%%%%>>>>>>>> 3    c d          00 H ( 4  5% 5*9/:W*X*X 5';-#!16;8=)   L', #',999+Q"illG+++++++++++++++'.{{3I'J'J$*1++l*C*C' (%GMM:N,O,ORYR_R_`gRhRh,h,h  )3?3  )()%gmmAAAA tCH~   X nd38nnnnXn:d3 ?&;:::X:10nc3hnnnn$ FFFFF =X ''''X'.2-1*/+0',#-(-(-( $d{-( $d{ -( $( -( %) -(!%-(-(-( tCH~tL11 2-(-(-(-(^ 7#7#*777766d3+;&<6WZ6666 ??d?s????*tCyt`cfjknfo`o4vvSvd vtvjnorjsvvvvI9)I90I9 I9  I9  $J I9DjI9I9I9I9\gk#',;,F2D2T!%$))-#'&*-1-1*/+0',#,0)XX++d9o=EV@WWX004 ?BTJ[E\\_ccX! X * X 0 X$JXX"X $JXDjXt X $d{X $d{X$(X %)!X"!%#X$%X&'X(#Tk)X, -XXXXt tCy S    %*48 ))c?)")'+Tk ) ))))^&*&* bk)#s(Od{  t  sCx * CCCCJ BBB[BBBBBrGr))Fr(rrry collectionsrcollections.abcrshutilrtypingrtokenizers.pre_tokenizersrrhuggingface_hubrrrr r r0r r}tokenizers.decodersr r/tokenizers.modelsr rtokenizers.trainersrrrrrrrrintegrations.ggmlrintegrations.mistral.constantsrmodeling_gguf_pytorch_utilsrtokenization_utils_baserrrrrrrutilsr r!r"r# get_loggerrrrSPECIAL_TOKENS_MAP_FILETOKENIZER_CONFIG_FILETIKTOKEN_VOCAB_FILEADDED_TOKENS_FILErr$r)PreTrainedTokenizerFastrArGrErFs  ######$$$$$$777777++++++--------//////111111666666********^^^^^^^^^^^^......000000555555DDDDDD======HGGGGGGGGGGG  H % %"3/'( !! (6EXYY,--mmmmm/mm.-mb+,rG