1@jlddlmZmZdZdZdZdZdZdZdZ d Z d Z d Z d Z d ZdZe e e e e eedZdS))data_gym_to_mergeable_bpe_ranksload_tiktoken_bpe <|endoftext|>z<|fim_prefix|>z<|fim_middle|>z<|fim_suffix|>z<|endofprompt|>zO'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\scPtdddd}ddt|tdid S) NzIhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpezLhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json@1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5@196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783)vocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashgpt2QPnameexplicit_n_vocabpat_strmergeable_ranksspecial_tokens)r r50k_pat_str ENDOFTEXTrs KC:\Projects\Historical\venv\Lib\site-packages\tiktoken_ext/openai_public.pyr r sG5bhY\ O!*$e,   cLtdd}ddt|tdidS)NzGhttps://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken@306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930 expected_hash r50k_baserrrrrrrs rrr!A'QXO !*$e,   rcLtdd}ddt|tdidS)NGhttps://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken@94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069r p50k_baseirrr rs rr%r%/r!rcxtdd}tdtdtdtdi}dt ||d S) Nr#r$rrr&ijik p50k_editrrrr)rr FIM_PREFIX FIM_MIDDLE FIM_SUFFIXrrrs rr(r(=sP'QXO  E:ujZ_`N*(   rc |tdd}tdtdtdtdt di}d d ||d S) NzIhttps://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken@223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7riiiii cl100k_basezu'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\sr))rrr*r+r, ENDOFPROMPTr-s rr0r0Ks^'SXO 6FFFV NP*(   rctdd}tdtdi}dgd}d|||d S) NzHhttps://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken@446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2dr? iR |)zi[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?zi[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?z \p{N}{1,3}z ?[^\s\p{L}\p{N}]+[\r\n/]*z \s*[\r\n]+z \s+(?!\S)z\s+ o200k_baser))rrr1join)rrrs rr6r6_sm'RXO f=Nhh     G*(   rct}d}|d}|d}i|dddddd d d d d ddddddddtddDz}||||dS)N o200k_harmonyrrri> r4i@ iA iB iC iD iE iF iG iH iI iJ iK iL )z<|startoftext|>rz<|reserved_200000|>z<|reserved_200001|>z <|return|>z <|constrain|>z<|reserved_200004|>z <|channel|>z <|start|>z<|end|>z <|message|>z<|reserved_200009|>z<|reserved_200010|>z<|reserved_200011|>z<|call|>ci|] }d|d| S)z <|reserved_z|>).0is r z!o200k_harmony..s&???A q   a???riM ir))r6range)base_encrrrrs rr9r9{s||H Dy!G01O # $!%%%%%%!" @?vv)>)>???#@N&*(   r)r rr%r(r0r6r9N) tiktoken.loadrrrr*r+r,r1rr rr%r(r0r6r9ENCODING_CONSTRUCTORSr;rrrCsLLLLLLLL      [            (8@ "r