§
    1@Äjª  ã                  óf   — d dl mZ d dlZd dlZd dlZdd„Zdd„Zddd„Z	 	 	 dd d„Zd!d„Z	dd"d„Z
dS )#é    )ÚannotationsNÚblobpathÚstrÚreturnÚbytesc                ó’  — d| vr>t          | dd¬¦  «        5 }|                     ¦   «         cd d d ¦  «         S # 1 swxY w Y   |                      d¦  «        r4dd l}|                     | ¦  «        }|                     ¦   «          |j        S 	 dd l}n"# t          $ r}t          d¦  «        |‚d }~ww xY w| 	                    | ¦  «        S )Nz://Úrbr   ©Ú	buffering)zhttp://zhttps://úOblobfile is not installed. Please install it by running `pip install blobfile`.)
ÚopenÚreadÚ
startswithÚrequestsÚgetÚraise_for_statusÚcontentÚblobfileÚImportErrorÚ
read_bytes)r   Úfr   Úrespr   Úes         ú>C:\Projects\Historical\venv\Lib\site-packages\tiktoken/load.pyÚ	read_filer      s.  € Ø�HÐÐÝ�(˜D¨AÐ.Ñ.Ô.ð 	°!Ø—6’6‘8”8ð	ð 	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð ×ÒÐ2Ñ3Ô3ð àˆˆˆà�|Š|˜HÑ%Ô%ˆØ×ÒÑÔÐØŒ|ÐðØˆˆˆˆøÝð ð ð ÝØ]ñ
ô 
àð	øøøøðøøøð ×Ò˜xÑ(Ô(Ð(s$   —8¸<¿<ÂB Â
B1ÂB,Â,B1ÚdataÚexpected_hashÚboolc                óZ   — t          j        | ¦  «                             ¦   «         }||k    S ©N)ÚhashlibÚsha256Ú	hexdigest)r   r   Úactual_hashs      r   Ú
check_hashr%      s(   € Ý”. Ñ&Ô&×0Ò0Ñ2Ô2€KØ˜-Ò'Ð'ó    ú
str | Nonec                óÊ  — d}dt           j        v rt           j        d         }nYdt           j        v rt           j        d         }n8dd l}t           j                             |                     ¦   «         d¦  «        }d}|dk    rt          | ¦  «        S t          j        |  	                    ¦   «         ¦  «         
                    ¦   «         }t           j                             ||¦  «        }t           j                             |¦  «        rxt          |dd¬	¦  «        5 }|                     ¦   «         }d d d ¦  «         n# 1 swxY w Y   |�t          ||¦  «        r|S 	 t          j        |¦  «         n# t           $ r Y nw xY wt          | ¦  «        }	|r&t          |	|¦  «        st#          d
| › d|› d�¦  «        ‚dd l}
	 t          j        |d¬¦  «         |dz   t)          |
                     ¦   «         ¦  «        z   dz   }t          |d¦  «        5 }|                     |	¦  «         d d d ¦  «         n# 1 swxY w Y   t          j        ||¦  «         n# t           $ r |r‚ Y nw xY w|	S )NTÚTIKTOKEN_CACHE_DIRÚDATA_GYM_CACHE_DIRr   zdata-gym-cacheFÚ r	   r
   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)Úexist_okú.z.tmpÚwb)ÚosÚenvironÚtempfileÚpathÚjoinÚ
gettempdirr   r!   Úsha1Úencoder#   Úexistsr   r   r%   ÚremoveÚOSErrorÚ
ValueErrorÚuuidÚmakedirsr   Úuuid4ÚwriteÚrename)r   r   Úuser_specified_cacheÚ	cache_dirr1   Ú	cache_keyÚ
cache_pathr   r   Úcontentsr;   Útmp_filenames               r   Úread_file_cachedrF   #   s	  € ØÐØ�rœzÐ)Ð)Ý”JÐ3Ô4ˆ	ˆ	Ø	¥¤Ð	+Ð	+Ý”JÐ3Ô4ˆ	ˆ	àˆˆˆå”G—L’L ×!4Ò!4Ñ!6Ô!6Ð8HÑIÔIˆ	Ø$Ðà�B‚€å˜Ñ"Ô"Ð"å”˜XŸ_š_Ñ.Ô.Ñ/Ô/×9Ò9Ñ;Ô;€Iå”—’˜i¨Ñ3Ô3€JÝ	„w‡~‚~�jÑ!Ô!ð 
Ý�*˜d¨aÐ0Ñ0Ô0ð 	°AØ—6’6‘8”8ˆDð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	àÐ ¥J¨t°]Ñ$CÔ$CÐ ØˆKð	ÝŒI�jÑ!Ô!Ð!Ð!øÝð 	ð 	ð 	ØˆDð	øøøõ ˜Ñ"Ô"€HØð 
�Z¨°-Ñ@Ô@ð 
ÝðI°hð Ið IÈ=ð Ið Ið Iñ
ô 
ð 	
ð
 €K€K€Kð	Ý
Œ�I¨Ð-Ñ-Ô-Ð-Ø! CÑ'­#¨d¯jªj©l¬lÑ*;Ô*;Ñ;¸fÑDˆÝ�, Ñ%Ô%ð 	¨Ø�GŠG�HÑÔÐð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	å
Œ	�, 
Ñ+Ô+Ð+Ð+øÝð ð ð àð 	Øð	ð 	ðøøøð
 €Os[   ÄD<Ä<E ÅE ÅE1 Å1
E>Å=E>Æ=AI ÈH/È#I È/H3È3I È6H3È7I ÉI ÉI FÚvocab_bpe_fileÚencoder_json_fileÚvocab_bpe_hashÚencoder_json_hashÚclobber_one_byte_tokensúdict[bytes, int]c                óŠ  ‡‡— d„ t          d¦  «        D ¦   «         }d„ |D ¦   «         Šd}t          d¦  «        D ]5}||vr/|                     |¦  «         |‰t          d|z   ¦  «        <   |dz  }Œ6t          |¦  «        dk    sJ ‚t	          | |¦  «                             ¦   «         }d„ |                     d¦  «        dd…         D ¦   «         }	dˆfd„Šd„ t          |¦  «        D ¦   «         }
~t          |
¦  «        }|	D ]$\  }}||
 ‰|¦  «         ‰|¦  «        z   <   |dz  }Œ%dd l}| 	                    t	          ||¦  «        ¦  «        }ˆfd„| 
                    ¦   «         D ¦   «         }|                     dd ¦  «         |                     dd ¦  «         |r#|D ] }t          |¦  «        dk    r||         |
|<   Œ!|
|k    sJ ‚|
S )Nc                óz   — g | ]8}t          |¦  «                             ¦   «         ¯#t          |¦  «        d k    ¯6|‘Œ9S )ú )ÚchrÚisprintable©Ú.0Úbs     r   ú
<listcomp>z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>a   s;   € ÐXÐXÐX˜Qµ°Q±´×1CÒ1CÑ1EÔ1EÐXÍ#ÈaÉ&Ì&ÐTWÊ-È-�qÈ-È-È-r&   é   c                ó.   — i | ]}t          |¦  «        |“ŒS © )rP   rR   s     r   ú
<dictcomp>z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>c   s    € Ð@Ð@Ð@¨1�S ™VœV QÐ@Ð@Ð@r&   r   é   c                óP   — g | ]#}t          |                     ¦   «         ¦  «        ‘Œ$S rX   )ÚtupleÚsplit)rS   Ú	merge_strs     r   rU   z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>n   s*   € ÐaÐaÐa¨y•%˜	ŸšÑ)Ô)Ñ*Ô*ÐaÐaÐar&   ú
éÿÿÿÿÚvaluer   r   r   c                ó:   •— t          ˆfd„| D ¦   «         ¦  «        S )Nc              3  ó(   •K  — | ]}‰|         V — Œd S r    rX   )rS   rT   Údata_gym_byte_to_bytes     €r   ú	<genexpr>zKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>q   s)   øè è € Ð=Ð=°!Ð*¨1Ô-Ð=Ð=Ð=Ð=Ð=Ð=r&   ©r   )ra   rd   s    €r   Údecode_data_gymz8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymp   s&   ø€ ÝÐ=Ð=Ð=Ð=°uÐ=Ñ=Ô=Ñ=Ô=Ð=r&   c                ó6   — i | ]\  }}t          |g¦  «        |“ŒS rX   rf   )rS   ÚirT   s      r   rY   z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>u   s&   € ÐFÐFÐF¡4 1 a•˜�s‘”˜QÐFÐFÐFr&   c                ó.   •— i | ]\  }} ‰|¦  «        |“ŒS rX   rX   )rS   ÚkÚvrg   s      €r   rY   z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>„   s)   ø€ ÐRÐRÐR±T°Q¸˜?˜?¨1Ñ-Ô-¨qÐRÐRÐRr&   s   <|endoftext|>s   <|startoftext|>)ra   r   r   r   )ÚrangeÚappendrP   ÚlenrF   Údecoder]   Ú	enumerateÚjsonÚloadsÚitemsÚpop)rG   rH   rI   rJ   rK   Úrank_to_intbyteÚnrT   Úvocab_bpe_contentsÚ
bpe_mergesÚ	bpe_ranksÚfirstÚsecondrr   Úencoder_jsonÚencoder_json_loadedrk   rd   rg   s                    @@r   Údata_gym_to_mergeable_bpe_ranksr   Y   sI  øø€ ð YÐX¥%¨¡+¤+ÐXÑXÔX€Oà@Ð@°Ð@Ñ@Ô@ÐØ	€AÝ�4‰[Œ[ð ð ˆØ�OÐ#Ð#Ø×"Ò" 1Ñ%Ô%Ð%Ø34Ð!¥# d¨Q¡h¡-¤-Ñ0Ø�‰FˆAøÝˆÑÔ 4Ò'Ð'Ð'Ð'õ *¨.¸.ÑIÔI×PÒPÑRÔRÐØaÐaÐ<N×<TÒ<TÐUYÑ<ZÔ<ZÐ[\Ð]_Ð[_Ô<`ÐaÑaÔa€Jð>ð >ð >ð >ð >ð >ð
 GÐF­9°_Ñ+EÔ+EÐFÑFÔF€IØõ 	ˆI‰Œ€AØ#ð ð ‰ˆˆvØFGˆ	�/�/ %Ñ(Ô(¨?¨?¸6Ñ+BÔ+BÑBÑCØ	ˆQ‰ˆˆà€K€K€Kð
 —:’:Õ.Ð/@ÐBSÑTÔTÑUÔU€LØRÐRÐRÐR¸\×=OÒ=OÑ=QÔ=QÐRÑRÔRÐà×ÒÐ,¨dÑ3Ô3Ð3Ø×ÒÐ.°Ñ5Ô5Ð5àð 6Ø$ð 	6ð 	6ˆAÝ�1‰vŒv˜Š{ˆ{Ø2°1Ô5�	˜!‘øàÐ+Ò+Ð+Ð+Ð+àÐr&   rz   Útiktoken_bpe_fileÚNonec                ó¨  — 	 dd l }n"# t          $ r}t          d¦  «        |‚d }~ww xY w|                     |d¦  «        5 }t          |                      ¦   «         d„ ¬¦  «        D ]T\  }}|                     t          j        |¦  «        dz   t          |¦  «         	                    ¦   «         z   dz   ¦  «         ŒU	 d d d ¦  «         d S # 1 swxY w Y   d S )Nr   r   r.   c                ó   — | d         S )NrZ   rX   )Úxs    r   ú<lambda>z#dump_tiktoken_bpe.<locals>.<lambda>›   s
   € À1ÀQÄ4€ r&   )Úkeyó    ó   
)
r   r   ÚBlobFileÚsortedrt   r>   Úbase64Ú	b64encoder   r6   )rz   r€   r   r   r   ÚtokenÚranks          r   Údump_tiktoken_bper�   “   sK  € ðØˆˆˆˆøÝð ð ð ÝØ]ñ
ô 
àð	øøøøðøøøð 
×	Ò	Ð,¨dÑ	3Ô	3ð Q°qÝ! )§/¢/Ñ"3Ô"3¸¸ÐHÑHÔHð 	Qð 	Q‰KˆE�4Ø�GŠG•FÔ$ UÑ+Ô+¨dÑ2µS¸±Y´Y×5EÒ5EÑ5GÔ5GÑGÈ%ÑOÑPÔPÐPÐPð	QðQð Qð Qñ Qô Qð Qð Qð Qð Qð Qð Qð Qøøøð Qð Qð Qð Qð Qð Qs"   ‚ ‡
&‘!¡&¿A:CÃCÃCc                ó&  — t          | |¦  «        }i }|                     ¦   «         D ]i}|sŒ	 |                     ¦   «         \  }}t          |¦  «        |t	          j        |¦  «        <   ŒB# t          $ r}t          d|›d| › �¦  «        |‚d }~ww xY w|S )NzError parsing line z in )rF   Ú
splitlinesr]   Úintr‹   Ú	b64decodeÚ	Exceptionr:   )r€   r   rD   ÚretÚliner�   rŽ   r   s           r   Úload_tiktoken_bper—   Ÿ   s¿   € åÐ 1°=ÑAÔA€HØ
€CØ×#Ò#Ñ%Ô%ð [ð [ˆØð 	Øð	[ØŸ*š*™,œ,‰KˆE�4Ý+.¨t©9¬9ˆC•Ô  Ñ'Ô'Ñ(Ð(øÝð 	[ð 	[ð 	[ÝÐR°4ÐRÐRÐ?PÐRÐRÑSÔSÐYZÐZøøøøð	[øøøà€Js   ­;A)Á)
BÁ3B	Â	B)r   r   r   r   )r   r   r   r   r   r   r    )r   r   r   r'   r   r   )NNF)rG   r   rH   r   rI   r'   rJ   r'   rK   r   r   rL   )rz   rL   r€   r   r   r�   )r€   r   r   r'   r   rL   )Ú
__future__r   r‹   r!   r/   r   r%   rF   r   r�   r—   rX   r&   r   ú<module>r™      sÔ   ðØ "Ð "Ð "Ð "Ð "Ð "à €€€Ø €€€Ø 	€	€	€	ð)ð )ð )ð )ð,(ð (ð (ð (ð
3ð 3ð 3ð 3ð 3ðr "&Ø$(Ø$)ð7ð 7ð 7ð 7ð 7ðt	Qð 	Qð 	Qð 	Qðð ð ð ð ð ð r&   