§
    1@Äj!  ã                  ón   — d Z ddlmZ ddlZddlZddlZ G d„ d¦  «        Z	 ddd„Z	 ddd„Zdd„Z	d„ Z
dS )zJThis is an educational implementation of the byte pair encoding algorithm.é    )ÚannotationsNc                  óf   — e Zd Zdd„Zddd„Zdd„Zdd„Zdd„Zed d„¦   «         Z	ed„ ¦   «         Z
dS )!ÚSimpleBytePairEncodingÚpat_strÚstrÚmergeable_ranksúdict[bytes, int]ÚreturnÚNonec               óš   — || _         || _        d„ |                     ¦   «         D ¦   «         | _        t	          j        |¦  «        | _        dS )zCreates an Encoding object.c                ó   — i | ]\  }}||“Œ	S © r   )Ú.0Útoken_bytesÚtokens      úFC:\Projects\Historical\venv\Lib\site-packages\tiktoken/_educational.pyú
<dictcomp>z3SimpleBytePairEncoding.__init__.<locals>.<dictcomp>   s   € Ð^Ð^Ð^Ñ0B°¸U˜ Ð^Ð^Ð^ó    N)r   r   ÚitemsÚ_decoderÚregexÚcompileÚ_pat)Úselfr   r   s      r   Ú__init__zSimpleBytePairEncoding.__init__   sI   € ð ˆŒà.ˆÔà^Ð^Ào×F[ÒF[ÑF]ÔF]Ð^Ñ^Ô^ˆŒÝ”M 'Ñ*Ô*ˆŒ	ˆ	ˆ	r   ÚcolourÚtextÚ	visualiseú
str | Noneú	list[int]c                óÊ   — | j                              |¦  «        }g }|D ]C}|                     d¦  «        }t          | j        ||¬¦  «        }|                     |¦  «         ŒD|S )z`Encodes a string into tokens.

        >>> enc.encode("hello world")
        [388, 372]
        úutf-8)r   )r   ÚfindallÚencodeÚ
bpe_encoder   Úextend)r   r   r   ÚwordsÚtokensÚwordÚ
word_bytesÚword_tokenss           r   r$   zSimpleBytePairEncoding.encode   sp   € ð ”	×!Ò! $Ñ'Ô'ˆØˆØð 	'ð 	'ˆDàŸš WÑ-Ô-ˆJÝ$ TÔ%9¸:ÐQZÐ[Ñ[Ô[ˆKØ�MŠM˜+Ñ&Ô&Ð&Ð&Øˆr   r(   Úbytesc                óF   ‡ — d                      ˆ fd„|D ¦   «         ¦  «        S )znDecodes a list of tokens into bytes.

        >>> enc.decode_bytes([388, 372])
        b'hello world'
        r   c              3  ó2   •K  — | ]}‰j         |         V — Œd S ©N©r   ©r   r   r   s     €r   ú	<genexpr>z6SimpleBytePairEncoding.decode_bytes.<locals>.<genexpr>-   s*   øè è € ÐAÐA°˜œ eÔ,ÐAÐAÐAÐAÐAÐAr   )Újoin©r   r(   s   ` r   Údecode_bytesz#SimpleBytePairEncoding.decode_bytes'   s,   ø€ ð �xŠxÐAÐAÐAÐA¸&ÐAÑAÔAÑAÔAÐAr   c                óV   — |                       |¦  «                             dd¬¦  «        S )u   Decodes a list of tokens into a string.

        Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace
        the invalid bytes with the replacement character "ï¿½".

        >>> enc.decode([388, 372])
        'hello world'
        r"   Úreplace©Úerrors)r5   Údecoder4   s     r   r:   zSimpleBytePairEncoding.decode/   s+   € ð × Ò  Ñ(Ô(×/Ò/°À	Ð/ÑJÔJÐJr   úlist[bytes]c                ó    ‡ — ˆ fd„|D ¦   «         S )zÁDecodes a list of tokens into a list of bytes.

        Useful for visualising how a string is tokenised.

        >>> enc.decode_tokens_bytes([388, 372])
        [b'hello', b' world']
        c                ó*   •— g | ]}‰j         |         ‘ŒS r   r0   r1   s     €r   ú
<listcomp>z>SimpleBytePairEncoding.decode_tokens_bytes.<locals>.<listcomp>B   s    ø€ Ð9Ð9Ð9¨�”˜eÔ$Ð9Ð9Ð9r   r   r4   s   ` r   Údecode_tokens_bytesz*SimpleBytePairEncoding.decode_tokens_bytes:   s   ø€ ð :Ð9Ð9Ð9°&Ð9Ñ9Ô9Ð9r   Útraining_dataÚ
vocab_sizeÚintc                óH   — t          | ||¬¦  «        }t          ||¬¦  «        S )z#Train a BPE tokeniser on some data!)ÚdatarA   r   ©r   r   )Ú	bpe_trainr   )r@   rA   r   r   s       r   ÚtrainzSimpleBytePairEncoding.trainD   s,   € õ $¨À:ÐW^Ð_Ñ_Ô_ˆÝ%¨gÀÐWÑWÔWÐWr   c                óŠ   — t          | t          ¦  «        rt          j        | ¦  «        } t	          | j        | j        ¬¦  «        S )NrE   )Ú
isinstancer   ÚtiktokenÚget_encodingr   Ú_pat_strÚ_mergeable_ranks)Úencodings    r   Úfrom_tiktokenz$SimpleBytePairEncoding.from_tiktokenJ   sF   € å�h¥Ñ$Ô$ð 	7ÝÔ,¨XÑ6Ô6ˆHÝ%ØÔ%°xÔ7Pð
ñ 
ô 
ð 	
r   N)r   r   r   r	   r
   r   ©r   )r   r   r   r   r
   r    )r(   r    r
   r,   )r(   r    r
   r   )r(   r    r
   r;   )r@   r   rA   rB   r   r   )Ú__name__Ú
__module__Ú__qualname__r   r$   r5   r:   r?   ÚstaticmethodrG   rO   r   r   r   r   r      sÄ   € € € € € ð+ð +ð +ð +ðð ð ð ð ð Bð Bð Bð Bð	Kð 	Kð 	Kð 	Kð:ð :ð :ð :ð ðXð Xð Xñ „\ðXð
 ð
ð 
ñ „\ð
ð 
ð 
r   r   r   r   r	   Úinputr,   r   r   r
   r    c                óò  ‡ — d„ |D ¦   «         }	 |r)|dv rt          |¦  «         n|dk    rt          |¦  «         d }d }t          t          |d d…         |dd …         ¦  «        ¦  «        D ]7\  }}‰                      |d         |d         z   ¦  «        }|�|�||k     r|}|}Œ8|€n3|€J ‚|d |…         ||         ||dz            z   gz   ||dz   d …         z   }ŒÊ|rt          ¦   «          ˆ fd	„|D ¦   «         }	|	S )
Nc                ó.   — g | ]}t          |g¦  «        ‘ŒS r   ©r,   ©r   Úbs     r   r>   zbpe_encode.<locals>.<listcomp>V   s    € Ð'Ð'Ð'˜A�U�A�3‰ZŒZÐ'Ð'Ð'r   T©r   ÚcolorÚsimpleéÿÿÿÿé   r   é   c                ó    •— g | ]
}‰|         ‘ŒS r   r   )r   Úpartr   s     €r   r>   zbpe_encode.<locals>.<listcomp>s   s   ø€ Ð6Ð6Ð6¨ˆo˜dÔ#Ð6Ð6Ð6r   )Úvisualise_tokensÚprintÚ	enumerateÚzipÚget)
r   rU   r   ÚpartsÚmin_idxÚmin_rankÚiÚpairÚrankr(   s
   `         r   r%   r%   S   sf  ø€ ð (Ð' Ð'Ñ'Ô'€Eð_àð 	ØÐ/Ð/Ð/Ý  Ñ'Ô'Ð'Ð'Ø˜hÒ&Ð&Ý�e‘”�ð ˆØˆÝ ¥ U¨3¨B¨3¤Z°°q°r°r´Ñ!;Ô!;Ñ<Ô<ð 	 ð 	 ‰GˆAˆtØ"×&Ò& t¨A¤w°°a´Ñ'8Ñ9Ô9ˆDØÐ XÐ%5¸Àº¸Ø�Ø�øð ÐØØÐ"Ð"Ð"ð �h�w�h” 5¨¤>°E¸'ÀA¹+Ô4FÑ#FÐ"GÑGÈ%ÐPWÐZ[ÑP[ÐP]ÐP]ÔJ^Ñ^ˆð/_ð2 ð Ý‰Œˆà6Ð6Ð6Ð6°Ð6Ñ6Ô6€FØ€Mr   rD   r   rA   rB   r   c                ó.  ‡— |dk     rt          d¦  «        ‚i }t          d¦  «        D ]}||t          |g¦  «        <   Œd„ t          j        || ¦  «        D ¦   «         }t          |¦  «        |k     �r$t          j        ¦   «         Š|D ]5}t          |d d…         |dd …         ¦  «        D ]}‰|xx         dz  cc<   ŒŒ6t          ‰ˆfd„¬¦  «        }	|	d         |	d         z   }
t          |¦  «        }|||
<   g }|D ]Ê}g }d}|t          |¦  «        dz
  k     rh||         ||dz            f|	k    r| 
                    |
¦  «         |d	z  }n | 
                    ||         ¦  «         |dz  }|t          |¦  «        dz
  k     °h|t          |¦  «        dz
  k    r| 
                    ||         ¦  «         | 
                    |¦  «         ŒË|}|r¹t          d
|	d         › d|	d         › �¦  «         t          d|
› dt          |¦  «        › d�¦  «         |dv r1t          d¦  «         t          d„ |d d…         D ¦   «         ¦  «         n1|dk    r+t          d¦  «         |d d…         D ]}t          |¦  «         Œt          d¦  «         t          |¦  «        |k     �°$|S )Né   z;vocab_size must be at least 256, so we can encode all bytesc                óL   — g | ]!}d „ |                      d¦  «        D ¦   «         ‘Œ"S )c                ó.   — g | ]}t          |g¦  «        ‘ŒS r   rX   rY   s     r   r>   z(bpe_train.<locals>.<listcomp>.<listcomp>ˆ   s    € Ð2Ð2Ð2˜��ˆs‰ŒÐ2Ð2Ð2r   r"   )r$   )r   r)   s     r   r>   zbpe_train.<locals>.<listcomp>‡   s@   € ð  ð  ð  Ø7;Ð2Ð2˜TŸ[š[¨Ñ1Ô1Ð2Ñ2Ô2ð ð  ð  r   r^   r_   c                ó   •— ‰|          S r/   r   )ÚxÚstatss    €r   ú<lambda>zbpe_train.<locals>.<lambda>“   s   ø€ °E¸!´H€ r   )Úkeyr   r`   z The current most common pair is z + zSo we made z our zth tokenr[   z9Now the first fifty words in our training data look like:c                ó   — g | ]	}|D ]}|‘ŒŒ
S r   r   )r   r)   r   s      r   r>   zbpe_train.<locals>.<listcomp>²   s%   € Ð!QÐ!QÐ!Q¨DÈDÐ!QÐ!QÀ5 %Ð!QÐ!QÐ!QÐ!Qr   é2   r]   z:Now the first twenty words in our training data look like:é   ú
)Ú
ValueErrorÚranger,   r   r#   ÚlenÚcollectionsÚCounterrf   ÚmaxÚappendrd   rc   )rD   rA   r   r   Úranksrk   r'   Úpiecerl   Úmost_common_pairr   r   Ú	new_wordsr)   Únew_wordrt   s                  @r   rF   rF   w   s,  ø€ ð �DÒÐÝÐVÑWÔWÐWØ€EÝ�4‰[Œ[ð ð ˆØˆ�e�Q�C‰jŒjÑÐð ð  Ý?D¼}ÈWÐVZÑ?[Ô?[ð ñ  ô  €Eõ
 ˆe‰*Œ*�zÒ
!Ñ
!åÔ#Ñ%Ô%ˆØð 	!ð 	!ˆEÝ˜E # 2 #œJ¨¨a¨b¨b¬	Ñ2Ô2ð !ð !�Ø�d��”˜qÑ ��‘�ð!õ ˜uÐ*<Ð*<Ð*<Ð*<Ð=Ñ=Ô=ÐØ& qÔ)Ð,<¸QÔ,?Ñ?ˆÝ�E‘
”
ˆà"ˆˆkÑð ˆ	Øð 	'ð 	'ˆDØˆHØˆAØ•c˜$‘i”i !‘mÒ#Ð#Ø˜”G˜T ! a¡%œ[Ð)Ð-=Ò=Ð=à—O’O KÑ0Ô0Ð0Ø˜‘F�A�Aà—O’O D¨¤GÑ,Ô,Ð,Ø˜‘F�Að •c˜$‘i”i !‘mÒ#Ð#ð •C˜‘I”I ‘MÒ!Ð!Ø—’  Q¤Ñ(Ô(Ð(Ø×Ò˜XÑ&Ô&Ð&Ð&Øˆð ð 
	ÝÐbÐ5EÀaÔ5HÐbÐbÐM]Ð^_ÔM`ÐbÐbÑcÔcÐcÝÐF ÐFÐFµ#°e±*´*ÐFÐFÐFÑGÔGÐGØÐ/Ð/Ð/ÝÐQÑRÔRÐRÝ Ð!QÐ!Q°E¸#¸2¸#´JÐ!QÑ!QÔ!QÑRÔRÐRÐRØ˜hÒ&Ð&ÝÐRÑSÔSÐSØ! # 2 #œJð  ð  �DÝ˜$‘K”K�K�KÝ�$‰KŒKˆKõW ˆe‰*Œ*�zÒ
!Ñ
!ðZ €Lr   Útoken_valuesr;   r   c                ó8  — d„ dD ¦   «         }d„ | D ¦   «         }d}d }|D ]k}||t          |¦  «        z           }||k    r#||dz   t          |¦  «        z           }||k    sJ ‚|}|t          |¦  «        z  }t          ||z   d¬¦  «         Œlt          d¦  «         d S )	Nc                ó   — g | ]}d |› d�‘Œ	S )z[48;5;Úmr   )r   rk   s     r   r>   z$visualise_tokens.<locals>.<listcomp>½   s$   € ÐPÐPÐP¨!Ð% Ð%Ð%Ð%ÐPÐPÐPr   )é§   é³   é¹   éM   éP   éD   é†   c                ó<   — g | ]}|                      d d¬¦  «        ‘ŒS )r"   r7   r8   )r:   )r   rs   s     r   r>   z$visualise_tokens.<locals>.<listcomp>Á   s(   € ÐVÐVÐVÀA˜AŸHšH W°Y˜HÑ?Ô?ÐVÐVÐVr   r   r_   Ú )Úendz[0m)r}   rd   )r‡   Ú
backgroundÚunicode_token_valuesÚrunning_lengthÚ
last_colorr   r\   s          r   rc   rc   ¼   sÔ   € ØPÐPÐ/OÐPÑPÔP€Jð WÐVÈÐVÑVÔVÐà€NØ€JØ%ð %ð %ˆØ˜>­C°
©O¬OÑ;Ô<ˆØ�JÒÐØ °Ñ 2µc¸*±o´oÑEÔFˆEØ˜JÒ&Ð&Ð&Ð&Øˆ
Ø�#˜e™*œ*Ñ$ˆÝˆe�e‰m Ð$Ñ$Ô$Ð$Ð$Ý	ˆ+ÑÔÐÐÐr   c                 ó²  — d} t          t          ¦  «        5 }|                     ¦   «         }d d d ¦  «         n# 1 swxY w Y   t                               |d| ¬¦  «        }t          d¦  «         |                     d¦  «        }|                     |¦  «        dk    sJ ‚|                     |¦  «        dk    sJ ‚| 	                    |¦  «        ddgk    sJ ‚|S )	NzN's|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+iX  )rA   r   zJThis is the sequence of merges performed in order to encode 'hello world':zhello worlds   hello worlds   hellos    world)
ÚopenÚ__file__Úreadr   rG   rd   r$   r:   r5   r?   )Úgpt2_patternÚfrD   Úencr(   s        r   Útrain_simple_encodingr    Ð   s  € à]ð õ 
�h‰Œð ˜1Ø�vŠv‰xŒxˆðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ !×
&Ò
& t¸À\Ð
&Ñ
RÔ
R€Cå	Ð
VÑWÔWÐWØ�ZŠZ˜Ñ&Ô&€FØ�:Š:�fÑÔ Ò.Ð.Ð.Ð.Ø×Ò˜FÑ#Ô# ~Ò5Ð5Ð5Ð5Ø×"Ò" 6Ñ*Ô*¨x¸Ð.CÒCÐCÐCÐCà€Js   —8¸<¿<rP   )r   r	   rU   r,   r   r   r
   r    )
rD   r   rA   rB   r   r   r   r   r
   r	   )r‡   r;   r
   r   )Ú__doc__Ú
__future__r   r~   r   rJ   r   r%   rF   rc   r    r   r   r   ú<module>r£      sà   ðØ PÐ Pà "Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð à €€€à €€€ðD
ð D
ð D
ð D
ð D
ñ D
ô D
ð D
ðP NVð!ð !ð !ð !ð !ðJ GOðBð Bð Bð Bð BðJð ð ð ð(ð ð ð ð r   