§
    |@Äj20  ã                   ó´  — d dl Z d dlZd dlZd dlmZmZ d dlmZmZ d dl	m
Z
mZmZmZ d dlZi dd“dd“d	d
“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd “d!d"“d#d$“d%d&“i d'd(“d)d*“d+d,“d-d.“d/d0“d1d2“d3d4“d5d6“d7d8“d9d:“d;d<“d=d>“d?d@“dAdB“dCdD“dEdF“dGdH“¥i dIdJ“dKdL“dMdN“dOdP“dQdR“dSdT“dUdV“dWdX“dYdZ“d[d\“d]d^“d_d`“dadb“dcdd“dedf“dgdh“didj“¥i dkdl“dmdn“dodp“dqdr“dsdt“dudv“dwdx“dydz“d{d|“d}d~“dd€“d�d‚“dƒd„“d…d†“d‡dˆ“d‰dŠ“d‹dŒ“¥i d�dŽ“d�d�“d‘d’“d“d”“d•d–“d—d˜“d™dš“d›dœ“d�dž“dŸd “d¡d¢“d£d¤“d¥d¦“d§d¨“d©dª“d«d¬“d­d®“¥d¯d°d±d²d³d´dµd¶d·d¸d¹dºd»d¼d½d¾œ¥Zi d¿„ e                     ¦   «         D ¦   «         ¥dÀddd¥dÁd§d�d7d7dƒdddÂœ¥Ze G dÃ„ dÄ¦  «        ¦   «         Z ed¬Å¦  «        dÑdÈedÉefdÊ„¦   «         Z ed¬Å¦  «        dÇdddËœdÌedÉedÍee         dÎee         dÏef
dÐ„¦   «         ZdS )Òé    N)Ú	dataclassÚfield)Úcached_propertyÚ	lru_cache)ÚDictÚListÚOptionalÚTupleÚenÚenglishÚzhÚchineseÚdeÚgermanÚesÚspanishÚruÚrussianÚkoÚkoreanÚfrÚfrenchÚjaÚjapaneseÚptÚ
portugueseÚtrÚturkishÚplÚpolishÚcaÚcatalanÚnlÚdutchÚarÚarabicÚsvÚswedishÚitÚitalianÚidÚ
indonesianÚhiÚhindiÚfiÚfinnishÚviÚ
vietnameseÚheÚhebrewÚukÚ	ukrainianÚelÚgreekÚmsÚmalayÚcsÚczechÚroÚromanianÚdaÚdanishÚhuÚ	hungarianÚtaÚtamilÚnoÚ	norwegianÚthÚthaiÚurÚurduÚhrÚcroatianÚbgÚ	bulgarianÚltÚ
lithuanianÚlaÚlatinÚmiÚmaoriÚmlÚ	malayalamÚcyÚwelshÚskÚslovakÚteÚteluguÚfaÚpersianÚlvÚlatvianÚbnÚbengaliÚsrÚserbianÚazÚazerbaijaniÚslÚ	slovenianÚknÚkannadaÚetÚestonianÚmkÚ
macedonianÚbrÚbretonÚeuÚbasqueÚisÚ	icelandicÚhyÚarmenianÚneÚnepaliÚmnÚ	mongolianÚbsÚbosnianÚkkÚkazakhÚsqÚalbanianÚswÚswahiliÚglÚgalicianÚmrÚmarathiÚpaÚpunjabiÚsiÚsinhalaÚkmÚkhmerÚsnÚshonaÚyoÚyorubaÚsoÚsomaliÚafÚ	afrikaansÚocÚoccitanÚkaÚgeorgianÚbeÚ
belarusianÚtgÚtajikÚsdÚsindhiÚguÚgujaratiÚamÚamharicÚyiÚyiddishÚloÚlaoÚuzÚuzbekÚfoÚfaroeseÚhtzhaitian creoleÚpsÚpashtoÚtkÚturkmenÚnnÚnynorskÚmtÚmalteseÚsanskritÚluxembourgishÚmyanmarÚtibetanÚtagalogÚmalagasyÚassameseÚtatarÚhawaiianÚlingalaÚhausaÚbashkirÚjavaneseÚ	sundaneseÚ	cantonese)ÚsaÚlbÚmyÚboÚtlÚmgÚasÚttÚhawÚlnÚhaÚbaÚjwÚsuÚyuec                 ó   — i | ]\  }}||“Œ	S © rÓ   )Ú.0ÚcodeÚlanguages      úBC:\Projects\Historical\venv\Lib\site-packages\whisper/tokenizer.pyú
<dictcomp>rØ   s   s   € Ð>Ð>Ð>™.˜$ €x�Ð>Ð>Ð>ó    rÅ   rÄ   )ÚburmeseÚ	valencianÚflemishÚhaitianÚletzeburgeschÚpushtoÚpanjabiÚ	moldavianÚmoldovanÚ	sinhaleseÚ	castilianÚmandarinc                   óü  — e Zd ZU dZej        ed<   eed<   dZe	e
         ed<   dZe	e
         ed<   dZee         ed<    ee¬	¦  «        Zee
ef         ed
<   d„ Zd„ Zdee         de
fd„Zdee         de
fd„Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Z edefd„¦   «         Z!d„ Z"edee         fd„¦   «         Z#edee
         fd„¦   «         Z$edee         fd„¦   «         Z%edee         fd„¦   «         Z&d ee         fd!„Z'd ee         fd"„Z(d ee         fd#„Z)dS )$Ú	TokenizerzIA thin wrapper around `tiktoken` providing quick access to special tokensÚencodingÚnum_languagesNrÖ   ÚtaskrÓ   Úsot_sequence)Údefault_factoryÚspecial_tokensc                 ó  — | j         j        D ]&}| j                              |¦  «        }|| j        |<   Œ'| j        d         }| j        d         }| j        d         }t	          t
                               ¦   «         ¦  «        d | j        …         }|g}| j        �3| 	                    |dz   | 
                    | j        ¦  «        z   ¦  «         | j        �$| j        dk    r|n|}| 	                    |¦  «         t	          |¦  «        | _        d S )Nú<|startoftranscript|>ú<|translate|>ú<|transcribe|>é   Ú
transcribe)rè   Úspecial_tokens_setÚencode_single_tokenrí   ÚtupleÚ	LANGUAGESÚkeysré   rÖ   ÚappendÚindexrê   rë   )	ÚselfÚspecialÚspecial_tokenÚsotÚ	translateró   Úlangsrë   Ú
task_tokens	            r×   Ú__post_init__zTokenizer.__post_init__Ž   s
  € Ø”}Ô7ð 	9ð 	9ˆGØ œM×=Ò=¸gÑFÔFˆMØ+8ˆDÔ Ñ(Ð(àÔ&Ð'>Ô?ˆØÔ,¨_Ô=ˆ	ØÔ-Ð.>Ô?ˆ
å•i—n’nÑ&Ô&Ñ'Ô'Ð(<¨$Ô*<Ð(<Ô=ˆØ�uˆØŒ=Ð$Ø×Ò  a¡¨%¯+ª+°d´mÑ*DÔ*DÑ DÑEÔEÐEØŒ9Ð Ø,0¬I¸Ò,EÐ,E˜j˜jÈ9ˆJØ×Ò 
Ñ+Ô+Ð+å! ,Ñ/Ô/ˆÔÐÐrÙ   c                 ó(   —  | j         j        |fi |¤ŽS ©N)rè   Úencode)rû   ÚtextÚkwargss      r×   r  zTokenizer.encode¡   s   € Ø#ˆtŒ}Ô# DÐ3Ð3¨FÐ3Ð3Ð3rÙ   Ú	token_idsÚreturnc                 óF   ‡ — ˆ fd„|D ¦   «         } ‰ j         j        |fi |¤ŽS )Nc                 ó*   •— g | ]}|‰j         k     ¯|‘ŒS rÓ   )Útimestamp_begin)rÔ   Útrû   s     €r×   ú
<listcomp>z$Tokenizer.decode.<locals>.<listcomp>¥   s&   ø€ ÐFÐFÐF˜1¨Q°Ô1EÒ-EÐ-E�QÐ-EÐ-EÐ-ErÙ   ©rè   Údecode©rû   r  r  s   `  r×   r  zTokenizer.decode¤   s9   ø€ ØFÐFÐFÐF 	ÐFÑFÔFˆ	Ø#ˆtŒ}Ô# IÐ8Ð8°Ð8Ð8Ð8rÙ   c                 ó(   —  | j         j        |fi |¤ŽS )zÆ
        Timestamp tokens are above other special tokens' id range and are ignored by `decode()`.
        This method decodes given tokens with timestamps tokens annotated, e.g. "<|1.08|>".
        r  r  s      r×   Údecode_with_timestampsz Tokenizer.decode_with_timestamps¨   s!   € ð
 $ˆtŒ}Ô# IÐ8Ð8°Ð8Ð8Ð8rÙ   c                 ó   — | j         j        S r  )rè   Ú	eot_token©rû   s    r×   ÚeotzTokenizer.eot¯   s   € àŒ}Ô&Ð&rÙ   c                 ó   — | j         d         S )Nrñ   ©rí   r  s    r×   ró   zTokenizer.transcribe³   s   € àÔ"Ð#3Ô4Ð4rÙ   c                 ó   — | j         d         S )Nrð   r  r  s    r×   rÿ   zTokenizer.translate·   ó   € àÔ" ?Ô3Ð3rÙ   c                 ó   — | j         d         S )Nrï   r  r  s    r×   rþ   zTokenizer.sot»   s   € àÔ"Ð#:Ô;Ð;rÙ   c                 ó   — | j         d         S )Nú<|startoflm|>r  r  s    r×   Úsot_lmzTokenizer.sot_lm¿   r  rÙ   c                 ó   — | j         d         S )Nú<|startofprev|>r  r  s    r×   Úsot_prevzTokenizer.sot_prevÃ   s   € àÔ"Ð#4Ô5Ð5rÙ   c                 ó   — | j         d         S )Nú<|nospeech|>r  r  s    r×   Ú	no_speechzTokenizer.no_speechÇ   s   € àÔ" >Ô2Ð2rÙ   c                 ó   — | j         d         S )Nú<|notimestamps|>r  r  s    r×   Úno_timestampszTokenizer.no_timestampsË   s   € àÔ"Ð#5Ô6Ð6rÙ   c                 ó   — | j         d         S )Nz<|0.00|>r  r  s    r×   r  zTokenizer.timestamp_beginÏ   s   € àÔ" :Ô.Ð.rÙ   c                 ób   — | j         €t          d¦  «        ‚|                      | j         ¦  «        S )zGReturns the token id corresponding to the value of the `language` fieldNz6This tokenizer does not have language token configured)rÖ   Ú
ValueErrorÚto_language_tokenr  s    r×   Úlanguage_tokenzTokenizer.language_tokenÓ   s1   € ð Œ=Ð ÝÐUÑVÔVÐVà×%Ò% d¤mÑ4Ô4Ð4rÙ   c                 ón   — | j                              d|› d�d ¦  «        x}r|S t          d|› d�¦  «        ‚)Nú<|ú|>z	Language z not found in tokenizer.)rí   ÚgetÚKeyError)rû   rÖ   Útokens      r×   r,  zTokenizer.to_language_tokenÛ   sM   € ØÔ'×+Ò+Ð,=°Ð,=Ð,=Ð,=¸tÑDÔDÐDˆ5ð 	ØˆLåÐE 8ÐEÐEÐEÑFÔFÐFrÙ   c                 óÞ   — g }| j                              ¦   «         D ]6\  }}|                     d¦  «        t          v r|                     |¦  «         Œ7t          |¦  «        d | j        …         S )Nú<|>)rí   ÚitemsÚstripr÷   rù   rö   ré   )rû   Úresultr3  Útoken_ids       r×   Úall_language_tokenszTokenizer.all_language_tokensá   sm   € àˆØ#Ô2×8Ò8Ñ:Ô:ð 	(ð 	(‰OˆE�8Ø�{Š{˜5Ñ!Ô!¥YÐ.Ð.Ø—’˜hÑ'Ô'Ð'øÝ�V‰}Œ}Ð1˜tÔ1Ð1Ô2Ð2rÙ   c                 óD   ‡ — t          ˆ fd„‰ j        D ¦   «         ¦  «        S )Nc              3   ój   •K  — | ]-}‰                      |g¦  «                             d ¦  «        V — Œ.dS )r5  N)r  r7  )rÔ   Ú_lrû   s     €r×   ú	<genexpr>z/Tokenizer.all_language_codes.<locals>.<genexpr>ë   s?   øè è € ÐWÐW¸�T—[’[ " Ñ&Ô&×,Ò,¨UÑ3Ô3ÐWÐWÐWÐWÐWÐWrÙ   )rö   r:  r  s   `r×   Úall_language_codeszTokenizer.all_language_codesé   s)   ø€ åÐWÐWÐWÐW¸dÔ>VÐWÑWÔWÑWÔWÐWrÙ   c                 óV   — t          t          | j        ¦  «        | j        gz   ¦  «        S r  )rö   Úlistrë   r(  r  s    r×   Ú#sot_sequence_including_notimestampsz-Tokenizer.sot_sequence_including_notimestampsí   s&   € å•T˜$Ô+Ñ,Ô,°Ô0BÐ/CÑCÑDÔDÐDrÙ   c                 ó\  — t          d¦  «        }|d                     ¦   «         z  }t          d¦  «        }t          d„ |D ¦   «         ¦  «        sJ ‚| j                             d¦  «        d         | j                             d¦  «        d         h}|t          |¦  «        z   D ]n}| j                             |¦  «        | j                             d|z   ¦  «        fD ]4}t          |¦  «        d	k    s||v r|                     |d         ¦  «         Œ5Œot          t          |¦  «        ¦  «        S )
u…  
        Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech
        annotations, to prevent sampling texts that are not actually spoken in the audio, e.g.

        - â™ªâ™ªâ™ª
        - ( SPEAKING FOREIGN LANGUAGE )
        - [DAVID] Hey there,

        keeping basic punctuations like commas, periods, question marks, exclamation points, etc.
        u#   "#()*+/:;<=>@[\]^_`{|}~ã€Œã€�ã€Žã€�uK   << >> <<< >>> -- --- -( -[ (' (" (( )) ((( ))) [[ ]] {{ }} â™ªâ™ª â™ªâ™ªâ™ªu   â™©â™ªâ™«â™¬â™­â™®â™¯c              3   óP   K  — | ]!}d t          |¦  «        cxk    odk    nc V — Œ"dS )i@&  i&  N)Úord)rÔ   Úcs     r×   r>  z.Tokenizer.non_speech_tokens.<locals>.<genexpr>  sD   è è € ÐEÐE°!�6�S ™VœVÐ-Ð-Ò-Ð- vÒ-Ð-Ð-Ð-ÐEÐEÐEÐEÐEÐErÙ   z -r   z 'ú rò   )
rA  ÚsplitÚsetÚallrè   r  ÚlenÚaddrö   Úsorted)rû   ÚsymbolsÚmiscellaneousr8  ÚsymbolÚtokenss         r×   Únon_speech_tokenszTokenizer.non_speech_tokensñ   s:  € õ Ð=Ñ>Ô>ˆØØZ×`Ò`ÑbÔbñ	
ˆõ Ð3Ñ4Ô4ˆÝÐEÐE°}ÐEÑEÔEÑEÔEÐEÐEÐEð ”-×&Ò& tÑ,Ô,¨QÔ/°´×1EÒ1EÀdÑ1KÔ1KÈAÔ1NÐOˆØ¥ ]Ñ 3Ô 3Ñ3ð 	*ð 	*ˆFà”×$Ò$ VÑ,Ô,Ø”×$Ò$ S¨6¡\Ñ2Ô2ðð *ð *�õ �v‘;”; !Ò#Ð# v°Ð'>Ð'>Ø—J’J˜v aœyÑ)Ô)Ð)øð*õ •V˜F‘^”^Ñ$Ô$Ð$rÙ   rQ  c                 óh   — | j         dv r|                      |¦  «        S |                      |¦  «        S )N>   r   r¥   rÅ   rG   r   rÑ   )rÖ   Úsplit_tokens_on_unicodeÚsplit_tokens_on_spaces)rû   rQ  s     r×   Úsplit_to_word_tokenszTokenizer.split_to_word_tokens  s:   € ØŒ=ÐAÐAÐAð ×/Ò/°Ñ7Ô7Ð7à×*Ò*¨6Ñ2Ô2Ð2rÙ   c                 ón  — |                       |¦  «        }d}g }g }g }d}|D ]�}|                     |¦  «         |                       |¦  «        }	||	vs"|||	                     |¦  «        z            |k    r>|                     |	¦  «         |                     |¦  «         g }|t          |	¦  «        z  }Œ‘||fS )Nu   ï¿½r   )r  rù   rú   rK  )
rû   rQ  Údecoded_fullÚreplacement_charÚwordsÚword_tokensÚcurrent_tokensÚunicode_offsetr3  Údecodeds
             r×   rT  z!Tokenizer.split_tokens_on_unicode  sâ   € Ø×2Ò2°6Ñ:Ô:ˆØ#ÐàˆØˆØˆØˆàð 	/ð 	/ˆEØ×!Ò! %Ñ(Ô(Ð(Ø×1Ò1°.ÑAÔAˆGð !¨Ð/Ð/Ø °·²Ð?OÑ1PÔ1PÑ PÔQØ#ò$ð $ð —’˜WÑ%Ô%Ð%Ø×"Ò" >Ñ2Ô2Ð2Ø!#�Ø¥# g¡,¤,Ñ.�øà�kÐ!Ð!rÙ   c                 óÔ  — |                       |¦  «        \  }}g }g }t          ||¦  «        D ]¸\  }}|d         | j        k    }|                     d¦  «        }	|                     ¦   «         t
          j        v }
|s|	s|
st          |¦  «        dk    r+|                     |¦  «         |                     |¦  «         Œ�|d         |z   |d<   |d          	                    |¦  «         Œ¹||fS )Nr   rG  éÿÿÿÿ)
rT  Úzipr  Ú
startswithr7  ÚstringÚpunctuationrK  rù   Úextend)rû   rQ  ÚsubwordsÚsubword_tokens_listrZ  r[  ÚsubwordÚsubword_tokensrü   Ú
with_spacerd  s              r×   rU  z Tokenizer.split_tokens_on_spaces7  s  € Ø(,×(DÒ(DÀVÑ(LÔ(LÑ%ˆÐ%ØˆØˆå'*¨8Ð5HÑ'IÔ'Ið 		7ð 		7Ñ#ˆG�^Ø$ QÔ'¨4¬8Ò3ˆGØ ×+Ò+¨CÑ0Ô0ˆJØ!Ÿ-š-™/œ/­VÔ-?Ð?ˆKØð 7˜*ð 7¨ð 7µs¸5±z´zÀQ²°Ø—’˜WÑ%Ô%Ð%Ø×"Ò" >Ñ2Ô2Ð2Ð2à! "œI¨Ñ/��b‘	Ø˜B”×&Ò& ~Ñ6Ô6Ð6Ð6à�kÐ!Ð!rÙ   )*Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtiktokenÚEncodingÚ__annotations__ÚintrÖ   r	   Ústrrê   rë   r
   r   Údictrí   r   r  r  r   r  r  r   r  ró   rÿ   rþ   r  r"  r%  r(  r  r-  r,  r:  r?  rB  rR  rV  rT  rU  rÓ   rÙ   r×   rç   rç   ƒ   s®  € € € € € € àSÐSàÔÐÐÑØÐÐÑØ"€Hˆh�sŒmÐ"Ð"Ñ"Ø€Dˆ(�3Œ-ÐÐÑØ!€L�%˜”*Ð!Ð!Ñ!Ø%* U¸4Ð%@Ñ%@Ô%@€N�D˜˜c˜”NÐ@Ð@Ñ@ð0ð 0ð 0ð&4ð 4ð 4ð9  S¤	ð 9¸ð 9ð 9ð 9ð 9ð9°°S´	ð 9Èð 9ð 9ð 9ð 9ð ð'�Sð 'ð 'ð 'ñ „_ð'ð ð5˜Cð 5ð 5ð 5ñ „_ð5ð ð4˜3ð 4ð 4ð 4ñ „_ð4ð ð<�Sð <ð <ð <ñ „_ð<ð ð4˜ð 4ð 4ð 4ñ „_ð4ð ð6˜#ð 6ð 6ð 6ñ „_ð6ð ð3˜3ð 3ð 3ð 3ñ „_ð3ð ð7˜sð 7ð 7ð 7ñ „_ð7ð ð/ ð /ð /ð /ñ „_ð/ð ð5 ð 5ð 5ð 5ñ „_ð5ðGð Gð Gð ð3 U¨3¤Zð 3ð 3ð 3ñ „_ð3ð ðX E¨#¤Jð Xð Xð Xñ „_ðXð ðE°U¸3´Zð Eð Eð Eñ „_ðEð ð!% 5¨¤:ð !%ð !%ð !%ñ „_ð!%ðF3¨4°¬9ð 3ð 3ð 3ð 3ð"¨d°3¬ið "ð "ð "ð "ð2"¨T°#¬Yð "ð "ð "ð "ð "ð "rÙ   rç   )ÚmaxsizeÚgpt2éc   Únameré   c                 óB  — t           j                             t           j                             t          ¦  «        d| › d�¦  «        }d„ d„ t          |¦  «        D ¦   «         D ¦   «         }t          |¦  «        }i }ddgd„ t          t           	                    ¦   «         ¦  «        d |…         D ¦   «         ¢d‘d	‘d
‘d‘d‘d‘d„ t          d¦  «        D ¦   «         ¢}|D ]}|||<   |dz  }Œt          j        t           j                             |¦  «        |d||¬¦  «        S )NÚassetsz	.tiktokenc                 óX   — i | ]'\  }}t          j        |¦  «        t          |¦  «        “Œ(S rÓ   )Úbase64Ú	b64decoderr  )rÔ   r3  Úranks      r×   rØ   z get_encoding.<locals>.<dictcomp>M  s?   € ð ð ð áˆE�4õ 	Ô˜ÑÔ¥ T¡¤ðð ð rÙ   c              3   óB   K  — | ]}|¯|                      ¦   «         V — Œd S r  )rH  )rÔ   Úlines     r×   r>  zget_encoding.<locals>.<genexpr>O  s/   è è € ÐNÐN¨TÈÐN˜DŸJšJ™LœLÐNÐNÐNÐNÐNÐNrÙ   z<|endoftext|>rï   c                 ó   — g | ]}d |› d�‘Œ	S )r/  r0  rÓ   )rÔ   Úlangs     r×   r  z get_encoding.<locals>.<listcomp>W  s    € Ð	KÐ	KÐ	K˜Dˆ-ˆtˆ-ˆ-ˆ-Ð	KÐ	KÐ	KrÙ   rð   rñ   r  r!  r$  r'  c                 ó"   — g | ]}d |dz  d›d�‘ŒS )r/  g{®Gáz”?z.2fr0  rÓ   )rÔ   Úis     r×   r  z get_encoding.<locals>.<listcomp>^  s+   € Ð	5Ð	5Ð	5 AÐ
ˆq�4‰xÐ
Ð
Ð
Ð
Ð	5Ð	5Ð	5rÙ   iÝ  rò   zJ's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+)rx  Úexplicit_n_vocabÚpat_strÚmergeable_ranksrí   )ÚosÚpathÚjoinÚdirnameÚ__file__ÚopenrK  rA  r÷   rø   Úrangero  rp  Úbasename)rx  ré   Ú
vocab_pathÚranksÚn_vocabrí   Úspecialsr3  s           r×   Úget_encodingr”  J  sr  € å”—’�bœgŸošo­hÑ7Ô7¸ÀdÐCUÐCUÐCUÑVÔV€Jðð àNÐNµT¸*Ñ5EÔ5EÐNÑNÔNðñ ô €Eõ �%‰jŒj€GØ€Nð 	Øðð 
LÐ	K¥D­¯ªÑ)9Ô)9Ñ$:Ô$:¸>¸M¸>Ô$JÐ	KÑ	KÔ	Kðð 	ð	ð
 	ðð 	ðð 	ðð 	ðð 	ðð 
6Ð	5­¨t©¬Ð	5Ñ	5Ô	5ð€Hð ð ð ˆØ 'ˆ�uÑØ�1‰ˆˆåÔÝŒW×Ò˜jÑ)Ô)Ø ØaØØ%ðñ ô ð rÙ   )ré   rÖ   rê   ÚmultilingualrÖ   rê   r	  c                ó   — |�F|                      ¦   «         }|t          vr)|t          v rt          |         }nt          d|› �¦  «        ‚| rd}|pd}|pd}nd}d }d }t	          ||¬¦  «        }t          ||||¬¦  «        S )NzUnsupported language: r•  r   ró   rv  )rx  ré   )rè   ré   rÖ   rê   )Úlowerr÷   ÚTO_LANGUAGE_CODEr+  r”  rç   )r•  ré   rÖ   rê   Úencoding_namerè   s         r×   Úget_tokenizerrš  n  s¸   € ð ÐØ—>’>Ñ#Ô#ˆØ�9Ð$Ð$ØÕ+Ð+Ð+Ý+¨HÔ5��å Ð!D¸(Ð!DÐ!DÑEÔEÐEàð Ø&ˆØÐ#˜tˆØÐ#�|ˆˆàˆØˆØˆå ¸mÐLÑLÔL€HåØ¨ÀÐPTðñ ô ð rÙ   )rv  rw  )r|  rˆ  rc  Údataclassesr   r   Ú	functoolsr   r   Útypingr   r   r	   r
   ro  r÷   r6  r˜  rç   rs  rr  r”  Úboolrš  rÓ   rÙ   r×   ú<module>rŸ     s  ðØ €€€Ø 	€	€	€	Ø €€€Ø (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ð .à €€€ðeØˆ)ðeàˆ)ðeð 	ˆ(ðeð 	ˆ)ð	eð
 	ˆ)ðeð 	ˆ(ðeð 	ˆ(ðeð 	ˆ*ðeð 	ˆ,ðeð 	ˆ)ðeð 	ˆ(ðeð 	ˆ)ðeð 	ˆ'ðeð 	ˆ(ðeð 	ˆ)ðeð  	ˆ)ð!eð" 	ˆ,ð#eð eð$ 	ˆ'ð%eð& 	ˆ)ð'eð( 	ˆ,ð)eð* 	ˆ(ð+eð, 	ˆ+ð-eð. 	ˆ'ð/eð0 	ˆ'ð1eð2 	ˆ'ð3eð4 	ˆ*ð5eð6 	ˆ(ð7eð8 	ˆ+ð9eð: 	ˆ'ð;eð< 	ˆ+ð=eð> 	ˆ&ð?eð@ 	ˆ&ðAeðB 	ˆ*ðCeðD 	ˆ+ðEeð eð eðF 	ˆ,ðGeðH 	ˆ'ðIeðJ 	ˆ'ðKeðL 	ˆ+ðMeðN 	ˆ'ðOeðP 	ˆ(ðQeðR 	ˆ(ðSeðT 	ˆ)ðUeðV 	ˆ)ðWeðX 	ˆ)ðYeðZ 	ˆ)ð[eð\ 	ˆ-ð]eð^ 	ˆ+ð_eð` 	ˆ)ðaeðb 	ˆ*ðceðd 	ˆ,ðeeðf 	ˆ(ðgeð eð eðh 	ˆ(ðieðj 	ˆ+ðkeðl 	ˆ*ðmeðn 	ˆ(ðoeðp 	ˆ+ðqeðr 	ˆ)ðseðt 	ˆ(ðueðv 	ˆ*ðweðx 	ˆ)ðyeðz 	ˆ*ð{eð| 	ˆ)ð}eð~ 	ˆ)ðeð@ 	ˆ)ðAeðB 	ˆ'ðCeðD 	ˆ'ðEeðF 	ˆ(ðGeðH 	ˆ(ðIeð eð eðJ 	ˆ+ðKeðL 	ˆ)ðMeðN 	ˆ*ðOeðP 	ˆ,ðQeðR 	ˆ'ðSeðT 	ˆ(ðUeðV 	ˆ*ðWeðX 	ˆ)ðYeðZ 	ˆ)ð[eð\ 	ˆ%ð]eð^ 	ˆ'ð_eð` 	ˆ)ðaeðb 	Ð
ðceðd 	ˆ(ðeeðf 	ˆ)ðgeðh 	ˆ)ðieðj 	ˆ)ðkeð eðl Ø
Ø
Ø
Ø
Ø
Ø
Ø
ØØ
Ø
Ø
Ø
Ø
ØðIeð eð e€	ðPØ>Ð>¨I¯OªOÑ,=Ô,=Ð>Ñ>Ô>ðàØØØØØØØØØØØðð ð Ð ð" ðC"ð C"ð C"ð C"ð C"ñ C"ô C"ñ „ðC"ðL €�4ÐÑÔð ð  �sð  °Cð  ð  ð  ñ Ôð ðF €�4ÐÑÔð Ø"Øðð ð Øðð ðð �sŒmð	ð
 �3Œ-ðð ðð ð ñ Ôðð ð rÙ   