jUddlmZddlmZddlmZddlmZmZm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZmZmZddlmZmZmZm Z m!Z!m"Z"m#Z#eezezezezZ$de%d<Gd d Z&ed d7dZ'de(dDZ)de%d<GddZ*Gdde*Z+Gdde*Z,Gdde*Z-Gdde*Z.Gdd e*Z/Gd!d"e*Z0Gd#d$e*Z1Gd%d&e*Z2Gd'd(e*Z3Gd)d*e*Z4ed d8d/Z5 d9d:d6Z6d S);) annotations) lru_cache) getLogger)COMMON_CJK_CHARACTERSCOMMON_SAFE_ASCII_CHARACTERSTRACECompatibleFamillyRange _ACCENTUATED_ARABIC_ARABIC_ISOLATED_FORM&_BASIC_LATIN_COMPATIBLE_RANGE_FAMILIES_CJK_COMPATIBLE_RANGE_FAMILIES#_COMPATIBLE_WITH_ANY_RANGE_FAMILIES_HANGUL_HALFWIDTH_KATAKANA _HIRAGANA _KATAKANA_LATIN _LIGATURE_RANGE_FAMILIES_SENTENCE_OPEN_PUNCTUATION _SUPERSCRIPT_THAI)_character_flags is_emoticonis_punctuation is_separator is_symbol remove_accent unicode_rangeint _GLYPH_MASKc8eZdZUdZdZded<ded<ded<ded<ded <ded <ded <ded <ded <ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded <ded!<ded"<d'd%Zd&S)(CharInfoz>Pre-computed character properties shared across all detectors.) character printablealphaupperlowerspacedigitis_ascii case_variableflags accentuatedlatinis_cjk is_katakanais_halfwidth_katakana is_arabic is_ligatureis_superscriptis_sentence_open_punctuationis_glyphpunctsymrangesepemoticonsafe common_cjk unaccentedstrr'boolr(r)r*r+r,r-r.r/r#r0r1r2r3r4r5r6r7r8r9r:r;r< str | Noner=r>r?r@rArBreturnNonec ||_t|}|dkrd|_d|_||_d|_d|_|tv|_d|_ d|_ d|_ d|_ d|_ d|_d|_d|_d|cxkrdkrWnnTd|_d|_d|_d|_d|_d|_d|_t0|_d|_d|_d|_n@d|cxkrdkrWnnTd|_d|_d|_d|_d|_d|_d|_t0|_d|_d|_d|_nd|cxkrd krRnnOd|_d|_d|_d|_d|_d|_d|_d |_d|_d|_d|_n}|d ksd |cxkrd krVnnSd|_d|_d|_d|_d|_|d k|_d|_d |_d|_d|_d|_n||_d|_d|_d|_d|_d|_d|_d |_d|_|jrt=|nd|_|jrt?|nd|_nd|_d|_||_| |_|!|_|"|_|#|_|$|_|j|jk|_tK|}|jrd|_ntM||_||_tO|tPz|_tO|t0z|_tO|tRz|_ tO|tTz|_ tO|tVz|_ tO|tXz|_ tO|tZz|_ tO|t\z|_tO|t^z|_tO|t`z|_|jr|jrtc||_n||_|j o|tdv|_|jr)t=||_t?||_nd|_d|_tg||_4tk||_6dS)zkCompute all properties for *character* (built once per codepoint, every branch assigns every slot).TFAZaz09r N)7r'ordr.r1rBr?rArr@r3r4r5r6r7r8r9r:r)r*r+r,r-r(r/rr0r2r;r< isprintablerr isalphaisupperislowerisspaceisdigitrrrDr rrrr rrrr$r!rr"r=rr>)selfr'or0s EC:\Projects\Muse2API\.venv\Lib\site-packages\charset_normalizer/md.py__init__zCharInfo.__init__ks#Y s77 DM$D 'DO!DM#DO!%AADIDK$D ).D &"DN$D "'D 05D -!DMQ}}}}"}}}}}! ! " " " !%%)"# ! "  qC! " ! " " !%%)"# ! "  qB" " " " ! !%%*" " "  bQ!\\\\r\\\\\" " " ! " !"b%*" " "  "+!6!6!8!8" " " " " %*" " :>.S^I666e 37>L9Y///u"DMDI&2244DN"**,,DJ"**,,DJ"**,,DJ"**,,DJ"**,,DJ!%tz!9D *)44Ez 7 % +I 6 6 DJ#EL$899D efn--DJut|,,DK#EI$566D )-e6I.I)J)JD &!%'/22DN#EI$566D "&u|';"<"NNNOOOKKKKKKKKKKKKKKKNNNJJJKKKLLLOOO&&&&NNNKKK III IIINNNJJJOOOH+H+H+H+H+H+r^r&N)maxsizer'rCrFc t|S)zBBuild (once per codepoint) and cache the CharInfo for *character*.)r&)r's r\ _char_inforhs I  r^cFg|]}tt|Sre)r&chr).0 _codepoints r\ rms3$$$",HS__$$$r^rIzlist[CharInfo]_ASCII_CHAR_INFOc>eZdZdZdZdd Zdd Zedd Zd S)MessDetectorPluginzy Base abstract class used for mess detection plugins. All detectors MUST extend and implement given methods. rer'rCinfor&rFrGct)z The main routine to be executed upon character. Insert the logic in witch the text would be considered chaotic. NotImplementedErrorrZr'rqs r\ feed_infozMessDetectorPlugin.feed_info s "!r^ct)zB Permit to reset the plugin to the initial state. rsrZs r\resetzMessDetectorPlugin.resets "!r^floatct)z Compute the chaos ratio based on what your feed() has seen. Must NOT be lower than 0.; No restriction gt 0. rsrxs r\ratiozMessDetectorPlugin.ratios "!r^Nr'rCrqr&rFrGrFrGrFrz) r_r`rarbrcrvrypropertyr|rer^r\rprpsm I"""""""" """X"""r^rpcBeZdZdZddZdd Zdd Zedd Zd S) TooManySymbolOrPunctuationPlugin_punctuation_count _symbol_count_character_count_last_printable_charrFrGc>d|_d|_d|_d|_dSNrrrxs r\r]z)TooManySymbolOrPunctuationPlugin.__init__*s('("#%&04!!!r^r'rCrqr&c|xjdz c_||jkrD|js=|jr|xjdz c_n%|js|jr|js|xjdz c_||_dS)1Optimized feed using pre-computed character info.rN) rrr@r;rr-r<r?rrus r\rvz*TooManySymbolOrPunctuationPlugin.feed_info1s " 1 1 1$) 1z (''1,'''Z (DH (T] (""a'""$-!!!r^c0d|_d|_d|_dSr)rrrrxs r\ryz&TooManySymbolOrPunctuationPlugin.reset=s "# !r^rzc^|jdkrdS|j|jz|jz }|dkr|ndS)Nr333333?)rrr)rZratio_of_punctuations r\r|z&TooManySymbolOrPunctuationPlugin.ratioBsK  A % %3  #d&8 8  !'"(AV'V(=(E(E$$3Nr^Nr~r}rrrer^r\rrNs{:I))))))))$$$$OOOXOOOr^rcBeZdZdZddZdd Zdd Zedd Zd S)UnprintablePlugin_unprintable_countr _has_escaperFrGc0d|_d|_d|_dSNrFrrxs r\r]zUnprintablePlugin.__init__ls '(%&!&r^r'rCrqr&c|dkrd|_|js#|js|dkr|dkr|xjdz c_|xjdz c_dS)rTurN)rr(r,rrrus r\rvzUnprintablePlugin.feed_infoqsq   #D  )J )V##X%%  # #q ( # # "r^c"d|_d|_dSr)rrrxs r\ryzUnprintablePlugin.resets"# r^rzcR|jdkrdS|jrdS|jdz|jz S)Nrr?r)rrrrxs r\r|zUnprintablePlugin.ratios;  A % %3   3'!+t/DDDr^Nr~r}rrrer^r\rris{II'''' # # # #!!!!EEEXEEEr^rcBeZdZdZddZdd Zdd Zedd Zd S)SuspiciousDuplicateAccentPlugin_successive_countr_last_latin_character_last_was_accentuatedrFrGc>d|_d|_d|_d|_dSrrrxs r\r]z(SuspiciousDuplicateAccentPlugin.__init__s(&'%&6:"+0"""r^r'rCrqr&c|xjdz c_|jV|jrO|jrH|jr|jjr|xjdz c_|j|jjkr|xjdz c_||_|j|_dSr)rrr1rr*rrBrus r\rvz)SuspiciousDuplicateAccentPlugin.feed_infos "  & 2  3* 3z ,d8> ,&&!+&&$"<"GGG&&!+&&%)"%)%5"""r^c>d|_d|_d|_d|_dSrrrxs r\ryz%SuspiciousDuplicateAccentPlugin.resets(!" !%)"%*"""r^rzc@|jdkrdS|jdz|jz S)Nrrr)rrrxs r\r|z%SuspiciousDuplicateAccentPlugin.ratios+  A % %3&*d.CCCr^Nr~r}rrrer^r\rrs}I1111 6 6 6 6++++ DDDXDDDr^rcBeZdZdZddZdd Zdd Zedd Zd S)SuspiciousRange"_suspicious_successive_range_countr_last_printable_seen_last_printable_rangerFrGc>d|_d|_d|_d|_dSrrrxs r\r]zSuspiciousRange.__init__s(78/%&04!15"""r^r'rCrqr&c0|xjdz c_|js|js|jrd|_d|_dS|j||_|j|_dS|j}|j}||ks| t||r|xjdz c_||_||_dSr) rr,r;r@rrr= is_suspiciously_successive_ranger)rZr'rqunicode_range_aunicode_range_bs r\rvzSuspiciousRange.feed_infos " :  ty (,D %)-D & F  $ ,(1D %)-D & F&*&@&*j o - -1H/QQ =771<77$-!%4"""r^c>d|_d|_d|_d|_dSr)rrrrrxs r\ryzSuspiciousRange.resets( !23/$(!%)"""r^rzcD|jdkrdS|jdz|jz }|S)NrRrr)rr)rZratio_of_suspicious_range_usages r\r|zSuspiciousRange.ratios8  B & &3  3a 7  !2"'/.r^Nr~r}rrrer^r\rrswI6666 55552**** ///X///r^rcBeZdZdZddZdd Zdd Zedd Zd S)SuperWeirdWordPlugin _word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchr_bad_character_count_buffer_length_buffer_last_char_buffer_last_char_accentuated_buffer_accent_count_buffer_glyph_count_buffer_upper_count_buffer_first_lower_buffer_has_non_ascii_buffer_last_char_ligature_buffer_has_internal_ligature_is_current_word_invalid_invalid_word_countrFrGcd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_d|_d|_d|_d|_dSrrrxs r\r]zSuperWeirdWordPlugin.__init__ s !$%() */!). %&)*!#$-138*)*!() () ). +0"05'38*.3%()   r^r'rCrqr&c |jr|jrd|_|j|_|jdkr |j|_|xjdz c_||_|jr|xj dz c_ |j sd|_ |j |_ |j r|xjdz c_|jr|xjdz c_n|js|jr|j rd|_dS|jsdS|js|jr|jrd|_d|_|js|js|jr|xjdz c_|j}|xj|z c_|dkr|j|z dkrd|_n|j r<|jr#|j |kr|xjdz c_d|_nV|jdkrd|_|xjdz c_n3|j r,|jr%|j |dz kr|xjdz c_d|_|dkr9|jr2|j dko |j |z dk}|s|xjdz c_d|_|jr'|xjdz c_|xj|z c_d |_|jr|xj dz c_ d |_d |_d|_d|_d |_ d|_d|_d|_ d |_d |_ d |_d |_dS|d vr5|j!s0|j"r+d|_|xjdz c_||_d |_ dSdSdSdS) rTrrNg?rF>_-<=>|~)#r)rrr7rr+rrr*rr.rr1rrr:rrr2r9r8rrr,r;r>rrrVrrrrr-r<)rZr'rq buffer_lengthprobable_camel_caseds r\rvzSuperWeirdWordPlugin.feed_info"s : . :592.2.>D +"a''+/:(   1 $  %.D "z .((A-((= 2-1*151AD . /))Q.))} 0((A-(((- 0tz 0TEU 0+/( F"  F  , 1   1$($F 1)-D %,0D ) :B 7B 7txB 7    !  !%!4M  ! !] 2 ! !!!,}<CC04D--65.668850MAA,,1,,04D---2204D-,,1,,,. 50 50MA4EEE ,,1,,04D-""t'?",q0H0=@CG% ,5,,1,,04D-( 2$$)$$))]:)),1), 6((A-((05-',D $"#D %)D "16D .()D %'(D $'(D $',D $).D &.3D +16D . . . @ @ @J A A)-D %   1 $  %.D "16D . . . A @ @ @ @ @r^cd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_d|_d|_d|_d|_dSr)rrrrrrrrrrrrrrrrrrrrxs r\ryzSuperWeirdWordPlugin.resets!%-2*$)!#(   !$%!#$ $%!#$ #$ #( %*"*/'-2*(-%#$   r^rzcb|jrdS|jdkr |jdkrdS|j|jz S)Nr rr)rrrrrrxs r\r|zSuperWeirdWordPlugin.ratiosC  # 3  r ! !d&>!&C&C3(4+@@@r^Nr~r}rrrer^r\rrsI,****0b7b7b7b7H%%%%*AAAXAAAr^rcFeZdZdZdZddZdd Zdd Zedd Z dS)CjkUncommonPluginz< Detect messy CJK text that probably means nothing. r_uncommon_countrFrGc"d|_d|_dSrrrxs r\r]zCjkUncommonPlugin.__init__s%&$%r^r'rCrqr&cX|xjdz c_|js|xjdz c_dSdSr)rrArrus r\rvzCjkUncommonPlugin.feed_infosD " &  A %  & &r^c"d|_d|_dSrrrxs r\ryzCjkUncommonPlugin.resets ! r^rzc|jdkrdSd|jz|jz dt|jdzz }td|S)Nrrr)rrmax)rZuncommon_form_usages r\r|zCjkUncommonPlugin.ratiosZ  1 $ $3 $ $t'< < T*B// /&1 3+,,,r^Nr~r}r r_r`rarbrcr]rvryrr|rer^r\rrs8I&&&&&&&&!!!! - - -X - - -r^rcFeZdZdZdZddZdd Zdd Zedd Z dS)SuspiciousKatakanaPluginzDDetect implausible halfwidth Katakana and uncommon CJK combinations._katakana_count_halfwidth_katakana_count _cjk_count_uncommon_cjk_countrFrGc>d|_d|_d|_d|_dSrrrxs r\r]z!SuspiciousKatakanaPlugin.__init__s'$%./& ()   r^r'rCrqr&c|jr)|xjdz c_|jr|xjdz c_dS|xjdz c_|js|xjdz c_dSdSr)r4rr5rrrArrus r\rvz"SuspiciousKatakanaPlugin.feed_infos     A % ) 4..!3.. F 1 *  $ $ ) $ $ $ $ * *r^c>d|_d|_d|_d|_dSrrrxs r\ryzSuspiciousKatakanaPlugin.resets' )*&#$   r^rzct|jdkr,|j|jkrd|jcxkr |jkrnndSdS)Nrrr)rrrrrxs r\r|zSuspiciousKatakanaPlugin.ratiosT  *a / /.$2FFFT_@@@@(@@@@@@3sr^Nr~r}rrrer^r\rrs}NNI**** * * * *%%%% Xr^rcBeZdZdZddZdd Zdd Zedd Zd S)ArchaicUpperLowerPlugin_buf_character_count_since_last_sep_successive_upper_lower_count#_successive_upper_lower_count_finalr_last_alpha_seen_upper_last_alpha_seen_lower_current_ascii_onlyrFrGcvd|_d|_d|_d|_d|_d|_d|_d|_dS)NFrTrrxs r\r]z ArchaicUpperLowerPlugin.__init__sG 45,23*890%&,1#,1#)-   r^r'rCrqr&c>|jo|j}| }|rg|jdkr\|jdkr#|js|js|xj|jz c_d|_d|_d|_|xjdz c_d|_dS|jr|j sd|_|jdkrJ|j r|j s|j r.|j r'|jr|xjdz c_d|_nd|_nd|_|xjdz c_|xjdz c_|j |_ |j |_ dS)rr@FrTNr)r)r/rr-r rrrrr.r*r r+r)rZr'rq is_concerned chunk_seps r\rvz!ArchaicUpperLowerPlugin.feed_infosu!Z>D,> **  =AA4:: ;0;8868823D .34D 0DI  ! !Q & ! !'+D $ F  # -DM -',D $  /! 3 3 "t: "  "#: "9%66!;66 %DII $DII!  " ,,1,,&*j#&*j###r^cvd|_d|_d|_d|_d|_d|_d|_d|_dS)NrFT)rrrrrr rr rxs r\ryzArchaicUpperLowerPlugin.reset=sG !/0,-.*340&+#&+# #'   r^rzc:|jdkrdS|j|jz S)Nrr)rrrxs r\r|zArchaicUpperLowerPlugin.ratioGs&  A % %37$:OOOr^Nr~r}rrrer^r\rrs~ I . . . .)1)1)1)1V((((PPPXPPPr^rcBeZdZdZddZddZdd Zedd Zd S)ArabicIsolatedFormPluginr_isolated_form_countrFrGc"d|_d|_dSrrrxs r\r]z!ArabicIsolatedFormPlugin.__init__Rs%&)*!!!r^c"d|_d|_dSrrrxs r\ryzArabicIsolatedFormPlugin.resetVs !$%!!!r^r'rCrqr&ch|xjdz c_|jtzr|xjdz c_dSdSr)rr0r rrus r\rvz"ArabicIsolatedFormPlugin.feed_infoZsJ " :- - +  % % * % % % % + +r^rzc>|jdkrdS|j|jz }|S)Nrrr)rZisolated_form_usages r\r|zArabicIsolatedFormPlugin.ratioas,  1 $ $3%)%>AV%V""r^Nr~r}r) r_r`rarcr]ryrvrr|rer^r\rrOsu/Ii... Ii & & &~ + )T*** )T***z / )T***:0Jy$///;/Ii...Ii....%/Ii...>/Ii... Jj j k j   j   j  j j j   / / / E 0R. $    % IdG $ $ $ $    Jj j k j   j   j  j j j   =/00  515 5 5HW 5 5!2 5 5    R<< JJuG0@"0EGG H H H JJuG.>suu.EGG H H HtUD$dD$O = =B JJu;;;; < < < < ! $ $$r^)r'rCrFr&)rrErrErFrD)rF)rrCr rzr!rDrFrz)7 __future__r functoolsrloggingrconstantrrr r r r r rrrrrrrrrrrrrrutilsrrrrr r!r"r$rdr&rhr=rnrprrrrrrrrrrrrIrer^r\rOs""""""".'>I- 9EA AAAAG+G+G+G+G+G+G+G+X 4 $$05c $$$ """"""""<)L)L)L)L)L'9)L)L)LXOOOOO1OOO6"E"E"E"E"E*"E"E"EJ)D)D)D)D)D&8)D)D)DX6/6/6/6/6/(6/6/6/rpApApApApA-pApApAf!-!-!-!-!-*!-!-!-H+++++1+++\TPTPTPTPTP0TPTPTPn#####1###8 4    HJOZ%Z%Z%Z%Z%Z%Z%r^