§
    >¯½j=  ã                  óò   — d Z ddlmZ ddlZddlZddlmZ ddlmZm	Z	m
Z
mZ  G d„ d¦  «        Z G d„ d	¦  «        Z ed
d
¬¦  «         G d„ d¦  «        ¦   «         Z G d„ d¦  «        Z G d„ d¦  «        ZdS )zÛ
High-Performance Two-Tier Chat Moderation & Anti-RMT Sentinel.
Tier 1: Synchronous Trie Profanity Filter with Unicode NFKC & Leetspeak folding (< 0.1ms).
Tier 2: Asynchronous Anti-RMT & Black Market Pattern Detection.
é    )ÚannotationsN)Ú	dataclass)ÚDictÚListÚSetÚTuplec                  ó   — e Zd ZdZdZdd„ZdS )ÚTrieNodez8Node structure for high-throughput string matching Trie.©ÚchildrenÚis_end_of_wordÚoriginal_wordÚreturnÚNonec                ó0   — i | _         d| _        d| _        d S )NFÚ r   ©Úselfs    ú/C:\Projects\FreeExile\server\chat\moderation.pyÚ__init__zTrieNode.__init__   s   € Ø-/ˆŒØ$)ˆÔØ"$ˆÔÐÐó    N©r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú	__slots__r   © r   r   r
   r
      s3   € € € € € ØBÐBØ?€Ið%ð %ð %ð %ð %ð %r   r
   c                  óþ   — e Zd ZU dZh d£Zdddddddddd	d
œ
Zded<   i dd“d	d“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd“d d!“dd"“dd#“d$d%“d&d'“d(d)d*œ¥Zded+<    ed,¦  «        Z	d<d=d2„Z
d>d5„Zd?d7„Zd?d8„Zd@d;„Zd-S )AÚSynchronousTrieFilterz‰
    Tier 1 Native Filter: Aho-Corasick inspired Trie with Leetspeak & Homoglyph normalization.
    Executes in < 0.1ms per message.
    >   õ   Ä‘mõ   Ä‘kmõ   cáº·cõ   lá»“nõ   buá»“iõ   Ä‘á»‹tÚccÚclÚdmÚsbÚcacÚcnmÚdcmÚditÚdkmÚlonÚnmdÚvclÚvklÚbuoiÚfuckÚscamÚshitÚbitchÚaÚoÚiÚeÚsÚtÚb)
ú@Ú0Ú1ú!Ú3ú$Ú4Ú5Ú7Ú8zDict[str, str]Ú
HOMOGLYPHSu3   [aÃ¡Ã áº£Ã£áº¡Äƒáº¯áº±áº³áºµáº·Ã¢áº¥áº§áº©áº«áº­@4]z[b8]Úcz[c]Údu   [dÄ‘]u"   [eÃ©Ã¨áº»áº½áº¹Ãªáº¿á»�á»ƒá»…á»‡3]u   [iÃ­Ã¬á»‰Ä©á»‹!1|]Úkz[k]Úlz[l1|]Úmz[m]Únz[n]u2   [oÃ³Ã²á»�Ãµá»�Ã´á»‘á»“á»•á»—á»™Æ¡á»›á»�á»Ÿá»¡á»£0]Úpz[p]Úrz[r]z[s$5]z[t7]Úuu    [uÃºÃ¹á»§Å©á»¥Æ°á»©á»«á»­á»¯á»±]Úvz[v]z[x]u   [yÃ½á»³á»·á»¹á»µ])ÚxÚyÚCHAR_VARIANTSz. _-/\+*~`'",;:()[]{}|NÚbanned_wordsúSet[str] | Noner   r   c                óœ   — t          ¦   «         | _        |�|n| j        }|D ]*}|                      |                      |¦  «        ¦  «         Œ+d S ©N)r
   ÚrootÚDEFAULT_BANNED_WORDSÚ_insertÚnormalize_fold)r   rX   ÚwordsÚwords       r   r   zSynchronousTrieFilter.__init__J   sZ   € Ý‘J”JˆŒ	Ø ,Ð 8��¸dÔ>WˆØð 	4ð 	4ˆDØ�LŠL˜×,Ò,¨TÑ2Ô2Ñ3Ô3Ð3Ð3ð	4ð 	4r   ra   Ústrc                ó’   — | j         }|D ].}||j        vrt          ¦   «         |j        |<   |j        |         }Œ/d|_        ||_        d S )NT)r\   r   r
   r   r   )r   ra   ÚnodeÚchars       r   r^   zSynchronousTrieFilter._insertP   sY   € ØŒyˆØð 	'ð 	'ˆDØ˜4œ=Ð(Ð(Ý&.¡j¤j�”˜dÑ#Ø”= Ô&ˆDˆDØ"ˆÔØ!ˆÔÐÐr   Útextc                ó|  ‡ — t          j        d|¦  «                             ¦   «         }|                     dd¦  «                             dd¦  «        }|                     dd¦  «        }d                     ˆ fd„|D ¦   «         ¦  «        }t          j        d|¦  «        }d                     d	„ |D ¦   «         ¦  «        S )
zPConverts Unicode NFKC, strips zero-width chars, folds homoglyphs and diacritics.ÚNFKCu   â€‹r   u   ï»¿u   Ä‘rL   c              3  óN   •K  — | ]}‰j                              ||¦  «        V — Œ d S r[   )rJ   Úget©Ú.0rK   r   s     €r   ú	<genexpr>z7SynchronousTrieFilter.normalize_fold.<locals>.<genexpr>a   s5   øè è € ÐGÐG°q˜œ×,Ò,¨Q°Ñ2Ô2ÐGÐGÐGÐGÐGÐGr   ÚNFDc              3  óJ   K  — | ]}t          j        |¦  «        d k    ¯|V — ŒdS )ÚMnN)ÚunicodedataÚcategory)rl   rK   s     r   rm   z7SynchronousTrieFilter.normalize_fold.<locals>.<genexpr>d   s7   è è € ÐIÐI˜Q­Ô)=¸aÑ)@Ô)@ÀDÒ)HÐ)H�qÐ)HÐ)HÐ)HÐ)HÐIÐIr   )rq   Ú	normalizeÚlowerÚreplaceÚjoin)r   rf   Ú
normalizedÚfoldedÚnfds   `    r   r_   z$SynchronousTrieFilter.normalize_foldY   s´   ø€ å Ô*¨6°4Ñ8Ô8×>Ò>Ñ@Ô@ˆ
à×'Ò'¨°"Ñ5Ô5×=Ò=¸hÈÑKÔKˆ
à×'Ò'¨¨cÑ2Ô2ˆ
à—’ÐGÐGÐGÐG¸JÐGÑGÔGÑGÔGˆåÔ# E¨6Ñ2Ô2ˆØ�wŠwÐIÐI #ÐIÑIÔIÑIÔIÐIr   c                óF   ‡ — d                      ˆ fd„|D ¦   «         ¦  «        S )z;Removes punctuation and separators used to obfuscate words.r   c              3  ó.   •K  — | ]}|‰j         v¯|V — Œd S r[   )ÚIGNORE_SEPARATORSrk   s     €r   rm   z4SynchronousTrieFilter.strip_noise.<locals>.<genexpr>h   s0   øè è € ÐJÐJ˜Q¨!°4Ô3IÐ*IÐ*I�qÐ*IÐ*IÐ*IÐ*IÐJÐJr   )rv   )r   rf   s   ` r   Ústrip_noisez!SynchronousTrieFilter.strip_noisef   s*   ø€ à�wŠwÐJÐJÐJÐJ $ÐJÑJÔJÑJÔJÐJr   Úraw_contentúTuple[bool, str]c                ó’  ‡ — |sd|fS ‰                       |¦  «        }‰                      |¦  «        }t          ¦   «         }t          t	          |¦  «        ¦  «        D ]j}‰ j        }t          |t	          |¦  «        ¦  «        D ]C}||         }||j        vr n/|j        |         }|j        r|                     |j	        ¦  «         ŒDŒk|sd|fS |}	d}
t          |t          d¬¦  «        }|D ]Z}ˆ fd„|D ¦   «         }d                     |¦  «        }d|› d�}t          j        |d|	t          j        ¬	¦  «        \  }}|d
k    rd}
|}	Œ[|
|	fS )z{
        Scans message content for profane or abusive keywords.
        Returns: (has_profanity, censored_content)
        FT)ÚkeyÚreversec                ój   •— g | ]/}‰j                              |t          j        |¦  «        ¦  «        ‘Œ0S r   )rW   rj   ÚreÚescape)rl   Úchr   s     €r   ú
<listcomp>z9SynchronousTrieFilter.scan_and_censor.<locals>.<listcomp>Œ   s4   ø€ ÐVÐVÐVÈ2˜TÔ/×3Ò3°B½¼	À"¹¼ÑFÔFÐVÐVÐVr   z[\W_]*z(?<![a-zA-Z0-9])z(?![a-zA-Z0-9])z***)Úflagsr   )r_   r}   ÚsetÚrangeÚlenr\   r   r   Úaddr   Úsortedrv   r„   ÚsubnÚ
IGNORECASE)r   r~   Únormalized_foldedÚ	condensedÚwords_in_condensedÚ	start_idxrd   Úcurr_idxre   ÚresultÚcensored_anyÚsorted_wordsra   Úpattern_partsÚpattern_bodyÚpatternÚ
new_resultÚcounts   `                 r   Úscan_and_censorz%SynchronousTrieFilter.scan_and_censorj   s—  ø€ ð
 ð 	&Ø˜+Ð%Ð%à ×/Ò/°Ñ<Ô<ÐØ×$Ò$Ð%6Ñ7Ô7ˆ	å'*¡u¤uÐõ �s 9™~œ~Ñ.Ô.ð 	?ð 	?ˆIØ”9ˆDÝ! )­S°©^¬^Ñ<Ô<ð ?ð ?�Ø  Ô*�Ø˜tœ}Ð,Ð,Ø�EØ”} TÔ*�ØÔ&ð ?Ø&×*Ò*¨4Ô+=Ñ>Ô>Ð>øøà!ð 	&Ø˜+Ð%Ð%ð ˆØˆåÐ0µcÀ4ÐHÑHÔHˆà ð 	$ð 	$ˆDØVÐVÐVÐVÐQUÐVÑVÔVˆMØ$Ÿ>š>¨-Ñ8Ô8ˆLØG¨,ÐGÐGÐGˆGÝ "¤¨°¸ÅbÄmÐ TÑ TÔ TÑˆJ˜Ø�qŠyˆyØ#�Ø#�øà˜VÐ#Ð#r   r[   )rX   rY   r   r   )ra   rb   r   r   )rf   rb   r   rb   )r~   rb   r   r   )r   r   r   r   r]   rJ   Ú__annotations__rW   r‰   r|   r   r^   r_   r}   r�   r   r   r   r    r       sõ  € € € € € € ðð ð
ð ð Ðð ØØØØØØØØØð"ð "€Jð ð ð ñ ð%ØÐCð%àˆWð%ð 	ˆVð%ð 	ˆXð	%ð
 	Ð2ð%ð 	Ð"ð%ð 	ˆVð%ð 	ˆXð%ð 	ˆVð%ð 	ˆVð%ð 	ÐBð%ð 	ˆVð%ð 	ˆVð%ð 	ˆXð%ð 	ˆWð%ð  	Ð0ð!%ð" 	ˆVð#%ð$ Ø!ð'%ð %ð %€Mð ð ð ñ ð, ˜Ð6Ñ7Ô7Ðð4ð 4ð 4ð 4ð 4ð"ð "ð "ð "ðJð Jð Jð JðKð Kð Kð Kð*$ð *$ð *$ð *$ð *$ð *$r   r    T)ÚslotsÚfrozenc                  óP   — e Zd ZU dZded<   ded<   ded<   ded<   ded	<   d
ed<   dS )ÚModerationAnalysisResultz+Output summary of two-tier chat moderation.ÚboolÚis_censoredrb   Úfiltered_contentÚis_rmtÚintÚ
risk_scoreÚshould_auto_mutezTuple[str, ...]Úviolation_reasonsN)r   r   r   r   rž   r   r   r   r¢   r¢   —   sZ   € € € € € € à5Ð5ØÐÐÑØÐÐÑØ€L€L�LØ€O€O�OØÐÐÑØ&Ð&Ð&Ñ&Ð&Ð&r   r¢   c                  óÊ   — e Zd ZdZd ej        d¦  «        fd ej        dej        ¦  «        fd ej        dej        ¦  «        fd ej        d	ej        ¦  «        fgZdd„ZdS )ÚAsyncSentinelRMTDetectorz£
    Tier 2 Asynchronous Sentinel: Real-Money Trading (RMT) and scam detector.
    Identifies phone numbers, social media handles, and bank transfer keywords.
    ÚPHONE_VNz(?:0|84|\+84)(?:3|5|7|8|9)\d{8}ÚSOCIAL_HANDLEz;(zalo|telegram|tele|fb\.com|facebook\.com)[\s\:\@\.\-_]*\w+ÚPAYMENT_KEYWORDzJ(chuyen khoan|atm|momo|ban vang|thu mua ngoc|ban acc|gdtg|shopacc|zalopay)ÚCRYPTO_PAYMENTz,(usdt|binance|crypto|vi dien tu|bitcoin|eth)r~   rb   Únormalized_contentr   úTuple[bool, int, List[str]]c                óL  — g }d}| j         D ]s\  }}|                     |¦  «        s|                     |¦  «        rD|                     |¦  «         |dk    r|dz  }ŒP|dk    r|dz  }Œ\|dk    r|dz  }Œh|dk    r|d	z  }Œtt          |¦  «        dk    }|t	          |d
¦  «        |fS )Nr   r­   é(   r®   é#   r¯   é2   r°   é-   éd   )ÚRMT_PATTERNSÚsearchÚappendr‹   Úmin)r   r~   r±   ÚreasonsÚscorerr   rš   r¦   s           r   Úevaluatez!AsyncSentinelRMTDetector.evaluate¯   sÛ   € ØˆØˆà!%Ô!2ð 
	 ð 
	 ÑˆH�gØ�~Š~˜kÑ*Ô*ð 	 ¨g¯nªnÐ=OÑ.PÔ.Pð 	 Ø—’˜xÑ(Ô(Ð(Ø˜zÒ)Ð)Ø˜R‘K�E�EØ Ò0Ð0Ø˜R‘K�E�EØÐ!2Ò2Ð2Ø˜R‘K�E�EØÐ!1Ò1Ð1Ø˜R‘K�Eøå�W‘” Ò!ˆØ•s˜5 #‘”¨Ð/Ð/r   N)r~   rb   r±   rb   r   r²   )	r   r   r   r   r„   Úcompiler�   r¹   r¿   r   r   r   r¬   r¬   ¢   s®   € € € € € ðð ð 
�Z�R”ZÐ BÑCÔCÐDØ	˜*˜"œ*Ð%cÐegÔerÑsÔsÐtØ	˜J˜BœJÐ'tÐvxô  wDñ  Eô  Eð  	FØ	˜:˜2œ:Ð&UÐWYÔWdÑeÔeÐfð	€Lð0ð 0ð 0ð 0ð 0ð 0r   r¬   c                  ó"   — e Zd ZdZd
d„Zdd„Zd	S )ÚChatModerationPipelinezBCoordinates Tier 1 Synchronous and Tier 2 Asynchronous moderation.r   r   c                óR   — t          ¦   «         | _        t          ¦   «         | _        d S r[   )r    Útrie_filterr¬   Úrmt_detectorr   s    r   r   zChatModerationPipeline.__init__Æ   s$   € Ý0Ñ2Ô2ˆÔÝ4Ñ6Ô6ˆÔÐÐr   r~   rb   r¢   c           
     óŠ  — | j                              |¦  «        \  }}| j                              | j                              |¦  «        ¦  «        }| j                             ||¦  «        \  }}}|}|r|dz  }|                     d¦  «         t          |d¦  «        }	|	dk    }
t          ||||	|
t          |¦  «        ¬¦  «        S )z7Processes a chat message through both moderation tiers.é   Ú	PROFANITYr¸   é<   )r¤   r¥   r¦   r¨   r©   rª   )
rÄ   r�   r}   r_   rÅ   r¿   r»   r¼   r¢   Útuple)r   r~   Ú
is_profaneÚcensored_contentrw   r¦   Ú	rmt_scorer½   Ú
total_riskÚ
final_riskÚshould_mutes              r   ÚprocesszChatModerationPipeline.processÊ   sÖ   € à'+Ô'7×'GÒ'GÈÑ'TÔ'TÑ$ˆ
Ð$ØÔ%×1Ò1°$Ô2B×2QÒ2QÐR]Ñ2^Ô2^Ñ_Ô_ˆ
à%)Ô%6×%?Ò%?ÀÈZÑ%XÔ%XÑ"ˆ�	˜7àˆ
Øð 	(Ø˜"ÑˆJØ�NŠN˜;Ñ'Ô'Ð'å˜ SÑ)Ô)ˆ
Ø  BÒ&ˆå'Ø"Ø-ØØ!Ø(Ý# G™nœnð
ñ 
ô 
ð 	
r   Nr   )r~   rb   r   r¢   )r   r   r   r   r   rÑ   r   r   r   rÂ   rÂ   Ã   sB   € € € € € ØLÐLð7ð 7ð 7ð 7ð
ð 
ð 
ð 
ð 
ð 
r   rÂ   )r   Ú
__future__r   r„   rq   Údataclassesr   Útypingr   r   r   r   r
   r    r¢   r¬   rÂ   r   r   r   ú<module>rÕ      sf  ððð ð #Ð "Ð "Ð "Ð "Ð "à 	€	€	€	Ø Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !Ø )Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )Ð )ð%ð %ð %ð %ð %ñ %ô %ð %ð{$ð {$ð {$ð {$ð {$ñ {$ô {$ð {$ð| €�˜dÐ#Ñ#Ô#ð'ð 'ð 'ð 'ð 'ñ 'ô 'ñ $Ô#ð'ð0ð 0ð 0ð 0ð 0ñ 0ô 0ð 0ðB
ð 
ð 
ð 
ð 
ñ 
ô 
ð 
ð 
ð 
r   