§
    l�’j‘)  ã                  óî   — d Z ddlmZ ddlmZ ddlmZ ddlZddl	m
Z
 dd	lmZmZ  e
j        e¦  «        Ze G d
„ d¦  «        ¦   «         Ze G d„ d¦  «        ¦   «         Zdd„Zd d„Zd!d„Zd"d„Zg d¢ZdS )#z<Template loading and validation for response_template dicts.é    )Úannotations)Ú	dataclass)ÚAnyNé   )Úloggingé   )ÚCONTENT_PARSERSÚvalidate_transform_stringsc                  ó’   — e Zd ZU ded<   ded<   ded<   ded<   ded	<   ded
<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   dS )ÚResponseTemplateFieldÚstrÚnamer   Úopen_reúlist[str] | NoneÚopen_literalsÚboolÚopen_literal_can_extendÚclose_reÚclose_literalsÚclose_literal_can_extendÚcontentÚdictÚcontent_argsÚrepeatsÚoptionalÚ	transformÚtransform_eachN)Ú__name__Ú
__module__Ú__qualname__Ú__annotations__© ó    ú`C:\Projects\DSCons\.venv\Lib\site-packages\transformers\utils\chat_parsing\response_templates.pyr   r      s™   € € € € € € à€I€I�IØ€L€L�LØ#Ð#Ð#Ñ#Ø!Ð!Ð!Ñ!Ø€M€M�MØ$Ð$Ð$Ñ$Ø"Ð"Ð"Ñ"Ø€L€L�LØÐÐÑØ€M€M�MØ€N€N�NØ€N€N�NØÐÐÑÐÐr#   r   c                  óR   — e Zd ZU ded<   ded<   ded<   dZded	<   dZd
ed<   dd„ZdS )ÚResponseTemplater   Údefaultsz dict[str, ResponseTemplateField]Úfieldsr   Ústart_anchor_reNz
str | NoneÚimplicitr   Ústart_anchor_literalsÚtextr   Úreturnc                óÖ   — d}| j                              |¦  «        D ]}|                     ¦   «         }Œ|€+| j        �dnd}t                               d|› d�¦  «         |S ||d…         S )a–  When parsing responses, we can't just parse the tokens generated
        by the model. This is because chat templates often include early parts of the message such as <start_of_turn>
        or <think> tokens in the prefill. Therefore, we take the whole chat as input, but truncate to the start
        of the most recent assistant message, which can include tokens from the template as well as the output.NÚstart_anchorÚstart_anchor_patternzresponse_template defines z_ but the anchor was not found in the prefix; the parser will process the entire prefix instead.)r)   ÚfinditerÚendr+   ÚloggerÚinfo)Úselfr,   Úlast_endÚmÚkinds        r$   Útruncate_past_last_anchorz*ResponseTemplate.truncate_past_last_anchor7   s•   € ð
  $ˆØÔ%×.Ò.¨tÑ4Ô4ð 	ð 	ˆAØ—u’u‘w”wˆHˆHØÐØ%)Ô%?Ð%K�>�>ÐQgˆDÝ�KŠKðT¨Tð Tð Tð Tñô ð ð ˆKØ�H�I�IŒÐr#   )r,   r   r-   r   )r   r   r    r!   r*   r+   r9   r"   r#   r$   r&   r&   /   sn   € € € € € € à€N€N�NØ,Ð,Ð,Ñ,ØÐÐÑØ€HÐÐÐÑØ.2ÐÐ2Ð2Ð2Ñ2ðð ð ð ð ð r#   r&   Úscoper   Úfieldr   Úliteral_keyÚpattern_keyr-   ú"tuple[Any, list[str] | None, bool]c           	     óä  ‡	— ||v r||v rt          | › d|› d|› d�¦  «        ‚||v �rm||         }t          |t          ¦  «        r|gŠ	n«t          |t          ¦  «        rm|st          | › d|› d�¦  «        ‚t	          d„ |D ¦   «         ¦  «        st          | › d|› d�¦  «        ‚t          t
                               |¦  «        ¦  «        Š	n)t          | › d|› dt          |¦  «        j        › �¦  «        ‚t          d	„ ‰	D ¦   «         ¦  «        rt          | › d|› d
�¦  «        ‚t          ‰	t          d¬¦  «        }t          ˆ	fd„‰	D ¦   «         ¦  «        }d                     d„ |D ¦   «         ¦  «        }t          j        |t          j        ¦  «        ‰	|fS ||v rX	 t          j        ||         t          j        ¦  «        ddfS # t          j        $ r}t          | › d|› d|› �¦  «        |‚d}~ww xY wdS )a¢  Compile a region's start/end anchor. An anchor can be:

    - a regex (`pattern_key`),
    - a single literal string (`literal_key`), or
    - a list of literal strings (`literal_key`), in which case any of these match

    All forms compile to a regex for matching, but we keep the literal strings around too: when present,
    the parser can emit regions faster, since it knows exactly how many trailing bytes might be a
    partial-match prefix (and that a finished literal match can never be extended by future bytes, except
    when a literal is a strict prefix of another in the same list).

    Returns `(compiled_re, literals, can_extend)`:

    - `compiled_re`: the compiled pattern.
    - `literals`: the literal strings, or `None` for a regex anchor.
    - `can_extend`: `True` iff one literal is a strict prefix of another, in which case a match at the
      buffer edge could still be lengthened by future input.
    z: cannot specify both 'z' and 'ú'z: 'z(' list must contain at least one literalc              3  ó@   K  — | ]}t          |t          ¦  «        V — Œd S ©N)Ú
isinstancer   ©Ú.0Úss     r$   ú	<genexpr>z"_compile_anchor.<locals>.<genexpr>e   s,   è è € Ð7Ð7¨a•z !¥SÑ)Ô)Ð7Ð7Ð7Ð7Ð7Ð7r#   z ' list must contain only stringsz+' must be a string or list of strings, got c              3  ó"   K  — | ]
}|d k    V — ŒdS )Ú Nr"   rD   s     r$   rG   z"_compile_anchor.<locals>.<genexpr>j   s&   è è € Ð)Ð)˜1ˆq�BŠwÐ)Ð)Ð)Ð)Ð)Ð)r#   z"' literals cannot be empty stringsT)ÚkeyÚreversec              3  óX   •K  — | ]$}‰D ]}||k    o|                      |¦  «        V — Œ Œ%d S rB   )Ú
startswith)rE   ÚaÚbÚliteralss      €r$   rG   z"_compile_anchor.<locals>.<genexpr>n   sC   øè è € ÐXÐX¸ÈxÐXÐXÈ!˜˜ašÐ3 A§L¢L°¡O¤OÐXÐXÐXÐXÐXÐXÐXr#   ú|c              3  ó>   K  — | ]}t          j        |¦  «        V — Œd S rB   )ÚreÚescaperD   s     r$   rG   z"_compile_anchor.<locals>.<genexpr>o   s*   è è € Ð9Ð9¨A�2œ9 Q™<œ<Ð9Ð9Ð9Ð9Ð9Ð9r#   NFz
: invalid z regex: )NNF)Ú
ValueErrorrC   r   ÚlistÚallr   ÚfromkeysÚtyper   ÚanyÚsortedÚlenÚjoinrS   ÚcompileÚDOTALLÚerror)
r:   r;   r<   r=   ÚrawÚorderedÚ
can_extendÚpatternÚerP   s
            @r$   Ú_compile_anchorrf   I   sk  ø€ ð& �eÐÐ ¨uÐ 4Ð 4Ý˜EÐ\Ð\¸+Ð\Ð\ÈkÐ\Ð\Ð\Ñ]Ô]Ð]Ø�eÐÑØ�KÔ ˆÝ�c�3ÑÔð 		xØ�uˆHˆHÝ˜�TÑ"Ô"ð 	xØð eÝ  EÐ!cÐ!c¨kÐ!cÐ!cÐ!cÑdÔdÐdÝÐ7Ð7°3Ð7Ñ7Ô7Ñ7Ô7ð ]Ý  EÐ![Ð![¨kÐ![Ð![Ð![Ñ\Ô\Ð\Ý�DŸMšM¨#Ñ.Ô.Ñ/Ô/ˆHˆHå ÐvÐv¨+ÐvÐvÕbfÐgjÑbkÔbkÔbtÐvÐvÑwÔwÐwÝÐ)Ð) Ð)Ñ)Ô)Ñ)Ô)ð 	[Ý ÐYÐY¨+ÐYÐYÐYÑZÔZÐZå˜¥s°DÐ9Ñ9Ô9ˆÝÐXÐXÐXÐX¸XÐXÑXÔXÑXÔXˆ
Ø—(’(Ð9Ð9°Ð9Ñ9Ô9Ñ9Ô9ˆÝŒz˜'¥2¤9Ñ-Ô-¨x¸ÐCÐCØ�eÐÐð	RÝ”:˜e KÔ0µ"´)Ñ<Ô<¸dÀEÐIÐIøÝŒxð 	Rð 	Rð 	RÝ ÐIÐI°ÐIÐIÀaÐIÐIÑJÔJÐPQÐQøøøøð	RøøøàÐs   Æ'G ÇG-ÇG(Ç(G-ÚspecÚNonec                ó&  — t          | t          ¦  «        s$t          dt          | ¦  «        j        › �¦  «        ‚|                      dd¦  «        }|dk    rt          d|› �¦  «        ‚t          | ¦  «        h d£z
  x}rt          dt          |¦  «        › �¦  «        ‚t          |                      di ¦  «        t          ¦  «        st          d¦  «        ‚|                      d	i ¦  «        }t          |t          ¦  «        r|st          d
¦  «        ‚dS )zíValidate the top-level response_template structure: it must be a dict of the expected version with
    only known keys, a dict of `defaults`, and a non-empty dict of `fields`. Per-field specs are checked
    separately in `_build_field`.z&response_template must be a dict, got Úversionr   z'Unsupported response_template version: >   r(   rj   r'   r/   r0   z#Unknown keys in response_template: r'   z)response_template.defaults must be a dictr(   z1response_template.fields must be a non-empty dictN)rC   r   rU   rY   r   ÚgetÚsetr[   )rg   rj   Úunknown_template_keysÚ
fields_raws       r$   Ú_validate_template_shapero   y   s$  € õ �d�DÑ!Ô!ð YÝÐWÅ$ÀtÁ*Ä*ÔBUÐWÐWÑXÔXÐXØ�hŠh�y !Ñ$Ô$€GØ�!‚|€|ÝÐLÀ7ÐLÐLÑMÔMÐMÝ # D¡	¤	Ð,uÐ,uÐ,uÑ uÐuÐð `ÝÐ^½vÐF[Ñ?\Ô?\Ð^Ð^Ñ_Ô_Ð_Ý�d—h’h˜z¨2Ñ.Ô.µÑ5Ô5ð FÝÐDÑEÔEÐEØ—’˜( BÑ'Ô'€JÝ�j¥$Ñ'Ô'ð N¨zð NÝÐLÑMÔMÐMðNð Nr#   r   c                óL  — h d£}d| › d�}t          |t          ¦  «        st          |› d�¦  «        ‚t          |¦  «        |z
  x}r!t          |› dt	          |¦  «        › �¦  «        ‚|                     dd¦  «        }|t          vr)t          |› d|› d	t	          t          ¦  «        › �¦  «        ‚t          ||d
d¦  «        \  }}}t          ||dd¦  «        \  }	}
}|                     d¦  «        }|                     dd¦  «        }t          |t          ¦  «        s&t          |› dt          |¦  «        j
        › �¦  «        ‚|r|€t          |› d�¦  «        ‚|�t          ||¦  «         ndt          ¦   «         }|�|t          |j        ¦  «        z  }|	�|t          |	j        ¦  «        z  }|r"t          |› dt	          |¦  «        › d�¦  «        ‚t          | ||||	|
|||                     di ¦  «        |                     dd¦  «        |                     dd¦  «        ||¬¦  «        S )zKValidate a single field spec and compile it into a `ResponseTemplateField`.>
   ÚopenÚcloser   r   r   r   r   Úopen_patternÚclose_patternr   zField 'r@   z must be a dictz: unknown keys r   r,   z: unknown content parser 'z'. Available: rq   rs   rr   rt   r   r   Fz%: transform_each must be a bool, got Nz5: transform_each is set but no transform was providedz5: open_pattern/close_pattern declares named group(s) zê, but the field has no 'transform'. Named captures are only surfaced through a 'transform' template (where they appear alongside 'content'). Either add a 'transform' that uses the captures, or remove the named groups from the pattern.r   r   r   T)r   r   r   r   r   r   r   r   r   r   r   r   r   )rC   r   rU   rl   r[   rk   r	   rf   r   rY   r   r
   Ú
groupindexr   )r   r;   Ú_ALLOWED_FIELD_KEYSr:   Úunknown_field_keysr   r   r   r   r   r   r   r   r   Úcaptured_namess                  r$   Ú_build_fieldry   ‹   sŸ  € ðð ð Ðð �dÐÐÐ€EÝ�e�TÑ"Ô"ð 4Ý˜EÐ2Ð2Ð2Ñ3Ô3Ð3Ý  ™ZœZÐ*=Ñ=Ð=Ðð PÝ˜EÐNÐNµ&Ð9KÑ2LÔ2LÐNÐNÑOÔOÐOØ�iŠi˜	 6Ñ*Ô*€GØ•oÐ%Ð%Ý˜EÐmÐm¸WÐmÐmÕTZÕ[jÑTkÔTkÐmÐmÑnÔnÐnÝ6EÀeÈUÐTZÐ\jÑ6kÔ6kÑ3€Gˆ]Ð3Ý9HÈÐPUÐW^Ð`oÑ9pÔ9pÑ6€HˆnÐ6Ø—	’	˜+Ñ&Ô&€IØ—Y’YÐ/°Ñ7Ô7€NÝ�n¥dÑ+Ô+ð iÝ˜EÐgÐgÍÈ^ÑH\ÔH\ÔHeÐgÐgÑhÔhÐhØð Z˜)Ð+Ý˜EÐXÐXÐXÑYÔYÐYØÐÝ" 5¨)Ñ4Ô4Ð4Ð4õ ™œˆØÐØ�c 'Ô"4Ñ5Ô5Ñ5ˆNØÐØ�c (Ô"5Ñ6Ô6Ñ6ˆNØð 	ÝØð =ð =Ý˜.Ñ)Ô)ð=ð =ð =ñô ð õ !ØØØ#Ø 7ØØ%Ø!9ØØ—Y’Y˜~¨rÑ2Ô2Ø—	’	˜) UÑ+Ô+Ø—’˜: tÑ,Ô,ØØ%ðñ ô ð r#   údict | ResponseTemplatec                ó  — t          | t          ¦  «        r| S t          | ¦  «         d„ | d                              ¦   «         D ¦   «         }d„ |                     ¦   «         D ¦   «         }t	          |¦  «        dk    r%t          dd                     |¦  «        › �¦  «        ‚t          d| dd	¦  «        \  }}}|€t          d
¦  «        ‚t          t          |  	                    di ¦  «        ¦  «        ||r|d         nd ||¬¦  «        S )Nc                ó6   — i | ]\  }}|t          ||¦  «        “ŒS r"   )ry   )rE   r   ra   s      r$   ú
<dictcomp>z*load_response_template.<locals>.<dictcomp>Ñ   s(   € ÐTÐTÐT±	°°cˆd•L  sÑ+Ô+ÐTÐTÐTr#   r(   c                ó&   — g | ]\  }}|j         ­|‘ŒS rB   )r   )rE   r   r;   s      r$   ú
<listcomp>z*load_response_template.<locals>.<listcomp>Ó   s#   € ÐWÐWÐW¡  eÀÄÐAV�tÐAVÐAVÐAVr#   r   zpAt most one field may omit 'open'/'open_pattern' (that field becomes the implicit-open / leftover sink). Found: z, Úresponse_templater/   r0   zGresponse_template must define 'start_anchor' or 'start_anchor_pattern'.r'   r   )r'   r(   r*   r)   r+   )
rC   r&   ro   Úitemsr\   rU   r]   rf   r   rk   )rg   r(   Úimplicit_fieldsr)   r+   Ú_s         r$   Úload_response_templater„   Ì   s2  € Ý�$Õ(Ñ)Ô)ð ØˆÝ˜TÑ"Ô"Ð"àTÐT¸TÀ(¼^×=QÒ=QÑ=SÔ=SÐTÑTÔT€FàWÐW¨v¯|ª|©~¬~ÐWÑWÔW€OÝ
ˆ?ÑÔ˜aÒÐÝðSØ6:·i²iÀÑ6PÔ6PðSð Sñ
ô 
ð 	
õ
 1@Ø˜T >Ð3Iñ1ô 1Ñ-€OÐ*¨Að ÐÝÐbÑcÔcÐcåÝ�d—h’h˜z¨2Ñ.Ô.Ñ/Ô/ØØ'6Ð@� Ô#Ð#¸DØ'Ø3ðñ ô ð r#   )r&   r   r„   )
r:   r   r;   r   r<   r   r=   r   r-   r>   )rg   r   r-   rh   )r   r   r;   r   r-   r   )rg   rz   r-   r&   )Ú__doc__Ú
__future__r   Údataclassesr   Útypingr   ÚregexrS   Úutilsr   Úcontent_parsersr	   r
   Ú
get_loggerr   r3   r   r&   rf   ro   ry   r„   Ú__all__r"   r#   r$   ú<module>rŽ      s^  ðð CÐ Bà "Ð "Ð "Ð "Ð "Ð "à !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à Ð Ð Ð à Ð Ð Ð Ð Ð Ø HÐ HÐ HÐ HÐ HÐ HÐ HÐ Hð 
ˆÔ	˜HÑ	%Ô	%€ð ðð ð ð ð ñ ô ñ „ðð  ðð ð ð ð ñ ô ñ „ðð2-ð -ð -ð -ð`Nð Nð Nð Nð$>ð >ð >ð >ðBð ð ð ð: RÐ
QÐ
Q€€€r#   