§
    @Äjy(  ã                   ó  — d dl mZmZ d dlZdgZdej        dej        fd„Z G d„ dej        j        ¦  «        Z	 G d	„ d
ej        j        ¦  «        Z
 G d„ dej        j        ¦  «        Z G d„ dej        j        ¦  «        ZdS )é    )ÚOptionalÚTupleNÚ	ConformerÚlengthsÚreturnc                 ó  — | j         d         }t          t          j        | ¦  «                             ¦   «         ¦  «        }t          j        || j        | j        ¬¦  «                             ||¦  «        |  	                    d¦  «        k    }|S )Nr   )ÚdeviceÚdtypeé   )
ÚshapeÚintÚtorchÚmaxÚitemÚaranger	   r
   ÚexpandÚ	unsqueeze)r   Ú
batch_sizeÚ
max_lengthÚpadding_masks       úLC:\Projects\Historical\venv\Lib\site-packages\torchaudio/models/conformer.pyÚ_lengths_to_padding_maskr   	   s~   € Ø”˜qÔ!€JÝ•U”Y˜wÑ'Ô'×,Ò,Ñ.Ô.Ñ/Ô/€JÝ”< 
°7´>ÈÌÐWÑWÔW×^Ò^Ø�Jñô à	×	Ò	˜1Ñ	Ô	ò€Lð Ðó    c                   ól   ‡ — e Zd ZdZ	 	 	 ddededededed	ed
dfˆ fd„Zdej	        d
ej	        fd„Z
ˆ xZS )Ú_ConvolutionModulea  Conformer convolution module.

    Args:
        input_dim (int): input dimension.
        num_channels (int): number of depthwise convolution layer input channels.
        depthwise_kernel_size (int): kernel size of depthwise convolution layer.
        dropout (float, optional): dropout probability. (Default: 0.0)
        bias (bool, optional): indicates whether to add bias term to each convolution layer. (Default: ``False``)
        use_group_norm (bool, optional): use GroupNorm rather than BatchNorm. (Default: ``False``)
    ç        FÚ	input_dimÚnum_channelsÚdepthwise_kernel_sizeÚdropoutÚbiasÚuse_group_normr   Nc                 ó.  •— t          ¦   «                              ¦   «          |dz
  dz  dk    rt          d¦  «        ‚t          j                             |¦  «        | _        t          j                             t          j                             |d|z  ddd|¬¦  «        t          j         	                    d¬¦  «        t          j                             |||d|dz
  dz  ||¬¦  «        |r!t          j         
                    d|¬¦  «        nt          j                             |¦  «        t          j                             ¦   «         t          j                             ||ddd|¬	¦  «        t          j                             |¦  «        ¦  «        | _        d S )
Nr   é   r   z<depthwise_kernel_size must be odd to achieve 'SAME' padding.)ÚstrideÚpaddingr!   )Údim)r%   r&   Úgroupsr!   )Ú
num_groupsr   )Úkernel_sizer%   r&   r!   )ÚsuperÚ__init__Ú
ValueErrorr   ÚnnÚ	LayerNormÚ
layer_normÚ
SequentialÚConv1dÚGLUÚ	GroupNormÚBatchNorm1dÚSiLUÚDropoutÚ
sequential)Úselfr   r   r   r    r!   r"   Ú	__class__s          €r   r,   z_ConvolutionModule.__init__   sq  ø€ õ 	‰Œ×ÒÑÔÐØ! AÑ%¨Ñ*¨aÒ/Ð/ÝÐ[Ñ\Ô\Ð\Ýœ(×,Ò,¨YÑ7Ô7ˆŒÝœ(×-Ò-ÝŒH�OŠOØØ�LÑ ØØØØð ñ ô õ ŒH�LŠL˜QˆLÑÔÝŒH�OŠOØØØ%ØØ.°Ñ2°qÑ8Ø#Øð ñ ô ð ð4�EŒH×Ò¨!¸,ÐÑGÔGÐGå”×%Ò% lÑ3Ô3ÝŒH�MŠM‰OŒOÝŒH�OŠOØØØØØØð ñ ô õ ŒH×Ò˜WÑ%Ô%ñ? 
ô  
ˆŒˆˆr   Úinputc                 ó®   — |                       |¦  «        }|                     dd¦  «        }|                      |¦  «        }|                     dd¦  «        S )z�
        Args:
            input (torch.Tensor): with shape `(B, T, D)`.

        Returns:
            torch.Tensor: output, with shape `(B, T, D)`.
        r   r$   )r0   Ú	transposer8   )r9   r;   Úxs      r   Úforwardz_ConvolutionModule.forwardM   sM   € ð �OŠO˜EÑ"Ô"ˆØ�KŠK˜˜1ÑÔˆØ�OŠO˜AÑÔˆØ�{Š{˜1˜aÑ Ô Ð r   ©r   FF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚfloatÚboolr,   r   ÚTensorr?   Ú__classcell__©r:   s   @r   r   r      sÂ   ø€ € € € € ð	ð 	ð  ØØ$ð-
ð -
àð-
ð ð-
ð  #ð	-
ð
 ð-
ð ð-
ð ð-
ð 
ð-
ð -
ð -
ð -
ð -
ð -
ð^!˜Uœ\ð !¨e¬lð !ð !ð !ð !ð !ð !ð !ð !r   r   c            	       óZ   ‡ — e Zd ZdZddedededdfˆ fd„Zd	ej        dej        fd
„Z	ˆ xZ
S )Ú_FeedForwardModulezÍPositionwise feed forward layer.

    Args:
        input_dim (int): input dimension.
        hidden_dim (int): hidden dimension.
        dropout (float, optional): dropout probability. (Default: 0.0)
    r   r   Ú
hidden_dimr    r   Nc                 ó   •— t          ¦   «                              ¦   «          t          j                             t          j                             |¦  «        t          j                             ||d¬¦  «        t          j                             ¦   «         t          j                             |¦  «        t          j                             ||d¬¦  «        t          j                             |¦  «        ¦  «        | _	        d S )NT)r!   )
r+   r,   r   r.   r1   r/   ÚLinearr6   r7   r8   )r9   r   rL   r    r:   s       €r   r,   z_FeedForwardModule.__init__d   sª   ø€ Ý‰Œ×ÒÑÔÐÝœ(×-Ò-ÝŒH×Ò˜yÑ)Ô)ÝŒH�OŠO˜I z¸ˆOÑ=Ô=ÝŒH�MŠM‰OŒOÝŒH×Ò˜WÑ%Ô%ÝŒH�OŠO˜J¨	¸ˆOÑ=Ô=ÝŒH×Ò˜WÑ%Ô%ñ
ô 
ˆŒˆˆr   r;   c                 ó,   — |                       |¦  «        S )z—
        Args:
            input (torch.Tensor): with shape `(*, D)`.

        Returns:
            torch.Tensor: output, with shape `(*, D)`.
        )r8   )r9   r;   s     r   r?   z_FeedForwardModule.forwardo   s   € ð �Š˜uÑ%Ô%Ð%r   )r   )rA   rB   rC   rD   r   rE   r,   r   rG   r?   rH   rI   s   @r   rK   rK   [   s�   ø€ € € € € ðð ð	
ð 	
 #ð 	
°3ð 	
Àð 	
ÐQUð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ð&˜Uœ\ð &¨e¬lð &ð &ð &ð &ð &ð &ð &ð &r   rK   c                   ó®   ‡ — e Zd ZdZ	 	 	 ddededededed	ed
eddfˆ fd„Zdej	        dej	        fd„Z
dej	        deej	                 dej	        fd„Zˆ xZS )ÚConformerLayera¢  Conformer layer that constitutes Conformer.

    Args:
        input_dim (int): input dimension.
        ffn_dim (int): hidden layer dimension of feedforward network.
        num_attention_heads (int): number of attention heads.
        depthwise_conv_kernel_size (int): kernel size of depthwise convolution layer.
        dropout (float, optional): dropout probability. (Default: 0.0)
        use_group_norm (bool, optional): use ``GroupNorm`` rather than ``BatchNorm1d``
            in the convolution module. (Default: ``False``)
        convolution_first (bool, optional): apply the convolution module ahead of
            the attention module. (Default: ``False``)
    r   Fr   Úffn_dimÚnum_attention_headsÚdepthwise_conv_kernel_sizer    r"   Úconvolution_firstr   Nc                 ó  •— t          ¦   «                              ¦   «          t          |||¬¦  «        | _        t          j                             |¦  «        | _        t          j                             |||¬¦  «        | _	        t          j         
                    |¦  «        | _        t          ||||d|¬¦  «        | _        t          |||¬¦  «        | _        t          j                             |¦  «        | _        || _        d S )N)r    T)r   r   r   r    r!   r"   )r+   r,   rK   Úffn1r   r.   r/   Úself_attn_layer_normÚMultiheadAttentionÚ	self_attnr7   Úself_attn_dropoutr   Úconv_moduleÚffn2Úfinal_layer_normrU   )	r9   r   rR   rS   rT   r    r"   rU   r:   s	           €r   r,   zConformerLayer.__init__‰   sè   ø€ õ 	‰Œ×ÒÑÔÐå& y°'À7ÐKÑKÔKˆŒ	å$)¤H×$6Ò$6°yÑ$AÔ$AˆÔ!Ýœ×4Ò4°YÐ@SÐ]dÐ4ÑeÔeˆŒÝ!&¤×!1Ò!1°'Ñ!:Ô!:ˆÔå-ØØ"Ø"<ØØØ)ð
ñ 
ô 
ˆÔõ ' y°'À7ÐKÑKÔKˆŒ	Ý %¤× 2Ò 2°9Ñ =Ô =ˆÔØ!2ˆÔÐÐr   r;   c                 ó–   — |}|                      dd¦  «        }|                      |¦  «        }|                      dd¦  «        }||z   }|S )Nr   r   )r=   r\   )r9   r;   Úresiduals      r   Ú_apply_convolutionz!ConformerLayer._apply_convolution¨   sO   € ØˆØ—’  1Ñ%Ô%ˆØ× Ò  Ñ'Ô'ˆØ—’  1Ñ%Ô%ˆØ˜5Ñ ˆØˆr   Úkey_padding_maskc                 ó¸  — |}|                       |¦  «        }|dz  |z   }| j        r|                      |¦  «        }|}|                      |¦  «        }|                      ||||d¬¦  «        \  }}|                      |¦  «        }||z   }| j        s|                      |¦  «        }|}|                      |¦  «        }|dz  |z   }|                      |¦  «        }|S )a
  
        Args:
            input (torch.Tensor): input, with shape `(T, B, D)`.
            key_padding_mask (torch.Tensor or None): key padding mask to use in self attention layer.

        Returns:
            torch.Tensor: output, with shape `(T, B, D)`.
        g      à?F)ÚqueryÚkeyÚvaluerb   Úneed_weights)rW   rU   ra   rX   rZ   r[   r]   r^   )r9   r;   rb   r`   r>   Ú_s         r   r?   zConformerLayer.forward°   sÿ   € ð ˆØ�IŠI�eÑÔˆØ�‰G�hÑˆàÔ!ð 	+Ø×'Ò'¨Ñ*Ô*ˆAàˆØ×%Ò% aÑ(Ô(ˆØ�~Š~ØØØØ-Øð ñ 
ô 
‰ˆˆ1ð ×"Ò" 1Ñ%Ô%ˆØ�‰LˆàÔ%ð 	+Ø×'Ò'¨Ñ*Ô*ˆAàˆØ�IŠI�a‰LŒLˆØ�‰G�hÑˆà×!Ò! !Ñ$Ô$ˆØˆr   r@   )rA   rB   rC   rD   r   rE   rF   r,   r   rG   ra   r   r?   rH   rI   s   @r   rQ   rQ   z   s  ø€ € € € € ðð ð( Ø$Ø"'ð3ð 3àð3ð ð3ð !ð	3ð
 %(ð3ð ð3ð ð3ð  ð3ð 
ð3ð 3ð 3ð 3ð 3ð 3ð>¨¬ð ¸¼ð ð ð ð ð$˜Uœ\ð $¸XÀeÄlÔ=Sð $ÐX]ÔXdð $ð $ð $ð $ð $ð $ð $ð $r   rQ   c                   ó˜   ‡ — e Zd ZdZ	 	 	 ddededededed	ed
edefˆ fd„Zdej	        dej	        de
ej	        ej	        f         fd„Zˆ xZS )r   a(  Conformer architecture introduced in
    *Conformer: Convolution-augmented Transformer for Speech Recognition*
    :cite:`gulati2020conformer`.

    Args:
        input_dim (int): input dimension.
        num_heads (int): number of attention heads in each Conformer layer.
        ffn_dim (int): hidden layer dimension of feedforward networks.
        num_layers (int): number of Conformer layers to instantiate.
        depthwise_conv_kernel_size (int): kernel size of each Conformer layer's depthwise convolution layer.
        dropout (float, optional): dropout probability. (Default: 0.0)
        use_group_norm (bool, optional): use ``GroupNorm`` rather than ``BatchNorm1d``
            in the convolution module. (Default: ``False``)
        convolution_first (bool, optional): apply the convolution module ahead of
            the attention module. (Default: ``False``)

    Examples:
        >>> conformer = Conformer(
        >>>     input_dim=80,
        >>>     num_heads=4,
        >>>     ffn_dim=128,
        >>>     num_layers=4,
        >>>     depthwise_conv_kernel_size=31,
        >>> )
        >>> lengths = torch.randint(1, 400, (10,))  # (batch,)
        >>> input = torch.rand(10, int(lengths.max()), input_dim)  # (batch, num_frames, input_dim)
        >>> output = conformer(input, lengths)
    r   Fr   Ú	num_headsrR   Ú
num_layersrT   r    r"   rU   c	           	      óÜ   •‡‡‡‡‡‡‡— t          ¦   «                              ¦   «          t          j                             ˆˆˆˆˆˆˆfd„t          |¦  «        D ¦   «         ¦  «        | _        d S )Nc                 ó<   •— g | ]}t          ‰‰‰‰‰‰‰¬ ¦  «        ‘ŒS ))r    r"   rU   )rQ   )	Ú.0rh   rU   rT   r    rR   r   rj   r"   s	     €€€€€€€r   ú
<listcomp>z&Conformer.__init__.<locals>.<listcomp>  sO   ø€ ð ð ð ð õ ØØØØ.Ø#Ø#1Ø&7ðñ ô ðð ð r   )r+   r,   r   r.   Ú
ModuleListÚrangeÚconformer_layers)
r9   r   rj   rR   rk   rT   r    r"   rU   r:   s
    ``` ````€r   r,   zConformer.__init__õ   s�   øøøøøøøø€ õ 	‰Œ×ÒÑÔÐå %¤× 3Ò 3ðð ð ð ð ð ð ð ð ð õ ˜zÑ*Ô*ðñ ô ñ!
ô !
ˆÔÐÐr   r;   r   r   c                 ó¨   — t          |¦  «        }|                     dd¦  «        }| j        D ]} |||¦  «        }Œ|                     dd¦  «        |fS )aX  
        Args:
            input (torch.Tensor): with shape `(B, T, input_dim)`.
            lengths (torch.Tensor): with shape `(B,)` and i-th element representing
                number of valid frames for i-th batch element in ``input``.

        Returns:
            (torch.Tensor, torch.Tensor)
                torch.Tensor
                    output frames, with shape `(B, T, input_dim)`
                torch.Tensor
                    output lengths, with shape `(B,)` and i-th element representing
                    number of valid frames for i-th batch element in output frames.
        r   r   )r   r=   rr   )r9   r;   r   Úencoder_padding_maskr>   Úlayers         r   r?   zConformer.forward  sd   € õ  8¸Ñ@Ô@Ðà�OŠO˜A˜qÑ!Ô!ˆØÔ*ð 	/ð 	/ˆEØ��aÐ-Ñ.Ô.ˆAˆAØ�{Š{˜1˜aÑ Ô  'Ð)Ð)r   r@   )rA   rB   rC   rD   r   rE   rF   r,   r   rG   r   r?   rH   rI   s   @r   r   r   ×   sã   ø€ € € € € ðð ðH Ø$Ø"'ð
ð 
àð
ð ð
ð ð	
ð
 ð
ð %(ð
ð ð
ð ð
ð  ð
ð 
ð 
ð 
ð 
ð 
ð8*˜Uœ\ð *°E´Lð *ÀUÈ5Ì<ÐY^ÔYeÐKeÔEfð *ð *ð *ð *ð *ð *ð *ð *r   )Útypingr   r   r   Ú__all__rG   r   r.   ÚModuler   rK   rQ   r   © r   r   ú<module>rz      s3  ðØ "Ð "Ð "Ð "Ð "Ð "Ð "Ð "à €€€ð ˆ-€ð e¤lð °u´|ð ð ð ð ðF!ð F!ð F!ð F!ð F!˜œœñ F!ô F!ð F!ðR&ð &ð &ð &ð &˜œœñ &ô &ð &ð>Zð Zð Zð Zð Z�U”X”_ñ Zô Zð ZðzN*ð N*ð N*ð N*ð N*�””ñ N*ô N*ð N*ð N*ð N*r   