j.ddlZddlZddlmZddlmcmZddlmZddl m Z m Z m Z ddl mZGddejZdZd ZGd d ejZGd d ejZGddejZGddejZGddejZGddejZGddejZGddejZGddejZeGddejZdS)N)store_hyperparameters)DropPath to_2tuple trunc_normal_)pad_to_multiplec8eZdZddejdffd ZdZxZS)MlpNct|p|}|p|}tj|||_||_tj|||_tj||_dSN) super__init__nnLinearfc1actfc2Dropoutdrop)self in_featureshidden_features out_features act_layerr __class__s \C:\Projects\Historical\venv\Lib\site-packages\spandrel/architectures/SwinIR/__arch/SwinIR.pyrz Mlp.__init__sw #2{ )8[9[/::9;;9_l;;Jt$$ c||}||}||}||}||}|Sr )rrrrrxs rforwardz Mlp.forward$sR HHQKK HHQKK IIaLL HHQKK IIaLLr)__name__ __module__ __qualname__rGELUrr" __classcell__rs@rr r s\'  %%%%%% rr c|j\}}}}||||z|||z||}|ddddddd|||}|S)z Args: x: (B, H, W, C) window_size (int): window size Returns: windows: (num_windows*B, window_size, window_size, C) rr)shapeviewpermute contiguous)r! window_sizeBHWCwindowss rwindow_partitionr9-sJAq!Q q!{"Kk1A;PQRRA !Q1a##..0055b+{TUVV  Nrct|jd||z|z |z z }||||z||z||d}|dddddd|||d}|S)z Args: windows: (num_windows*B, window_size, window_size, C) window_size (int): Window size H (int): Height of image W (int): Width of image Returns: x: (B, H, W, C) rr.r*rr+r,r-)intr/r0r1r2)r8r3r5r6r4r!s rwindow_reverser<>s GM! A 3k A BCCA  1 Q+-{K  A !Q1a##..0055aArBBA HrcFeZdZdZ d fd Zd dZdefdZd ZxZ S) WindowAttentionaWindow based multi-head self attention (W-MSA) module with relative position bias. It supports both of shifted and non-shifted window. Args: dim (int): Number of input channels. window_size (tuple[int]): The height and width of the window. num_heads (int): Number of attention heads. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 proj_drop (float, optional): Dropout ratio of output. Default: 0.0 TNr ct||_||_||_||z}|p|dz|_t jtj d|dzdz d|dzdz z||_ tj |jd} tj |jd} tj tj | | g} tj| d} | dddddf| dddddfz } | ddd} | dddddfxx|jddz z cc<| dddddfxx|jddz z cc<| dddddfxxd|jdzdz zcc<| d}|d|t j||dz||_t j||_t j|||_t j||_t3|j d t jd |_dS) Ngr+rr*r.relative_position_indexrbias{Gz?std)dim)rrrFr3 num_headsscaler Parametertorchzerosrelative_position_bias_tablearangestackmeshgridflattenr1r2sumregister_bufferrqkvr attn_dropproj proj_droprSoftmaxsoftmax)rrFr3rGqkv_biasqk_scalerTrVhead_dimcoords_hcoords_wcoordscoords_flattenrelative_coordsr@rs rrzWindowAttention.__init___s &")#/4 -/L K[^+a/A A4F4JKY W W- - ) < 0 344< 0 344U^Xx,@AABBvq11 111aaa: &4 )C C *11 q!  *,,  111a   D$4Q$7!$;;   111a   D$4Q$7!$;;   111a   A(8(;$;a$??   "1"5"5b"9"9 68OPPP9S#'999I..Ic3'' I..d7TBBBBzb))) rc|j\}}}||||d|j||jzddddd}|d|d|d} }}||jz}||ddz} |j|j d |j d|j dz|j d|j dzd} | ddd } | | dz} ||jd} | || z| |j||| d dz} | d|j||} | | } n| | } || } | | zdd|||}||}||}|S)z Args: x: input features with shape of (num_windows*B, N, C) mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None rr+rr*r,r.)r/rSreshaperGr1rH transposerLr@r0r3r2 unsqueezerXrTrUrV) rr!maskB_Nr7rSqkvattnrelative_position_biasnWs rr"zWindowAttention.forwards; 7Aq HHQKK WRAt~qDN/B C C WQ1a # # F F F1  N1;;r2&&&!%!B  ( - -b 1 1" $  Q $"21"5 5  Q $"21"5 5    "8!?!? q!" " *,, ,66q999  AB99R2Xr4>1a@@4>>DDillD99RA66D<<%%DD<<%%D~~d## AX A & & . .r1a 8 8 IIaLL NN1  rreturnc6d|jd|jd|jS)Ndim=, window_size= , num_heads=)rFr3rGrs r extra_reprzWindowAttention.extra_reprs'\dh\\d.>\\DN\\\rcd}|||jzdz|jzz }||j|z|j|jzz|zz }||j|z|z|j|jzzz }|||jz|jzz }|S)Nrr)rFrG)rrhflopss rrwzWindowAttention.flopss TX!DH,, !#tx4>'ABQFF !#a'48t~+EFF TX(( r)TNr r r ) r#r$r%__doc__rr"strrurwr'r(s@rr>r>Qs  $.*.*.*.*.*.*`0000d]C]]]]       rr>c jeZdZdZddddddddejejf fd Zd Zd Z d e fd Z d Z xZ S)SwinTransformerBlockaSwin Transformer Block. Args: dim (int): Number of input channels. input_resolution (tuple[int]): Input resulotion. num_heads (int): Number of attention heads. window_size (int): Window size. shift_size (int): Shift size for SW-MSA. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. drop (float, optional): Dropout rate. Default: 0.0 attn_drop (float, optional): Attention dropout rate. Default: 0.0 drop_path (float, optional): Stochastic depth rate. Default: 0.0 act_layer (nn.Module, optional): Activation layer. Default: nn.GELU norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm r@TNr c  t||_||_||_||_||_||_t|j|jkr d|_t|j|_d|jcxkr |jks nJd| ||_ t|t|j|||| | |_ | dkrt| ntj|_| ||_t%||z}t'||| | |_|jdkr||j}nd}|d|dS)Nrz shift_size must in 0-window_size)r3rGrYrZrTrVr )rrrr attn_mask)rrrFinput_resolutionrGr3 shift_size mlp_ratiominnorm1r>rrlrrIdentity drop_pathnorm2r;r mlpcalculate_maskrR)rrFrrGr3rrrYrZrrTrr norm_layermlp_hidden_dimrrs rrzSwinTransformerBlock.__init__s  0"&$" t$ % %)9 9 9DO"4#899D   3 3 3 34#3 3 3 3 3 3 - 4 3 3 Z__ # !$"233    1:C),,,R[]]Z__ S9_--*     ?Q  ++D,ABBIII [)44444rc |\}}tjd||df}td|j t|j |j t|j df}td|j t|j |j t|j df}d}|D]}|D]} ||dd|| ddf<|dz }t ||j} | d|j|jz} | d| dz } | | dkd| dkd} | S)Nr*rr.r+gYr ) rJrKslicer3rr9r0re masked_fill) rx_sizer5r6img_maskh_slicesw_slicescnthw mask_windowsrs rrz#SwinTransformerBlock.calculate_mask s1;1a|,, !d&& ' ' 4##do%5 6 6 4?"D ) )  !d&& ' ' 4##do%5 6 6 4?"D ) )    A  '*Aq!!!$q ( d&  $((T-=@P-PQQ  **1-- 0F0Fq0I0II )))q.&AAMM NC  rc|\}}|j\}}}|}||}|||||}|jdkr&t j||j |j fd} n|} t | |j} | d|j|jz|} |j|kr| | |j } nB| | | | |j } | d|j|j|} t| |j||} |jdkr$t j| |j|jfd}n| }||||z|}|||z}|||||z}|S)Nr)r*r+)shiftsdimsr.)rf)r/rr0rrJrollr9r3rrlrrtodevicer<rrr) rr!rr5r6r4Lr7shortcut shifted_x x_windows attn_windowss rr"zSwinTransformerBlock.forward?s1'1a JJqMM FF1aA   ?Q   DO+do-=>VIII% t'  NN  4#33Q   F * *99%LL 99 3 3F ; ; > >qx H H%L $((T-=t?OQRSS "<1A1aHH  ?Q   4?DO"D6AAA FF1a!eQ   t~~a(( ( txx 1 6677 7rroc fd|jd|jd|jd|jd|jd|j S)Nrq, input_resolution=rsrrz , shift_size=z , mlp_ratio=)rFrrGr3rrrts rruzSwinTransformerBlock.extra_reprusu h48 h h0E h hSWSa h h+ h h:>/ h hW[We h h rc(d}|j\}}||j|z|zz }||z|jz |jz }|||j|j|jzzz }|d|z|z|jz|jz|jzz }||j|z|zz }|S)Nrr+)rrFr3rlrwr)rrwr5r6rns rrwzSwinTransformerBlock.flops{s$1 A!! UT% %(8 8 diood&69I&IJJJJ QTX%04>AA A!! r)r#r$r%rxrr& LayerNormrrr"ryrurwr'r(s@rr{r{s. '<959595959595v>444l C          rr{cHeZdZdZejffd ZdZdefdZ dZ xZ S) PatchMergingzPatch Merging Layer. Args: input_resolution (tuple[int]): Resolution of input feature. dim (int): Number of input channels. norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm ct||_||_t jd|zd|zd|_|d|z|_dS)Nr,r+FrA)rrrrFrr reductionnorm)rrrFrrs rrzPatchMerging.__init__sa  01s7AG%@@@Jq3w'' rc6|j\}}|j\}}}|||zks Jd|dzdkr |dzdksJd|d|d|||||}|ddddddddddf}|ddddddddddf}|ddddddddddf} |ddddddddddf} tj||| | gd }||d d |z}||}||}|S) z x: B, H*W, C zinput feature has wrong sizer+rzx size (*z) are not even.Nr*r.r,)rr/r0rJcatrr) rr!r5r6r4rr7x0x1x2x3s rr"zPatchMerging.forwardsx$1'1aAEzzz9zzz1uzza!eqjjj*KQ*K*K*K*K*Kjj( FF1aA   qqq!$Q$1aaa  qqq!$Q$1aaa  qqq!$Q$1aaa  qqq!$Q$1aaa  Ir2r2& + + FF1b!a%  IIaLL NN1  rroc&d|jd|jS)Nzinput_resolution=z, dim=rrFrts rruzPatchMerging.extra_reprsJ4#8JJJJJrc||j\}}||z|jz}||dz|dzzdz|jzdz|jzz }|S)Nr+r,rrr5r6rws rrwzPatchMerging.flopssR$1A  !q&Q!V$q(483a7$(BB r r#r$r%rxrrrr"ryrurwr'r(s@rrrs:<((((((.KCKKKKrrc XeZdZdZddddddejddf fd ZdZd efd Z d Z xZ S) BasicLayera.A basic Swin Transformer layer for one stage. Args: dim (int): Number of input channels. input_resolution (tuple[int]): Input resolution. depth (int): Number of blocks. num_heads (int): Number of attention heads. window_size (int): Local window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. drop (float, optional): Dropout rate. Default: 0.0 attn_drop (float, optional): Attention dropout rate. Default: 0.0 drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. r}TNr Fc J t|_|_||_||_t j   f dt|D|_ | |  |_ dSd|_ dS)Nc g|]H}t |dzdkrdn dz ttr|n IS)r+r) rFrrGr3rrrYrZrrTrr)r{ isinstancelist) .0irTrFrrrrrrGrZrYr3s r z'BasicLayer.__init__..s   "!%%5' +%&UaZZqqkQ6F'%%'!)T22#ill")   r)rFr) rrrFrdepthuse_checkpointr ModuleListrangeblocks downsample)rrFrrrGr3rrYrZrrTrrrrrs `` ````````` rrzBasicLayer.__init__s"  0 ,m              "u#      .  !(j cjDOOO#DOOOrc|jD],}|jrtj|||} |||}-|j||}|Sr )rr checkpointr)rr!rblks rr"zBasicLayer.forwardsc; # #C" #)#q&99C6NN ? &""Arroc6d|jd|jd|jS)Nrqrz, depth=)rFrrrts rruzBasicLayer.extra_reprs)^dh^^43H^^RVR\^^^rcd}|jD]}||z }|j||jz }|SNr)rrwr)rrwrs rrwzBasicLayer.flopssQ; ! !C SYY[[ EE ? & T_**,, ,E rrr(s@rrrs4 <4#4#4#4#4#4#l_C____rrc ReZdZdZddddddejddddd f fd Zd Zd ZxZ S) RSTBaResidual Swin Transformer Block (RSTB). Args: dim (int): Number of input channels. input_resolution (tuple[int]): Input resolution. depth (int): Number of blocks. num_heads (int): Number of attention heads. window_size (int): Local window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. drop (float, optional): Dropout rate. Default: 0.0 attn_drop (float, optional): Attention dropout rate. Default: 0.0 drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. img_size: Input image size. patch_size: Patch size. resi_connection: The convolutional block before residual connection. r}TNr Fr,1convct||_||_t ||||||||| | | | | ||_|dkrt j||ddd|_n|dkrt j t j||dzdddt j dd t j|dz|dzddd t j dd t j|dz|ddd|_t||d |d |_ t||d |d |_dS) N)rFrrrGr3rrYrZrrTrrrrrrr*3convr,皙?Tnegative_slopeinplacerimg_size patch_sizein_chans embed_dimr)rrrFrrresidual_grouprConv2dconv Sequential LeakyReLU PatchEmbed patch_embed PatchUnEmbed patch_unembed)rrFrrrGr3rrYrZrrTrrrrrrresi_connectionrs rrz RSTB.__init__2sz(  0(-#!!)   " g % % #sAq!44DII  ' '  #saxAq11 C>>> #(C1HaA66 C>>> #(CAq11 DI&!    *!    rc ||||||||zSr )rrrr)rr!rs rr"z RSTB.forwardxsS    $,,T-@-@F-K-KVTTUU    rcd}||jz }|j\}}|||z|jz|jzdzz }||jz }||jz }|S)Nr )rrwrrFrrrrwr5r6s rrwz RSTB.flopss $**,,,$1 Q!DH,q00 !''))) #))+++ r) r#r$r%rxrrrr"rwr'r(s@rrrs: <%D D D D D D L   rrc2eZdZdZ d fd ZdZd ZxZS) rawImage to Patch Embedding Args: img_size (int): Image size. Default: 224. patch_size (int): Patch token size. Default: 4. in_chans (int): Number of input image channels. Default: 3. embed_dim (int): Number of linear projection output channels. Default: 96. norm_layer (nn.Module, optional): Normalization layer. Default: None rr,r`Ncptt|}t|}|d|dz|d|dzg}||_||_||_|d|dz|_||_||_||||_ dSd|_ dSNrr*) rrrrrpatches_resolution num_patchesrrrrrrrrrrrs rrzPatchEmbed.__init__s X&&z** QK:a= ( QK:a= ( ! $"4-a03Ea3HH  "  !" 9--DIIIDIIIrc|ddd}|j||}|S)Nr+r*)rPrdrr s rr"zPatchEmbed.forwards= IIaLL " "1a ( ( 9  ! ArcLd}|j\}}|j|||z|jzz }|Sr)rrrrs rrwzPatchEmbed.flopss4}1 9 QUT^+ +E rrr,rrNr#r$r%rxrr"rwr'r(s@rrrslPT. rrc2eZdZdZ d fd ZdZd ZxZS) rayImage to Patch Unembedding Args: img_size (int): Image size. Default: 224. patch_size (int): Patch token size. Default: 4. in_chans (int): Number of input image channels. Default: 3. embed_dim (int): Number of linear projection output channels. Default: 96. norm_layer (nn.Module, optional): Normalization layer. Default: None rr,rrNc:tt|}t|}|d|dz|d|dzg}||_||_||_|d|dz|_||_||_dSr) rrrrrrrrrrs rrzPatchUnEmbed.__init__s X&&z** QK:a= ( QK:a= ( ! $"4-a03Ea3HH  "rc|j\}}}|dd||j|d|d}|S)Nr*r+r)r/rdr0r)rr!rr4HWr7s rr"zPatchUnEmbed.forwardsE72q KK1   " "1dnfQi K Krc d}|Sr)rrws rrwzPatchUnEmbed.flopss  rrrr(s@rrrslPT######$ rrc"eZdZdZfdZxZS)UpsamplezUpsample module. Args: scale (int): Scale factor. Supported scales: 2^n and 3. num_feat (int): Channel number of intermediate features. c >g}||dz zdkrtttj|dD]W}|t j|d|zddd|t jdXno|dkrV|t j|d|zddd|t jdntd|dtj |dS) Nr*rr+r,rrzscale z/ is not supported. Supported scales: 2^n and 3.) rr;mathlogappendrr PixelShuffle ValueErrorrr)rrHnum_featm_rs rrzUpsample.__init__s!  UQY A % %3txq112233 - -8Q\1aCCDDD++,,,, -aZZ HHRYxXq!Q?? @ @ @ HHR_Q'' ( ( ( (RRRR  !r)r#r$r%rxrr'r(s@rrrsB         rrc*eZdZdZdfd ZdZxZS)UpsampleOneStepa*UpsampleOneStep module (the difference with Upsample is that it always only has 1conv + 1pixelshuffle) Used in lightweight SR to save parameters. Args: scale (int): Scale factor. Supported scales: 2^n and 3. num_feat (int): Channel number of intermediate features. Nc ||_||_g}|tj||dz|zddd|tj|t j|dS)Nr+rr*)rrrrrrrr)rrHr num_out_chrrrs rrzUpsampleOneStep.__init__sz  0  8eQh*%