FPh4qRddlZddlmZddlZddlmZddlmcmZddl m cm Z ddl m Z Gddej jZGddej ZGdd ej ZGd d ej ZGd d ej ZGddej ZGddej j ZGddej ZGddej ZGddej ZGddej Zy)N)Tuple) to_2tuplec$eZdZdZdfd ZxZS) Conv2d_BNzTA sequential container that performs 2D convolution followed by batch normalization.c t ||jdtjj |||||||dtjj |} tjjj| j|tjjj| jd|jd| y)zInitializes the MBConv model with given input channels, output channels, expansion ratio, activation, and drop path. cF)biasrbnN) super__init__ add_moduletorchnnConv2d BatchNorm2dinit constant_weightr ) selfabksstridepaddilationgroupsbn_weight_initr __class__s vC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\ultralytics/models/sam/modules/tiny_encoder.pyr zConv2d_BN.__init__s  UXX__Q2vsHf[`_ab XX ! !! $   >:  + b!)r!rr!r!r!)__name__ __module__ __qualname____doc__r __classcell__rs@rrrs^ " "r rc(eZdZdZfdZdZxZS) PatchEmbedzREmbeds images into patches and projects them into a specified embedding dimension.c Lt|t|}|ddz|ddzf|_|jd|jdz|_||_||_|}tjt||dzddd|t|dz|ddd|_ y)zInitialize the PatchMerging class with specified input, output dimensions, resolution and activation function. rr!N) r r rpatches_resolution num_patchesin_chans embed_dimr Sequentialrseq)rr0r1 resolution activationimg_sizenrs rr zPatchEmbed.__init__)s $-j$9#+A;!#3Xa[A5E"F22158O8OPQ8RR  " == hQ1a 0 L a1faAq ) r c$|j|S)zNRuns input tensor 'x' through the PatchMerging model's sequence of operations.)r3rxs rforwardzPatchEmbed.forward:sxx{r r"r#r$r%r r;r&r's@rr)r)&s\ "r r)c(eZdZdZfdZdZxZS)MBConvzVMobile Inverted Bottleneck Conv (MBConv) layer, part of the EfficientNet architecture.ct|||_t||z|_||_t ||jd|_||_t |j|jddd|j|_ ||_ t |j|dd|_ ||_ tj|_y)zInitializes a convolutional layer with specified dimensions, input resolution, depth, and activation function. r!)rr-rrrr)rrN)r r r0int hidden_chans out_chansrconv1act1conv2act2conv3act3rIdentity drop_path)rr0rD expand_ratior5rLrs rr zMBConv.__init__Bs   < 78"x):):qA L t00$2C2CRSYZcgctctu L t00)RUV L r c|}|j|}|j|}|j|}|j|}|j |}|j |}||z }|j |S)z7Implements the forward pass for the model architecture.)rErFrGrHrIrLrJ)rr:shortcuts rr;zMBConv.forwardXsm JJqM IIaL JJqM IIaL JJqM NN1  X yy|r r<r's@rr>r>?s`', r r>c(eZdZdZfdZdZxZS) PatchMergingzNMerges neighboring patches in the feature map and projects to a new dimension.ct|||_||_||_||_t ||ddd|_|dvrdnd}t ||d|d||_t ||ddd|_ y)zInitializes the ConvLayer with specific dimension, input resolution, depth, activation, drop path, and other optional parameters. r!r)i@ii@r,r-)rN) r r input_resolutiondimout_dimactrrErGrI)rrSrTrUr5stride_crs rr zPatchMerging.__init__hs{  0 <sGQ15 ?21wHaP wAq9 r c|jdk(r@|j\}}t|}|j|||dj dddd}|j |}|j |}|j|}|j |}|j|}|jdjddS)zfApplies forward pass on the input utilizing convolution and activation layers, and returns the result.r-rr!r,) ndimrSlenviewpermuterErVrGrIflatten transpose)rr:HWBs rr;zPatchMerging.forwardws 66Q;((DAqAAq!Q#++Aq!Q7A JJqM HHQK JJqM HHQK JJqMyy|%%a++r r<r's@rrQrQesX :,r rQc4eZdZdZ dfd ZdZxZS) ConvLayerz Convolutional Layer featuring multiple MobileNetV3-style inverted bottleneck convolutions (MBConv). Optionally applies downsample operations to the output, and provides support for gradient checkpointing. c @t |||_||_||_||_t jt|D cgc]&} t||| |t|tr|| n|(c} |_ |d|_ y||||||_ ycc} w)a Initializes the ConvLayer with the given dimensions and settings. Args: dim (int): The dimensionality of the input and output. input_resolution (Tuple[int, int]): The resolution of the input image. depth (int): The number of MBConv layers in the block. activation (Callable): Activation function applied after each convolution. drop_path (Union[float, List[float]]): Drop path rate. Single float or a list of floats for each MBConv. downsample (Optional[Callable]): Function for downsampling the output. None to skip downsampling. use_checkpoint (bool): Whether to use gradient checkpointing to save memory. out_dim (Optional[int]): The dimensionality of the output. None means it will be the same as `dim`. conv_expand_ratio (float): Expansion ratio for the MBConv layers. NrTrUr5)r r rTrSdepthuse_checkpointr ModuleListranger> isinstancelistblocks downsample) rrTrSrgr5rLrnrhrUconv_expand_ratioirs rr zConvLayer.__init__s4  0 ,mmU|%%$! ! *9d ; !    $%%& #-"4$* #w:;O%%s+Bc|jD],}|jrtj||n||}.|j|S|j|S)z^Processes the input through a series of convolutional layers and returns the activated output.rmrh checkpointrnrr:blks rr;zConvLayer.forwardM;;C151D1D %%c1-#a&AOO+qC1CCr )rANFN@r<r's@rrdrds&,O\Dr rdcFeZdZdZddej dffd ZdZxZS)Mlpz Multi-layer Perceptron (MLP) for transformer architectures. This layer takes an input with in_features, applies layer normalization and two fully-connected layers. NrAc&t||xs|}|xs|}tj||_tj |||_tj |||_||_tj||_ y)zjInitializes Attention module with the given parameters including dimension, key_dim, number of heads, etc.N) r r r LayerNormnormLinearfc1fc2rVDropoutdrop)r in_featureshidden_features out_features act_layerrrs rr z Mlp.__init__so #2{ )8[LL- 99[/:99_l;;JJt$ r c|j|}|j|}|j|}|j|}|j |}|j|S)zRApplies operations on input x and returns modified x, runs downsample if not None.)r|r~rVrrr9s rr;z Mlp.forwardsQ IIaL HHQK HHQK IIaL HHQKyy|r ) r"r#r$r%rGELUr r;r&r's@rryrys% 59tWYW^W^eg %r rycdeZdZdZ dfd Zej dfd ZdZxZ S) Attentionag Multi-head attention module with support for spatial awareness, applying attention biases based on spatial resolution. Implements trainable attention biases for each unique offset between spatial positions in the resolution grid. Attributes: ab (Tensor, optional): Cached attention biases for inference, deleted during training. c t|t|trt |dk(sJ||_|dz|_||_||zx|_}t||z|_ t||z|z|_ ||_ |j|dzz}tj||_tj |||_tj |j||_t't)j*t-|dt-|d}t |} i} g} |D]W} |D]P} t/| d| dz t/| d| dz f}|| vrt | | |<| j1| |RYt2jj5t3j6|t | |_|j;dt3j<| j?| | dy) ac Initializes the Attention module. Args: dim (int): The dimensionality of the input and output. key_dim (int): The dimensionality of the keys and queries. num_heads (int, optional): Number of attention heads. Default is 8. attn_ratio (float, optional): Attention ratio, affecting the dimensions of the value vectors. Default is 4. resolution (Tuple[int, int], optional): Spatial resolution of the input feature map. Default is (14, 14). Raises: AssertionError: If `resolution` is not a tuple of length 2. r,grr!attention_bias_idxsF) persistentN) r r rktupler[ num_headsscalekey_dimnh_kdrBddh attn_ratiorr{r|r}qkvprojrl itertoolsproductrjabsappendr Parameterzerosattention_biasesregister_buffer LongTensorr\)rrTrrrr4rhpointsNattention_offsetsidxsp1p2offsetrs rr zAttention.__init__s* *e,ZA1EEE"_  $y00 UZ')*j7*+i7$ GGeai LL% 99S!$IIdggs+ i''jm(RST KBbebem,c"Q%"Q%-.@A!22034E0F%f- -f56  !& 2 25;;y#N_J`3a b 2E4D4DT4J4O4OPQST4Ubghr ct|||rt|dr|`y|jdd|j f|_y)zNSets the module in training mode and handles attribute 'ab' based on the mode.abN)r trainhasattrrrr)rmoders rrzAttention.trains?  d GD$'++At/G/G,GHDGr cD|j\}}}|j|}|j|}|j|||jdj |j |j |jgd\}}}|jdddd}|jdddd}|jdddd}|jj|jj|_ ||jddz|jz|jr|jdd|j fn |jz} | j#d} | |zjddj%|||j&}|j)|S) zcPerforms forward pass over the input tensor 'x' by applying normalization and querying keys/values.rYr-)rTrr,r!N)shaper|rr\rsplitrrr]rtordevicer_rtrainingrsoftmaxreshaperr) rr:rbr_rqkvattns rr;zAttention.forward%sa''1a IIaLhhqk((1a4::DLL$,,X\X^X^;_ef:g1a IIaAq ! IIaAq ! IIaAq !''**T2299:Q[[R((DJJ6GK}}&&q$*B*B'BCZ^ZaZac|||# AX A & . .q!TWW =yy|r )r+)r)T) r"r#r$r%r rno_gradrr;r&r's@rrrs= 0idU]]_IIr rcVeZdZdZdddddej ffd ZdZdefd Z xZ S) TinyViTBlockzOTinyViT Block that applies self-attention and a local convolution to the input.rwrAr-c t|||_||_||_|dkDsJd||_||_tj|_ ||zdk(sJd||z} ||f} t|| |d| |_ t||z} | } t|| | ||_|dz}t|||d|||_y ) a Initializes the TinyViTBlock. Args: dim (int): The dimensionality of the input and output. input_resolution (Tuple[int, int]): Spatial resolution of the input feature map. num_heads (int): Number of attention heads. window_size (int, optional): Window size for attention. Default is 7. mlp_ratio (float, optional): Ratio of mlp hidden dim to embedding dim. Default is 4. drop (float, optional): Dropout rate. Default is 0. drop_path (float, optional): Stochastic depth rate. Default is 0. local_conv_size (int, optional): The kernel size of the local convolution. Default is 3. activation (torch.nn, optional): Activation function for MLP. Default is nn.GELU. Raises: AssertionError: If `window_size` is not greater than 0. AssertionError: If `dim` is not divisible by `num_heads`. rz"window_size must be greater than 0z"dim must be divisible by num_headsr!)rr4)rrrrr,r@N)r r rTrSr window_size mlp_ratiorrKrLrrrBrymlpr local_conv)rrTrSrrrrrLlocal_conv_sizer5head_dimwindow_resolutionmlp_hidden_dimmlp_activationrrs rr zTinyViTBlock.__init__?s<  0"QD DD&"Y!#I%II#)#(+6c8Y1Qbc S9_-#3R`gkl"#CPS\_`r c >|j\}}|j\}}}|||zk(sJd|}||jk(r"||jk(r|j|}n|j ||||}|j||jzz |jz}|j||jzz |jz} |dkDxs| dkD} | rt j |ddd| d|f}||z|| z} } | |jz} | |jz}|j || |j||j|jddj|| z|z|j|jz|}|j|}|j || ||j|j|jddj|| | |}| r|ddd|d|fj}|j |||}||j|z}|jddj||||}|j|}|j |||jdd}||j|j|zS)z~Applies attention-based transformation or padding to input 'x' before passing it through a local convolution. zinput feature has wrong sizerr,r-Nr!) rSrrrr\Frr_r contiguousrLrr)rr:r`rarbLCres_xpad_bpad_rpaddingpHpWnHnWs rr;zTinyViTBlock.forwardvs$$1''1aAEz999z   Q$*:*:%: ! Aq!Q"A%%D,<,<(<<@P@PPE%%D,<,<(<<@P@PPEai,519GEE!aAua78YE Bt'''Bt'''Bq"d..D4D4D#)Aq/''!b&2+t?O?ORVRbRb?bde*f  ! Aq"b$"2"2D4D4DaHRRSTVWX``abdfhjlmnAa!RaRiL++-q!QA DNN1% % KK1  % %aAq 1 OOA  FF1aO % %a +4>>$((1+...r returnc d|jd|jd|jd|jd|j S)zReturns a formatted string representing the TinyViTBlock's parameters: dimension, input resolution, number of attentions heads, window size, and MLP ratio. dim=, input_resolution=z , num_heads=z, window_size=z , mlp_ratio=)rTrSrrrrs r extra_reprzTinyViTBlock.extra_reprsTdhhZ243H3H2IVZVdVdUef"../|DNN;KM Mr r"r#r$r%rrr r;strrr&r's@rrr<s9Y 775an(/TMCMr rcZeZdZdZddddddej dffd ZdZd efd Z xZ S) BasicLayerz>A basic TinyViT layer for one stage in a TinyViT architecture.rwrANFr-cJt|||_||_||_| |_t jt|Dcgc]+}t||||||t|tr||n|| |  -c}|_ | d|_ y| ||| | |_ ycc}w)a Initializes the BasicLayer. Args: dim (int): The dimensionality of the input and output. input_resolution (Tuple[int, int]): Spatial resolution of the input feature map. depth (int): Number of TinyViT blocks. num_heads (int): Number of attention heads. window_size (int): Local window size. mlp_ratio (float, optional): Ratio of mlp hidden dim to embedding dim. Default is 4. drop (float, optional): Dropout rate. Default is 0. drop_path (float | tuple[float], optional): Stochastic depth rate. Default is 0. downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default is None. use_checkpoint (bool, optional): Whether to use checkpointing to save memory. Default is False. local_conv_size (int, optional): Kernel size of the local convolution. Default is 3. activation (torch.nn, optional): Activation function for MLP. Default is nn.GELU. out_dim (int | None, optional): The output dimension of the layer. Default is None. Raises: ValueError: If `drop_path` is a list of float but its length doesn't match `depth`. ) rTrSrrrrrLrr5Nrf)r r rTrSrgrhrrirjrrkrlrmrn)rrTrSrgrrrrrLrnrhrr5rUrprs rr zBasicLayer.__init__sJ  0 ,mmU| %%$! !1#'#*4Y*E)A,9 /% $ %% & #-"4$* #w:;O %%s0B c|jD],}|jrtj||n||}.|j|S|j|S)zQPerforms forward propagation on the input tensor and returns a normalized tensor.rrrts rr;zBasicLayer.forwardrvr rcTd|jd|jd|jS)zWReturns a string representation of the extra_repr function with the layer's parameters.rrz, depth=)rTrSrgrs rrzBasicLayer.extra_reprs/dhhZ243H3H2IRVR\R\Q]^^r rr's@rrrs?H 77;OzD _C_r rcjeZdZdZd dededdffd ZdejdejfdZ xZ S) LayerNorm2dz6A PyTorch implementation of Layer Normalization in 2D. num_channelsepsrNct|tjt j ||_tjt j||_||_ y)zKInitialize LayerNorm2d with the number of channels and an optional epsilon.N) r r rrronesrrr r)rrrrs rr zLayerNorm2d.__init__sG ll5::l#;< LL\!:; r r:c|jdd}||z jdjdd}||z tj||jzz }|j ddddf|z|j ddddfzS)z5Perform a forward pass, normalizing the input tensor.r!T)keepdimr,N)meanpowrsqrtrrr )rr:uss rr;zLayerNorm2d.forwards FF1dF # UKKN  4  0 UejjTXX. .{{1dD=)A- !T4-0HHHr )gư>) r"r#r$r%rBfloatr rTensorr;r&r's@rrrs>@SuII%,,Ir rceZdZdZdddgdgdgdgdd d d d d dd ffd ZdZdZejjdZ dZ dZ xZ S)TinyViTa The TinyViT architecture for vision tasks. Attributes: img_size (int): Input image size. in_chans (int): Number of input channels. num_classes (int): Number of classification classes. embed_dims (List[int]): List of embedding dimensions for each layer. depths (List[int]): List of depths for each layer. num_heads (List[int]): List of number of attention heads for each layer. window_sizes (List[int]): List of window sizes for each layer. mlp_ratio (float): Ratio of MLP hidden dimension to embedding dimension. drop_rate (float): Dropout rate for drop layers. drop_path_rate (float): Drop path rate for stochastic depth. use_checkpoint (bool): Use checkpointing for efficient memory usage. mbconv_expand_ratio (float): Expansion ratio for MBConv layer. local_conv_size (int): Local convolution kernel size. layer_lr_decay (float): Layer-wise learning rate decay. Note: This implementation is generalized to accept a list of depths, attention heads, embedding dimensions and window sizes, which allows you to create a "stack" of TinyViT models of varying configurations. r-i)`ii)r,r,r,)r-r )rrrrrwrAg?F?ct|||_||_||_t ||_||_tj}t||d|||_ |jj}||_ tjd| t|Dcgc]}|j!}}tj"|_t'|j D]}t)|||dd|dk(r|dz n|zz|dd|dk(r|dz n|zzf|||t|d|t|d|dz||j dz krt*nd| |t-|dzt |dz |}|dk(rt/dd| i|}n!t1d|||||j| | d |}|j$j3|tj4|d |_|dkDrtj8|d |ntjj;|_|j?|j@|jC|tjDtjF|d d dd tId tjFd d ddd tId |_%ycc}w)a Initializes the TinyViT model. Args: img_size (int, optional): The input image size. Defaults to 224. in_chans (int, optional): Number of input channels. Defaults to 3. num_classes (int, optional): Number of classification classes. Defaults to 1000. embed_dims (List[int], optional): List of embedding dimensions for each layer. Defaults to [96, 192, 384, 768]. depths (List[int], optional): List of depths for each layer. Defaults to [2, 2, 6, 2]. num_heads (List[int], optional): List of number of attention heads for each layer. Defaults to [3, 6, 12, 24]. window_sizes (List[int], optional): List of window sizes for each layer. Defaults to [7, 7, 14, 7]. mlp_ratio (float, optional): Ratio of MLP hidden dimension to embedding dimension. Defaults to 4. drop_rate (float, optional): Dropout rate. Defaults to 0. drop_path_rate (float, optional): Drop path rate for stochastic depth. Defaults to 0.1. use_checkpoint (bool, optional): Whether to use checkpointing for efficient memory usage. Defaults to False. mbconv_expand_ratio (float, optional): Expansion ratio for MBConv layer. Defaults to 4.0. local_conv_size (int, optional): Local convolution kernel size. Defaults to 3. layer_lr_decay (float, optional): Layer-wise learning rate decay. Defaults to 1.0. r)r0r1r4r5r,r-r!N)rTrSrgrLrnrhrUr5ro)rrrrrrYF) kernel_sizer )rrr )&r r r6 num_classesdepthsr[ num_layersrrrr) patch_embedr.rlinspacesumitemrilayersrjdictrQminrdrrr{ norm_headr}rKheadapply _init_weightsset_layer_lr_decayr2rrneck)rr6r0r embed_dimsrr window_sizesr drop_ratedrop_path_raterhmbconv_expand_ratiorlayer_lr_decayr5r.r:dpri_layerkwargslayerrs rr zTinyViT.__init__sH   & f+"WW %x0:1 191;= "--@@"4"'>3v;!OP!OAqvvx!OPmmo T__-Gw'"4Q"7AQX\]Q]'A+cjIAoBIIjnk:SXS[S[SdSdSf  4%%& /MM II2     II      KQs"J<cz |}t|j}t|Dcgc] }|||z dz zc} d |jj fdd|j D][}|j D]}|j  fddz |j<|jj  fd]|k(sJ|j|jfD]}|j fd|jD] \}} || _ d } |j | ycc}w) zASets the learning rate decay for each layer in the TinyViT model.r!c<|jD] }||_ y)zTSets the learning rate scale for each layer in the model based on the layer's depth.N) parameterslr_scale)mrps r _set_lr_scalez1TinyViT.set_layer_lr_decay.._set_lr_scales\\^" $r c|dS)Nrrr:r! lr_scaless rz,TinyViT.set_layer_lr_decay..sq)A,)Gr rc|S)Nrr:r!rpr$s rr%z,TinyViT.set_layer_lr_decay..smAy|&Dr Nc"|dz S)Nr!rr's rr%z,TinyViT.set_layer_lr_decay..sq)APQEBR1Sr c|dS)NrYrr#s rr%z,TinyViT.set_layer_lr_decay..smAy}=r ch|jD]}t|drJ|jy)zNChecks if the learning rate scale attribute is present in module's parameters.rN)rr param_name)rr s r_check_lr_scalez3TinyViT.set_layer_lr_decay.._check_lr_scales+\\^q*-;q||;-$r ) rrrjrr rrmrnr r named_parametersr+) rr decay_ratergrprblockrrr r,r!r$s ` @@rrzTinyViT.set_layer_lr_decays# DKK -))+DAqAL, < ?#7JsD8ct|tjr8|j+tjj |jdyyt|tj rUtjj |jdtjj |jdyy)zRInitializes weights for linear layers and layer normalization in the given module.Nrr)rkrr}r rrr{r)rrs rrzTinyViT._init_weightss| a #vv!!!!&&!," 2<< ( GG  affa ( GG  ahh ,)r cdhS)zQReturns a dictionary of parameter names where weight decay should not be applied.rrrs rno_weight_decay_keywordsz TinyViT.no_weight_decay_keywordss ###r cb|j|}|jd|}d}t|t|jD]}|j|}||}|j \}}}|j |dd|}|j dddd}|j|S)zKRuns the input through the model layers and returns the transformed output.rr!@r-r,)rrrjr[sizer\r]r)rr:start_irprrbrrs rforward_featureszTinyViT.forward_featuress   Q  DKKN1 wDKK 01AKKNEaA2&&(1a FF1b"a  IIaAq !yy|r c$|j|S)zQExecutes a forward pass on the input tensor through the constructed model layers.)r7r9s rr;zTinyViT.forwards$$Q''r )r"r#r$r%r rrrjitignorer2r7r;r&r's@rrrsn6& "k Z!$F - YY$$ (r r)rtypingrrtorch.nnrtorch.nn.functional functionalrtorch.utils.checkpointutilsrsultralytics.utils.instancerr2rModuler)r>rQrdryrrrrrrr rrCs ++0 "## "2#RYY#L ,299 ,F9D 9Dx"))8YYxiM299iMXH_H_VI"))I$K(biiK(r