
    Ph                    ~   d Z ddlZddlmZ ddlmc mZ ddlmc mc m	c m
c mZ ddlmc mc m
Z ddlmc mc m
c mZ ddlmZ ddlmZ ddlmZmZ ddlmZmZmZmZ ddlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6m7Z7 ddl8m9Z9 dd	l:m;Z; 	 dd
l<m=Z=m>Z>m?Z?m@Z@ ddlAmBZBmCZC ddlDmEZE ddlFmGZG dZHddlJZJddlKZKddlLZLddlMZMddlNZNddlOZOddlPZQddlRmSZS ddlTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[ ddl\m]Z^  G d d      Z_d Z`d Zaej                  j                         ZcecfdZd G d d      ZeddZfd Zgd Zhd Zid Zjd ZkddZld  Zmd! Znd" Zod# Zpd$ Zqd% Zrd& Zsd' Ztd( Zu	 ddlvZvdZw eOj                  ew d)      Zyd* ZzeQj                  dfd+Z| G d, d-e      Z} G d. d/e}      Z~ G d0 d1ej                  j                        Z G d2 d3ej                  j                        Z G d4 d5ej                  j                        Z G d6 d7ej                        Z G d8 d9ej                  j                        Z G d: d;ej                  j                        Z G d< d=ej                  j                        Z G d> d?ej                  j                        Z G d@ dAej                  j                        Z G dB dCej                  j                        Z G dD dEej                  j                        Z G dF dGej                  j                        Z G dH dIej                  j                        Z G dJ dKej                  j                        Z G dL dMej                  j                        Z G dN dOej                  j                        Z G dP dQej                  j                        Z G dR dSej                        Z G dT dUej                  j                        Z G dV dWej                  j                        Z G dX dYej                  j                        Z G dZ d[ej                  j                        Z G d\ d]ej                  j                        Z G d^ d_ej                  j                        Z G d` daej                  j                        Z G db dcej                  j                        Z G dd deej                  j                        Z G df dgej                  j                        Z G dh diej                  j                        Z G dj dkej                  j                        Z G dl dmej                  j                        Z G dn doej                  j                        Z G dp dqej                  j                        Z G dr dsej                  j                        Z G dt duej                  j                        Z G dv dwej                  j                        Z G dx dyej                  j                        Z G dz d{ej                  j                        Z G d| d}ej                  j                        Z G d~ dej                  j                        Z G d dej                        Z G d dej                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                        Z G d dej                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d de      Z G d dej                        Z G d dej                        Z G d dej                        Z G d dej                        Z G d dej                        Z G d dejx                        Z G d dej                        Z G d dej                        Z G d dej                        Z G d dej                  j                        Z G d dej                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                  j                        Z G d dej                        Z G dĄ dej                        Z G dƄ dej                        Z G dȄ dɫ      Zy# eI$ r dZHY 	aw xY w# eI$ r dZwY w xY w)zImporting this file includes common utility methods and base clases for
checking quantization api and properties of resulting modules.
    N)_FusedModule)TestCaseTEST_WITH_ROCM)	QuantTypedefault_dynamic_qat_qconfigdefault_embedding_qat_qconfig%default_symmetric_qnnpack_qat_qconfig)QuantWrapper	QuantStubDeQuantStubdefault_qconfigdefault_dynamic_qconfigdefault_per_channel_qconfigQConfigdefault_observerdefault_weight_observerpropagate_qconfig_convertget_default_qconfigquantize_dynamic_jitquantize_jit!float_qparams_weight_only_qconfigget_default_qat_qconfigPerChannelMinMaxObserverdefault_dynamic_quant_observerquantizeQConfigMappingget_default_qconfig_mappingget_default_qat_qconfig_mapping))get_default_dynamic_quant_module_mappings$get_default_qconfig_propagation_listget_default_qat_module_mappings)override_quantized_engine)_load_for_lite_interpreter)
prepare_fxprepare_qat_fx
convert_fxconvert_to_reference_fx)NSSingleResultValuesType
NSSubgraph)Node)GraphModuleTF)	FileCheck)CallableTupleDictAnyUnionTypeOptionalc                   X    e Zd ZdZd Zed        Zed        Zed        Zd Z	d Z
d Zy	)
NodeSpecz( Used for checking GraphModule Node
    c                      || _         || _        y)z
        op: call_function | call_module
        target:
          for call_function, target would be a function
          for call_module, target would be the type of PyTorch module
        N)optarget)selfr8   r9   s      vC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/testing/_internal/common_quantization.py__init__zNodeSpec.__init__B   s         c                     t        d|      S )Ncall_functionr6   clsr9   s     r;   r?   zNodeSpec.call_functionL   s    00r=   c                     t        d|      S )Ncall_methodr@   rA   s     r;   rD   zNodeSpec.call_methodP       v..r=   c                     t        d|      S )Ncall_moduler@   rA   s     r;   rG   zNodeSpec.call_moduleT   rE   r=   c                 D    t        | j                  | j                  f      S N)hashr8   r9   r:   s    r;   __hash__zNodeSpec.__hash__X   s    TWWdkk*++r=   c                     t        |t              st        S | j                  |j                  k(  xr | j                  |j                  k(  S rI   )
isinstancer6   NotImplementedr8   r9   )r:   others     r;   __eq__zNodeSpec.__eq__[   s7    %*!!ww%(("Bt{{ell'BBr=   c                 ^    t        | j                        dz   t        | j                        z   S )N )reprr8   r9   rK   s    r;   __repr__zNodeSpec.__repr__a   s#    DGG}s"T$++%666r=   N)__name__
__module____qualname____doc__r<   classmethodr?   rD   rG   rL   rQ   rU    r=   r;   r6   r6   ?   sX     1 1 / / / /,C7r=   r6   c                  X    t         j                  j                         r
t        sddgS dgS )Ncpucuda)torchr^   is_availabler   r[   r=   r;   get_supported_device_typesra   d   s&    #jj557E6?[UZT[[r=   c                     |D ]  } | | }	 y)z
    Default evaluation function takes a torch.utils.data.Dataset or a list of
    input Tensors and run the model on the dataset
    Nr[   )model
calib_datainpoutputs       r;   test_only_eval_fnrg   g   s    
  r=   c                    t         j                  j                  | j                         d      }d\  }}}t	        d      D ]  }| j                          |D ]  \  }}	|j                           | |      }
 ||
|	      }|j                          |j                          ||j                         z  }t        j                  |
d      \  }}||	j                  d      z  }|||	k(  j                         j                         z  }  |||fS )zh
    Default train function takes a torch.utils.data.Dataset and train the model
    on the dataset
    gMbP?lr)r   r   r   
      r   )r_   optimAdam
parametersrangetrain	zero_gradbackwardstepitemmaxsizesum)rc   
train_dataloss_fn	optimizer
train_losscorrecttotalidatar9   rf   loss_	predicteds                 r;   test_only_train_fnr   p   s    
   !1!1!3 >I!(J2Y&LD&!4[F66*DMMONN$))+%J 99VQ/LAyV[[^#E	V+0027799G '  w%%r=   c                   ,    e Zd ZdZddZd ZddZd Zy)	AverageMeterz1Computes and stores the average and current valuec                 @    || _         || _        | j                          y rI   )namefmtreset)r:   r   r   s      r;   r<   zAverageMeter.__init__   s    	

r=   c                 <    d| _         d| _        d| _        d| _        y Nr   )valavgrx   countrK   s    r;   r   zAverageMeter.reset   s    
r=   c                     || _         | xj                  ||z  z  c_        | xj                  |z  c_        | j                  | j                  z  | _        y rI   )r   rx   r   r   )r:   r   ns      r;   updatezAverageMeter.update   s<    C!G

a
88djj(r=   c                 ~    d| j                   z   dz   | j                   z   dz   } |j                  di | j                  S )Nz{name} {valz} ({avgz})r[   )r   format__dict__)r:   fmtstrs     r;   __str__zAverageMeter.__str__   s;    )I5@4Gv}}-t}}--r=   N)z:frl   )rV   rW   rX   rY   r<   r   r   r   r[   r=   r;   r   r      s    ;
).r=   r   c                    t        j                         5  t        |      }|j                  d      }| j	                  |ddd      \  }}|j                         }|j                  |j                  dd      j                  |            }g }|D ]X  }	|d|	 j                  d      j                         j                  dd      }
|j                  |
j                  d|z               Z |cddd       S # 1 sw Y   yxY w)zNComputes the accuracy over the k top predictions for the specified values of kr   rl   TN)keepdimg      Y@)r_   no_gradrv   rw   topkteqview	expand_asfloatrx   appendmul_)rf   r9   r   maxk
batch_sizer   predr}   resk	correct_ks              r;   accuracyr      s    	4y[[^
++dAtT24vvx''&++a,66t<=A((,22488D8IIJJy~~ej&89:   
s   CC33C<c                 ~   | j                          d}|D ]  \  }}t        j                         }	t        dd       |dz  }|j                  |      |j                  |      }} | |      }
 ||
|      }|j	                          |j                          |j                          t        |
|d      \  }}||k\  s y  y )Nr   . )endrl   )rl      )r   )rq   timeprinttorr   rs   rt   r   )rc   	criterionr{   data_loaderdevicentrain_batchescntimager9   
start_timerf   r   acc1acc5s                 r;   train_one_epochr      s    	KKM
C$vYY[
crq(&))F*;vu(ff6:
d.  % r=   c                     dt         j                  d<   dt         j                  d<   t        j                  d| |       y )N	localhostMASTER_ADDR12355MASTER_PORTgloorank
world_size)osenvirondistinit_process_groupr   s     r;   	ddp_setupr      s2     +BJJ} 'BJJ} 	F*Er=   c                  ,    t        j                          y rI   )r   destroy_process_groupr[   r=   r;   ddp_cleanupr      s     r=   c                 \   t        | |       |j                          t        j                  j                  j                  || g      }|j                  |        |}t        j                  j                  |j                         d      }t        |t        |t        | d       t                y )N)
device_idsg-C6?ri   rl   )r   r^   r_   nnparallelDistributedDataParallelr   rm   SGDro   r   r   datasetr   )r   r   preparedmodel_with_ddpr{   s        r;   run_ddpr      s}    dJMMOxx  88tf8UHKKN 9 9 ;GINIy'4KMr=   c                 0    t        | t               d       y )NTinplace)r   r    )modules    r;   convert_dynamicr      s    F=?Nr=   c                     t        | |       y rI   )r   )rc   qconfig_dicts     r;   prepare_dynamicr      s    ul+r=   c                 4   ||z  }||z  }d\  }}t        j                  ||| |f|z         }|||z
  j                         z  }t        j                  |||t         j                        }||z  }|	|z  }	|d | }|	d | }	d\  }}t        j                  ||||f|z         }t        j                  dd|f      }|rddt        |      z  z   }t        j                  |t         j                        }t        j                  |	t         j                        } |j                  | |j                          |j                  | z
  j                         z  }||z  |j                         z  }t        j                  ||j                         |j                         dt         j                        }nb|d   ||	d   z
  j                         z  }||d   z  |j                         z  }t        j                  ||d   |	d   t         j                        }|||||
r|fS d fS )	Nr      scale
zero_pointdtype)r   r   rk   )r   rl   r   r   )r_   randintr   quantize_per_tensorquint8lentensorreshapequantize_per_channeldoublelongqint8)r   in_channels_per_groupinput_feature_map_sizeout_channels_per_groupgroupskernel_sizeX_scaleX_zero_pointW_scaleW_zero_pointuse_biasuse_channelwisein_channelsout_channelsX_value_minX_value_maxX_initXX_qW_value_minW_value_maxW_initb_initW_shapeW_scales_tensorW_zero_points_tensorWbW_qs                                r;   _make_conv_test_inputr     s/   
 (&0K)F2L!'[+]][	["%;;=F 	6L(//11A

#
#	\GC $G,.Lm|$G.L ")[+ ]][	,.<>F ]]1b</2FD3{#333,,wekkB$||LL#O##W-LLN91997CCUUWMo%6((%%')=)B)B)Da++ AJ&<?299;;gaj 6<<>1''WQZLO5;;P sAsA44t44r=   c                     d\  }}t        j                  |||      }| ||z
  j                         z  }t        j                  || |t         j                        }||fS )Nr   r   )r_   r   r   r   r   )r   r   sizesr  r  r  r  r  s           r;   !_make_conv_add_extra_input_tensorr    sd    !'[+]]F
 	*$++--A

#
#	:U\\CCc6Mr=   c                      dt         t              r6dt        j                  j                  j
                  vrd _         _         S t        j                          fd       }|S )NzrQuantized operations require FBGEMM. FBGEMM is only optimized for CPUs with instruction set support AVX2 or newer.fbgemmTc                      dt         j                  j                  j                  vrt	        j
                         | i | y )Nr  r_   backends	quantizedsupported_enginesunittestSkipTestargskwargsfnreasons     r;   wrapperzskipIfNoFBGEMM.<locals>.wrapper$  9    5>>33EEE##F++r=   
rN   typer_   r  r  r  __unittest_skip____unittest_skip_why__	functoolswrapsr   r"  r!  s   ` @r;   skipIfNoFBGEMMr+    sd     BF"d5>>33EEE#'B '-B$	__R   
 Nr=   c                      dt         t              r6dt        j                  j                  j
                  vrd _         _         S t        j                          fd       }|S )N%Quantized operations require QNNPACK.qnnpackTc                      dt         j                  j                  j                  vrt	        j
                         | i | y Nr.  r  r  s     r;   r"  z skipIfNoQNNPACK.<locals>.wrapper4  s9    ENN44FFF##F++r=   r$  r*  s   ` @r;   skipIfNoQNNPACKr1  ,  sa    4F"dENN44FFF#'B '-B$	__R   
 Nr=   c                      dt         t              r6dt        j                  j                  j
                  vrd _         _         S t        j                          fd       }|S )Nr-  r.  Tc                      dt         j                  j                  j                  vrt	        j
                        t        d      5   | i | d d d        y # 1 sw Y   y xY wr0  )r_   r  r  r  r  r  r#   r  s     r;   r"  z#withQNNPACKBackend.<locals>.wrapperN  sM    ENN44FFF##F++&y1 211s   	AA#r$  r*  s   ` @r;   withQNNPACKBackendr4  D  sc     5F"dENN44FFF#'B '-B$	__R    Nr=   c                      dt         t              r6dt        j                  j                  j
                  vrd _         _         S t        j                          fd       }|S )Nz$Quantized operations require ONEDNN.onednnTc                      dt         j                  j                  j                  vrt	        j
                         | i | y )Nr6  r  r  s     r;   r"  zskipIfNoONEDNN.<locals>.wrapper_  r#  r=   r$  r*  s   ` @r;   skipIfNoONEDNNr8  W  sa    3F"d5>>33EEE#'B '-B$	__R   
 Nr=   c                      dt         t              r8t        j                  j                  j                         sd _         _         S t        j                          fd       }|S )Nz*Quantized operations require BF16 support.Tc                      t         j                  j                  j                         st	        j
                         | i | y rI   )r_   opsmkldnn_is_mkldnn_bf16_supportedr  r  r  s     r;   r"  z#skipIfNoONEDNNBF16.<locals>.wrappero  s7    yy99;##F++r=   )
rN   r%  r_   r;  r<  r=  r&  r'  r(  r)  r*  s   ` @r;   skipIfNoONEDNNBF16r>  g  s_    9F"dyy99;#'B '-B$	__R   
 Nr=   c                      dt         t              r6dt        j                  j                  j
                  vrd _         _         S t        j                          fd       }|S )Nz!Quantized operations require X86.x86Tc                      dt         j                  j                  j                  vrt	        j
                         | i | y )Nr@  r  r  s     r;   r"  zskipIfNoX86.<locals>.wrapper  s9    00BBB##F++r=   r$  r*  s   ` @r;   skipIfNoX86rB  w  sa    0F"d00BBB#'B '-B$	__R   
 Nr=   c                      dt         t              r$t        j                         sd _         _         S t        j                          fd       }|S )Nzdynamo doesn't support.Tc                  h    t        j                         st        j                         | i | y rI   )torchdynamois_dynamo_supportedr  r  r  s     r;   r"  z&skipIfNoDynamoSupport.<locals>.wrapper  s-    ..0##F++r=   )rN   r%  rE  rF  r&  r'  r(  r)  r*  s   ` @r;   skipIfNoDynamoSupportrG    sU    &F"d..0#'B '-B$	__R   
 Nr=   zno torchvisionc                     |r t         j                  j                  | |      S t         j                  j                  |       S rI   )r_   jittracescript)rc   tracingr   s      r;   get_script_modulerM    s-    +2599??5$'O		8H8H8OOr=   c                     t        j                  | j                  d   dz   f|      }| |dd t        j                  t        j
                  ||            }|r|dd S |dd S )z6
    Convert lengths to offsets for embedding_bag
    r   rl   r   Nr   )npzerosshaper_   
from_numpycumsum)r   offset_typeuse_begin_offsettts       r;   lengths_to_offsetsrW    sb     
1771:>#;	7BBqrF			"))Bk:	;B#2wab6Mr=   c            
       p    e Zd Z fdZd Zd Zd Zd!dZd Zd Z	d	 Z
d
 Zd Zd Zd Zd Zd Zd Zd"dZd Zd Z	 	 d#dZ	 	 	 d$dZd%dZerRdeeeeef   f   deeeeeef   eeef   f   f   dededdf
dZdeeeeeee f   f   f   ddfdZ!	 	 	 	 	 	 	 	 	 	 	 d&dZ"e#jH                  fd Z% xZ&S )'QuantizationTestCasec                    t         |           t        d      D cg c])  }t        j                  ddt        j
                        g+ c}| _        t        d      D cg c]O  }t        j                  ddt        j
                        t        j                  dddt        j                        gQ c}| _	        t        d      D cg c]*  }t        j                  dddt        j
                        g, c}| _
        t        d      D cg c]+  }t        j                  ddddt        j
                        g- c}| _        t        d      D cg c],  }t        j                  dddddt        j
                        g. c}| _        t        d      D cg c]P  }t        j                  dddt        j
                        t        j                  ddd	t        j                        gR c}| _        t        d      D cg c]Q  }t        j                  ddddt        j
                        t        j                  ddd	t        j                        gS c}| _        t        d      D cg c]R  }t        j                  dddddt        j
                        t        j                  ddd	t        j                        gT c}| _        | j                  | j                  | j                  d
| _        t"        j$                  t"        j&                  g| _        t"        j*                  t"        j$                  t"        j&                  g| _        y c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w )N   r   r   r   rl   )r[     rk   r   )rl   r[  r\  )supersetUprp   r_   randr   rd   r   r   ry   img_data_1dimg_data_2dimg_data_3dimg_data_1d_trainimg_data_2d_trainimg_data_3d_trainimg_data_dictr   STATICQATstatic_quant_typesDYNAMICall_quant_types)r:   r   	__class__s     r;   r^  zQuantizationTestCase.setUp  s   JOPQ(S(QEJJq!5;;?@(Sw|}~w  AwrsEJJq!5;;?qRSUYafakakAlmw  A%*1X/%- #ZZ1bDE%-/ &+1X/%- #ZZ1b"EKKHI%-/ &+1X/%- #ZZ1aAU[[IJ%-/ ,18"5+3a $)::aBekk#J#(==At5::#N#P+3"5
 ,18"5+3a $)::aB%++#N#(==At5::#N#P+3"5
 ,18"5+3a $)::aAq!5;;#O#(==At5::#N#P+3"5 #'"2"2"&"2"2"&"2"24
 $-#3#3Y]]"C ) 1 193C3CY]]S3 T A///"5"5"5s3   .M AM	/M0M1M#AM(AM--AM2c                 p    | j                  t        |d             | j                  t        |d             y)zChecks the module does not contain child
            modules for quantization preparation, e.g.
            quant, dequant and observer
        quantdequantN)assertFalsehasattrr:   r   s     r;   checkNoPrepModulesz'QuantizationTestCase.checkNoPrepModules  s.    
 	1234r=   c                     | j                  t        |d             |j                         D ]  }| j                  |        y)z3Checks the module does not contain qconfig
        qconfigN)rp  rq  childrencheckNoQconfig)r:   r   childs      r;   rw  z#QuantizationTestCase.checkNoQconfig  s8     	34__&E& 'r=   c                     | j                  t        |d             | j                  t        |d             | j                  t        |d             y)zChecks the module contains child
            modules for quantization preparation, e.g.
            quant, dequant and observer
        r   rn  ro  N)
assertTruerq  rr  s     r;   checkHasPrepModulesz(QuantizationTestCase.checkHasPrepModules  s=    
 	1201	23r=   Nc           	         |
t               }|i }|j                  di       }d }t        |d      r|j                   ||      r1t	        |t
        j                  j                        st        |      |v st        |      |j                         v rbt	        |t
        j                  j                  j                        s4| j                  t        |d      dt        t        |            z   dz          t        |      t               j!                         vrst        |      |j!                         vrWt	        |t"              sF|j%                         D ]2  }t        |      t        j&                  fv r | j)                  |||       4 yyyy)zrChecks the module or module's leaf descendants
            have observers in preparation for quantization
        N%float_to_observed_custom_module_classc                 V    d}| j                         D ]  \  }}|dk7  s|dz  } |dk(  S )Nr   activation_post_processrl   )named_children)r   submodule_name_countr   r   s       r;   is_leaf_modulez;QuantizationTestCase.checkObservers.<locals>.is_leaf_module  s@    #$ !002a44(A-( 3 (1,,r=   ru  r  zmodule: z do not have observer)r!   getrq  ru  rN   r_   r   
Sequentialr%  keysaoquantizationr   rz  strr"   valuesr   rv  DropoutcheckObservers)r:   r   propagate_qconfig_listprepare_custom_config_dict&float_to_observed_module_class_mappingr  rx  s          r;   r  z#QuantizationTestCase.checkObservers  sZ    ")%I%K"%-)+&1K1O1OPwy{1|.	- 69%&..*DF#Jvuxx?R?R,SV 66<AFFHH&%(("7"7"C"CDOOGF,EF&T&\)::=TTV <>@GGII<ELLNN&,/*;2::,.##E+AC]^ + 0 O Jr=   c                     | j                  t        |j                        t        j                         | j                  t        |j
                        t        j                         y)zZChecks that mod has nn.Quantize and
            nn.DeQuantize submodules inserted
        N)assertEqualr%  rn  nnqQuantizero  
DeQuantizer:   mods     r;   checkQuantDequantz&QuantizationTestCase.checkQuantDequant  s=     	cii#,,7ckk*CNN;r=   c                     | j                  t        |j                        t        j                         | j                  |       y)zChecks that mod has been swapped for an nnq.Linear
            module, the bias is qint32, and that the module
            has Quantize and DeQuantize submodules
        N)r  r%  r   r  Linearr  r  s     r;   checkWrappedQuantizedLinearz0QuantizationTestCase.checkWrappedQuantizedLinear  s/    
 	cjj)3::6s#r=   c                 V    | j                  t        |      t        j                         y rI   )r  r%  r  r  r  s     r;   checkQuantizedLinearz)QuantizationTestCase.checkQuantizedLinear  s    cCJJ/r=   c                     | j                  t        |      t        j                         | j                  |j                  j
                  |       ycChecks that mod has been swapped for an nnqd.Linear
            module, the bias is float.
        N)r  r%  nnqdr  _packed_paramsr   r:   r  r   s      r;   checkDynamicQuantizedLinearz0QuantizationTestCase.checkDynamicQuantizedLinear  s8     	cDKK0++1159r=   c                     | j                  t        |      t        j                         | j                  |j                  j
                  |       yr  )r  r%  nniqd
LinearReLUr  r   r  s      r;   checkDynamicQuantizedLinearReluz4QuantizationTestCase.checkDynamicQuantizedLinearRelu   s:     	cE$4$45++1159r=   c                     |j                         }t        j                         }t        j                  ||       |j                  d       t        j                  |      }|j                  |        || } || } fd}	 |	||       t        j                         }t        j                  ||       |j                  d       t        j                  |      }
 |
| } |	||       y )Nr   c                    j                  | d   |d          t        | d   t              r=j                  | d   d   |d   d          j                  | d   d   |d   d          y j                  | d   |d          y )Nr   rl   )r  rN   tuple)ref_outload_outr:   s     r;   check_outputszEQuantizationTestCase.check_eager_serialization.<locals>.check_outputs2  s    WQZ!5'!*e,  AA?  AA?  Xa[9r=   )
state_dictioBytesIOr_   saveseekloadload_state_dict)r:   	ref_modelloaded_modelx
model_dictr  loaded_dictr  r  r  loadeds   `          r;   check_eager_serializationz.QuantizationTestCase.check_eager_serialization'  s    ))+
JJL

:q!	q	jjm$$[1Q-#	: 	gx(JJL

9a 	q	A1:gx(r=   c                     |j                         }|j                         }| j                  ||j                         z  t	                      | j                  ||j                         z  t	                      y rI   )
get_weightget_biasr  r  set)r:   r  weight_keys	bias_keysweightbiass         r;   check_weight_bias_apiz*QuantizationTestCase.check_weight_bias_apiB  sX    %%'!!#v{{}4ce<TYY[0#%8r=   c                    t         j                  dt         j                  di}| j                  t	        |      |       |j
                  D ]5  }| j                  |j                  j                         d   d   ||          7 y)zfChecks that mod has been swapped for an nnqd.LSTM type
            module, the bias is float.
        quantized_dynamicquantized_fp16r   N)r_   r   float16r  r%  _all_weight_valuesparam__getstate__r:   r  reference_module_typer   wt_dtype_mappacked_paramss         r;   checkDynamicQuantizedLSTMz.QuantizationTestCase.checkDynamicQuantizedLSTMH  sp     %8%--IYZc$9: 33M]00==?B1E|TYGZ[ 4r=   c                 j    | j                  t        |      t        j                  j                         y rI   )r  r%  r_   r   r  r  s     r;   checkLinearz QuantizationTestCase.checkLinearQ  s    cEHHOO4r=   c                     t         j                  dt         j                  di}| j                  t	        |      |       t        |d      rE|j                  D ]5  }| j                  |j                  j                         d   d   ||          7 yy)r  r  r  r  r   N)	r_   r   r  r  r%  rq  r  r  r  r  s         r;   checkDynamicQuantizedModulez0QuantizationTestCase.checkDynamicQuantizedModuleT  s     %8%--IYZc$9:3,-!$!7!7  !4!4!A!A!CA!Fq!I<X]K^_ "8 .r=   c                     t         j                  j                  |      }| j                  ||||       t         j                  j	                  ||d         }| j                  ||||       y r   )r_   rI  rK  _checkScriptablerJ  )r:   orig_modrd   check_save_loadscriptedtraceds         r;   checkScriptablez$QuantizationTestCase.checkScriptable^  sX    99##H-h*oN :a=9h
OLr=   c                    | j                  |||       t        j                         }t        j                  j                  ||       |j                  d       t        j                  j                  |      }|r| j                  |||       y y r   )"_checkModuleCorrectnessAgainstOrigr  r  r_   rI  r  r  r  )r:   r  
script_modrd   r  buffer
loaded_mods          r;   r  z%QuantizationTestCase._checkScriptableg  sl    //*jQ 		z6*AYY^^F+
 33Hj*U r=   c                 J    |D ]  } || } || }| j                  ||         y rI   )r  )r:   r  test_modrd   re   
ref_outputscripted_outputs          r;   r  z7QuantizationTestCase._checkModuleCorrectnessAgainstOrigu  s.    C!3J&nO_j9 r=   c
           	         |rt        dt        |             dt        t        j                  j
                  j                        i}
|r|j                         }|rd|	t        n|	i}
t        |||d         j                         }|rt        d|j                         i }i }dD ]`  }|r t        ||
|      ||<    ||   |      ||<   %t        j                  |      }t        ||
t        |gd|      ||<    ||   |d    ||<   b |r2t        d	|d
   j                         t        d|d   j                         |rM| j!                  |d
   |d          t#               j%                  |      j'                  |d   j                         |d   S )NzTesting:r   r   zinput graph:)TF)debugF)r   r  zdebug graph:Tznon debug graph:)r   r  r   r_   r  r  engineevalr   rM  graphr   copydeepcopyr   rg   r  r-   checkrun)r:   r   inputsquantized_oprL  r  r  	eval_modedynamicru  r   rc   modelsoutputsinputs_copys                  r;   checkGraphModeOpz%QuantizationTestCase.checkGraphModeOp|  s   *c&k*/0H0H0O0OPQ[[]F7? 7PWXL!&'6!9=BBD.%++."E 4ULPU Vu!.v!6 #mmF3 ,<):[MSX!!u "/q	!: # .&,"4"45$fUm&9&9:WT]GEN; Kl+F5M//0e}r=   c           	      <   i }g }t        |j                  d            }|j                  j                  D ]  }d}	|j                  dk(  s|j                  dk(  r!t        |j                  |j                        }	n;|j                  dk(  r,t        |j                  t        ||j                                 }	|	|j                  |	       |	|v r||	xx   dz  cc<   d||	<    |#| j                  ||v dt        |      z   d	z          ||j                         D ]  \  }}
|
d
k7  ri| j                  ||v dt        |      z   dz          | j                  ||   |
k(  dt        |      z   dz   t        |
      z   dz   t        ||         z          t| j                  ||vdt        |      z   dz           |id
}|D ]   }	|t        |      k(  r y|	||   k(  s|dz  }" | j                  |t        |      k(  d| j                  |d      z   dz   t        |      z          yy)a   Check if GraphModule contains the target node
        Args:
            graph_module: the GraphModule instance we want to check
            expected_node, expected_node_occurrence, expected_node_list:
               see docs for checkGraphModeFxOp
        Fremove_duplicateNr?   rD   rG   rl   znode:z not found in the graph moduler   zCheck failed for node:z
 not foundz Expected occurrence:z Found occurrence:z! expected no occurrence but foundzCheck failed for graph:)	print_strzExpected ordered list:)dictnamed_modulesr  nodesr8   r6   r9   r%  r   rz  r  itemsr   printGraphModule)r:   graph_moduleexpected_nodeexpected_node_occurrenceexpected_node_listnodes_in_graph	node_listmodulesnoder   
occurrence	cur_indexs               r;   checkGraphModuleNodesz*QuantizationTestCase.checkGraphModuleNodes  sc    	|1151IJ &&,,DAww/)TWW-ETWWdkk2M)TWWd74;;+?&@A}  #&"1%*%()N1% - $OOM^;Ws=GY=Y<>= > $/-E-K-K-M)z?OO%703}3EE$%& OO&}5C03}3EE/025j/B,-/2>-3P/QRS OO%^;03}3EE;<= .N" )I$6 77*955NI	 
 OOS!344)%%le%DE() &'() *r=   c                    t        |j                  d            }g }|j                  j                  D ]  }dj	                  t        t        |j                  |j                  |j                  |j                  |j                  g            }|j                  dk(  r'|dt        t        ||j                                 z   z  }|j                  |        dj	                  |      }|rt        |       |S )NFr  rS   rG   z module type: 
)r  r  r  r  joinmaprT   r8   r   r9   r  r  r%  r   r   )r:   r  r  r  
node_infosr   	node_infostr_to_prints           r;   r  z%QuantizationTestCase.printGraphModule  s    |1151IJ
##))ATADD!&&!((AFFAHH+U!VWItt}$-T'!((:K5L0MMM	i(	 *
 yy,,r=   matched_subgraph_pairsexpected_typesgm_agm_breturnc                 @   dt         dt        dt        t        t        f   fd}| j                  t        |      t        |      k(  dt        |       dt        |              |j                         D ]  \  }}|\  }}	|\  }
}|	\  }}||   \  }} ||j                  |      } ||j                  |      } ||j                  |      } ||j                  |      }|
|u xr ||u xr
 ||u xr ||u }| j                  |dj                  ||
|||f||||f              y)	a  
            Verifies that the types specified in expected_types match
            the underlying objects pointed to by the nodes in matched_subgraph_pairs.

            An example successful test case:

              matched_subgraph_pairs = {'x0': (graph_a_conv_0_node, graph_b_conv_0_node)}
              expected_types = {'x0': (nn.Conv2d, nnq.Conv2d)}

            The function tests for key equivalence, and verifies types with
            instance checks.
            r  gmr  c                     | j                   dk(  r!t        || j                        }t        |      S | j                   dv sJ | j                  S )NrG   )r?   rD   )r8   getattrr9   r%  )r  r  r  s      r;   _get_underlying_op_typez]QuantizationTestCase.assert_types_for_matched_subgraph_pairs.<locals>._get_underlying_op_type  sG     77m+!"dkk2C9$77&FFFF;;&r=   z-Expected length of results to match, but got  and z(Type mismatch at {}: expected {}, got {}N)r+   r,   r2   r.   r  rz  r   r  
start_nodeend_noder   )r:   r  r  r  r  r  r   vexpected_types_aexpected_types_bexp_type_start_aexp_type_end_aexp_type_start_bexp_type_end_b
subgraph_a
subgraph_bact_type_start_aact_type_start_bact_type_end_aact_type_end_btypes_matchs                        r;   'assert_types_for_matched_subgraph_pairsz<QuantizationTestCase.assert_types_for_matched_subgraph_pairs  st   ('' +'x}%' OO*+s>/BB?DZ@[?\\abeftbuavw ',,.1562 "23C0 .3C0 .)?)B&
J#::;P;PRV#W #::;P;PRV#W !89L9Ld!S!89L9Ld!S/3CC 7#~57%)997 $~5  >EE)>;K^\)>;K^\^ /r=   act_compare_dictc                    |j                         D ]F  \  }}|j                         D ],  \  }}| j                  t        |      dk(  d| d       |j                         \  }}t	        t        ||               D ]  }||   |   }	||   |   }
| j                  |	d   |	d   k(  d| d| d| d       | j                  t        |	d         t        |
d         k(  d| d| d| d	       |t
        j                  j                  k(  xr d
|	d   v xs d
|
d   v }|st	        t        |	d               D ]r  }|	d   |   }|
d   |   }t        |t        j                        r;| j                  |j                  |j                  k(  d| d| d| dd| dz          it        |t              rE|d   }|d   }| j                  |j                  |j                  k(  d| d| d| dd| dz          t        |t              sJ dt        |              t        |      dk(  sJ t        |d         dk(  sJ |d   j                  |d   j                  k(  sJ |d   d   j                  |d   d   j                  k(  sJ |d   d   j                  |d   d   j                  k(  rsJ  |	d   }|
d   }|	d   }|
d   }|	d   t
        j                  j                  k(  r*| j                  ||k(         | j                  ||k(         |	d   t
        j                   j                  k(  s| j                  ||k7         | j                  ||k7          / I y)a  
            Verifies that the act_compare_dict (output of Numeric Suite APIs) is valid:
            1. for each layer, results are recorded for two models
            2. number of seen tensors match
            3. shapes of each pair of seen tensors match
            r[  zLayer z) does not have exactly two model results.r%  z, r  z do not have the same type.r  z- do not have the same number of seen Tensors.conv1dprev_node_target_typerS   zhave a shape mismatch at idx r   r   zunhandled type rl   ref_node_nameprev_node_nameN)r  rz  r   r  rp   r)   WEIGHTvaluerN   r_   TensorrQ  listr  r%  NODE_OUTPUT
NODE_INPUT)r:   r,  
layer_nameresult_type_to_dataresult_type
layer_datamodel_name_0model_name_1res_idxlayer_data_0layer_data_1is_weight_functional_conv1didxvalues_0values_1ref_node_name_0ref_node_name_1prev_node_name_0prev_node_name_1s                      r;   assert_ns_compare_dict_validz1QuantizationTestCase.assert_ns_compare_dict_valid/  s    4D3I3I3K/
//B/H/H/J+KOOJ1, ,UVX 2<1B.L,#(Z-E)F#G'1,'?'H'1,'?'H(0L4HH$ZL<.l^Snoq X 67X 678$ZL<.l^  TA  BC (+C+J+J+P+PP  (L9P,Q Q !R (L9P,Q Q	 4  ;',Sh1G-H'I+7+A#+F+7+A#+F#-h#E$(OO(0(..(H*0B|nER^Q__`(a*GuA(N)O%P &0$%?/7{H/7{H$(OO(0(..(H*0B|nER^Q__`(a*GuA(N)O%P
 ,6h+F %K*9$x.9I(J%K+F+.x=A+=$=+=+.x{+;q+@$@+@+3A;+<+<@Q@Q+Q$Q+Q+3A;q>+?+?8A;q>CWCW+W$W+W+3A;q>+?+?8A;q>CWCW+W$W+W- (J2 +7*G*6*G+78H+I(+78H+I('/3K3W3W3]3]] OOO?O,OP OOO?O,OP)&15M5X5X5^5^^ OOO?O,OP OOO?O,OPq $H 0K 4Lr=   c                 <   t        |      t        k(  r|d   }|t        j                  k(  r>t	        t
        j                  j                  j                        }|j                          n|t        j                  k(  r>t        t
        j                  j                  j                        }|j                          n/t        }t               j                  |      }|j                          |t        j                  k(  rt         }nt"        }|	Ft        |	      t        t$        fv sJ d       t'        |	t              r|	}nt        j(                  |	      } ||||||      }|t        j*                  k(  s ||  |r8t-                t-        d|       t-        d|       t-                t-        d|       | j/                  ||
||       t1        j2                  |      }t5        t1        j2                  |            }t7        t1        j2                  |            } || } || }t1        j2                  |      }t1        j2                  |      }|r|n|}|r1t-                t-        d|       | j9                  |       t-                | j/                  ||||       |||||dS )	a   Quantizes model with graph mode quantization on fx and check if the
                quantized model contains the quantized_node

                Args:
                    model: floating point torch.nn.Module
                    inputs: one positional sample input arguments for model
                    expected_node: NodeSpec
                        e.g. NodeSpec.call_function(torch.quantize_per_tensor)
                    expected_node_occurrence: a dict from NodeSpec to
                        expected number of occurrences (int)
                        e.g. {NodeSpec.call_function(torch.quantize_per_tensor) : 1,
                                NodeSpec.call_method('dequantize'): 1}
                    expected_node_list: a list of NodeSpec, used to check the order
                        of the occurrence of Node
                        e.g. [NodeSpec.call_function(torch.quantize_per_tensor),
                                NodeSpec.call_module(nnq.Conv2d),
                                NodeSpec.call_function(F.hardtanh_),
                                NodeSpec.call_method('dequantize')]
                    is_reference: if True, enables reference mode
                    print_debug_info: if True, prints debug info
                    custom_qconfig_dict: overrides default qconfig_dict
                    prepare_expected_node: same as expected_node, but for prepare
                    prepare_expected_node_occurrence: same as
                        expected_node_occurrence, but for prepare
                    prepare_expected_node_list: same as expected_node_list, but
                        for prepare

                Returns:
                    A dictionary with the following structure:
                   {
                       "prepared": ...,  # the prepared model
                       "quantized": ...,  # the quantized non-reference model
                       "quantized_reference": ...,  # the quantized reference model
                       "result": ...,  # the result for either quantized or
                                       # quantized_reference model depending on the
                                       # is_reference argument
                   }
            r   z8custom_qconfig_dict should be a QConfigMapping or a dict)example_inputsprepare_custom_configbackend_configzquant type:
zoriginal model:
zprepared model:
zquantized model:
)r   r  quantized_referencequantized_outputquantized_reference_output)r%  r5  r   rh  r   r_   r  r  r  rq   rg  r   r  r   r   
set_globalr&   r%   r  rN   	from_dictrj  r   r  r  r  r'   r(   r  )r:   rc   r  
quant_typer  r   r  is_referenceprint_debug_infocustom_qconfig_dictprepare_expected_node prepare_expected_node_occurrenceprepare_expected_node_listrL  rM  qconfig_mappingru  preparer   prepared_copyqgraphqgraph_referenceresultresult_referenceqgraph_copyqgraph_reference_copyqgraph_to_checks                              r;   checkGraphModeFxOpz'QuantizationTestCase.checkGraphModeFxOpy  sE   n F|t#Y]]*"A%..BZBZBaBa"by///"=enn>V>V>]>]"^

1"0"2"="=g"F

Y]]*($ #./0^T4JJ ONOJ1>B&9O&4&>&>?R&SO%&;-	/H
 !2!22&!oz2)51)84&&/02LN !MM(3Mh 78F6t}}X7NOV_F/8--/K$(MM2B$C!2>.FO*O<%%o6&&0HJ\^ -!,+@(.2B	D Dr=   c	                    |r||g}	n|g}	|j                         }
t        j                         }t        j                  |
|       |j                  d       t        j                  |      }t        j                  j                  j                  }|
D ]{  }t        |
|   t        j                  j                        s+t        ||   t        j                  j                        sJ  ||
|         } |||         }| j                  ||       } |rt        j                  ||dd|      }nt        j                   |||      }| j#                  |||	       |j%                  |       | j                   ||j&                  j(                         ||j&                  j(                               | j+                  ||	gd       |r%t        j,                  j                  ||ddd      }n!t        j,                  j!                  ||	      }|r<t/        j0                  |t        j2                  d
      }t5        t6        |      |_        t;        |        ||	  |r"t        j                  j=                  |      }d}n!t        j                   j=                  |      }d} ||	  | j?                  |tA        |      v        y )Nr   Trx   )num_embeddingsembedding_diminclude_last_offsetmoder   )rf  rg  r   )r  Frf  rg  rh  scale_grad_by_freqri  rf  rg  )r   qschemech_axis
activationr  QuantizedEmbeddingBagQuantizedEmbedding)!r  r  r  r_   r  r  r  r;  r  embedding_bag_unpackrN   _CScriptObjectr  r  EmbeddingBag	Embeddingr  r  r  _packed_weightr  r   r   	with_args per_channel_affine_float_qparamsr   r   ru  r   
from_floatrz  r  )r:   qembrf  rg  indicesoffsetsset_qconfig
is_emb_bagr   r  emb_dictr  r  embedding_unpackkey
emb_weightloaded_weightloaded_qembfloat_embeddingfloat_qparams_observerq_embeddingbagexpected_names                         r;   checkEmbeddingSerializationz0QuantizationTestCase.checkEmbeddingSerialization  sk    w'FYF??$JJL

8Q	q	jjm 99..CCC(3-)>)>?!+c"2EHH4I4IJJJ-hsm<
 0S1A B  ];  **.Xe?C%W\^K --~UbjopK&&t[&A##K0)$*=*=*L*LM)+*D*D*S*ST	V
 	TF8TB #hh33>anHLafmr 4 tO $hh00^k0lO%=%G%GePUPvPvPQ&S" '.9W5K'MO# 	(  --88IN3M ]]55oFN0M^)<<=r=   NN)F)FFTTFN)NNNT)NNNFFNNNNNN)'rV   rW   rX   r^  rs  rw  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  HAS_FXr0   r  r/   r*   r.   r,   r+  r1   rI  rd  r_   r   r  __classcell__rl  s   @r;   rY  rY    sw   T:5'4!_F<$0::)69\5`MV: SXLP*\ %)#	>)@ 6	$(eJ
4J.K)K$L6	 !eE(H2D,EuXW_M_G`,`&a!ab6	 	6	
 6	 6	pH	Q"3S$sCx.-@(A#ABH	Q H	Q^ #)-#'"!&$(&*15+/&*#y	Dz DI<<?>r=   rY  c                       e Zd Zdeej
                  j                     fdZdej
                  j                  dej                  fdZ	y)QuantizationLiteTestCasemodel_classc                     d}t        |      5  t        j                  j                  j	                  |      } |di |}t        |t        | j                  g      }d d d        |S # 1 sw Y   S xY w)Nr.  r[   )r#   r_   r  r  r   r   rg   rd   )r:   r  r  qengineru  rc   s         r;   _create_quantized_modelz0QuantizationLiteTestCase._create_quantized_model7  se    &w/hh++??HG)&)EU$57HIE 0
  0
 s   AA&&A0rc   inputc                    d}t        |      5  t        j                  j                  |      } ||      }d}t	        d|dz         D ]  }	 t        j                  |j                               }|j                  d       t        |      }	 |	|      }
t        j                  j                  ||
       |	j                  |      }t        j                  j                  ||       |	j                  d|      }t        j                  j                  ||        n d d d        y # t        $ r}||k(  r|Y d }~d }~ww xY w# 1 sw Y   y xY w)Nr.  r   rl   r   forward)r#   r_   rI  rK  rp   r  r  $_save_to_buffer_for_lite_interpreterr  r$   testingassert_closer  
run_methodAssertionError)r:   rc   r  r  script_modulescript_module_result	max_retryretryr  mobile_modulemobile_module_resultmobile_module_forward_resultmobile_module_run_method_resultes                 r;   _compare_script_and_mobilez3QuantizationLiteTestCase._compare_script_and_mobileA  s%    &w/!II,,U3M#0#7 Iq)a-0!ZZ(Z(Z(\]FKKN$>v$FM+8+?(MM../CEYZ3@3H3H3O0MM../CEab6C6N6NyZ_6`3MM../CEde ) 1 0/( & !	) 	!) 0/s6   <D>C
D!D>!	D;*D61D>6D;;D>>EN)
rV   rW   rX   r3   r_   r   Moduler  r4  r  r[   r=   r;   r  r  6  s<    43H */((//*/,,r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )SingleLayerLinearModelc                     t         |           t        j                  j	                  dd      j                  t        j                        | _        y Nr   r   )r]  r<   r_   r   r  r   r   fc1r:   rl  s    r;   r<   zSingleLayerLinearModel.__init__e  s6    88??1a(++%+++>r=   c                 (    | j                  |      }|S rI   r  r:   r  s     r;   r  zSingleLayerLinearModel.forwardi      HHQKr=   r  .c                 0    t        j                  dd      fS Nrl   r   r_   r_  rK   s    r;   get_example_inputsz)SingleLayerLinearModel.get_example_inputsm      

1a ""r=   	rV   rW   rX   r<   r  r/   r1   r  r  r  s   @r;   r  r  d  s!    ?#E#s(O #r=   r  c                   <     e Zd Zd fd	Zd Zdeedf   fdZ xZS )AnnotatedSingleLayerLinearModelc                    t         |           t        j                  j                  j                  |      | _        t        t        j                  j                  dd      j                  t        j                              | _        y r  )r]  r<   r_   r  r  r   ru  r
   r   r  r   r   r  r:   r  rl  s     r;   r<   z(AnnotatedSingleLayerLinearModel.__init__q  sW    xx,,@@I1 5 8 8u{{ 8 KLr=   c                 (    | j                  |      }|S rI   r  r  s     r;   r  z'AnnotatedSingleLayerLinearModel.forwardv  r  r=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z2AnnotatedSingleLayerLinearModel.get_example_inputsz  r  r=   r  r  r  s   @r;   r  r  p  s"    M
#E#s(O #r=   r  c                   <     e Zd Zd fd	Zd Zdeedf   fdZ xZS )SingleLayerLinearDynamicModelc                    t         |           t        j                  j                  j                  |      | _        t        j                  j                  dd      j                  t        j                        | _        y r  )r]  r<   r_   r  r  r   ru  r   r  r   r   r  r  s     r;   r<   z&SingleLayerLinearDynamicModel.__init__~  sR    xx,,@@I88??1a(++%+++>r=   c                 (    | j                  |      }|S rI   r  r  s     r;   r  z%SingleLayerLinearDynamicModel.forward  r  r=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z0SingleLayerLinearDynamicModel.get_example_inputs  r  r=   r  r  r  s   @r;   r  r  }  s!    ?
#E#s(O #r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )LinearAddModelc                 .   t         |           t        j                  j	                  dd      j                  t        j                        | _        t        j                  j	                  dd      j                  t        j                        | _        y Nr      r   	r]  r<   r_   r   r  r   r   r  fc2r  s    r;   r<   zLinearAddModel.__init__  ^    88??1a(++%+++>88??1a(++%+++>r=   c                 v    | j                  |      }t        j                  |d      }| j                  |      }|S Nr   )r  r_   addr  r  s     r;   r  zLinearAddModel.forward  s0    HHQKIIaOHHQKr=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z!LinearAddModel.get_example_inputs  r  r=   r  r  s   @r;   r  r    s!    ?
#E#s(O #r=   r  c                   $     e Zd Z fdZd Z xZS )RNNDynamicModelc                 Z   t         |           t        | _        |dk(  rCt        j
                  j                  dd      j                  t        j                        | _	        |dk(  rDt        j
                  j                  dd      j                  t        j                        | _	        y y )NGRUr[  r   LSTM)r]  r<   r   ru  r_   r   r  r   r   r  r  r:   mod_typerl  s     r;   r<   zRNNDynamicModel.__init__  sz    .uxx||Aq),,5;;,?DHvxx}}Q*--EKK-@DH r=   c                 (    | j                  |      }|S rI   r  r  s     r;   r  zRNNDynamicModel.forward  r  r=   rV   rW   rX   r<   r  r  r  s   @r;   r  r    s    Ar=   r  c                   $     e Zd Z fdZd Z xZS )RNNCellDynamicModelc                    t         |           t        | _        |dk(  rCt        j
                  j                  dd      j                  t        j                        | _	        |dk(  rCt        j
                  j                  dd      j                  t        j                        | _	        |dk(  rEt        j
                  j                  ddd      j                  t        j                        | _	        |dk(  rFt        j
                  j                  ddd	      j                  t        j                        | _	        y y )
NGRUCellr[  r   LSTMCellRNNReLUrelu)nonlinearityRNNTanhtanh)r]  r<   r   ru  r_   r   r  r   r   r  r  RNNCellr  s     r;   r<   zRNNCellDynamicModel.__init__  s    .y xx''1-00u{{0CDHz!xx((A.111DDHy xx''16'BEEEKKEXDHy xx''16'BEEEKKEXDH !r=   c                 (    | j                  |      }|S rI   r  r  s     r;   r  zRNNCellDynamicModel.forward  r  r=   r  r  s   @r;   r  r    s    
Yr=   r  c                   &     e Zd Zd fd	Zd Z xZS )LSTMwithHiddenDynamicModelc                    t         |           t        j                  j                  j                  |      | _        t        j                  j                  dd      j                  t        j                        | _        y )Nr[  r   )r]  r<   r_   r  r  r   ru  r   r  r   r   lstmr  s     r;   r<   z#LSTMwithHiddenDynamicModel.__init__  sR    xx,,@@IHHMM!Q'***=	r=   c                 4    | j                  ||      \  }}||fS rI   )r  )r:   r  hids      r;   r  z"LSTMwithHiddenDynamicModel.forward  s    1c"3#vr=   r  r  r  s   @r;   r  r    s    >
r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )	ConvModelc                     t         |           t        j                  j	                  dddd      j                  t        j                        | _        y Nr\  r   Fr  r   )r]  r<   r_   r   Conv2dr   r   convr  s    r;   r<   zConvModel.__init__  s<    HHOOAq!%O8;;%++;N	r=   c                 (    | j                  |      }|S rI   r  r  s     r;   r  zConvModel.forward      IIaLr=   r  .c                 4    t        j                  dddd      fS Nrl   r\  r   r  rK   s    r;   r  zConvModel.get_example_inputs      

1aA&((r=   r  r  s   @r;   r  r    s"    O)E#s(O )r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )ConvTransposeModelc                     t         |           t        j                  j	                  dddd      j                  t        j                        | _        y r  )r]  r<   r_   r   ConvTranspose2dr   r   r  r  s    r;   r<   zConvTransposeModel.__init__  s?    HH,,Q15,ADD5;;DW	r=   c                 (    | j                  |      }|S rI   r  r  s     r;   r  zConvTransposeModel.forward  r  r=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z%ConvTransposeModel.get_example_inputs  r  r=   r  r  s   @r;   r  r    s"    X)E#s(O )r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )AnnotatedConvModelc                 F   t         |           t        j                  j                  j                  |      | _        t        j                  j                  dddd      j                  t        j                        | _        t               | _        t               | _        y r  )r]  r<   r_   r  r  r   ru  r   r  r   r   r  r   rn  r   ro  r  s     r;   r<   zAnnotatedConvModel.__init__  sj    xx,,@@IHHOOAq!%O8;;%++;N	[
"}r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   rn  r  ro  r  s     r;   r  zAnnotatedConvModel.forward  .    JJqMIIaLLLOr=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z%AnnotatedConvModel.get_example_inputs  r  r=   r  r  s   @r;   r  r    !    %)E#s(O )r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )AnnotatedConvTransposeModelc                 F   t         |           t        j                  j                  j                  |      | _        t        j                  j                  dddd      j                  t        j                        | _        t               | _        t               | _        y r  )r]  r<   r_   r  r  r   ru  r   r  r   r   r  r   rn  r   ro  r  s     r;   r<   z$AnnotatedConvTransposeModel.__init__  sm    xx,,@@IHH,,Q15,ADD5;;DW	[
"}r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   r  r  s     r;   r  z#AnnotatedConvTransposeModel.forward  r  r=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z.AnnotatedConvTransposeModel.get_example_inputs  r  r=   r  r  s   @r;   r
  r
    r  r=   r
  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )ConvBnModelc                 2   t         |           t        j                  j	                  dddd      j                  t        j                        | _        t        j                  j                  d      j                  t        j                        | _	        y r  )
r]  r<   r_   r   r  r   r   r  BatchNorm2dbnr  s    r;   r<   zConvBnModel.__init__  sd    HHOOAq!%O8;;%++;N	((&&q),,5;;,?r=   c                 J    | j                  |      }| j                  |      }|S rI   r  r  r  s     r;   r  zConvBnModel.forward  !    IIaLGGAJr=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  zConvBnModel.get_example_inputs  r  r=   r  r  s   @r;   r  r    s"    @

)E#s(O )r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )AnnotatedConvBnModelc                    t         |           t        | _        t        j
                  j                  dddd      j                  t        j                        | _	        t        j
                  j                  d      j                  t        j                        | _        t               | _        t               | _        y r  )r]  r<   r   ru  r_   r   r  r   r   r  r  r  r   rn  r   ro  r  s    r;   r<   zAnnotatedConvBnModel.__init__
  s}    &HHOOAq!%O8;;%++;N	((&&q),,5;;,?[
"}r=   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S rI   )rn  r  r  ro  r  s     r;   r  zAnnotatedConvBnModel.forward  s;    JJqMIIaLGGAJLLOr=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z'AnnotatedConvBnModel.get_example_inputs  r  r=   r  r  s   @r;   r  r  	  s!    %)E#s(O )r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )ConvBnReLUModelc                 h   t         |           t        j                  j	                  dddd      j                  t        j                        | _        t        j                  j                  d      j                  t        j                        | _	        t        j                  d      | _        y Nr\  r   Fr  r   Tr   )r]  r<   r_   r   r  r   r   r  r  r  ReLUr  r  s    r;   r<   zConvBnReLUModel.__init__  st    HHOOAq!%O8;;%++;N	((&&q),,5;;,?GGD)	r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   r  r  r  r  s     r;   r  zConvBnReLUModel.forward#  s.    IIaLGGAJIIaLr=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z"ConvBnReLUModel.get_example_inputs)  r  r=   r  r  s   @r;   r  r    s!    *)E#s(O )r=   r  c                   B     e Zd Zd fd	Zd Zd Zdeedf   fdZ xZ	S )AnnotatedConvBnReLUModelc                     t         |           t        j                  j                  j                  |      | _        t        j                  j                  dddd      j                  t        j                        | _        t        j                  j                  d      j                  t        j                        | _        t        j                  d      | _        t!               | _        t%               | _        y r  )r]  r<   r_   r  r  r   ru  r   r  r   r   r  r  r  r   r  r   rn  r   ro  r  s     r;   r<   z!AnnotatedConvBnReLUModel.__init__-  s    xx,,@@IHHOOAq!%O8;;%++;N	((&&q),,5;;,?GGD)	[
"}r=   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S rI   )rn  r  r  r  ro  r  s     r;   r  z AnnotatedConvBnReLUModel.forward6  sH    JJqMIIaLGGAJIIaLLLOr=   c                     | j                   r0t        j                  j                  j	                  | g dgd       y t        j                  j                  j                  | g dgd       y )Nr"  Tr   trainingr_   r  r  fuse_modules_qatfuse_modulesrK   s    r;   
fuse_modelz#AnnotatedConvBnReLUModel.fuse_model>  sQ    ==HH!!224:P9Q[_2`HH!!..t6L5MW[.\r=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z+AnnotatedConvBnReLUModel.get_example_inputsE  r  r=   r  )
rV   rW   rX   r<   r  r-  r/   r1   r  r  r  s   @r;   r%  r%  ,  s'    %])E#s(O )r=   r%  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )TwoLayerConvModelc                 :   t         |           t        j                  j	                  dddd      j                  t        j                        | _        t        j                  j	                  dddd      j                  t        j                        | _        y )Nr\  r   Fr  r   rl   )	r]  r<   r_   r   r  r   r   conv1conv2r  s    r;   r<   zTwoLayerConvModel.__init__I  sj    XX__Q15_9<<5;;<O
XX__Q15_9<<5;;<O
r=   c                 J    | j                  |      }| j                  |      }|S rI   r2  r3  r  s     r;   r  zTwoLayerConvModel.forwardN  !    JJqMJJqMr=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z$TwoLayerConvModel.get_example_inputsS  r  r=   r  r  s   @r;   r0  r0  H  s"    P

)E#s(O )r=   r0  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )TwoLayerLinearModelc                 .   t         |           t        j                  j	                  dd      j                  t        j                        | _        t        j                  j	                  dd      j                  t        j                        | _        y r  r  r  s    r;   r<   zTwoLayerLinearModel.__init__W  r  r=   c                 J    | j                  |      }| j                  |      }|S rI   r  r  r  s     r;   r  zTwoLayerLinearModel.forward\  !    HHQKHHQKr=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z&TwoLayerLinearModel.get_example_inputsa  r  r=   r  r  s   @r;   r9  r9  V  s!    ?

#E#s(O #r=   r9  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )LinearModelWithSubmodulec                 v    t         |           t               | _        t	        j
                  dd      | _        y r  )r]  r<   r9  submr   r  fcr  s    r;   r<   z!LinearModelWithSubmodule.__init__e  s)    ')	))Aq/r=   c                 J    | j                  |      }| j                  |      }|S rI   )rB  rC  r  s     r;   r  z LinearModelWithSubmodule.forwardj  r  r=   r  .c                 6    | j                   j                         S rI   )rB  r  rK   s    r;   r  z+LinearModelWithSubmodule.get_example_inputso      yy++--r=   r  r  s   @r;   r@  r@  d  s!    "

.E#s(O .r=   r@  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )AnnotatedTwoLayerLinearModelc                    t         |           t        j                  j	                  dd      j                  t        j                        | _        t        t        j                  j	                  dd      j                  t        j                              | _	        t        j                  j                  j                  d      | j                  _        y )Nr   r  r   r  )r]  r<   r_   r   r  r   r   r  r
   r  r  r  r   ru  r  s    r;   r<   z%AnnotatedTwoLayerLinearModel.__init__s  s    88??1a(++%+++>1 5 8 8u{{ 8 KL 8800DDXNr=   c                 J    | j                  |      }| j                  |      }|S rI   r<  r  s     r;   r  z$AnnotatedTwoLayerLinearModel.forwardy  r=  r=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z/AnnotatedTwoLayerLinearModel.get_example_inputs~  r  r=   r  r  s   @r;   rH  rH  r  s"    O
#E#s(O #r=   rH  c                   $     e Zd Z fdZd Z xZS )ActivationsTestModelc                 6   t         |           t        j                  j                  j                  d      | _        t        j                  j                  j                         | _        t        j                  j                         j                  t        j                        | _        t        j                  j                         j                  t        j                        | _        t        j                  j                  j!                         | _        y )Nr  r   )r]  r<   r_   r  r  r   ru  r   rn  r   	Hardswishr   r   	hardswishELUelur   ro  r  s    r;   r<   zActivationsTestModel.__init__  s    xx,,@@JXX**446
++-00u{{0C88<<>$$5;;$7xx,,88:r=   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S rI   )rn  rP  rR  ro  r  s     r;   r  zActivationsTestModel.forward  s<    JJqMNN1HHQKLLOr=   r  r  s   @r;   rM  rM    s    ;r=   rM  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )LinearReluModelc                     t         |           t        j                  j	                  dd      j                  t        j                        | _        t        j                  j                         | _	        y r  )
r]  r<   r_   r   r  r   r   rC  r   r  r  s    r;   r<   zLinearReluModel.__init__  sG    ((//!Q'***=HHMMO	r=   c                 F    | j                  | j                  |            }|S rI   r  rC  r  s     r;   r  zLinearReluModel.forward      IIdggaj!r=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z"LinearReluModel.get_example_inputs  r  r=   r  r  s   @r;   rU  rU    s!    $
#E#s(O #r=   rU  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )LinearReluLinearModelc                 t   t         |           t        j                  j	                  dd      j                  t        j                        | _        t        j                  j                         | _	        t        j                  j	                  dd      j                  t        j                        | _
        y r  r]  r<   r_   r   r  r   r   r  r   r  r  r  s    r;   r<   zLinearReluLinearModel.__init__  o    88??1a(++%+++>HHMMO	88??1a(++%+++>r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   r  r  r  r  s     r;   r  zLinearReluLinearModel.forward  .    HHQKIIaLHHQKr=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z(LinearReluLinearModel.get_example_inputs  r  r=   r  r  s   @r;   r\  r\    s!    ?#E#s(O #r=   r\  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )LinearReluAddModelc                 t   t         |           t        j                  j	                  dd      j                  t        j                        | _        t        j                  j                         | _	        t        j                  j	                  dd      j                  t        j                        | _
        y r  r^  r  s    r;   r<   zLinearReluAddModel.__init__  r_  r=   c                     | j                  |      }| j                  |      }t        j                  |d      }| j	                  |      }t        j
                  j                         | _        |S r  r  r  r_   r  r  r   r   r  s     r;   r  zLinearReluAddModel.forward  N    HHQKIIaLIIaOHHQKHHMMO	r=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z%LinearReluAddModel.get_example_inputs  r  r=   r  r  s   @r;   re  re    s!    ?#E#s(O #r=   re  c                   <     e Zd Zd fd	Zd Zdeedf   fdZ xZS )LinearBnLeakyReluModelc                     t         |           t        j                  dd      | _        t        j
                  d      | _        t        j                  d      | _        || _	        y )Nr   g{Gz?)
r]  r<   r   r  linearBatchNorm1dbn1d	LeakyReLU
leaky_reluwith_bn)r:   rs  rl  s     r;   r<   zLinearBnLeakyReluModel.__init__  sF    ii1oNN1%	,,t,r=   c                     | j                  |      }| j                  r| j                  |      }| j                  |      }|S rI   )rn  rs  rp  rr  r  s     r;   r  zLinearBnLeakyReluModel.forward  s6    KKN<<		!AOOAr=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z)LinearBnLeakyReluModel.get_example_inputs  r  r=   r  r  r  s   @r;   rl  rl    s!    #E#s(O #r=   rl  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )LinearTanhModelc                     t         |           t        j                  dd      | _        t        j
                         | _        y r  )r]  r<   r   r  rn  Tanhr  r  s    r;   r<   zLinearTanhModel.__init__  s,    ii1oGGI	r=   c                 J    | j                  |      }| j                  |      }|S rI   )rn  r  r  s     r;   r  zLinearTanhModel.forward  s!    KKNIIaLr=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z"LinearTanhModel.get_example_inputs  r  r=   r  r  s   @r;   rw  rw    s!    

#E#s(O #r=   rw  c                   F     e Zd Z	 	 	 	 	 d fd	Zd Zdeedf   fdZ xZS )ConvBnAddReluModelc                 >   t         |           t        j                  ddd      | _        t        j                  ddd      | _        t        j                  d      | _        t        j                         | _	        || _
        || _        || _        || _        || _        y )Nr   )r[  r[  )r]  r<   r   r  r  r3  r  r  r   r  rs  	with_relutwo_conv	left_convuse_torch_add)r:   rs  r  r  r  r  rl  s         r;   r<   zConvBnAddReluModel.__init__  sy     	IIaF+	YYq!V,
..#GGI	" "*r=   c                 @   | j                   r| j                  r| j                  rEt        j                  | j                  | j                  |            | j                  |            }nt        j                  | j                  |      | j                  |            }n| j                  r4| j                  | j                  |            | j                  |      z   }n| j                  |      | j                  |      z   }n|| j                  r| j                  rh| j                  r6t        j                  | j                  | j                  |            |      }n"t        j                  | j                  |      |      }n| j                  r5t        j                  || j                  | j                  |                  }nt        j                  || j                  |            }n| j                  rE| j                  r$| j                  | j                  |            |z   }nY| j                  |      |z   }nD| j                  r$|| j                  | j                  |            z   }n|| j                  |      z   }| j                  r| j                  |      }|S rI   )r  r  rs  r_   r  r  r  r3  r  r  r  )r:   x1x2r  s       r;   r  zConvBnAddReluModel.forward  s   ==!!<<		$''$))B-"8$**R.IA		$))B-B@A<<		".B?A		"

26A!!>>||!IIdggdiim&<bA!IIdiimR8||!IIb$''$))B-*@A!IIb$))B-8>>|| GGDIIbM2R7 IIbMB.||2!772.>>		!Ar=   r  .c                 b    t        j                  dddd      t        j                  dddd      fS )Nrl   r   r\  r[  r  rK   s    r;   r  z%ConvBnAddReluModel.get_example_inputs  s+    

1aA&

1aA(>??r=   )TTTTTr  r  s   @r;   r}  r}    s3    #+"%N@E#s(O @r=   r}  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )ConvReluModelc                     t         |           t        j                  j	                  ddd      j                  t        j                        | _        t        j                  j                         | _	        y )Nr\  r   r   )
r]  r<   r_   r   r  r   r   rC  r   r  r  s    r;   r<   zConvReluModel.__init__  sI    ((//!Q*--EKK-@HHMMO	r=   c                 F    | j                  | j                  |            }|S rI   rX  r  s     r;   r  zConvReluModel.forward$  rY  r=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z ConvReluModel.get_example_inputs(  r  r=   r  r  s   @r;   r  r    s!    $
)E#s(O )r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )ConvReluConvModelc                 x   t         |           t        j                  j	                  ddd      j                  t        j                        | _        t        j                  j                         | _	        t        j                  j	                  ddd      j                  t        j                        | _
        y Nr\  r   r   rl   r]  r<   r_   r   r  r   r   r  r   r  r  r  s    r;   r<   zConvReluConvModel.__init__-  s    88??1a+..U[[.AHHMMO	88??1a+..U[[.Ar=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   ra  r  s     r;   r  zConvReluConvModel.forward3  rb  r=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z$ConvReluConvModel.get_example_inputs9  r  r=   r  r  s   @r;   r  r  ,  s"    B)E#s(O )r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )ConvReluAddModelc                 x   t         |           t        j                  j	                  ddd      j                  t        j                        | _        t        j                  j                         | _	        t        j                  j	                  ddd      j                  t        j                        | _
        y r  r  r  s    r;   r<   zConvReluAddModel.__init__>  r  r=   c                     | j                  |      }| j                  |      }t        j                  |d      }| j	                  |      }t        j
                  j                         | _        |S r  rh  r  s     r;   r  zConvReluAddModel.forwardD  ri  r=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z#ConvReluAddModel.get_example_inputsL  r  r=   r  r  s   @r;   r  r  =  s"    B)E#s(O )r=   r  c                   $     e Zd Z fdZd Z xZS )NormalizationTestModelc                 l   t         |           t        j                  j                  j                         | _        t        j                  j                  dd      j                  t        j                        | _        t        j                  j                  d      | _        t        j                  j                  dd      | _        t        j                  j!                  d      | _        t        j                  j%                  d      | _        t        j                  j)                  d      | _        y )Nr   r  r   r[  )r]  r<   r_   r  r  r   rn  r   r  r   r   r  	LayerNorm
layer_norm	GroupNorm
group_normInstanceNorm1dinstance_norm1dInstanceNorm2dinstance_norm2dInstanceNorm3dinstance_norm3dr  s    r;   r<   zNormalizationTestModel.__init__P  s    XX**446
88??1a(++%+++>((,,Q/((,,Q2$xx66q9$xx66q9$xx66q9r=   c                 p   | j                  |      }| j                  |      }| j                  |      }| j                  |j	                  d      j                  ddd            }| j                  |      }| j                  |j	                  d            }| j                  |j	                  d            }|S )Nr   rl   r\  )	rn  r  r  r  	unsqueezerepeatr  r  r  r  s     r;   r  zNormalizationTestModel.forwardZ  s    JJqMHHQKOOAOOAKKO221a;<  #  R1  R1r=   r  r  s   @r;   r  r  O  s    :r=   r  c                   $     e Zd Z fdZd Z xZS )NestedModelc                     t         |           t               | _        t	               | _        t        j                  j                  dd      j                  t        j                        | _        y r  )r]  r<   rU  sub1r9  sub2r_   r   r  r   r   fc3r  s    r;   r<   zNestedModel.__init__e  sJ    #%	')	88??1a(++%+++>r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   r  r  r  r  s     r;   r  zNestedModel.forwardk  .    IIaLIIaLHHQKr=   r  r  s   @r;   r  r  d  s    ?r=   r  c                   $     e Zd Z fdZd Z xZS )AnnotatedNestedModelc                    t         |           t               | _        t	               | _        t        t        j                  j                  dd      j                  t        j                              | _        t        | j                  _        t        | j
                  j                        | j
                  _        |dk(  r t         | j
                  j                  _        y t        | j
                  j                  _        y )Nr   r   r  )r]  r<   rU  r  r9  r  r
   r_   r   r  r   r   r  r   ru  r  r   r  s     r;   r<   zAnnotatedNestedModel.__init__r  s    #%	')	1 5 8 8u{{ 8 KL*$TYY]]3		h$?DIIMM!$3DIIMM!r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   r  r  s     r;   r  zAnnotatedNestedModel.forward~  r  r=   r  r  s   @r;   r  r  q  s    
4r=   r  c                   $     e Zd Z fdZd Z xZS )AnnotatedSubNestedModelc                 \   t         |           t               | _        t	        t                     | _        t	        t        j                  j                  dd      j                  t        j                              | _        t        | j                  _        t        | j                  _        y r  )r]  r<   rU  r  r
   r9  r  r_   r   r  r   r   r  r   ru  r  s    r;   r<   z AnnotatedSubNestedModel.__init__  sm    #%	 !4!67	1 5 8 8u{{ 8 KL*+		r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   r  r  s     r;   r  zAnnotatedSubNestedModel.forward  r  r=   r  r  s   @r;   r  r    s    ,r=   r  c                   $     e Zd Z fdZd Z xZS ) AnnotatedCustomConfigNestedModelc                    t         |           t               | _        t	               | _        t        t        j                  j                  dd      j                  t        j                              | _        t        | j                  _        t        | j
                  _        t        j                  t        j                   d}t#        t%        j&                  di |t(              }|| j
                  j*                  _        t        | j
                  j*                        | j
                  _        t        | j
                  j,                        | j
                  _        y )Nr   r   )r   rm  ro  r[   )r]  r<   rU  r  r9  r  r
   r_   r   r  r   r   r  r   ru  r   per_tensor_affiner   r   ry  r   r  r  )r:   custom_optionscustom_qconfigrl  s      r;   r<   z)AnnotatedCustomConfigNestedModel.__init__  s    #%	')	1 5 8 8u{{ 8 KL*+		 \\..
 !,<,F,F,X,X(?A .		$TYY]]3		$TYY]]3		r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   r  r  s     r;   r  z(AnnotatedCustomConfigNestedModel.forward  r  r=   r  r  s   @r;   r  r    s    4&r=   r  c                   $     e Zd Z fdZd Z xZS )QuantSubModelc                 J   t         |           t               | _        t	        t                     | _        t        | j                  _        t        j                  j                  dd      j                  t        j                        | _        t        | j                  _        y r  )r]  r<   rU  r  r
   r9  r  r   ru  r_   r   r  r   r   r  r  s    r;   r<   zQuantSubModel.__init__  sh    #%	 !4!67	+		88??1a(++%+++>*r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   r  r  s     r;   r  zQuantSubModel.forward  r  r=   r  r  s   @r;   r  r    s    +r=   r  c                   *     e Zd Z fdZd Zd Z xZS )InnerModulec                    t         |           t        j                  j	                  dd      j                  t        j                        | _        t        j                  j                         | _	        t        j                  j	                  dd      j                  t        j                        | _
        t        j                  j                         | _        y r  )r]  r<   r_   r   r  r   r   r  r   relu1r  relu2r  s    r;   r<   zInnerModule.__init__  s    88??1a(++%+++>XX]]_
88??1a(++%+++>XX]]_
r=   c           	      ~    | j                  | j                  | j                  | j                  |                        S rI   )r  r  r  r  r  s     r;   r  zInnerModule.forward  s+    zz$((4::dhhqk#:;<<r=   c                 4   g }t        | j                               }t        |      D ]  \  }\  }}t        |t        j
                  j                        s.|t        |      dz
  k\  r nLt        ||dz      d   t        j
                  j                        so|j                  |||dz      d   g        | j                  r-t        j                  j                  j                  | |d       y t        j                  j                  j                  | |d       y )Nrl   r   Tr   )r5  r  	enumeraterN   r_   r   r  r   r   r   r*  r  r  r+  r,  )r:   fusable_layersr  rB  current_namelayers         r;   r,  zInnerModule.fuse_modules  s    d1134*3N*C&C&,%1#n-11nS1W5a8%((--H"))<+9#'+B1+E+G H +D ==HH!!224QU2VHH!!..t^T.Rr=   )rV   rW   rX   r<   r  r,  r  r  s   @r;   r  r    s    %=Sr=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )FunctionalLinearc                     t         |           t        j                  d      | _        t        j
                  d      | _        y )N)r   r   r   )r]  r<   r_   r_  r  rP  r  r  s    r;   r<   zFunctionalLinear.__init__  s-    jj(KKN	r=   c                 X    t        j                  || j                  | j                        S rI   )Frn  r  r  r  s     r;   r  zFunctionalLinear.forward  s    xx4;;		22r=   r  .c                 0    t        j                  dd      fS r  r  rK   s    r;   r  z#FunctionalLinear.get_example_inputs  r  r=   r  r  s   @r;   r  r    s!    #
3#E#s(O #r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS ) SingleLayerFunctionalLinearModelc                 @    t         |           t               | _        y rI   )r]  r<   r  linear1r  s    r;   r<   z)SingleLayerFunctionalLinearModel.__init__  s    ')r=   c                 (    | j                  |      }|S rI   )r  r  s     r;   r  z(SingleLayerFunctionalLinearModel.forward  s    LLOr=   r  .c                 6    | j                   j                         S rI   r  r  rK   s    r;   r  z3SingleLayerFunctionalLinearModel.get_example_inputs      ||..00r=   r  r  s   @r;   r  r    s!    *1E#s(O 1r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )TwoLayerFunctionalLinearModelc                 ^    t         |           t               | _        t               | _        y rI   r]  r<   r  r  linear2r  s    r;   r<   z&TwoLayerFunctionalLinearModel.__init__  "    ')')r=   c                 J    | j                  |      }| j                  |      }|S rI   )r  r  r  s     r;   r  z%TwoLayerFunctionalLinearModel.forward  s!    LLOLLOr=   r  .c                 6    | j                   j                         S rI   r  rK   s    r;   r  z0TwoLayerFunctionalLinearModel.get_example_inputs  r  r=   r  r  s   @r;   r  r    s!    *

1E#s(O 1r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )FunctionalLinearAddModelc                 ^    t         |           t               | _        t               | _        y rI   r  r  s    r;   r<   z!FunctionalLinearAddModel.__init__  r  r=   c                 v    | j                  |      }t        j                  |d      }| j                  |      }|S r  )r  r_   r  r  r  s     r;   r  z FunctionalLinearAddModel.forward  s0    LLOIIaOLLOr=   r  .c                 6    | j                   j                         S rI   r  rK   s    r;   r  z+FunctionalLinearAddModel.get_example_inputs	  r  r=   r  r  s   @r;   r  r    s!    *
1E#s(O 1r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )FunctionalLinearReluModelc                 @    t         |           t               | _        y rI   )r]  r<   r  rn  r  s    r;   r<   z"FunctionalLinearReluModel.__init__  s    &(r=   c                 R    | j                  |      }t        j                  |      }|S rI   )rn  r  r  r  s     r;   r  z!FunctionalLinearReluModel.forward  s!    KKNFF1Ir=   r  .c                 6    | j                   j                         S rI   )rn  r  rK   s    r;   r  z,FunctionalLinearReluModel.get_example_inputs  s    {{--//r=   r  r  s   @r;   r  r    s!    )
0E#s(O 0r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )FunctionalLinearReluLinearModelc                     t         |           t               | _        t	        j
                         | _        t               | _        y rI   )r]  r<   r  r  r   r   r  r  r  s    r;   r<   z(FunctionalLinearReluLinearModel.__init__  s/    ')GGI	')r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   )r  r  r  r  s     r;   r  z'FunctionalLinearReluLinearModel.forward   s.    LLOIIaLLLOr=   r  .c                 6    | j                   j                         S rI   r  rK   s    r;   r  z2FunctionalLinearReluLinearModel.get_example_inputs&  r  r=   r  r  s   @r;   r  r    s!    *1E#s(O 1r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )FunctionalConv2dc                     t         |           t        j                  dddd      | _        t        j                  d      | _        d| _        d| _        d| _        d| _	        y )Nr\  rl   rl   )r   r   rl   )
r]  r<   r_   r_  r  r  stridepaddingdilationr   r  s    r;   r<   zFunctionalConv2d.__init__*  sO    jjAq!,JJqM	r=   c           	          t        j                  || j                  | j                  | j                  | j
                  | j                  | j                        S rI   )r  conv2dr  r  r  r  r  r   r  s     r;   r  zFunctionalConv2d.forward3  s8    xx4;;		4;;dmm]a]h]hiir=   r  .c                 4    t        j                  dddd      fS r  r  rK   s    r;   r  z#FunctionalConv2d.get_example_inputs6  r  r=   r  r  s   @r;   r  r  )  s"    j)E#s(O )r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )SingleLayerFunctionalConvModelc                 @    t         |           t               | _        y rI   )r]  r<   r  r2  r  s    r;   r<   z'SingleLayerFunctionalConvModel.__init__:  s    %'
r=   c                 (    | j                  |      }|S rI   )r2  r  s     r;   r  z&SingleLayerFunctionalConvModel.forward>  s    JJqMr=   r  .c                 6    | j                   j                         S rI   r2  r  rK   s    r;   r  z1SingleLayerFunctionalConvModel.get_example_inputsB      zz,,..r=   r  r  s   @r;   r   r   9  s!    (/E#s(O /r=   r   c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )TwoLayerFunctionalConvModelc                 ^    t         |           t               | _        t               | _        y rI   )r]  r<   r  r2  r3  r  s    r;   r<   z$TwoLayerFunctionalConvModel.__init__F  s"    %'
%'
r=   c                 J    | j                  |      }| j                  |      }|S rI   r5  r  s     r;   r  z#TwoLayerFunctionalConvModel.forwardK  r6  r=   r  .c                 6    | j                   j                         S rI   r  rK   s    r;   r  z.TwoLayerFunctionalConvModel.get_example_inputsP  r  r=   r  r  s   @r;   r  r  E  s!    (

/E#s(O /r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )FunctionalConvReluModelc                 @    t         |           t               | _        y rI   )r]  r<   r  r  r  s    r;   r<   z FunctionalConvReluModel.__init__T  s    $&	r=   c                 R    | j                  |      }t        j                  |      }|S rI   )r  r  r  r  s     r;   r  zFunctionalConvReluModel.forwardX  s!    IIaLFF1Ir=   r  .c                 6    | j                   j                         S rI   )r  r  rK   s    r;   r  z*FunctionalConvReluModel.get_example_inputs]  rF  r=   r  r  s   @r;   r  r  S  s!    '
.E#s(O .r=   r  c                   :     e Zd Z fdZd Zdeedf   fdZ xZS )FunctionalConvReluConvModelc                     t         |           t               | _        t	        j
                         | _        t               | _        y rI   )r]  r<   r  r2  r   r   r  r3  r  s    r;   r<   z$FunctionalConvReluConvModel.__init__a  s/    %'
GGI	%'
r=   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rI   )r2  r  r3  r  s     r;   r  z#FunctionalConvReluConvModel.forwardg  s.    JJqMIIaLJJqMr=   r  .c                 6    | j                   j                         S rI   r  rK   s    r;   r  z.FunctionalConvReluConvModel.get_example_inputsm  r  r=   r  r  s   @r;   r  r  `  s!    (/E#s(O /r=   r  c                   .     e Zd ZdZ fdZd Zd Z xZS )SkipQuantModelVWe can skip quantization by explicitly
    setting qconfig of a submodule to None
    c                     t         |           t               | _        t        j
                  j                  dd      j                  t        j                        | _	        y r  )
r]  r<   r  subr_   r   r  r   r   rC  r  s    r;   r<   zSkipQuantModel.__init__t  s?    =((//!Q'***=r=   c                 B    | j                  | j                  |            S rI   rC  r  r  s     r;   r  zSkipQuantModel.forwardy      wwtxx{##r=   c                 8    | j                   j                          y rI   )r  r,  rK   s    r;   r,  zSkipQuantModel.fuse_modules|  s    r=   rV   rW   rX   rY   r<   r  r,  r  r  s   @r;   r  r  p  s    >
$ r=   r  c                   .     e Zd ZdZ fdZd Zd Z xZS )AnnotatedSkipQuantModelr  c                 V   t         |           t        j                  j                  j                  |      | _        t        t                     | _	        t        j                  j                  dd      j                  t        j                        | _        d | j                  _        y r  )r]  r<   r_   r  r  r   ru  r
   r  r  r   r  r   r   rC  r  s     r;   r<   z AnnotatedSkipQuantModel.__init__  sk    xx,,@@I.((//!Q'***=r=   c                 B    | j                  | j                  |            S rI   r  r  s     r;   r  zAnnotatedSkipQuantModel.forward  r  r=   c                 L    | j                   j                  j                          y rI   )r  r   r,  rK   s    r;   r,  z$AnnotatedSkipQuantModel.fuse_modules  s    $$&r=   r  r  s   @r;   r   r     s    $'r=   r   c                   (     e Zd ZdZ fdZd Z xZS )QuantStubModelBA Module with manually inserted `QuantStub` and `DeQuantStub`
    c                 @   t         |           t        j                  j                  j                  d      | _        t               | _        t               | _
        t        j                  j                  dd      j                  t        j                        | _        y )Nr.  r   r   )r]  r<   r_   r  r  r   ru  r   rn  r   ro  r   r  r   r   rC  r  s    r;   r<   zQuantStubModel.__init__  sd    xx,,@@K[
"}((//!Q'***=r=   c                 h    | j                  |      }| j                  |      }| j                  |      S rI   )rn  rC  ro  r  s     r;   r  zQuantStubModel.forward  s*    JJqMGGAJ||Ar=   rV   rW   rX   rY   r<   r  r  r  s   @r;   r%  r%    s    >r=   r%  c                   (     e Zd ZdZ fdZd Z xZS )ManualLinearQATModelr&  c                    t         |           t        j                  j                  j                  |      | _        t               | _        t               | _
        t        j                  j                  dd      j                  t        j                        | _        t        j                  j                  dd      j                  t        j                        | _        y Nr   rl   r   rk   )r]  r<   r_   r  r  r   ru  r   rn  r   ro  r   r  r   r   r  r  r  s     r;   r<   zManualLinearQATModel.__init__  s    xx,,DDWM[
"}88??1a(++%+++>88??1b),,5;;,?r=   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      S rI   )rn  r  r  ro  r  s     r;   r  zManualLinearQATModel.forward  s7    JJqMHHQKHHQK||Ar=   r)  r  s   @r;   r+  r+    s    @r=   r+  c                   (     e Zd ZdZ fdZd Z xZS )ManualDropoutQATModelr&  c                    t         |           t        j                  j                  j                  |      | _        t               | _        t               | _
        t        j                  j                  dd      j                  t        j                        | _        t        j                  j!                  d      | _        y )Nr   rl   r         ?)r]  r<   r_   r  r  r   ru  r   rn  r   ro  r   r  r   r   r  r  dropoutr  s     r;   r<   zManualDropoutQATModel.__init__  sz    xx,,DDWM[
"}88??1a(++%+++>xx'',r=   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      S rI   )rn  r  r3  ro  r  s     r;   r  zManualDropoutQATModel.forward  s7    JJqMHHQKLLO||Ar=   r)  r  s   @r;   r0  r0    s    -r=   r0  c                   *     e Zd ZdZd fd	Zd Z xZS )ManualLinearDynamicQATModelz1A Module that uses a dynamic QAT by default.
    c                 L   t         |           |xs t        | _        t        j
                  j                  dd      j                  t        j                        | _	        t        j
                  j                  dd      j                  t        j                        | _
        y r-  )r]  r<   r   ru  r_   r   r  r   r   r  r  r:   ru  rl  s     r;   r<   z$ManualLinearDynamicQATModel.__init__  sk    ="=88??1a(++%+++>88??1b),,5;;,?r=   c                 J    | j                  |      }| j                  |      }|S rI   r<  r  s     r;   r  z#ManualLinearDynamicQATModel.forward  r=  r=   rI   r)  r  s   @r;   r6  r6    s    @r=   r6  c                   *     e Zd ZdZd fd	Zd Z xZS )ManualConvLinearQATModelzpA module with manually inserted `QuantStub` and `DeQuantStub`
    and contains both linear and conv modules
    c                 X   t         |           |r|n(t        j                  j                  j                  d      | _        t               | _        t               | _
        t        j                  j                  ddd      j                  t        j                        | _        t        j                  j!                  dd      j                  t        j                        | _        t        j                  j!                  dd      j                  t        j                        | _        y )Nr.  r\  rl   )r   r   @   rk   )r]  r<   r_   r  r  r   ru  r   rn  r   ro  r   r  r   r   r  r  r  r  r8  s     r;   r<   z!ManualConvLinearQATModel.__init__  s    ")wuxx/D/D/\/\]f/g[
"}HHOOAqaO8;;%++;N	88??2r*--EKK-@88??2r*--EKK-@r=   c                     | j                  |      }| j                  |      }|j                  dd      j                         }| j	                  |      }| j                  |      }| j                  |      S )Nr   r=  )rn  r  r   
contiguousr  r  ro  r  s     r;   r  z ManualConvLinearQATModel.forward  s\    JJqMIIaLFF2rN%%'HHQKHHQK||Ar=   rI   r)  r  s   @r;   r;  r;    s    Ar=   r;  c                   "     e Zd ZdZ fdZ xZS )ManualConvLinearSymmQATModelzhSame as ManualConvLinearQATModule but with Symmetric Quantization.
    Supported only with qnnpack.
    c                 ,    t         |   t               y rI   )r]  r<   r	   r  s    r;   r<   z%ManualConvLinearSymmQATModel.__init__  s    >?r=   )rV   rW   rX   rY   r<   r  r  s   @r;   rA  rA    s    @ @r=   rA  c                        e Zd Z fdZ	 	 ddej
                  deej
                     deej
                     fdZ xZS )ManualEmbeddingBagLinearc                 T   t         |           t        j                  ddd      | _        t
        | j                  _        t               | _        t               | _
        t        j                  dd      j                  t        j                        | _        t!        d      | _        y )Nrk      rx   )rf  rg  ri  rl   r   r.  )r]  r<   r   rv  embr   ru  r   rn  r   ro  r  r   r_   r   rn  r   r  s    r;   r<   z!ManualEmbeddingBagLinear.__init__  sp    ??"BUS8[
"}iiA&)))<.y9r=   r  r~  per_sample_weightsc                     | j                  |||      }| j                  |      }| j                  |      }| j                  |      S rI   )rG  rn  rn  ro  )r:   r  r~  rH  r  s        r;   r  z ManualEmbeddingBagLinear.forward  s=    HHUG%78JJqMKKN||Ar=   r  )	rV   rW   rX   r<   r_   r4  r4   r  r  r  s   @r;   rD  rD    sC    : NR=AU\\ HU\\4J $,U\\$:r=   rD  c                   \     e Zd ZdZd fdZdej                  dej                  fdZ xZS )DeFusedEmbeddingBagLineara  A module to simulate QAT embedding bag with a linear layer,
    this module uses a separate embedding and bagging op, similar
    to that which is described in the EmbeddingBag documentation.

    https://pytorch.org/docs/stable/generated/torch.nn.EmbeddingBag.html
    r  c                 |   t         |           t        j                  dd      | _        t
        | j                  _        t        j                  | _	        t               | _        t               | _        t        j                  dd      j                  t        j                         | _        t%        d      | _        y )Nrk   rF  rl  rl   r   r.  )r]  r<   r   rw  rG  r   ru  r_   rx   
bagging_opr   rn  r   ro  r  r   r   rn  r   r  s    r;   r<   z"DeFusedEmbeddingBagLinear.__init__  sy    <<rD8))[
"}iiA&)))<.y9r=   r  c                     | j                  | j                  |      d      }| j                  |      }| j                  |      }| j	                  |      S Nrl   dim)rM  rG  rn  rn  ro  )r:   r  r  s      r;   r  z!DeFusedEmbeddingBagLinear.forward  sD    OODHHUOO3JJqMKKN||Ar=   r  N)	rV   rW   rX   rY   r<   r_   r4  r  r  r  s   @r;   rK  rK    s(    :U\\ ell r=   rK  c                   $     e Zd Z fdZd Z xZS )SubModelForFusionc                 
   t         |           t        j                  dddd       j	                  t
        j                        | _        t        j                  d      j	                  t
        j                        | _	        y )Nr[  rl   r  r   )
r]  r<   r   r  r   r_   r   r  r  r  r  s    r;   r<   zSubModelForFusion.__init__  sX    IIaAD1445;;4G	..#&&U[[&9r=   c                 J    | j                  |      }| j                  |      }|S rI   r  r  s     r;   r  zSubModelForFusion.forward  r  r=   r  r  s   @r;   rT  rT    s    :
r=   rT  c                   $     e Zd Z fdZd Z xZS )SubModelWithoutFusionc                    t         |           t        j                  dddd       j	                  t
        j                        | _        t        j                  d      j	                  t
        j                        | _	        y )Nr[  rl   r  r   Fr   )
r]  r<   r   r  r   r_   r   r  r   r  r  s    r;   r<   zSubModelWithoutFusion.__init__!  sX    IIaAD1445;;4G	GGE*--EKK-@	r=   c                 B    | j                  | j                  |            S rI   )r  r  r  s     r;   r  zSubModelWithoutFusion.forward&  s    yy1&&r=   r  r  s   @r;   rX  rX     s    A
'r=   rX  c                   $     e Zd Z fdZd Z xZS )ModelForFusionc                    t         |           t        j                  dddd       j	                  t
        j                        | _        t        j                  d      j	                  t
        j                        | _	        t        j                  d      j	                  t
        j                        | _        t               | _        t               | _        t        j                   dd	      j	                  t
        j                        | _        t%               | _        t)               | _        || _        t        j.                  ddd
d       j	                  t
        j                        | _        t        j                  d      j	                  t
        j                        | _        t        j4                  d      j	                  t
        j                        | _        t        j                  d      j	                  t
        j                        | _        t        j:                  ddd      j	                  t
        j                        | _        t        j>                  d      j	                  t
        j                        | _         t        j                  d      j	                  t
        j                        | _!        d | j                  _        d | j"                  _        y )Nr\  r[  rl   r  r   Tr   $   rk   )rl   rl   rl   F)"r]  r<   r   r  r   r_   r   r2  r  bn1r   r  rT  r  rX  r  r  rC  r   rn  r   ro  ru  Conv3dr3  r  BatchNorm3dbn2relu3Conv1dconv3ro  bn3relu4r8  s     r;   r<   zModelForFusion.__init__*  s   YYq!QT255EKK5H
>>!$''ekk':WWT*--EKK-@
%'	)+	))B#&&U[[&9[
"}YYq!YT:==EKK=P
WWU+..U[[.A
>>!$''ekk':WWT*--EKK-@
YYq!Q'***=
>>!$''ekk':WWT*--EKK-@
 		r=   c                    |j                  d      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|j                  d      }|j                  d      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }|j                  dd      j                         }| j                  |      }| j                  |      }| j!                  |      }| j#                  |      }| j%                  |      }| j                  |      }|S )Nr[  r   r^  )squeezern  re  rf  rg  r  r2  r_  r  r  ro  r  r   r?  rC  r3  r  rb  rc  r:   r  ys      r;   r  zModelForFusion.forward@  s   IIaLJJqMJJqMHHQKJJqMKKNKKNJJqMHHQKJJqMIIaLLLOIIaLIIb"((*GGAJJJqMJJqMHHQKJJqMLLOr=   r  r  s   @r;   r\  r\  )  s    ,r=   r\  c                        e Zd Z fdZ xZS )
ConvBNReLUc           	          t         |   t        j                  ddddd      t        j                  d      t        j
                  d             y )Nr\  rl   Fr  r   )r]  r<   r   r  r  r   r  s    r;   r<   zConvBNReLU.__init__X  s<    IIaAqu-NN1GGE"	
r=   )rV   rW   rX   r<   r  r  s   @r;   rm  rm  W  s    
 
r=   rm  c                   $     e Zd Z fdZd Z xZS )ModelWithSequentialFusionc                    t         |           t        j                  ddd      | _        t        j
                  d      | _        g }t        d      D ]  }|j                  t                       t        j                  | | _        t        j                  dd      t        j
                  d      g}t        j                  | | _        t        j                         | _        t               | _        t#               | _        y )Nr\  rl   Fr   ,  rk   )r]  r<   r   r  r2  r   r  rp   r   rm  r  featuresr  
classifierseqr   rn  r   ro  )r:   layersr   headrl  s       r;   r<   z"ModelWithSequentialFusion.__init__`  s    YYq!Q'
WWU+
qAMM*,' v.		#r"BGGE$:;--.==?[
"}r=   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }t	        j
                  |d      }| j                  |      }| j                  |      }| j                  |      }|S )N)r   rr  )	rn  r2  r  rs  r_   r   rt  ru  ro  r  s     r;   r  z!ModelWithSequentialFusion.forwardn  su    JJqMJJqMJJqMMM!MM!./OOAHHQKLLOr=   r  r  s   @r;   rp  rp  _  s    %	r=   rp  c                   $     e Zd Z fdZd Z xZS )ModelForFusionWithBiasc                    t         |           t        j                  dddd      j	                  t
        j                        | _        t        j                  d      j	                  t
        j                        | _	        t        j                  d      j	                  t
        j                        | _        t        j                  dddd      j	                  t
        j                        | _        t        j                  d      j	                  t
        j                        | _        t               | _        t!               | _        y )	Nr\  r[  r   Tr  r   r   rl   )r]  r<   r   r  r   r_   r   r2  r  r_  r   r  r3  rb  r   rn  r   ro  r  s    r;   r<   zModelForFusionWithBias.__init__z  s    YYq!QT255EKK5H
>>!$''ekk':WWT*--EKK-@
YYq!QT255EKK5H
>>!$''ekk':[
"}r=   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }|S rI   )rn  r2  r_  r  r3  rb  ro  r  s     r;   r  zModelForFusionWithBias.forward  sb    JJqMJJqMHHQKJJqMJJqMHHQKLLOr=   r  r  s   @r;   rz  rz  y  s    %r=   rz  c                   $     e Zd Z fdZd Z xZS )ModelForLinearBNFusionc                 X   t         |           t        j                  dd      | _        t        j
                  d      | _        t        j                  j                  | j                  j                         t        j                  j                  | j                  j                         y )N   rk   )r]  r<   r   r  rC  ro  r  inituniform_r  r  r  s    r;   r<   zModelForLinearBNFusion.__init__  sd    ))B#..$
(
&r=   c                 B    | j                  | j                  |            S rI   )r  rC  r  s     r;   r  zModelForLinearBNFusion.forward  s    wwtwwqz""r=   r  r  s   @r;   r~  r~    s    '#r=   r~  c                       e Zd Zd Zd Zy)DummyObserverc                      y)N)g      ?r   r[   rK   s    r;   calculate_qparamszDummyObserver.calculate_qparams  s    r=   c                     |S rI   r[   r  s     r;   r  zDummyObserver.forward  s    r=   N)rV   rW   rX   r  r  r[   r=   r;   r  r    s    r=   r  c                   $     e Zd Z fdZd Z xZS )ModelForConvTransposeBNFusionc                 f   t         |           t        j                  ddd      | _        t        j
                  d      | _        t        j                  ddd      | _        t        j                  d      | _
        t        j                  ddd      | _        t        j                  d      | _        y )Nr\  rl   )r]  r<   r   ConvTranspose1dr2  ro  r_  r  r3  r  rb  ConvTranspose3dre  ra  rf  r  s    r;   r<   z&ModelForConvTransposeBNFusion.__init__  s    ''1a0
>>!$''1a0
>>!$''1a0
>>!$r=   c                    | j                  |      }| j                  |      }|j                  d      }| j                  |      }| j	                  |      }|j                  d      }| j                  |      }| j                  |      }|S )Nr[  )r2  r_  r  r3  rb  re  rf  r  s     r;   r  z%ModelForConvTransposeBNFusion.forward  so    JJqMHHQKKKNJJqMHHQKKKNJJqMHHQKr=   r  r  s   @r;   r  r    s    %	r=   r  c                   $     e Zd Z fdZd Z xZS )ModelWithFunctionalsc                     t         |           t        j                         | _        t        j                         | _        t        j                         | _        t        j                         | _        y rI   )r]  r<   r  FloatFunctionalmycatmyadd
myadd_relumymatmulr  s    r;   r<   zModelWithFunctionals.__init__  sN    ((*
((*
--/++-r=   c                     | j                   j                  |||g      }| j                  j                  ||      }| j                  j                  ||      }| j                  j                  ||j                        }|S rI   )	r  catr  r  r  add_relur  matmulT)r:   r  rk  zwus         r;   r  zModelWithFunctionals.forward  se    JJNNAq!9%JJNN1a OO$$Q*MM  ACC(
 r=   r  r  s   @r;   r  r    s    .	r=   r  c                   *     e Zd Z fdZd Zd Z xZS )
ResNetBasec                 
   t         |           t        j                  }d}t        j                  ||dd      | _         ||      | _        t        j                         | _        t        j                         | _	        t        j                  j                         | _        t        j                  j                         | _        t        j                   d      | _        t        j                  j%                  |d      | _        y )Nr\  r  Fr  rl   )r]  r<   r   r  r  r2  r_  r   r  r  r_   Identity
downsampler  r  myopAdaptiveAvgPool2davgpoolr  rC  r:   
norm_layerinplanesrl  s      r;   r<   zResNetBase.__init__  s    ^^
YYx6F
h'WWY
WWY
((++-LL002	++F3((//(A.r=   c                 X   | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  j                  ||      }| j                  |      }| j                  |      }t        j                  |d      }| j                  |      }|S )Nrl   )r2  r_  r  r  r  r  r  r  r_   flattenrC  r:   r  outidentitys       r;   r  zResNetBase.forward  s    jjmhhsmjjo??1%iimmC*jjoll3mmC#ggcl
r=   c                     | j                   r0t        j                  j                  j	                  | g dgd       y t        j                  j                  j                  | g dgd       y )N)r2  r_  r  Tr   r)  rK   s    r;   r-  zResNetBase.fuse_model  sQ    ==HH!!224:S9T^b2cHH!!..t6O5PZ^._r=   )rV   rW   rX   r<   r  r-  r  r  s   @r;   r  r    s    /
`r=   r  c                   $     e Zd Z fdZd Z xZS )ModelMultipleOpsc                 x   t         |           t        j                  }d}t        j                  ||dd      | _        t        j                  ||dd      | _         ||      | _        t        j                         | _	        t        j                         | _
        t        j                  j                         | _        t        j                  j                         | _        t        j                  j                         | _        t        j$                  d      | _        t        j(                  dd      | _        y Nr\  r  Fr  )r   r   rF     )r]  r<   r   r  r  r2  r3  r_  r   r  r  r_   r  r  r  r  skip_addr  r  r  r  rC  r  s      r;   r<   zModelMultipleOps.__init__  s    ^^
YYx6F
YYx6F
h'WWY
WWY
((++-446<<//1++F3))B"r=   c                    | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  j                  ||      }| j                  |      }| j                  |      }| j                  |      }t        j                  j                  j                  |dd      }| j                  j                  ||g      }|j                  dd      }| j                  |      }|S Nr[  r   rF  )r2  r_  r  r  r  r  r  r  r3  r_   r   
functional
max_pool2dr  r   rC  r  s       r;   r  zModelMultipleOps.forward 	  s    jjmhhsmjjo??1%mmX.jjoll3jjohh!!,,S!Q7hhllC:&kk"i(ggcl
r=   r  r  s   @r;   r  r    s    #r=   r  c                   $     e Zd Z fdZd Z xZS )ModelMultipleOpsNoAvgPoolc                 2   t         |           t        j                  }d}t        j                  ||dd      | _        t        j                  ||dd      | _         ||      | _        t        j                         | _	        t        j                         | _
        t        j                  j                         | _        t        j                  j                         | _        t        j                  d      | _        t        j"                  dd      | _        y r  )r]  r<   r   r  r  r2  r3  r_  r   r  r  r  r  r  r  	MaxPool2dmaxpoolr  rC  r  s      r;   r<   z"ModelMultipleOpsNoAvgPool.__init__	  s    ^^
YYx6F
YYx6F
h'WWY
WWY
446<<//1||F+))B"r=   c                    | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  j                  ||      }| j                  |      }| j                  |      }| j                  |      }t        j                  j                  j                  |dd      }| j                  j                  ||g      }|j                  dd      }| j                  |      }|S r  )r2  r_  r  r3  r  r  r  r  r_   r   r  r  r  r   rC  )r:   r  r  skips       r;   r  z!ModelMultipleOpsNoAvgPool.forward"	  s    jjmhhsmjjozz!}mmT*jjoll3jjohh!!,,S!Q7hhllC:&kk"i(ggcl
r=   r  r  s   @r;   r  r  	  s    #r=   r  c                   $     e Zd Z fdZd Z xZS )EmbeddingBagModulec                 t    t         |           t        j                  j	                  ddddd      | _        y )Nrk   rF  TFrx   rj  )r]  r<   r_   r   rv  rG  r  s    r;   r<   zEmbeddingBagModule.__init__2	  s7    88(("=AV[bg ) ir=   c                 (    | j                  |||      S rI   rG  )r:   r}  r~  rH  s       r;   r  zEmbeddingBagModule.forward7	  s    xx*<==r=   r  r  s   @r;   r  r  1	  s    i
>r=   r  c                   $     e Zd Z fdZd Z xZS )EmbeddingModulec                 n    t         |           t        j                  j	                  dd      | _        y Nrk   rF  rl  r]  r<   r_   r   rw  rG  r  s    r;   r<   zEmbeddingModule.__init__;	  s)    88%%Rr%Jr=   c                 $    | j                  |      S rI   r  r:   r}  s     r;   r  zEmbeddingModule.forward?	  s    xx  r=   r  r  s   @r;   r  r  :	  s    K!r=   r  c                   $     e Zd Z fdZd Z xZS )EmbeddingWithStaticLinearc                 4   t         |           t        j                  j	                  dd      | _        t        j                  j                  dd      | _        t        | j
                  _	        t        | _	        t               | _        t               | _        y )Nrk   rF  rl  r   r[  )r]  r<   r_   r   rv  rG  r  rC  r   ru  r   r   rn  r   ro  r  s    r;   r<   z"EmbeddingWithStaticLinear.__init__C	  sd    88(("(M((//!Q'<&[
"}r=   c                     | j                  ||      }| j                  |      }| j                  |      }| j                  |      }t	        j
                  |g|gz   d      }|S rO  )rG  rn  rC  ro  r_   r  )r:   r}  r~  	linear_inrG  q_xrC  rs  s           r;   r  z!EmbeddingWithStaticLinear.forwardL	  sX    hhw(jj#WWS\\\"99bTSE\q1r=   r  r  s   @r;   r  r  B	  s    %r=   r  c                   p     e Zd Zd fdZdej
                  dej
                  dej
                  fdZ xZS )DenseTopMLPr  c                 
   t         |           t        j                  t        j                  ||            | _        t        j                  t        j                  ||z   |      t        j                  ||            | _        y rI   )r]  r<   r   r  r  	dense_mlptop_mlp)r:   	dense_dim	dense_outrg  
top_out_intop_out_outrl  s         r;   r<   zDenseTopMLP.__init__V	  s_    IIi+
 }}IIi-/<IIj+.
r=   sparse_featuredensec                     | j                  |      }t        j                  |g|gz   d      }| j                  |      }|S rO  )r  r_   r  r  )r:   r  r  dense_featurers  r  s         r;   r  zDenseTopMLP.forwarda	  s?    
 u-99m_/??QGll8$
r=   rR  )rV   rW   rX   r<   r_   r4  r  r  r  s   @r;   r  r  T	  s5    	
		 ||	 
		r=   r  c                   $     e Zd Z fdZd Z xZS )EmbBagWrapperc                 \    t         |           t        j                  ||d      | _        y )Nrx   )ri  )r]  r<   r   rv  emb_bag)r:   rf  rg  rl  s      r;   r<   zEmbBagWrapper.__init__o	  s"    ~}5Qr=   c                 &    | j                  ||      S rI   )r  )r:   r}  r~  s      r;   r  zEmbBagWrapper.forwards	  s    ||GW--r=   r  r  s   @r;   r  r  n	  s    R.r=   r  c                        e Zd ZdZdZdZdZdZdZdZ	d fdZ
dej                  d	ej                  d
ej                  dej                  fdZ xZS )SparseNNModelrk   r   r   r[  rl   r  c                     t         |           t        | j                  | j                        | _        t        | j                  | j                  | j                  | j                  | j                        | _        y rI   )r]  r<   r  _NUM_EMBEDDINGS_EMBEDDING_DIMmodel_sparser  
_DENSE_DIM_DENSE_OUTPUT_TOP_OUT_IN_TOP_OUT_OUT	dense_topr  s    r;   r<   zSparseNNModel.__init__	  s]    )$*>*>@S@ST$OOT//1D1DdFVFVr=   sparse_indicessparse_offsetsr  c                 N    | j                  ||      }| j                  ||      }|S rI   )r  r  )r:   r  r  r  r  r  s         r;   r  zSparseNNModel.forward	  s+     **>>Jnn^U3
r=   rR  )rV   rW   rX   r  r  r  r  r  r  _TOP_MLP_DIMr<   r_   r4  r  r  r  s   @r;   r  r  v	  sf    ONJMKLL

 
 ||	

 

r=   r  c                      e Zd Z G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z	 G d	 d
ej                  j
                        Z
 G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z G d dej                  j
                        Z G d d ej                  j
                        Z G d! d"ej                  j
                        Z G d# d$ej                  j
                        Z G d% d&ej                  j
                        Zy')(TestHelperModulesc                   $     e Zd Z fdZd Z xZS )%TestHelperModules.Conv2dPropAnnotatonc                     t         |           t        j                  j	                  ddd      | _        t        j                  j                  dd      | _        y Nr\  )r]  r<   r_   r   r  r  r  rn  r  s    r;   r<   z.TestHelperModules.Conv2dPropAnnotaton.__init__	  s<    G1a0DI((//!Q/DKr=   c                     | j                  |      }|j                  dd      }t        j                  j                  j                  |dd      }| j                  |      }|S )Nr   r\  g      r2  )r  r   r_   r   r  hardtanhrn  r  s     r;   r  z-TestHelperModules.Conv2dPropAnnotaton.forward	  sN    		!Ar1A##,,Qc:AAAHr=   r  r  s   @r;   Conv2dPropAnnotatonr  	  s    	0
	r=   r  c                   $     e Zd Z fdZd Z xZS ))TestHelperModules.Conv2dWithObsSharingOpsc                     t         |           t        j                  j	                  ddd      | _        t        j                  j                         | _        t        j                  j                  d      | _	        y Nr\  r  )
r]  r<   r_   r   r  r  Hardtanhr  r  adaptive_avg_pool2dr  s    r;   r<   z2TestHelperModules.Conv2dWithObsSharingOps.__init__	  sQ    G1a0DI!HH--/DM',xx'A'A&'ID$r=   c                     | j                  |      }| j                  |      }| j                  |      }t        j                  |      }|S rI   )r  r  r  r_   meanr  s     r;   r  z1TestHelperModules.Conv2dWithObsSharingOps.forward	  s?    		!A((+Aa A

1AHr=   r  r  s   @r;   Conv2dWithObsSharingOpsr  	  s    	J	r=   r  c                   $     e Zd Z fdZd Z xZS ),TestHelperModules.Conv2dWithTwoLinearPermutec                    t         |           t        j                  j	                  ddd      | _        t        j                  j                  ddd      | _        t        j                  j                  dd      | _        y )Nr\     r  Fr  	r]  r<   r_   r   r  r  r  r  r  r  s    r;   r<   z5TestHelperModules.Conv2dWithTwoLinearPermute.__init__	  V    G2q1DI 88??2qu?=DL 88??1a0DLr=   c                     | j                  |      }t        j                  |d      }| j                  | j	                  |            S Nr   r[  r\  rl   )r  r_   permuter  r  r:   r  conv_outpermute_outs       r;   r  z4TestHelperModules.Conv2dWithTwoLinearPermute.forward	  s7    yy|H--,?K<<[ 9::r=   r  r  s   @r;   Conv2dWithTwoLinearPermuter  	      	1	;r=   r  c                   $     e Zd Z fdZd Z xZS )%TestHelperModules.Conv2dWithTwoLinearc                    t         |           t        j                  j	                  ddd      | _        t        j                  j                  ddd      | _        t        j                  j                  dd      | _        y )Nr\  r	  r=  r  Fr  r
  r  s    r;   r<   z.TestHelperModules.Conv2dWithTwoLinear.__init__	  r  r=   c                     | j                  |      }t        j                  |d      }| j                  | j	                  |            S )N)r[  r=  )r  r_   r   r  r  )r:   r  r  reshape_outs       r;   r  z-TestHelperModules.Conv2dWithTwoLinear.forward	  s7    yy|H--':K<<[ 9::r=   r  r  s   @r;   Conv2dWithTwoLinearr  	  r  r=   r  c                   $     e Zd Z fdZd Z xZS )$TestHelperModules.ConvLinearWPermutec                     t         |           t        j                  j	                  ddd      | _        t        j                  j                  dd      | _        y )Nr\  r  )r]  r<   r_   r   r  r  r  r  r  s    r;   r<   z-TestHelperModules.ConvLinearWPermute.__init__	  s<    G1a0DI 88??1a0DLr=   c                 r    | j                  |      }t        j                  |d      }| j                  |      S r  )r  r_   r  r  r  s       r;   r  z,TestHelperModules.ConvLinearWPermute.forward	  s.    yy|H--,?K<<,,r=   r  r  s   @r;   ConvLinearWPermuter  	  s    	1
	-r=   r  c                   $     e Zd Z fdZd Z xZS )!TestHelperModules.TwoLinearModulec                     t         |           t        j                  j	                  ddd      | _        t        j                  j	                  dd      | _        y )Nr  r	  Fr  )r]  r<   r_   r   r  r  r  r  s    r;   r<   z*TestHelperModules.TwoLinearModule.__init__	  s>    G 88??1bu?=DL 88??2q1DLr=   c                 B    | j                  | j                  |            S rI   )r  r  r  s     r;   r  z)TestHelperModules.TwoLinearModule.forward	  s    <<Q00r=   r  r  s   @r;   TwoLinearModuler!  	  s    	2
	1r=   r$  c                   $     e Zd Z fdZd Z xZS )TestHelperModules.ConvMaxPool2dc                     t         |           t        j                  j	                  ddd      | _        t        j                  j                  dd      | _        y )Nr[  rl   )r]  r<   r_   r   r  r  r  poolr  s    r;   r<   z(TestHelperModules.ConvMaxPool2d.__init__	  s>    G1a0DI**1a0DIr=   c                 J    | j                  |      }| j                  |      }|S rI   )r  r(  r  s     r;   r  z'TestHelperModules.ConvMaxPool2d.forward	  s!    		!A		!AHr=   r  r  s   @r;   ConvMaxPool2dr&  	  s    	1
	r=   r*  c                   $     e Zd Z fdZd Z xZS )+TestHelperModules.ConvWithAdaptiveAvgPool2dc                     t         |           t        j                  j	                  ddd      | _        t        j                  j                  d      | _        y r   )r]  r<   r_   r   r  r  r  r  r  s    r;   r<   z4TestHelperModules.ConvWithAdaptiveAvgPool2d.__init__	  s=    G1a0DI',xx'A'A&'ID$r=   c                 J    | j                  |      }| j                  |      }|S rI   )r  r  r  s     r;   r  z3TestHelperModules.ConvWithAdaptiveAvgPool2d.forward	  s$    		!A((+AHr=   r  r  s   @r;   ConvWithAdaptiveAvgPool2dr,  	  s    	J
	r=   r/  c                   &     e Zd Zd fd	Zd Z xZS ) TestHelperModules.ConvWithBNReluc                    t         |           t        j                  j                  t        j                  j
                  d}t        j                  j                  t        j                  j                  d} ||   ddd|      | _        |r ||   d      | _	        n#t        j                  j                         | _	        |r$t        j                  j                         | _        y t        j                  j                         | _        y )N)rl   r[  r\  r  )r]  r<   r_   r   rd  r  ro  r  r  r  r  r   r  )r:   r  rQ  r  r  convsbnsrl  s          r;   r<   z)TestHelperModules.ConvWithBNRelu.__init__	  s    GEHHOO<Ehh**uxx/C/CDC"c
1a6DI"#c(1+((++-!HHMMO	!HH--/	r=   c                 h    | j                  |      }| j                  |      }| j                  |      S rI   r"  r  s     r;   r  z(TestHelperModules.ConvWithBNRelu.forward
  s*    		!A
A99Q<r=   )r[  TTr  r  s   @r;   ConvWithBNRelur1  	  s    	0	 r=   r6  c                   *     e Zd Z fdZd Zd Z xZS )"TestHelperModules.Conv2dThenConv1dc                     t         |           t        j                  j	                  ddd      | _        t        j                  j                  ddd      | _        y r  )r]  r<   r_   r   rd  r.  r  r  r  s    r;   r<   z+TestHelperModules.Conv2dThenConv1d.__init__
  s>    G((//!Q2DK((//!Q2DKr=   c                 l    | j                  |      }|j                  d      }| j                  |      }|S r   )r  ri  r.  r  s     r;   r  z*TestHelperModules.Conv2dThenConv1d.forward
  s.    AA		!AAAHr=   c                 4    t        j                  dddd      fS r  )r_   randnrK   s    r;   rK  z1TestHelperModules.Conv2dThenConv1d.example_inputs
  s    KK1a+--r=   )rV   rW   rX   r<   r  rK  r  r  s   @r;   Conv2dThenConv1dr8  
  s    	3
		.r=   r=  c                   $     e Zd Z fdZd Z xZS )TestHelperModules.Conv2dWithCatc                     t         |           t        j                  j	                  ddd      | _        t        j                  j	                  ddd      | _        y r  r]  r<   r_   r   r  r2  r3  r  s    r;   r<   z(TestHelperModules.Conv2dWithCat.__init__
  >    GAq1DJAq1DJr=   c                 |    | j                  |      }| j                  |      }t        j                  ||gd      }|S rO  r2  r3  r_   r  )r:   r  rk  r  s       r;   r  z'TestHelperModules.Conv2dWithCat.forward
  s5    

1A

1A		1a&a(AHr=   r  r  s   @r;   Conv2dWithCatr?  
  s    	2
	r=   rE  c                   $     e Zd Z fdZd Z xZS )"TestHelperModules.Conv2dWithTwoCatc                     t         |           t        j                  j	                  ddd      | _        t        j                  j	                  ddd      | _        y r  rA  r  s    r;   r<   z+TestHelperModules.Conv2dWithTwoCat.__init__"
  rB  r=   c                     | j                  |      }| j                  |      }t        j                  ||gd      }||z   }t        j                  ||g      }|S rO  rD  r:   r  r  x3x4rk  r  r  s           r;   r  z*TestHelperModules.Conv2dWithTwoCat.forward'
  sP    BBBB		2r(*ARA		1a&!AHr=   r  r  s   @r;   Conv2dWithTwoCatrG  !
  s    	2
	r=   rM  c                       e Zd Zd Zy)TestHelperModules.ThreeAddc                 $    ||z   }||z   }||z   }|S rI   r[   rJ  s           r;   r  z"TestHelperModules.ThreeAdd.forward0
  s"    RARAAAHr=   NrV   rW   rX   r  r[   r=   r;   ThreeAddrO  /
  s    	r=   rR  c                   $     e Zd Z fdZd Z xZS )!TestHelperModules.EmbeddingModulec                 n    t         |           t        j                  j	                  dd      | _        y r  r  r  s    r;   r<   z*TestHelperModules.EmbeddingModule.__init__7
  s)    Gxx))2)NDHr=   c                 $    | j                  |      S rI   r  r  s     r;   r  z)TestHelperModules.EmbeddingModule.forward;
  s    88G$$r=   r  r  s   @r;   r  rT  6
  s    	O	%r=   r  c                   $     e Zd Z fdZd Z xZS )+TestHelperModules.EmbeddingConvLinearModulec                    t         |           t        j                  j	                  dd      | _        t        j                  j                  ddd      | _        t        j                  j                  dd      | _	        y )Nrk   r  rl  r	  )rl   r\  )
r]  r<   r_   r   rw  rG  r  r  r  rn  r  s    r;   r<   z4TestHelperModules.EmbeddingConvLinearModule.__init__?
  sW    Gxx))1)MDH2v6DI((//"a0DKr=   c                    | j                  |      }t        j                  |d      }t        j                  |d      }| j	                  |      }t        j                  |d      }t        j
                  |d      }| j                  |      S )Nr   rP  )r   r\  rl   r[  r  )rG  r_   r  r  r  ri  rn  )r:   r}  
embeddingsr  s       r;   r  z3TestHelperModules.EmbeddingConvLinearModule.forwardE
  sm    '*J;Jz<@Jyy,H}}X|<H}}X15H;;x((r=   r  r  s   @r;   EmbeddingConvLinearModulerX  >
  s    	1	)r=   r\  c                       e Zd Zd Zy)TestHelperModules.AddInplaceAddc                     ||z   }||z  }|S rI   r[   rj  s      r;   r  z'TestHelperModules.AddInplaceAdd.forwardO
      AAFAHr=   NrQ  r[   r=   r;   AddInplaceAddr^  N
      	r=   ra  c                       e Zd Zd Zy)TestHelperModules.MulInplaceMulc                     ||z  }||z  }|S rI   r[   rj  s      r;   r  z'TestHelperModules.MulInplaceMul.forwardU
  r`  r=   NrQ  r[   r=   r;   MulInplaceMulrd  T
  rb  r=   rf  c                       e Zd Zd Zy)TestHelperModules.AddMulScalarc                 .    |dz   }|dz  }|dz  }|dz  }|S r  r[   r  s     r;   r  z&TestHelperModules.AddMulScalar.forward[
  s+    AAAAFAFAHr=   NrQ  r[   r=   r;   AddMulScalarrh  Z
  s    	r=   rj  c                   $     e Zd Z fdZd Z xZS )+TestHelperModules.ConvBnReLU2dAndLinearReLUc                     t         |           t        j                  d      | _        t
        j                  j                  ddd      | _        t
        j                  j                         | _
        y )NT)r  r\  r  Fr  )r]  r<   r  r6  conv_bn_relur_   r   r  rn  r   r  r  s    r;   r<   z4TestHelperModules.ConvBnReLU2dAndLinearReLU.__init__c
  sO    G 1 @ @d @ KD((//!QU/;DKDIr=   c                 v    | j                  |      }t        j                  |d      }| j                  |      }|S r  )rn  r_   r  rn  )r:   r  r  
linear_outs       r;   r  z3TestHelperModules.ConvBnReLU2dAndLinearReLU.forwardi
  s6    !!!$A--<8K[1Jr=   r  r  s   @r;   ConvBnReLU2dAndLinearReLUrl  b
  s    	(	r=   rq  N)rV   rW   rX   r_   r   r  r  r  r  r  r  r$  r*  r/  r6  r=  rE  rM  rR  r  r\  ra  rf  rj  rq  r[   r=   r;   r  r  	  s~   ehhoo %((// 
;UXX__ 
;
;ehhoo 
;	-UXX__ 	-1%((// 1	 		EHHOO 	   *.588?? .
 
588?? 588?? %%((// %)EHHOO )   uxx EHHOO r=   r  )r   rI   )rY   r_   torch.nnr   torch.nn.functionalr  r  'torch.ao.nn.intrinsic.quantized.dynamicr  	intrinsicr  r  r  torch.ao.nn.quantizedr  torch.ao.nn.quantized.dynamicr  torch.ao.nn.intrinsicr   torch.distributeddistributedr   $torch.testing._internal.common_utilsr   r   torch.ao.quantizationr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   +torch.ao.quantization.quantization_mappingsr    r!   r"   (torch.testing._internal.common_quantizedr#   torch.jit.mobiler$   !torch.ao.quantization.quantize_fxr%   r&   r'   r(   torch.ao.ns.fx.ns_typesr)   r*   torch.fx.graphr+   torch.fxr,   r  ImportErrorr  r  r(  r   r   r  numpyrO  torch.testingr-   typingr.   r/   r0   r1   r2   r3   r4   torch._dynamo_dynamorE  r6   ra   rg   CrossEntropyLoss_default_loss_fnr   r   r   r   r   r   r   r   r   r  r  r+  r1  r4  r8  r>  rB  rG  torchvisionHAS_TORCHVISIONskipIfskip_if_no_torchvisionrM  int64rW  rY  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r%  r0  r9  r@  rH  rM  rU  r\  re  rl  rw  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r   r%  r+  r0  r6  r;  rA  rD  rK  rT  rX  r\  r  rm  rp  rz  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r[   r=   r;   <module>r     s       7 7 7 # # , , .   I Q Q Q Q Q Q
 
 8  M#$F  	   	   # D D D ##7 #7J\ 88,,. 2B &,. .0 $F!O,35j
 0&    O )_)<>NO P ')hh 	G
>8 G
>R)3 )\
#UXX__ 
##ehhoo ##EHHOO ##RYY #ehhoo %((// " 
) 
)
) 
)) )")%((// )")%((// ))588?? )&)ehhoo ) )uxx )8) )#%((// #.ryy .#588?? #588??  #ehhoo ##EHHOO # # #$#UXX__ #$#ehhoo #:@ :@z)EHHOO )) )")uxx )$UXX__ *%((// 588?? &ehhoo uxx 4EHHOO S%((// S6
#uxx 
#
1uxx 
11EHHOO 11uxx 10		 01bii 1 )uxx ) 
/UXX__ 
//%((// /.bii ./")) /  UXX__  'ehhoo '$UXX__ 588?? "EHHOO "%((// uxx *@#; @ryy "		 .			 	'BII ',RYY ,\
 
		 4RYY *	#RYY 	#EHHOO BII ,588?? 0` `Buxx F <> >!ehhoo ! $")) 4.BII .BII :Z ZGK  FT  Os$   *"h# 2h1 #h.-h.1h<;h<