
    Ph\                         d Z ddlmZmZ ddlZddlmZmZ ddlmZ ddl	m
Z
 ddlmZ d	gZ ed
ej                        Z G d d	e
      Zy)zDTracks the running statistics per mini-batch instead of micro-batch.    )TypeVarcastN)Tensornn)
batch_norm)
_BatchNorm   )is_recomputingDeferredBatchNormTModule)boundc                        e Zd ZU dZeed<   eed<   eed<   eed<   eed<   	 	 	 	 ddeded	ed
ededdf fdZ	deddfdZ
dedefdZddZdedefdZeddededefd       Z xZS )r   z\A BatchNorm layer tracks multiple micro-batches to update running statistics per mini-batch.sumsum_squaresrunning_meanrunning_varnum_batches_trackednum_featuresepsmomentumaffinechunksreturnNc                    t         |   ||||d       | j                  dt        j                  | j
                               | j                  dt        j                  | j                               d| _        d| _        || _	        y )NT)track_running_statsr   r   r   )
super__init__register_buffertorch
zeros_liker   r   countertrackedr   )selfr   r   r   r   r   	__class__s         tC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/pipeline/sync/batchnorm.pyr   zDeferredBatchNorm.__init__    su     	sHfRVWUE$4$4T5F5F$GH]E,<,<T=M=M,NO    inputc                 b    |j                         dk  rt        d|j                         z        y )N   z*expected at least 3D input (got %dD input))dim
ValueError)r#   r'   s     r%   _check_input_dimz"DeferredBatchNorm._check_input_dim1   s,    99;!IEIIKWXX r&   c                 (   dg}|j                  t        d|j                                      t        j                         5  | xj
                  |j                  |      z  c_        | xj                  |dz  j                  |      z  c_        ddd       |j                         j                         |j                  d      z  }| xj                  |z  c_	        | xj                  dz  c_
        | j                  | j                  k(  S # 1 sw Y   |xY w)z#Tracks statistics of a micro-batch.r   r)   Nr	   )extendranger*   r   no_gradr   r   sizenumelr!   r"   r   )r#   r'   r*   r1   s       r%   _trackzDeferredBatchNorm._track6   s     c

5EIIK()]]_HH		#&H! 0 0 55  zz|!!#uzz!}4||t{{** _s   ADDc                 L   d}| xj                   dz  c_         | j                  dt        | j                         z  }n| j                  }| j                  | j                  z  }| j
                  | j                  z  |dz  z
  }|}| xj                  d|z
  z  c_        | xj                  ||z  z  c_        | xj                  d|z
  z  c_        | xj                  ||z  z  c_        | j                  j                          | j
                  j                          d| _        d| _	        y)z.Update the running statistics of a mini-batch.        r	   Ng      ?r)   r   )
r   r   floatr   r!   r   r   r   zero_r"   )r#   exponential_average_factormeanvarms        r%   _commitzDeferredBatchNorm._commitF   s    %("  A% == ),uT5M5M/N)N&)-&xx$,,&-	9 'QU"TAX%AE!C!G# r&   c           
      `   | j                   sEt        || j                  | j                  | j                  | j
                  dd| j                        S t               s#| j                  |      }|r| j                          t        |d d | j                  | j
                  dd| j                        S )NFr5   )r   r   weightbiastrainingr   r   T)
r@   r   r   r   r>   r?   r   r
   r3   r<   )r#   r'   tracked_enoughs      r%   forwardzDeferredBatchNorm.forward`   s    }}!.. ,,{{YYHH	 	  "[[/N  ;;	
 		
r&   modulec                    t        |t              r|j                  |u rt        t        |      S |}t        |t
              r|j                  rt        |j                  |j                  |j                  |j                  |      }|j                  r8|j                  d|j                         |j                  d|j                         |j                  d|j                         |j                  d|j                          |j                  d|j"                         |j%                         D ]'  \  }}|j'                  || j)                  ||             ) t        t        |      S )aU  Converts a :class:`nn.BatchNorm` or underlying :class:`nn.BatchNorm`s into :class:`DeferredBatchNorm`::

            from torchvision.models.resnet import resnet101
            from torchpipe.batchnorm import DeferredBatchNorm
            model = resnet101()
            model = DeferredBatchNorm.convert_deferred_batch_norm(model)

        r>   r?   r   r   r   )
isinstancer   r   r   r   r   r   r   r   r   r   register_parameterr>   r?   r   r   r   r   named_children
add_moduleconvert_deferred_batch_norm)clsrC   r   module_outputnamechilds         r%   rI   z-DeferredBatchNorm.convert_deferred_batch_norm   s    f/0V]]f5L((#)fj)f.H.H-f.A.A6::v`f`m`mouvM}}006==I00E)).&:M:MN))-9K9KL))*?A[A[\!002KD%$$T3+J+J5RX+YZ 3 G]++r&   )gh㈵>g?Tr	   )r   N)r	   )__name__
__module____qualname____doc__r   __annotations__intr6   boolr   r,   r3   r<   rB   classmethodr   rI   __classcell__)r$   s   @r%   r   r      s    f	K
   	
   
"Yf Y Y
+F +t + 4"
V "
 "
H , ,# ,g , ,r&   )rQ   typingr   r   r   r   r   torch.nn.functionalr   torch.nn.modules.batchnormr   
checkpointr
   __all__Moduler   r    r&   r%   <module>r^      sE    K     * 1 &
 )299
-H,
 H,r&   