
    Phb                     6   d dl Z d dlmZmZmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ d dlmZ ddlmZmZ dZ G d d	e      Z ee
      	 ddej(                  deee	j                  j(                        dej(                  fd       Zdej(                  defdZy)    N)AnyDictIterableListOptionalSetTuple)_State)DistributedDataParallel   )_get_registrycontract c                   x    e Zd Zd fdZefdej                  deej                     deej                     de	ddf
dZ
dej                  deej                     ddfd	Zdej                  d
eedf   dee	ef   defdZdej                  deej"                     dej"                  dej"                  fdZ xZS )_ReplicateStatereturnNc                     t         |           t        j                         | _        d| _        t        j                         | _        g | _        y )NF)super__init__nnParameterListmodulehas_initialized_param_list_param_names)self	__class__s    rC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/_composable/replicate.pyr   z_ReplicateState.__init__   sA    !#!1!1!3%*-/-=-=-? (*    r   ignored_modulesignored_paramsprefixc                 b   t        |      ry ||v ry |t        k7  r| dnt        }|j                  d      D ]D  \  }}||vs| j                  j	                  |       | j
                  j	                  | |        F |j                         D ]  \  }}	| j                  |	||| |          y )N.F)recurse)r"   )_is_fully_sharded_ROOT_MODULE_PREFIXnamed_parametersr   appendr   named_children_collect_params)
r   r   r    r!   r"   recurse_prefixnpnamechild_modules
             r   r+   z_ReplicateState._collect_params   s     V$ _$ #&99vhaL?R 	 ++E+:DAq&  ''*!!((N+;A3)?@ ;
 #)"7"7"9D,  ()$0	 !  #:r   c                    t        |      rt        d      | j                  ry d| _        || _        |D ch c]  }|j	                         D ]  }|  }}}| j                  |||       |j                  | j                  d       |j                  | j                         d|v rV|d   ;|d   }t        |t        j                        r|j                  dk(  rd |d<   n|g|d<   nd |d<   |j                  d       t        | j                   fi || _        t%        j&                  | j"                        t(        j+                  | j                        _        y c c}}w )NzGCannot apply `replicate()` on a Module already managed by `fully_shard`T)with_kwargs	device_idcpu
device_ids)r&   RuntimeErrorr   r   
parametersr+   register_forward_pre_hookforward_pre_hookregister_forward_hookforward_post_hook
isinstancetorchdevicetypepopr   r   _ddpweakrefref	replicatestate_ddp_weakref)r   r   r    kwargsmr.   r!   r3   s           r   initz_ReplicateState.init8   s?    V$Y  #%4M_alln!n!_MV_nE(()>)>D(Q$$T%;%;<& 
 k".";/	i69>>U;R+/F<( -6;F<('+|$JJ{#+D,<,<GG	4;KK		4J	$13 Ns   Eargs.rG   c                 :     | j                   j                  |i |S N)rA   _pre_forward)r   r   rJ   rG   s       r   r9   z _ReplicateState.forward_pre_hookc   s      &tyy%%t6v66r   inputoutputc                 8    | j                   j                  |      S rL   )rA   _post_forward)r   r   rN   rO   s       r   r;   z!_ReplicateState.forward_post_hookh   s     yy&&v..r   )r   N)__name__
__module____qualname__r   r'   r   Moduler   	Parameterstrr+   rI   r	   r   r   r9   r=   Tensorr;   __classcell__)r   s   @r   r   r      s   * *		 RYY BLL)	
  
@)K		)K RYY)K
 
)KV7ii7',S#X7@DS#X7	7
/		/ U\\"/ 	/
 
/r   r   )	state_clsr   r    r   c                 2   t         j                  j                  d       d|v r=t        |d   t        t         j
                  f      st        dt        |d                |i }nt        |      }t        j                  |       j                  | |fi | | S )zReplicates a module

    Args:
        module (torch.nn.Module): module to replicate

    Example::
        >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d)
        >>> module = nn.Linear(3, 3)
        >>> replicate(module)
    ztorch.distributed.replicater3   z6Expected device_id to be int or torch.device, but got )r=   _C_log_api_usage_oncer<   intr>   r6   r?   setrD   rE   rI   )r   r    rG   s      r   rD   rD   q   s      
HH  !>? f&-U\\/BC{ 3457 
 o.OOF  CFCMr   c                 &    t        |       }|yd|v S )z+Check if module is marked with fully_shard.Ffully_shard)r   )r   registrys     r   r&   r&      s     V$HH$$r   rL   )rB   typingr   r   r   r   r   r   r	   r=   torch.nnr   #torch.distributed._composable_stater
   torch.nn.parallelr   r   r   r'   r   rU   rD   boolr&    r   r   <module>ri      s     B B B   6 5 - `/f `/F 
O$ <@ II huxx78  YY	  % F%bii %D %r   