
    Ph1                        d dl Z d dlZd dlmZmZ d dlZd dlmc mc m	Z
 d dlmZ d dlmZmZmZmZmZ d dlmZmZmZmZmZmZmZ d dlmZ ddlmZ dZ ej@                         d	ed
e!fd       Z"dedejF                  ddfdZ$dedejF                  ddfdZ%e jL                  dedejF                  defd       Z'dede!de!de!de!ddfdZ(e jL                  dejF                  dede!de!de!de!fd       Z)e jL                  dejF                  dede!de!de!de!de!fd       Z*e jL                  dejF                  de!de!de!de!de!fd       Z+dedejF                  ddfdZ,dedejF                  ddfdZ-y)    N)cast	Generator)
_FSDPState_has_fsdp_params_module_handleHandleTrainingStateTrainingState)"_get_fsdp_root_states_with_modules
_lazy_init%_reset_flat_param_grad_info_if_needed_reshard_reshard_grads_unshard_unshard_grads)	_p_assert   )FlatParamHandle_flat_paramhandlewriteback_gradc                     dt         j                  dt         j                  f fd} | j                        } j                  j                  d|j	                          j                  |       |rf j                  }|W j                  j                  J  | j                  j                        }|d|j	                          j                  |       yyy)aP  
    For the handle, writes back the this rank's shard of the unsharded
    flattened parameter to the sharded flattened parameter. If
    ``writeback_grad=True``, then writes back to the sharded gradient as
    well.

    Precondition: The handle's ``FlatParameter`` 's data points to the
    padded unsharded flattened parameter.
    flat_param_or_gradreturnc                     j                   r0t        j                  | j                  j                        \  }}|S | S )N)uses_sharded_strategyr   _get_unpadded_shardrank
world_size)r   shard_r   s      vC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/fsdp/_unshard_param_utils.py
_get_shardz-_writeback_to_local_shard.<locals>._get_shard.   sD    '' '::"!!HE1
 L "!    N)torchTensor
flat_param_local_shardnumelcopy_sharded_gradgrad)r   r   r"   param_shardexisting_grad
grad_shards   `     r!   _writeback_to_local_shardr/      s    "u|| " " V../K
""#8[%6%6%89??L++$$$))555#F$5$5$:$:;J.J,,./55jA % r#   statemoduler   c                     t        | |      rCt        t        j                  |j                        j
                  j                  t        d       yy)a  
    De-registers the flattened parameter from the wrapped module, hiding it
    from ``nn.Module`` methods.

    We do not use ``del`` because we want ``FLAT_PARAM`` to always be an
    attribute but dynamically change whether it is visible to ``nn.Module``
    methods.
    N)r   r   nnModuler1   _parameterspop
FLAT_PARAM)r0   r1   s     r!   _deregister_flat_paramr8   G   s8     v&RYY&2266z4H 'r#   c                     t        | |      }t        | |      r@|j                  t        t        j
                  |j                        j                  t        <   yy)a2  
    Registers the flattened parameter to the wrapped module, making it
    visible to ``nn.Module`` methods.

    We do not use :meth:`nn.Module.register_parameter` because we want
    ``FLAT_PARAM`` to always be an attribute but dynamically change whether
    it is visible to ``nn.Module`` methods.
    N)	r   r   r&   r   r3   r4   r1   r5   r7   r0   r1   r   s      r!   _register_flat_paramr;   U   sD     E6*Fv&AGARARRYY&22:> 'r#   c              #     K   t        | |      }|sd yt        | |       	 |j                         5  d ddd       |j                  st	        | |       yy# 1 sw Y   #xY w# |j                  st	        | |       w w xY ww)ax  
    Assumes that the flattened parameter is unsharded. When in the context,
    de-registers the flattened parameter and unflattens the original
    parameters as ``nn.Parameter`` views into the flattened parameter.
    After the context, re-registers the flattened parameter and restores
    the original parameters as ``Tensor`` views into the flattened
    parameter.
    N)r   r8   unflatten_as_params_use_orig_paramsr;   r:   s      r!   _unflatten_as_paramsr?   d   sz      E6*Fuf-	4++- . **$UF3 + .- **$UF3 +s2    BA& AA&  BA#A& &BB	writeback
rank0_onlyoffload_to_cpu
with_gradsc           	         |r-|s| j                   st        d| d| j                    d| d      |r-| j                  r!| j                  j                  st        d      |r|rt        d      |r|st	        j
                  d       y y y )Nzwith_grads=z, use_orig_params=z, offload_to_cpu=z is not supported yetz5offload_to_cpu=True and NO_SHARD is not supported yetz7writeback=True and rank0_only=True is not supported yetzoffload_to_cpu=True and rank0_only=False may result in theunsharded parameters being redundantly copied to CPU memory for GPUs sharing the same CPU memory, which risks CPU OOM. We recommend using offload_to_cpu=True with rank0_only=True.)r>   NotImplementedError_handler   warningswarn)r0   r@   rA   rB   rC   s        r!   _validate_unshard_params_argsrI   {   s     ~U-C-C!* &$556 7,- .#$
 	
 %--1T1T!C
 	
 Z "E
 	
 jH	
 )~r#   c              #     K   t        |||||       |j                  j                          t        ||       }d}|r|j                  t
        j                  k7  r|}|sd y|j                  t
        j                  k(  sJ d|j                          t
        j                  |_        t        |       |j                         }|j                  j                         }	t        |||	|	       |rt        |       |rD|j                  dk7  r5t        |||       |rt        |       	 d t
        j                  |_        yt!        j"                         5 }
|r+|j$                  r|
j'                  |j)                                |j*                  s|
j'                  t-        ||              	 d |
j/                          |rt1        ||       t        |||       |rt        |       t
        j                  |_        	 ddd       y# t
        j                  |_        w xY w# |
j/                          |rt1        ||       t        |||       |rt        |       t
        j                  |_        w xY w# 1 sw Y   yxY ww)zl
    This unshards the parameters for a single FSDP state ``state`` that
    corresponds to ``module``.
    Nz/Expects the handle training to be IDLE but got r   )rI   _device_handlesynchronizer   _training_stater   SUMMON_FULL_PARAMSIDLEr   needs_unshardcurrent_streamr   r   r   r   r   
contextlib	ExitStackr   enter_contextto_cpur>   r?   closer/   )r1   r0   r@   rA   rB   rC   maybe_handler   free_unsharded_flat_paramcomputation_streamstacks              r!   _unshard_fsdp_state_paramsr[      s'     "y*nj 
$$&!%0LF((,?,R,RR 	"5":"::R	89O9O8PQR: 1CCF)&1 & 4 4 6 --<<>UF.0BCvejjAo 9:6"	>%8%=%=F" !!#u&">">##FMMO4
 ))##$8$GH	B-fjA(AB"6*)<)A)A&' $# &9%=%=F"  -fjA(AB"6*)<)A)A&' $#sQ   D!I;$H (*I;AI/(H,AI/:	I;HI;AI,,I//I84I;recursec              #     K   t        |||||       |rft        j                         5 }t        t	        j
                  |        D ]&  \  }}|j                  t        ||d||||             ( d ddd       yt        ||        |j                  t        j                  k(  rt        d      |j                  t        j                  k(  rt        d      t        | |||||      5  	 t        j                  |_	        d t        j                  |_	        	 ddd       y# 1 sw Y   yxY w# t        j                  |_	        w xY w# 1 sw Y   yxY ww)z
    This is a helper for :func:`_unshard_params` that recursively calls
    :func:`_unshard_fsdp_state_params` on FSDP states if ``recurse=True``.
    NOTE: This runs lazy initialization.
    Fr1   r0   r\   r@   rA   rB   rC   Nz:Cannot manually unshard parameters during forward/backwardzECannot manually unshard parameters when already unsharding parameters)r1   r0   r@   rA   rB   rC   )rI   rR   rS   ziptraversal_utils_get_fsdp_states_with_modulesrT   _unshard_params_recurser   training_stater	   FORWARD_BACKWARDAssertionErrorrN   r[   rO   )	r1   r0   r\   r@   rA   rB   rC   rZ   fsdp_modules	            r!   rb   rb      sR     "y*nj !!#u '* >>vF'"{ ##+*# %"+#-'5#-
' % $& 	uf}===H
 	
 
		!A!A	AS
 	
 
$%
	6#0#C#CE #0#5#5E 
 
; $& 	, $1#5#5E 
 
sN   &EA	D!1A6E'E)D-E	E!D*&E-EEEEc              #      K   t        |       \  }}t        j                         5 }t        ||      D ]&  \  }	}
|j	                  t        |
|	|||||             ( d ddd       y# 1 sw Y   yxY ww)z~
    This unshards FSDP-managed parameters for all modules with FSDP applied in
    the module tree rooted at ``module``.
    r^   N)r
   rR   rS   r_   rT   rb   )r1   r\   r@   rA   rB   rC   root_fsdp_statesroot_fsdp_modulesrZ   root_fsdp_stateroot_fsdp_modules              r!   _unshard_paramsrl   (  s      +MV*T''				514/2
-O- '+)#')#1)
2
 	 
   ! 
   s   #A4:A(	A4(A1-A4c                     t        | |      }|syt        |j                  d| j                   d|j                          |j                          t	        | |       y)zO
    Deregisters the original parameters; registers the ``FlatParameter``.
    Nz)Inconsistent `_use_orig_params` -- FSDP: z	 handle: )r   r   r>   _deregister_orig_paramsr;   r:   s      r!   rn   rn   I  sd     E6*F
3E4J4J3K L**+	-
 ""$'r#   c                     t        | |      }|syt        | |       |j                  |j                        r!|j	                          |j                          y|j                  d       y)zO
    Deregisters the ``FlatParameter``; registers the original parameters.
    NT)	as_params)r   r8   
is_shardedr&   _use_sharded_views_use_sharded_grad_views_use_unsharded_viewsr:   s      r!   _register_orig_paramsru   Y  s]     E6*F5&)**+!!#&&(##d#3r#   ).rR   rG   typingr   r   r$   'torch.distributed.fsdp._traversal_utilsdistributedfsdp_traversal_utilsr`   torch.nnr3   $torch.distributed.fsdp._common_utilsr   r   r   r   r	   %torch.distributed.fsdp._runtime_utilsr
   r   r   r   r   r   r   torch.distributed.utilsr   r   r   r7   no_gradboolr/   r4   r8   r;   contextmanagerr?   rI   r[   rb   rl   rn   ru    r#   r!   <module>r      s     "  A A     . (
 $B$B$B $BNI* Ibii ID IS
 SBII S$ S 4
 4BII 4) 4 4,


 
 	

 
 

B IBIIIBIB IB 	IB
 IB IB IBX ;6II;6;6 ;6 	;6
 ;6 ;6 ;6 ;6| II  	
   @(: (ryy (T ( 4 4RYY 44 4r#   