
    Ph-                        d dl mZmZmZmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlm Z m!Z!m"Z" d d	l#m$Z$ d d
l%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+ d dl,m-Z-  ee      ddddddddddddde
j\                  deej^                     dee-   dee+   dee*   dee)   deeej                  j\                        deee0ejb                  f      deee
j\                  gdf      de2de2deeeej                  jf                        eeej                  j\                        f   de
j\                  fd       Z4y)    )CallableIterableOptionalUnionN)contract)_get_module_state_insert_module_state)
_FSDPState)_annotate_modules_for_dynamo)
_init_buffer_state_init_core_state_init_device_handle_init_ignored_module_states_init_param_handle_from_module_init_prefetching_state_init_process_group_state_init_runtime_state_init_state_dict_stateHYBRID_SHARDING_STRATEGIES)_register_post_forward_hook_register_pre_forward_hook_register_root_pre_forward_hook)_register_all_state_dict_hooks)
_auto_wrap)BackwardPrefetch
CPUOffloadMixedPrecisionShardingStrategy)_Policy)	state_clsF)process_grouppolicystrategymixed_precisioncpu_offloadignored_modules	device_idparam_init_fnsync_module_statesforward_prefetchignored_statesmoduler!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   returnc                   t         j                  j                  d       |t        |t              st        d|       t        j                  |       }t        || ||      }t        || |j                  |      }t        | |j                  d       t        ||||      }|X||||||||	|
|d
}|t        v r|j                  |j                   f|d<   t#        | ||j                  |j                  |t               t%        ||xs t&        j(                  ||dddd      }t+        |      }t-        |t.        j0                  |
      }t3        ||       }t5        || |||	      }t7        |      }t9        |       t;        ||        t=        ||        t?        ||        tA        | |       | jC                         D ])  }||jD                  v stG        |      tA        ||       + | S )	zM
    Applies ``FullyShardedDataParallel` (FSDP) semantics to ``module``.
    ztorch.distributed.fully_shardzExpects a `_Policy` but got T)
r!   r#   r$   r%   r&   r'   r(   r)   r*   r+   r!      )limit_all_gathersuse_orig_paramsbackward_prefetch_limitforward_prefetch_limit)r*   )$torch_C_log_api_usage_once
isinstancer   
ValueErrorfully_shardstater   r   _ignored_paramsr   _ignored_modulesr   r   r!   _inter_node_pgr   r   r   
FULL_SHARDr   r   r   BACKWARD_PREr   r   r   r   r   r   r   r	   modules_fully_sharded_module_to_handler   )r,   r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r:   root_kwargs	submodules                  tC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/_composable/fully_shard.pyr9   r9   '   s   * 
HH  !@A*VW"=7x@AAf%E'vWEvu/D/DiPE )?)?F%e]HfME* .&."*"4 0,
 11,1,?,?AUAU+VK(""!!	
 /$// ! 	E  &E#,,?OE uf-E*vy-1CE #5)E"5)uf-v.#E62 '^^%	>>>!),4 E2 & M    )5typingr   r   r   r   r4   torch.distributeddistributeddisttorch.nnnn&torch.distributed._composable.contractr   #torch.distributed._composable_stater   r	   $torch.distributed.fsdp._common_utilsr
   $torch.distributed.fsdp._dynamo_utilsr   "torch.distributed.fsdp._init_utilsr   r   r   r   r   r   r   r   r   r   %torch.distributed.fsdp._runtime_utilsr   r   r   (torch.distributed.fsdp._state_dict_utilsr   "torch.distributed.fsdp._wrap_utilsr   torch.distributed.fsdp.apir   r   r   r   torch.distributed.fsdp.wrapr   ModuleProcessGroupintdevicebool	Parameterr9    rE   rD   <module>r]      s   6 6     ; W ; M   
 T 9  0 
J 26 $+/04(,;?48;?$" 	UIIU D--.U W	U
 '(U n-U *%U huxx78U c5<</01U Hbii[$%678U U U %((,,-.%((//9R0SSU  YY!U  UrE   