
    Ph              	          d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZ d dlZd dlmc mc mZ d dlmZmZmZ  e j0                  e      Z G d d      Zd	ej8                  j:                  d
eee	eee   f   f   fdZy)    N)defaultdict)contextmanager)Enum)DictIteratorListSetTuple)_apply_to_modules_get_module_fsdp_stateclean_tensor_namec                       e Zd ZU  G d dee      Z ee      Ze	eef   e
d<    e       Zee   e
d<   edd       Zeededed   fd	              Zed
eddfd       Zy)SimpleProfilerc                   $    e Zd ZdZdZdZdZdZdZy)SimpleProfiler.Typeall
all_gatherall_gather_object
reshardingH2DD2HN)	__name__
__module____qualname__ALL	ALLGATHERALLGATHER_OBJ
RESHARDINGr   r        nC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/fsdp/_debug_utils.pyTyper      s      	+!
r    r"   results	profilingreturnNc                 l    | j                   j                          | j                  j                          y )N)r#   clearr$   )clss    r!   resetzSimpleProfiler.reset   s"    r    profile_typec              #     K   || j                   vs
J | d       | j                   j                  |       t        j                         }	 d  t        j                         }| j                  |xx   ||z
  z  cc<   | j                   j                  |       y # t        j                         }| j                  |xx   ||z
  z  cc<   | j                   j                  |       w xY ww)Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r$   addtime	monotonicr#   remove)r(   r*   beginends       r!   profilezSimpleProfiler.profile$   s      3==0 	
n  	
0 	,' 	/.."CKK%u4%MM  . .."CKK%u4%MM  .s    AC'B A
C'AC$$C'msgc                 x    t         j                  d|t        | j                               | j	                          y )Nz%s %s)loggerwarningstrr#   r)   )r(   r3   s     r!   dump_and_resetzSimpleProfiler.dump_and_reset6   s$    wS%56		r    )r%   N)r   r   r   r7   r   r"   r   floatr#   r   __annotations__setr$   r	   classmethodr)   r   r   r2   r8   r   r    r!   r   r      s    sD  !,E 2GT#u*2%Is3x  /3 /8D> /  /     r    r   modelr%   c                     d }d }dg}i }t        | ||| j                         D cg c]  \  }}|	 c}}||      S c c}}w )a  
    It is used for composable fully_shard() code path, it returns
      1. sharded module tree info: each line reprents a submodule name that contats the
    submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
    the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
    level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
    is like this:
        [CompositeModel] FULLY SHARDED
            l1[Linear]
            u1[UnitModule] FULLY SHARDED
                u1.l1[Linear]
                u1.seq[Sequential]
                    u1.seq.0[ReLU]
                    u1.seq.1[Linear]
                    u1.seq.2[ReLU]
                u1.l2[Linear]
            u2[UnitModule] FULLY SHARDED
                u2.l1[Linear]
                u2.seq[Sequential]
                    u2.seq.0[ReLU]
                    u2.seq.1[Linear]
                    u2.seq.2[ReLU]
                u2.l2[Linear]
            l2[Linear]
      2. a dict mapping from the concated module FQN and class name to a list of its managed
    original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
            {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
             'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
             'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
            }
    All FQNs are prefixed starting from ``model``.

    Args:
        model (torch.nn.Module): Root module (which may or may not be passed to
                                 composable `fully_shard()`).
    c                 <   |dz  }t        |      dkD  r|d   dk(  r|d d n|}|dz   | j                  j                  z   dz   }d|z  |z   }t        |       }	|	|dxx   |dz   z  cc<   y |	j                  j                  | d       }
|
r|dxx   |d	z   dz   z  cc<   n|dxx   |dz   z  cc<   |
rl|
j                  }t        |t        j                        sJ |j                  D cg c]  }t        ||z          }}||v r||   j                  |       y |||<   y y c c}w )
N   r   .[] 
z FULLY SHARDED)len	__class__r   r   _fully_sharded_module_to_handleget
flat_param
isinstanceflat_param_fileFlatParameter_fqnsr   extend)moduleprefix
tree_levelsharded_tree_infosharded_module_name_to_fqns
num_spacestrimed_prefixprefixed_module_nameprinted_prefixed_module_namestatehandleparamnameglobal_fqnss                 r!   	module_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnd   sc     !^
K!Or
c0AF3BK 	  -s2V5E5E5N5NNQTT'*Z'7:N'N$&v.=a $@4$GG 66::64Ha ,/??$F  a $@4$GG %%Ee_%B%BCCC=B[[=HT!&4-0[   $'BB+,@AHHUDO+,@A s   "Dc                     | d   |fS )Nr   r   )rT   rU   s     r!   	return_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fn   s     #%@@@r     )r   named_parameters)r=   r_   ra   rT   rU   key_s          r!   1_get_sharded_module_tree_with_module_name_to_fqnsrf   <   sg    P"PHA
 	$ 9; 11343a34#  	5s   =) loggingr-   collectionsr   
contextlibr   enumr   typingr   r   r   r	   r
   torch"torch.distributed.fsdp._flat_paramdistributedfsdp_flat_paramrM   $torch.distributed.fsdp._common_utilsr   r   r   	getLoggerr   r5   r   nnModuler7   rf   r   r    r!   <module>ru      s      # %  3 3  < <  
		8	$& &R[88??[
3S$s)^$$%[r    