
    Ph             
       t	   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	m
Z
 d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZ d dlZd dlZd dlZd dlZd dlmZ  d dl!mc m"c m#c m$Z$ d dl%mc m"c m#c m&Z' d dl(mc m"c m#c m)Z* d dl+m,Z, d dl-m,c m.Z/ d dl0m1Z2 d d	l0m3Z4 d d
l5m6Z6m7Z7 d dl8m9Z:m;Z<m=Z>m?Z@ d dlAmBZB d dlCmDZDmEZEmFZFmGZG d dlHmIZImJZJ d dlKmLZL d dlMmNZNmOZO d dlPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZc d dldmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZm d dlnmc moc mpZp d dlqmrZr 	 d dlsZsdZtej                  dk(  rd dlwZwnd dlxZx G d de,j                        Zz G d d      Z{ e{d      Z|de|_}         e{ ej                  dd            Zddddid e|ed!dd"dd#g d$ggZe j                  j                  e j                  j                  e j                  j                  e j                  j
                  gZe j                  j                  e j                  j                  e j                  j                  e j                  j
                  gZe j                  j                  e j                  j                  e j                  j                  e j                  j
                  gZd%Z e	d&e      Z G d' d(e      Z emet d)      Zej                  d*   Z ej                  d+d,      Zd-Zd.d/iZd0 Zd1Zd2Zd3Zd4Zd5Z G d6 d7e      Z G d8 d9e,j                        Z G d: d;e,j                        Z G d< d=e,j                        Z G d> d?e,j                        Z G d@ dAe,j                        Z G dB dCe,j                        Z G dD dEe,j                        Z G dF dGe,j                        Z G dH dIe,j                        Z G dJ dKe,j                        Z e       Z e       Z edL      Z e,jN                  d"dMN      ZdO ZdPZd-dQdQdRZdS ZdT ZedU        ZedV        Zdej`                  dfdWZdej`                  fdXZdY ZdZ Z G d[ d\      Z G d] d^eQ      Z G d_ d`      Z eeejp                         y# eu$ r dZtY tw xY w)a    N)
namedtupleOrderedDict)contextmanagernullcontext)	dataclass)	timedelta)reduce)Union
NamedTupleCallableAny)TEST_MASTER_ADDR)TEST_MASTER_PORT)
GradScalerautocast)post_localSGD_hookpowerSGD_hookdefault_hooksquantization)_apply_optimizer_in_backward)get_world_size_get_default_groupAllreduceOptionsGroupMember)$_verify_param_shape_across_processes_sync_module_states)DistributedDataParallel)_dump_DDP_relevant_env_vars_MixedPrecision)MultiProcessTestCase
TEST_SKIPSinit_multigpu_helperinitialize_temp_directoriescleanup_temp_dirsimple_sparse_reduce_testsskip_if_rocmskip_if_small_worldsizeskip_if_odd_worldsizeskip_if_lt_x_gpunccl_skip_if_lt_x_gpuskip_if_no_gpurequire_n_gpus_for_nccl_backendrequires_nccl_versioncaptured_outputwith_nccl_blocking_waitwith_dist_debug_levelsverify_ddp_error_loggedDistTestCases)	instantiate_parametrized_testsIS_MACOS
IS_WINDOWSFILE_SCHEMA	IS_FBCODENO_MULTIPROCESSING_SPAWNIS_SANDCASTLEskip_but_pass_in_sandcastleskip_but_pass_in_sandcastle_if)DistributedSamplerTFwin32c                   $     e Zd Z fdZd Z xZS )NetWithBuffersc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        | j                  dt        j                  dd             y N
   Fbias   buffer   	super__init__nnLinearabregister_buffertorchrandnself	__class__s    C:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/testing/_internal/distributed/distributed_test.pyrJ   zNetWithBuffers.__init__e   sQ    2r.2qu-Xu{{1a'89    c                 x    | j                   j                  d       | j                  | j                  |            S NrE   )rF   add_rN   rM   rS   xs     rU   forwardzNetWithBuffers.forwardk   s+    vvdffQi  rV   __name__
__module____qualname__rJ   r\   __classcell__rT   s   @rU   r?   r?   d   s    :!rV   r?   c                       e Zd Zd Zd Zy)Fooc                     || _         y N)r[   rZ   s     rU   rJ   zFoo.__init__q   s	    rV   c                     d }| j                   j                         D ]  \  }}|j                   |   } |||      r y y)Nc                 l    t        | t        j                        rt        j                  | |      S | |k(  S rf   )
isinstancerP   Tensorequal)valueothers     rU   eqzFoo.__eq__.<locals>.eqv   s+    %.{{5%00E>!rV   FT)__dict__items)rS   rm   rn   attrrl   other_values         rU   __eq__z
Foo.__eq__u   sD    	"
  ==..0KD%...Ke[) 1 rV   N)r^   r_   r`   rJ   rs    rV   rU   rd   rd   p   s    
rV   rd   rB   rE         nested)key1key2key3foorG   string)rv      rw   rM   rN   r   c                   J    e Zd ZU ej                  ed<   ej                  ed<   y)TestNamedTupleInput_1rM   rN   N)r^   r_   r`   rP   tensor__annotations__rt   rV   rU   r   r      s    ||O||OrV   r   zno torchvisionBACKENDINIT_METHODenv://i,  test_DistributedDataParallel  c                    t        |t        j                  j                        r|j	                         n|j
                  }|D cg c]:  }|j                  j                  |       s|j                  j                  |       r|< c}S c c}w rf   )	ri   rP   profilerprofileeventsfunction_eventsnameendswith
startswith)
event_namer   
event_listevents       rU   get_profiling_eventr      s{     h 6 67 	%%  &%%JJ
+uzz/D/DZ/P 	:  s   ?B:Expected to have finished reduction in the prior iterationz:passing the keyword argument `find_unused_parameters=True`z.Since `find_unused_parameters=True` is enabledz:`forward` function outputs participate in calculating losszMset the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAILc                       e Zd ZU eed<   ej                  ed<   eej                  e
f   ed<   eed<   dZeed<   dZeed<   dZeed	<   y)
DDPUnevenTestInputr   modelinpsync_intervalFthrow_on_early_terminationNhookstate)r^   r_   r`   strr   rK   Moduler
   rP   r   tupleintr   boolr   r   r   r   rt   rV   rU   r   r      sJ    
I99	u||U"	##',,D(E3rV   r   c                   $     e Zd Z fdZd Z xZS )_FC2c                     t         |           t        j                  ddd      | _        d| j                  j
                  _        y )NrB   2   TrC   F)rI   rJ   rK   rL   fcrD   requires_gradrR   s    rU   rJ   z_FC2.__init__   s2    ))B.%*"rV   c                 (    | j                  |      }|S rf   r   rZ   s     rU   r\   z_FC2.forward   s    GGAJrV   r]   rb   s   @rU   r   r      s    +
rV   r   c                   $     e Zd Z fdZd Z xZS )Netc                 d   t         |           t        j                  ddd      | _        t               | _        t        j                  ddd      | _        t        j                         | _	        t        j                  t        j                  ddg      j                         d      | _        y )NrG   rB   FrC   r   rv   r   )rI   rJ   rK   rL   fc1r   fc2fc3ReLUrelu	ParameterrP   r   longno_grad_paramrR   s    rU   rJ   zNet.__init__   sx    99Q/699R/GGI	\\LL!Q %%'u
rV   c                     | j                  | j                  |            }| j                  | j                  |            }| j                  |      }t	        j
                  |d      S )NrE   dim)r   r   r   r   FsoftmaxrZ   s     rU   r\   zNet.forward   sL    IIdhhqk"IIdhhqk"HHQKyy""rV   r]   rb   s   @rU   r   r      s    
#rV   r   c                   $     e Zd Z fdZd Z xZS )LargeNetc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        y )N  i  FrC   r   rI   rJ   rK   rL   r   r   rR   s    rU   rJ   zLargeNet.__init__  s6    99T4e499T3U3rV   c                 J    | j                  |      }| j                  |      }|S rf   r   r   rZ   s     rU   r\   zLargeNet.forward  s!    HHQKHHQKrV   r]   rb   s   @rU   r   r     s    4
rV   r   c                   $     e Zd Z fdZd Z xZS )Taskc                 ~    t         |           t        j                  t	        j
                  dd            | _        y NrG   )rI   rJ   rK   r   rP   onesprR   s    rU   rJ   zTask.__init__  s)    ejjA./rV   c                      | j                   |z   S rf   )r   rZ   s     rU   r\   zTask.forward  s    vvzrV   r]   rb   s   @rU   r   r     s    0rV   r   c                   &     e Zd Zd fd	Zd Z xZS )BatchNormNetc                     t         |           t        j                  ddd      | _        t        j
                  d|      | _        t        j                  ddd      | _        y )NrG   (   FrC   rv   affine)rI   rJ   rK   rL   r   BatchNorm1dbnr   )rS   r   rT   s     rU   rJ   zBatchNormNet.__init__  sH    99Q/..6299R/rV   c                     t        j                  | j                  |      d      }| j                  |      }t        j                  |d      }| j	                  |      }t        j                  |d      S )N)rv   rB   )r   r   rE   r   )rP   reshaper   r   r   r   r   rZ   s     rU   r\   zBatchNormNet.forward"  sU    MM$((1+{3GGAJMM!X&HHQKyy""rV   Tr]   rb   s   @rU   r   r     s    0#rV   r   c                   $     e Zd Z fdZd Z xZS )UnusedParamTwoLinLayerNetc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        t        j                  ddd      | _        y )NrB   FrC   r}   )rI   rJ   rK   rL   rM   rN   crR   s    rU   rJ   z"UnusedParamTwoLinLayerNet.__init__+  sJ    2r.2r.1ae,rV   c                 N    | j                  |      }| j                  |      }||fS rf   r~   rS   r[   rM   rN   s       rU   r\   z!UnusedParamTwoLinLayerNet.forward1  %    FF1IFF1I1vrV   r]   rb   s   @rU   r   r   *  s    -rV   r   c                   $     e Zd Z fdZd Z xZS )DictOutputModulec                 @    t         |           t               | _        y rf   )rI   rJ   r   modulerR   s    rU   rJ   zDictOutputModule.__init__8  s    /1rV   c                 `    | j                  |      }|d   |d   z   j                         }||dS )Nr   rE   )predictionsloss)r   sum)rS   r[   r   r   s       rU   r\   zDictOutputModule.forward<  s:    kk!nAQ/446&
 	
rV   r]   rb   s   @rU   r   r   7  s    2
rV   r   c                   $     e Zd Z fdZd Z xZS )TwoLinLayerNetc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        y )NrB   FrC   rE   )rI   rJ   rK   rL   rM   rN   rR   s    rU   rJ   zTwoLinLayerNet.__init__F  s6    2r.2qu-rV   c                 N    | j                  |      }| j                  |      }||fS rf   r~   r   s       rU   r\   zTwoLinLayerNet.forwardK  r   rV   r]   rb   s   @rU   r   r   E  s    .
rV   r   c                   *     e Zd ZdZd fd	Zd Z xZS )EmbeddingNetDifferentParamsz{
    A module containing an embedding with different dimension or different # of
    parameters depending on the rank.
    c                     t         |           |s|dk(  rdnd}t        j                  d|      | _        t        j
                  |d      | _        |rt        j
                  ddd      | _        y y )	Nr   r   r   rB   )num_embeddingsembedding_dimrE   FrC   )rI   rJ   rK   	Embedding	embeddingrL   linlin2)rS   rankdiff_num_paramsr   rT   s       rU   rJ   z$EmbeddingNetDifferentParams.__init__W  s]    .$!)R}U99]A.		!QU3DI rV   c                 F    | j                  |      }| j                  |      S rf   )r   r   rZ   s     rU   r\   z#EmbeddingNetDifferentParams.forward_  s    NN1xx{rV   F)r^   r_   r`   __doc__rJ   r\   ra   rb   s   @rU   r   r   Q  s    
4rV   r   c                   $     e Zd Z fdZd Z xZS )ControlFlowToyModelc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        y NrB   FrC   )rI   rJ   rK   rL   lin1r   rR   s    rU   rJ   zControlFlowToyModel.__init__e  s6    IIb"51	IIb"51	rV   c                     t        j                  |t        j                  dd|j                              }|r3| j	                  t        j                  | j                  |                  S t        j                  | j                  |            S )N   rB   device)rP   rk   r   r   r   r   r   r   )rS   r[   use_second_layers      rU   r\   zControlFlowToyModel.forwardj  s^     ;;q%**RAHH*MN99QVVDIIaL12266$))A,''rV   r]   rb   s   @rU   r   r   d  s    2
(rV   r   r   Gz?)momentumc                 X    | j                  d      d   }|t        v r	t        |   S t        S )N.r   )splitCUSTOMIZED_TIMEOUTDEFAULT_TIMEOUT)test_id	test_names     rU   get_timeoutr	  y  s/    c"2&I&&!),,rV   <   r}   )test_ddp_uneven_inputs test_ddp_model_diff_across_rankstest_ddp_has_finalizedc                     d }t         | vrt        dt          d|        S  |t        j                  t                     st        dt          d      S d S )Nc                    | t         j                  j                  k(  rt        j                         S | t         j                  j                  k(  rt        j
                         S | t         j                  j                  k(  rt        j                         S | t         j                  j                  k(  rt        j                         S | t        j                  d   v ryy)NpluginTF)distBackendGLOOis_gloo_availableNCCLis_nccl_availableMPIis_mpi_availableUCCis_ucc_availabler2   backend_featurebackends    rU   checkz+require_backend_is_available.<locals>.check  s    dll'''))++dll'''))++dll&&&((**dll&&&((**m33H==rV   zTest requires backend z to be one of z to be availablec                     | S rf   rt   funcs    rU   <lambda>z.require_backend_is_available.<locals>.<lambda>      rV   )r   r:   r  r  )backendsr  s     rU   require_backend_is_availabler%    sc     h*$WI^H:F
 	
 g&'*$WI-=>
 	
 rV   c                 b    t        t        j                  d         | k  rt        d| z        S d S )N
WORLD_SIZEzTest requires world size of %dc                     | S rf   rt   r   s    rU   r"  z$require_world_size.<locals>.<lambda>  r#  rV   )r   osenvironr:   )
world_sizes    rU   require_world_sizer,    s5    
2::l#$z1*,z9
 	
 rV   c               #     K   t         j                  d   } t         j                  j                  | d      }t	        |d      5 }	 t
        j                  dk(  r8t        j                  |j                         t        j                  d       d  n6t        j                  |j                         t        j                         d  t
        j                  dk(  r4t        j                  |j                         t        j                  d       n2t        j                  |j                         t        j                         |j!                          	 d d d        y # t
        j                  dk(  r4t        j                  |j                         t        j                  d       n2t        j                  |j                         t        j                         |j!                          w xY w# 1 sw Y   y xY ww)NTEMP_DIRlockfilewr=   rE   )r)  r*  pathjoinopensysplatformmsvcrtlockingfilenoLK_RLCKfcntlflockLOCK_EXLK_UNLCKLOCK_UNclose)r.  r/  lfs      rU   _lockrA    s    zz*%Hww||Hj1H	h		||w&ryy{FNNA>BIIK7||w&ryy{FOOQ?BIIK7HHJ 
	 ||w&ryy{FOOQ?BIIK7HHJ 
	s9   A G2G&BEB	G&	G2BG##G&&G/+G2c               #     K   t        j                         dk(  r-t        j                         \  } }t	        j
                  |        nd }|g}t        j                  |       |d   }	 | t        j                         dk(  rt	        j                  |       y y # t        j                         dk(  rt	        j                  |       w w xY wwNr   )r  get_ranktempfilemkstempr)  r?  broadcast_object_listremove)fdr   object_lists      rU   _rank_temp_filerK    s     }}!##%D
&K{+q>D
==?aIIdO  4==?aIIdO  s   A$C'B +.C/CCc                     || }|(t        j                  | | | |      j                  |      S t        j                  | | | |      j                  |      j                  |      S Ndtype)rP   emptyfill_cuda)sizerl   rO  	device_ids       rU   _build_tensorrU    s]    }{{4t59??FF{{4t59??FKKIVVrV   c                     || }t        j                  t        |       D cg c]  }| c}|      j                  |      S c c}w )N)rS  rO  )rP   rP  rangerQ  )r   dim_sizerl   rO  _s        rU   _build_multidim_tensorrZ    s>    };;uSz:z!Xz:%HNNuUU:s   	Ac                  V    t         j                  j                  j                  d      S NT)record_shapes)rP   autogradr   r   rt   rV   rU   _create_autograd_profilerr_    s     >>""***>>rV   c                      t         j                  j                  t         j                  j                  j                  gd      S NT)
activitiesr]  )rP   r   r   ProfilerActivityCPUrt   rV   rU   _create_torch_profilerre    s;    >>!!NN++//
 	 "  rV   c                   2    e Zd ZdZed        Zedd       Zy)Barrierr   c                    d| _         t        j                  j                  t        j                  d   d      }t        j
                  |      D ]5  }t        j                  t        j                  j                  ||             7 y )Nr   r.  barrier)
barrier_idr)  r1  r2  r*  listdirunlink)clsbarrier_dirf_names      rU   initzBarrier.init  sT    ggll2::j#99Ejj-FIIbggll;78 .rV   Nc                    |t        j                         }| xj                  dz  c_        t        j                  j                  t        j                  d   d      }t        t        j                               }t        j                  j                  ||      }t               5  t        |d      5 }|j                  t        | j                               d d d        d d d        t        j                         }	 d}t               5  t        j                  |      D ]a  }	t        t        j                  j                  ||	            5 }|j                         }
t        |
      | j                  k\  r|dz  }d d d        c 	 d d d        ||k(  ry t        j                         |z
  |kD  rt!        d      t        j"                  d       # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   nxY w)NrE   r.  ri  r0  r   zbarrier timeout皙?)r  r   rj  r)  r1  r2  r*  r   getpidrA  r3  writetimerk  readr   RuntimeErrorsleep)rm  wait_fortimeoutrn  pidbarrier_filef
start_timearrivedro  datas              rU   synczBarrier.sync  sl   **,H!ggll2::j#99E"))+ww||K5WlC(ACNN+, )  YY[
G jj5Fbggll;?@A vvxt96#qLG A@ 6  ("yy{Z''1"#455JJsO 	 )( W A@ sI   $G1%GGAG/	.G#7
G/G	GG #G,(G//G8NrB   )r^   r_   r`   rj  classmethodrp  r  rt   rV   rU   rg  rg    s-    J9 9  rV   rg  c                   l     e Zd Ze fd       Z fdZ fdZed        Zed        Z	ed        Z
 xZS )TestDistBackendc                 b    t        t              t        j                  d<   t        |           y )NMASTER_ADDR)r   r  r)  r*  rI   
setUpClass)rm  rT   s    rU   r  zTestDistBackend.setUpClass  s!    $'$4

=!rV   c                     t         |           t                t        j	                          | j
                  j                  g| _        y rf   )rI   setUpr#   rg  rp  r  __wrapped__skip_return_code_checksrR   s    rU   r  zTestDistBackend.setUp   s4    #% )-(C(C(O(O'P$rV   c                 6    t                t        | 	          y rf   )r$   rI   tearDownrR   s    rU   r  zTestDistBackend.tearDown*  s    rV   c                 *    t          | j                   S rf   )r6   	file_namerS   s    rU   init_methodzTestDistBackend.init_method.  s    t~~.//rV   c                    t         dk(  rDt        j                  j                         s&t	        j
                  t        d   j                          | |      }||_        ||_	        t        j                  j                         rgt        j                  j                         t        |j                        k  r3t	        j
                  t        d|j                      j                         	 t        j                  |t              }t!        |      }t#        j$                  |j&                  t         t        |j                        |j                  |       |j-                          |j/                  ||       |j-                          t#        j0                          t	        j
                  d       y # t(        $ r=}d|j*                  d   v r&t	        j
                  t        d   j                          d }~ww xY w)	Nncclno_cudaz
multi-gpu-secondsr  r  r+  r   rz  	recompiler   backend_unavailable)r   rP   rR  is_availabler4  exitr!   	exit_coder   r  device_countr   r+  CUSTOM_PG_TIMEOUTgetdefault_pg_timeoutr   r  init_process_groupr  rw  args_barrierrun_testdestroy_process_group)	rm  r   r  r  piperS   pg_timeout_secondsrz  es	            rU   _runzTestDistBackend._run2  si   fUZZ%<%<%>HHZ	*4459~	"::""$)@)@)BSOOF
 *
 HHZ*T__,= >?IIJ	!2!6!6yBT!U(:;G## ,,t/YY  	i&""$  	affQi'$9:DDE		s   *A+F1 1	G7:8G22G7c                 (    t         j                  d   S )Nr'  r)  r*  r  s    rU   r+  zTestDistBackend.world_sizeZ  s    zz,''rV   )r^   r_   r`   r  r  r  r  propertyr  r  r+  ra   rb   s   @rU   r  r    s]     
Q 0 0 # #N ( (rV   r  c                        e Zd Z G d d      Zy)DistributedTestc                   T   e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Z eedk7  d       eej)                  d       d      d               Ze eedk7  d      d               Z eedk7  d      d        Zd Z eej8                  d          ed       ed      d                      Z eej8                  d          ed       e jB                  edk(  d      d                      Z" eeej8                  d   vde d        ed!       ed      d"                      Z# eeej8                  d   vde d       e$d#               Z% eeej8                  d   vde d        ed!       ed!      d$                      Z& eeej8                  d   vde d        ed!       ed!      d%                      Z' eeej8                  d   vde d        ed!       ed!      d&                      Z( eeej8                  d   vde d       e$d'               Z) eeej8                  d   vde d        ed!       ed!      d(                      Z* eeej8                  d   vde d        ed      d)               Z+ eeej8                  d   vde d        ed      d*               Z, ed      d+        Z- eeej8                  d   vde d        ed      d,               Z. eeej8                  d   vde d        ed!       ed!      d-                      Z/e$ eed.k7  xs e0xs e1d/      d0               Z2e$ eed.k7  xs e0xs e1d/      d1               Z3e$ eed.k7  d2       e4d3d4      d5                      Z5e$ eed.k7  d2       e4d3d4      d6                      Z6e$ eed.k7  d2       e4d3d4      d7                      Z7e$e eed.k7  d2       e4d3d4      d8                             Z8 eedk7  d9      d:        Z9 eedk7  d9      d;        Z: eed.k7  d2       e4d3d4      d<               Z; eed.k7  d2       e4d3d4      d=               Z< eed.k7  d2       e4d3d4      d>               Z=e$ eed.k7  d?       e4d3d4      ddA                     Z>e$ eed.k7  d?       e4d3d4      dB                      Z?e$ eed.k7  d?       e4d3d4      dC                      Z@e$ eed.k7  d?       e4d3d4       ee0dD       eeAxs eBdE      dF                                    ZCdG ZD eed.k(  dH      dI        ZE eed.k(  dJ      dK        ZF eed.k(  dJ       ee0dD       eeAxs eBdE      dL                      ZGdM ZH eeej                  dN   v e dO      dP        ZJ eeej                  dN   v e dO      dQ        ZK eeej                  dN   v e dO       ee0dR       eeAxs eBdE      dS                      ZLdT ZM eed.k(  dJ      dU        ZN eed.k(  dJ      dV        ZO eed.k(  dJ       ee0dR       eeAxs eBdE      dW                      ZPdX ZQ eed.k(  dY      dZ        ZR eed.k(  dY      d[        ZS eed.k(  dY       ee0dR       eeAxs eBdE      d\                      ZT eed.k(  d]      d^        ZU	 	 	 dd`ZV eed.k(  da      db        ZW eedk7  xr ed.k7  dc      e$dd               ZXe eed.k(  da      de               ZY eed.k(  da      df        ZZ eed.k7  dg      e$dh               Z[	 	 ddiZ\ eed.k(  da       eeej                  dj   v e dk      dl               Z] eed.k7  dm       eeej                  dj   v e dk      e$dn                      Z^ eed.k(  da       eeej                  dj   v e dk      do               Z_ eed.k(  da       eeej                  dj   v e dk      dp               Z` eed.k(  da       eeej                  dj   v e dk      dq               Za eed.k(  da       eeej                  dj   v e dk      edr                      Zb eed.k(  da       eeej                  dj   v e dk      eds                      Zc eed.k(  da       eeej                  dj   v e dk      edt                      Zd eed.k(  da       eeej                  dj   v e dk      edu                      Ze eed.k(  da       eeej                  dj   v e dk      dv               Zf eed.k(  da       eeej                  dj   v e dk      dw               Zg eed.k(  da       eeej                  dj   v e dk      dx               Zh eed.k(  da       eeej                  dj   v e dk      dy               Zi	 	 ddzZj eed.k(  da       eeej                  dj   v e dk      d{               Zk eed.k7  dm       eeej                  dj   v e dk      e$d|                      Zl eed.k7  d}       eeej                  dj   v e dk      e$d~                      Zm	 ddZn eed.k7  d      e$d               Zoe$ eej8                  d         d               Zpdd@d_d@ddZqd_d@erj                  d_fdZt eed.k(  da      d        Zu eed.k(  da      d        Zv eedk7  xr ed.k7  d      e$d               Zw eedk7  xr ed.k7  d      e$d               Zx eed.k(  da      d        Zy eed.k(  da      d        Zz eedk7  xr ed.k7  d      e$d               Z{ eed.k(  da      d        Z| eed.k(  da      d        Z} eed.k(  da      d        Z~e eed.k(  da      d               Ze eed.k(  da      d               Ze eed.k(  da      d               Ze eed.k(  da      d               Z eed.k(  da      d        Z eed.k(  da      d        Z eed.k(  da      d        Z eed.k(  da      d        Zd Z eedk7  d      d        Z eedk7  d      e$d               Zed        Zed        Zed        Zed        Z eed.k(  da      d        Z	 	 ddZ edh      d        Z edh      d        Z edh      d        Z edh      d        Ze edh      d               Ze edh      d               Ze edh      d               Ze edh      d               Z edh      d        Z edh      d        Z edh      d        Z edh      d        Zd_d@erj                  fdZ eed.k(  da       eedk(  d      d               Z eed.k(  da       eedk(  d      d               Z eed.k7  d      e$d               Z eed.k(  da       eedk(  d      d               Z eed.k7  d      e$d               Z eed.k(  da       eedk(  d      ed                      Z eed.k(  da       eedk(  d      d               Z	 ddZ eed.k(  da       eedk(  d      d               Z eed.k(  da       eedk(  d      d               Z eed.k7  d      e$d               Z eed.k(  da       eedk(  d      ed                      Z eed.k(  da       eedk(  d      d               Zd_d@erj                  fdZ eed.k(  da      d        Z eed.k7  d«      e$dÄ               Z eed.k(  da      dĄ        Z eed.k7  d«      e$dń               Ze eed.k(  da      dƄ               Z eed.k(  da      dǄ        Z eed.k7  dȫ      e$dɄ               Z	 ddʄZ eed.k7  d˫      e$d̄               Z eed.k7  d˫      e$d̈́               Zd΄ Zerj                  fdτZ eeej                  d   v e dѝ      d҄        Z eeej                  d   v e dѝ      dӄ        Ze eeej                  d   v e dѝ      dԄ               Z eeej                  d   v e dѝ      dՄ        Z eeej                  d   v e dѝ      dք        Zd_d@erj                  fdׄZd_d@erj                  fd؄Zd_d@erj                  fdلZ eedk7  d۫      d܄        Z eed.k7  dݫ      e$dބ               Z eedk7  d۫      d߄        Z eed.k7  dݫ      e$d               Z eedk7  d۫      d        Z eed.k7  dݫ      e$d               Z eedk7  d۫      d        Z eed.k7  dݫ      e$d               Z eedk7  d      d        Z eed.k7  d      ed               Z eedk7  d      d        Z eed.k7  d      ed               Z eedk7  d۫      ed               Z eed.k7  dݫ      e$ed                      Z eedk7  d۫      ed               Z eed.k7  dݫ      e$ed                      Z eedk7  d      ed               Z eed.k7  dݫ      eed                      Z eedk7  d۫      d        Z eed.k7  dݫ      e$d               Z eedk7  d۫      d        Z eed.k7  dݫ      e$d               Z eedk7  d      d        Z eed.k7  d      ed               Z	 ddZe$ eedk(  d       eedk(  xr e1d      d                      Zee$ eedk(  d      d                      Zee$ eedk(  d      d                      Z eeej                  d   v e d      d        Ze eeej                  d   v e d      d                Z eeej                  d   v e d      d        Zߐd Zd Zd Z	 ddZd Z	 	 	 	 	 ddZ	 	 	 	 ddZdÐd	Z eed.k(  d
      d        Z eed.k(  d
      d        Z eeej8                  d   vde d      d        Z eeej8                  d   vde d       e eej                  d               d               Z eed.k(  d      d        Z eeej8                  d   vde d       e eej                  d               d               Z eeej8                  d   vde d       e eej                  d               d               Zd Z	  ed      d        Z	 dĐdZ ed      d        Z ed      d        Z ed      d        ZdefdZ ed      d         Zd! Z ed      d"        Z ed      d#        Z ed      d$        Z ed      d%        ZdŐd&Z  eeej8                  d   vde d       e eej                  d               d'               Z eeej8                  d   vde d       e eej                  d               d(               Z eeej8                  d   vde d       e eej                  d               d)               Z eeej8                  d   vde d       eed*       e eej                  d               d+                      Z	 dÐd,Z	 dÐd-Z	 dƐd.Z eedk7  xr ed.k7  xr edk7  d/       e	ed      d0               Z
 eedk7  xr ed.k7  xr edk7  d/       e	ed      d1               Z eedk7  xr ed.k7  xr edk7  d/       e	ed      d2               Z eedk7  xr ed.k7  xr edk7  d/       e	ed      d3               Z eedk7  xr ed.k7  xr edk7  d/       e	ed      d4               Z eeej8                  d   vde d      e$d5               ZdÐd6Z eeej8                  d   vde d      e$d7               Z	 	 dǐd8Zd9 Zd: Zd; Zd< Zd= Z ed       eeej8                  d   vde d      d>               Z ed       eeej8                  d   vde d      d?               Zd@ Z ed!      e eeej8                  d   vde d      dA                      Z ed!      e eeej8                  d   vde d      dB                      Z ed       eeej8                  d   vde d      dC               Z eeej8                  d   vde d      e$dD               ZdE Z  eeej8                  d   vde d      e$dF               Z! eeej8                  d   vde d      e$dG               Z" eeej8                  d   vde d      e$dH               Z# eeej8                  d   vde d      e$ ed      dI                      Z$ eeej8                  d   vde d      e$dJ               Z% eeej8                  d   vde d      e$dK               Z& eeej8                  d   vde d      e$dL               Z'dM Z( eed.k(  d
      dN        Z) eeej8                  d   vde d      e$dO               Z* eed.k(  d
      dP        Z+e,dQ        Z-e.j^                  d@fdRZ0 ed.h       ed      dS               Z1 ed.h       ed      dT               Z2 ed.h       e eej                  d               dU               Z3 ed.h       ed      dV               Z4 eeej8                  d   vde d       e eej                  d               dW               Z5ddXZ6 eej8                  d          e7 eej                  d         ej                  dY          e8g dZ[      d\                      Z9 eej8                  d          e7 eej                  d         ej                  dY          e8g d][      d^                      Z:dd_Z; eedk(  d       eej8                  d          e8g d][      d`                      Z< eedk(  d       eej8                  d          e8g d][      da                      Z=db Z> ed       eeej8                  d   vde d      dc               Z? ed       eeej8                  d   vde d      dd               Z@de ZA eej8                  d          ed      df               ZB eej8                  d          ed       ee0dR       eeAxs eBdE      dg                             ZC ed       eeej8                  d   vde d      dh               ZDdi ZE ed       eeej8                  d   vde d      dj               ZF ed       eeej8                  d   vde d      dk               ZG ed       eeej8                  d   vde d      dl               ZH ed       eeej8                  d   vde d      dm               ZIddnZJ eej8                  d          e7 eej                  d         ej                  dY          e8dog[       e j                  dp      dq                             ZL eej8                  d          e7 eej                  d         ej                  dY          e8dog[      dr                      ZMdÐdsZN eej8                  d          ed      dt               ZO e8g dZ[       eej8                  d          ed      du                      ZP eej8                  d          ed      dv               ZQ eej8                  d          ed      dw               ZR eej8                  d          ed      dx               ZS e8g dZ[       eej8                  d          ed      dy                      ZT eej8                  d          ed      dz               ZU e8g dZ[       eej8                  d          ed      d{                      ZV edh      d|        ZWd} ZXd~ ZY eej8                  d          ed      d               ZZ eej8                  d          ed      d               Z[	 dÐdZ\d Z] eej8                  d          eedk(  xr e1d       ed      d                      Z^ eej8                  d          eedk(  xr e1d       ed      d                      Z_d Z` eej8                  d          eedk(  xr e1d       ed      d                      Za eej8                  d          eedk(  xr e1d       ed      d                      Zbd Zc eeej8                  d   vde d       ed      d               Zd eeej8                  d   vde d       ed      d               Ze eeej8                  d   vde d       ed      d               Zf	 dȐdZg eej8                  d          ed      d               Zh edh       eeAxs eBd      d               Zi edh      d        Zjd Zkel eej8                  d          e eej                  d               d                      Zmel eej8                  d          e eej                  d               d                      Zn edh      d        Zo edh      e eeAxs eBd      d                      Zp edh      ed               Zq eej8                  d          e8dg[       ed      d                      Zr eeej8                  d   vde d       e8dg[       ed      d                      Zsd Zt e8g dZ[       eej8                  d          ed      d                      Zu e8g dZ[       eej8                  d          ed      d                      Zv eeej8                  d   vde d       ed      d               Zw eeej8                  d   vde d       ed       e j                  dp      d                      Zx ed       eeej8                  d   vde d      d               Zy ed       eeej8                  d   vde d      d               Zz ed       eeej8                  d   vde d      d               Z{ ed       eeej8                  d   vde d      d               Z|d Z} ed       eeej8                  d   vde d      d               Z~ ed       eeej8                  d   vde d      d               Zd Z ed       eeej8                  d   vde d      d               Z ed       eeej8                  d   vde d      d               Z ed       eeej8                  d   vde d      d               Z ed       eeej8                  d   vde d      d               Z ed       eeej8                  d   vde d       e j                  d      d                      Z ed       eed.k7  d      d               Zd Z ed!       ed!       eeej8                  d   vde d      d                      Z ed!       ed!       eeej8                  d   vde d      d                      Z ed       eeej8                  d   vde d      d               Z ed       eed.k7  xr edk7  d      d               Z ed       eed.k7  xr edk7  d      d               Z ed       eeej8                  d   vde d      d               Z eej8                  d          ed      d               Zd Z eeej8                  d   vde d       e eej                  d                eedk(  xr e1d      d                      Z eej8                  d          ed      d               Z ed       ed       eeej8                  d   vde d      d                      Zy@(  DistributedTest._DistTestBasec                 .    t        j                  |i | y rf   )rg  r  )rS   r  kwargss      rU   r  z&DistributedTest._DistTestBase._barriera  s    LL$)&)rV   c                 z    ddg}t        j                  |fi |}t        j                         }||vrg d |fS |||fS )NrE   rG   )r  	new_grouprD  rS   r  groupgroup_idr   s        rU   _init_group_testz.DistributedTest._DistTestBase._init_group_testd  sK    FE~~e6v6H==?D5 D$''8T**rV   c                     t        t        dt        j                                     }t        j                  di |}t        j
                         }|||fS )Nr   rt   )listrW  r  r   r  rD  r  s        rU   _init_full_group_testz3DistributedTest._DistTestBase._init_full_group_testm  sE    q$"5"5"789E~~//H==?D8T**rV   c                     t        t        dt        j                                     }t        j                  j
                  }t        j                         }|||fS rC  )r  rW  r  r   r  WORLDrD  rS   r  r  r   s       rU   _init_global_testz/DistributedTest._DistTestBase._init_global_tests  sC    q$"5"5"789Ezz''H==?D8T**rV   c                    t        |j                  j                               }|j                  j                         D ]  \  }}| j                  |||           t	        |j                               }t	        |j                               }t        ||      D ]  \  }}	t        t        j                               D 
cg c]  }
t        j                  |       }}
t        j                  ||       t        t        j                               D 
cg c]  }
t        j                  |	       }}
|D ]  }| j                  ||        t        j                  ||	       |D ]  }| j                  ||	         y c c}
w c c}
w rf   )dictr   named_buffersassertEqualr  buffersziprW  r  r   rP   
empty_like
all_gather)rS   m1m2m1_buf_dictr   buf
m1_buffers
m2_buffersbuf1buf2rY  gathered_bufsgathered_bufs_m2rN   s                 rU   _verify_buffers_equalz3DistributedTest._DistTestBase._verify_buffers_equaly  sK   ryy6689KYY446	c  k$&78 7 bjjl+Jbjjl+J #J
 ;t49$:M:M:O4P!4PqE$$T*4P  ! t449$:M:M:O4P$4PqE$$T*4P ! $ 'A$$Q- ' 0$7)A$$Q- * !<!$s   >E:E?c                 2   t               5 \  }}t                |j                         j                         }d d d        d }g d}|D ]  } ||      }| j	                  |        g d}|D ]  } ||      }| j                  |        y # 1 sw Y   WxY w)Nc                 `    d|  d| t         j                  v rt         j                  |     S d S )Nzenv:=N/Ar  vars    rU   format_linezRDistributedTest._DistTestBase.test_dump_DDP_relevant_env_vars.<locals>.format_line  s4    cU!sbjj7HBJJsO#TUUe#TUUrV   )r  MASTER_PORTr'  NCCL_TOPO_DUMP_FILETORCH_NCCL_ASYNC_ERROR_HANDLING)xxxyyyzzz)r.   r   getvalue
splitlinesassertInassertNotIn)rS   outrY  linesr  varsr  lines           rU   test_dump_DDP_relevant_env_varsz=DistributedTest._DistTestBase.test_dump_DDP_relevant_env_vars  s     "hsA+-113 #VD "3'dE* D
 "3'  u- 1 #"s   ,BBc                    t         j                  j                  t         j                  d   d      }t	        t        j
                               }t        j                         }t        t         j                  j                  ||      d      5 }|j                  t	        t        j                                      d d d        | j                          t               }t        j                  |      D ]\  }t        t         j                  j                  ||            5 }|j                  t        |j!                                      d d d        ^ | j#                  t%        |      |       | j                          t        j                         dk(  rMt        j                  |      D ]5  }t        j&                  t         j                  j                  ||             7 | j                          y # 1 sw Y   8xY w# 1 sw Y   xY w)Nr.  test_dirr0  r   )r)  r1  r2  r*  r   rs  r  r   r3  rt  rD  r  setrk  addr   rv  r  lenrl  )rS   r  r{  num_processesr}  	all_ranksro  s          rU   test_get_rankz+DistributedTest._DistTestBase.test_get_rank  sH   ww||BJJz$:JGHbiik"C //1Mbggll8S1371DMMO,- 8 MMOI**X."'',,x89QMM#affh-0 :9 / S^];MMO}}!# jj2FIIbggll8V<= 3 MMO# 87 :9s   -G0)G=0G:=H	c                    t        j                         dkD  rddg}nddg}t        j                  |      }t        j	                         }| j                  t        j                         |       t        j                         |v r&| j                  t        j                  |      |       y | j                  t        d      5  t        j                  |       d d d        y # 1 sw Y   y xY w)NrG   rE   r   zInvalid process group specified)
r  r   r  r   lowerr  get_backendrD  assertRaisesRegex
ValueError)rS   r  r  backend_strs       rU   test_get_backendz.DistributedTest._DistTestBase.test_get_backend  s    ""$q(AA~~e,H!--/KT--/=}}%'  !1!1(!;[I++ A $$X.  s    CC(c                 h   t         j                         }| j                  t        j                  t         j                               |       | j                  t        j                  t               |       | j                  t              5  t        j                  d        d d d        | j                  t              5  t        j                  d       d d d        | j                  t              5  t        j                  dg       d d d        y # 1 sw Y   qxY w# 1 sw Y   JxY w# 1 sw Y   y xY w)Nru   gloo)r   r  r  r  r  upperassertRaisesr  )rS   r  s     rU   test_Backend_enum_classz5DistributedTest._DistTestBase.test_Backend_enum_class  s    mmoGT\\'--/:GDT\\'2G<"":.T" /"":.Q /"":.fX& /.	 /.....s$   
D=D0D(DD%(D1c                     t        j                         dkD  rddg}nddg}t        j                  |      }| j                          t        j                  |       y )NrG   rE   r   )r  r   r  r  r  rS   r  r  s      rU   test_destroy_groupz0DistributedTest._DistTestBase.test_destroy_group  sJ    ""$q(AA~~e,HMMO&&x0rV   c                    t        j                         dkD  rddg}nddg}t        j                  |      }t        j                         |v r^| j	                  t        j                  |      d       | j                  t        j                  |      t        t        d            v        y | j	                  t        j                  |      d       | j	                  t        j                  |      d       y )NrG   rE   r   r   )r  r   r  rD  r  
assertTruer  rW  r  s      rU   test_get_rank_size_groupz6DistributedTest._DistTestBase.test_get_rank_size_group  s    ""$q(AA~~e,H}}%'  !4!4X!>Bh 74a> IJ  !4!4X!>C  x!8"=rV   c                 v    | j                         \  }}}| j                          t        j                  |       y rf   )r  r  r  r  rS   rY  r  s      rU   test_destroy_full_groupz5DistributedTest._DistTestBase.test_destroy_full_group  s-    !779NAxMMO&&x0rV   c                    | j                         \  }}}| j                  t        j                  |      t        j                                | j                  t        j                  |      t        j                                y rf   )r  r  r  r   rD  r  s      rU   test_get_rank_size_full_groupz;DistributedTest._DistTestBase.test_get_rank_size_full_group   sY    !779NAxT00:D<O<O<QRT]]84dmmoFrV   c                    t        j                  |      }|dk(  rt        j                         |j                         z   }t        j                         t         j
                  j                  k(  r| j                  t        d      }n| j                  t        d      }|5  t        j                  |       d d d        | j                  t        j                         |d       y y # 1 sw Y   1xY w)Nr   zfailed to pass monitoredBarrierz (Timed out|closed|timeout) rr  )delta)r  rD  ru  total_secondsget_debug_level
DebugLevelDETAILr  	Exceptionri  assertGreaterAlmostEqual)rS   r  rz  
local_rankexpected_timeexception_ctxs         rU   _test_barrier_timeoutz3DistributedTest._DistTestBase._test_barrier_timeout  s    x0J Q $		g.C.C.E E '')T__-C-CC$($:$:!#D%M %)$:$:!#A%M #LL* #--diik=PS-T	 #]s   C$$C-r  z#Only gloo backend supports timeoutsfile://zRequires file:// initialization method. Both tcp:// and env:// rely on the TCP store for which reinitialization has proven racy.c                 |   t        j                          | j                  t        t        j
                  d                t        d      }t        j                  t        t        t        t        j
                  d         | j                  |       | j                  t         j                  j                  |       y )Nr'  ry  rE   r  r  )r  r  r  r   r)  r*  r   r  r   r   r   r  r  r  )rS   rz  s     rU   test_barrier_timeout_globalz9DistributedTest._DistTestBase.test_barrier_timeout_global  s     &&( MM3rzz,'?#@MA  *G##'rzz,78YY &&tzz'7'7ArV   c                 r    t        d      }| j                  |      \  }}}|| j                  ||       y y )Nr}   r  rz  )r   r  r  rS   rz  rY  r  s       rU   test_barrier_timeout_groupz8DistributedTest._DistTestBase.test_barrier_timeout_group6  sB    
  *G!2272CNAx#**8W= $rV   c                 r    t        d      }| j                  |      \  }}}|| j                  ||       y y )NrE   r  r  )r   r  r  r  s       rU   test_barrier_timeout_full_groupz=DistributedTest._DistTestBase.test_barrier_timeout_full_group@  sB      *G!777HNAx#**8W= $rV   c                    t         dk(  rd}n't         dk(  rd}nt         t        j                  d   v rd} |      \  }}}|y |dk(  r | j                  |j	                         d       |dk(  r | j                  |j	                         d       | j                  ||t        j                  |                | j                  t        |      t        j                  |             t        j                  |      }t        j                  j                  |       t        d|      j                         }t        j                  ||d   |       | j                  t        dd      |j                  d	             y )
Nr  r  r  r  rG   rl   r   srcr  cpu)r   r2   r  r  _get_backend_namer  r  rD  r  r   rP   rR  
set_devicerU  	broadcastto)rS   initializernew_backendr  r  r   
group_rankr   s           rU   _test_group_override_backendz:DistributedTest._DistTestBase._test_group_override_backendL  s6   & $F"$M99(CC$$/$D!E8Tf$ : : <fEf$ : : <fET5x)@#ABSZ)<)<X)FG x0JJJ!!*- #1J7<<>FNN6uQxx@]1A6		%8HIrV   gpuru   rG   c                 :    | j                  | j                         y rf   )r+  r  r  s    rU   test_backend_groupz0DistributedTest._DistTestBase.test_backend_groupi  s     --d.C.CDrV   uccz:broken, see https://github.com/pytorch/pytorch/pull/113620c                 :    | j                  | j                         y rf   )r+  r  r  s    rU   test_backend_full_groupz5DistributedTest._DistTestBase.test_backend_full_groupo  s     --d.H.HIrV   subgroupzThe z< backend does not support creating subgroups on CUDA devicesrv   c                 \   d}t        j                  |      \  }}t        j                         }| j                  |j	                         |       | j                  t        |      ||z         | j                  t        j                  |             |D ]  }t        j                  |        y r   )	r  new_subgroupsr   r  rS  r  assertFalse_rank_not_in_groupr  )rS   subgroup_sizecur_subgroup	subgroupsr+  r2  s         rU   test_new_subgroupsz0DistributedTest._DistTestBase.test_new_subgroupsu  s     M&*&8&8&G#L),,.J\..0-@S^Z--GHT44\BC%**84 &rV   c                     | j                  t        d      5  t        j                  d       d d d        y # 1 sw Y   y xY w)Nzmust not exceedd   r  r  r  r4  r  s    rU   0test_new_subgroups_group_size_exceeds_world_sizezNDistributedTest._DistTestBase.test_new_subgroups_group_size_exceeds_world_size  s0     ''
4EF""3' GFF   6?c                     | j                  t        d      5  t        j                  d       d d d        y # 1 sw Y   y xY w)Nz0The world size must be divisible by 'group_size'ru   r=  r  s    rU   9test_new_subgroups_world_size_not_divisible_by_group_sizezWDistributedTest._DistTestBase.test_new_subgroups_world_size_not_divisible_by_group_size  s8     ''N ""1%  r?  c                    | j                         \  }}}t        t        j                         t              }||   d   }t        j
                  ddgddgg      \  }}|dk\  r| j                  |       np| j                  |j                         d       | j                  t        |      d       |dk(  s|dk(  r| j                  ||d          n| j                  ||d          |D ]  }t        j                  |        y )Nr   rG   rE   ru   ranks_per_subgroup_listrv   )r  r"   r  r   r   new_subgroups_by_enumerationassertIsNoner  rS  r  r  )	rS   r  r  r   rank_to_GPUrT  r8  r9  r2  s	            rU   !test_new_subgroups_by_enumerationz?DistributedTest._DistTestBase.test_new_subgroups_by_enumeration  s     %)$:$:$<!E8T.t/B/B/DgNK#D)!,I&*&G&G*+Q!Q(8'#L) A~!!,/  !2!2!4a8  Y3>Y!^$$\9Q<@$$\9Q<@%**84 &rV   c                 $   | j                         \  }}}t        t        j                         t              }||   d   }t        |      }| j                  t        d      5  t        j                  ddg|dgg       d d d        y # 1 sw Y   y xY w)Nr   NThe new group's rank should be within the world_size set by init_process_grouprE   rG   rC  )r  r"   r  r   r   r  rw  rE  )rS   r  r  r   rG  rT  r+  s          rU   ?test_new_subgroups_by_enumeration_input_rank_exceeds_world_sizez]DistributedTest._DistTestBase.test_new_subgroups_by_enumeration_input_rank_exceeds_world_size  s     %)$:$:$<!E8T.t/B/B/DgNK#D)!,I'1J''` 11./Vj!_,E	  s    BBc                     | j                         \  }}}| j                  t        d      5  t        j                  ddgddgg       d d d        y # 1 sw Y   y xY w)NrJ  r   rC  )r  r  r  r  rE  r  s       rU   5test_new_subgroups_by_enumeration_negative_input_rankzSDistributedTest._DistTestBase.test_new_subgroups_by_enumeration_negative_input_rank  s]     %)$:$:$<!E8T''` 11.0"XBx,@	  s   AAc                     | j                  t        d      5  t        j                  dgddgddgg       d d d        y # 1 sw Y   y xY w)Nz$Rank 1 has appeared in both subgroupr   rE   rG   ru   rC  )r  r  r  rE  r  s    rU   &test_new_subgroups_overlap_not_allowedzDDistributedTest._DistTestBase.test_new_subgroups_overlap_not_allowed  sL     ''B 11./S1a&1a&,A  s	   ?Ac           	         t        j                         }t        t        j                         t              }||   d   }t        j                  t        j                  dddd      t        j                         t        j                  ddd            j                  |      }|j                         D ]&  }t        j                  |j                        |_        ( t        j                   |j                         d        |j                         D ];  }| j#                  |j                  t        j                  |j                               = |j                         D ])  }t        j                  |j                        |z  |_        + t        j$                  ddgd	
      }t        j                   |j                         |       t        j&                  |      sR|j                         D ]>  }| j#                  |j                  t        j                  |j                        dz         @ y |j                         D ]>  }| j#                  |j                  t        j                  |j                        |z         @ y )Nr   ru   rE   )kernel_sizepaddingr}   FrC   )paramsprocess_groupr  ranksr        ?)r  rD  r"   r   r   rK   
SequentialConv2dr   rL   rR  
parametersrP   	ones_liker  model_averaging_utilsaverage_parametersr  r  r6  )rS   r   rG  rT  r   r   
group_nccls          rU   test_average_parametersz5DistributedTest._DistTestBase.test_average_parameters  s    ==?D.t/B/B/DgNK#D)!,IMM		!QAq9			!QU+ d9o	  %%'0 (!44'') %%'  )@A ( %%'047 (q!ffEJ!44'') **:6))+A$$QVVU__QVV-Ds-JK , ))+A$$QVVU__QVV-Dt-KL ,rV   c                    t        j                         }t        j                         }t        |t              }||   d   }t        j                  ddd      j                  |      }t        |j                               }t        j                  |j                        |z  }t        j                  |j                        t        t        |            z  |z  }d}	dD ]  }
t        j                   |	|
      }t        dd	      D ]  }t#        j$                  |      |_        |j                         D ]&  }t        j                  |j                        |_        ( |j)                  |j                                ||
k\  r(||
z
  |	z  dk(  r| j+                  |j                  |       | j+                  |j                  |         y )
Nr   rE   r}   FrC   rv               periodwarmup_stepsr   )r  rD  r   r"   r   rK   rL   rR  nextr]  rP   r^  r  r   rW  	averagersPeriodicModelAveragercopydeepcopygradr`  r  rS   r   r+  rG  rT  r   paramr   expected_avg_tensorrj  rk  averagersteprV  s                 rU   test_periodic_model_averagerz:DistributedTest._DistTestBase.test_periodic_model_averager  su    ==?D,,.J.z7CK#D)!,IIIa/44Y?E))+,E__UZZ047F

+c%
2C.DDzQ   F 0$::! "!RLD!%v!6EJ"'"2"2"4&+ooejj&A #5 //0@0@0BC|+1D0NRS0S((5HI ((V< )	 !1rV   c                    t        j                         }t        j                         }t        |t              }||   d   }t        j                  ddd      j                  |      }t        |j                               }t        j                  j                  |j                         d      }d}d	D ]  }	t        j                  ||	
      }
t        dd      D ]v  }|j                   D ]Z  }|d   D ]P  }t        j"                  |j$                        |z  |_        t        j"                  |j$                        |z  |_        R \ |
j)                  |j                          ||	k\  r||	z
  |z  dk(  r{|j                   D ]j  }|d   D ]`  }|j&                  | j+                  |j$                  t        j"                  |j$                        t-        t        |            z  |z         b l |j                   D ]U  }|d   D ]K  }|j&                  | j+                  |j$                  t        j"                  |j$                        |z         M W y  y )Nr   rE   r}   FrC   rr  lrrv   rd  ri  r   rV  )r  rD  r   r"   r   rK   rL   rR  rl  r]  rP   optimSGDrm  rn  rW  param_groupsr^  r  rq  r`  r  r   )rS   r   r+  rG  rT  r   rs  optrj  rk  ru  rv  param_grouprV  s                 rU   (test_periodic_model_averager_param_groupzFDistributedTest._DistTestBase.test_periodic_model_averager_param_group4  s    ==?D,,.J.z7CK#D)!,IIIa/44Y?E))+,E++//%"2"2"4/=CF 0$::! "!RLD'*'7'7&1(&;F*///%***E*LFK*///%***E*LFK '< (8
 //0@0@A|+1D0NRS0S+.+;+;K*5h*?#);;#6$, $ 0 0$)JJ$)OOEJJ$?&)%
*;&<%=&0%1!" +@ ,< ,/+;+;K*5h*?#);;#6$, $ 0 0$)JJ

0Kd0R!" +@ ,<+ )	 !1rV   c                    t        j                         }t        j                         }t        |t              }||   d   }t        j                  ddd      j                  |      }t        |j                               }t        j                  |j                        |z  }t        j                  |j                        t        t        |            z  |z  }d}	dD ]	  }
t        j                   t#        |	|fg      |
      }t%        j&                  |	|
	      }t        dd
      D ]  }t)        j*                  |      |_        |j                         D ]&  }t        j                  |j                        |_        ( |j/                  |j                                ||
k\  r(||
z
  |	z  dk(  r| j1                  |j                  |       | j1                  |j                  |         y )Nr   rE   r}   FrC   rv   rd  period_group_size_dictrk  ri  r   )r  rD  r   r"   r   rK   rL   rR  rl  r]  rP   r^  r  r   rW  hierarchicalSGDHierarchicalModelAveragerr   rm  rn  ro  rp  rq  r`  r  rr  s                 rU   Ntest_1_level_hierarchical_model_averager_equivalent_to_periodic_model_averagerzlDistributedTest._DistTestBase.test_1_level_hierarchical_model_averager_equivalent_to_periodic_model_averagera  s    ==?D,,.J.z7CK#D)!,IIIa/44Y?E))+,E__UZZ047F

+c%
2C.DDzQ   F 0*DD ,78L7M+N!-	 %::! "!RLD!%v!6EJ"'"2"2"4&+ooejj&A #5 //0@0@0BC|+1D0NRS0S((5HI ((V< ) !1rV   c                    t        j                         }t        j                         }t        |t              }||   d   }t        j                  ddd      j                  |      }t        |j                               }t        j                  |j                        |z  }d}d}	d}
d}d}d	}t        |
|	f||f||fg      }t        j                  ||
      }|j                   |
   }|j                   |   }t        j"                  |      }t        j"                  |      }||	z  |	z  t%        j&                  t)        t+        |	                  z   j-                         }||z  |z  t%        j&                  t)        t+        |                  z   j-                         }| j/                  ||       | j/                  ||       t        j                  |j                        t1        |      z  |	z  }t        j                  |j                        t1        |      z  |z  }t        j                  |j                        t1        t+        |            z  |z  }t+        dd      D ]  }t3        j4                  |      |_        |j                         D ]&  }t        j                  |j                        |_        ( |j9                  |j                                |dk(  s|dk(  r| j/                  |j                  |       |dk(  s|dk(  r| j/                  |j                  |       |dk(  s|dk(  s
|dk(  s|dk(  r| j/                  |j                  |       | j/                  |j                  |        y )Nr   rE   r}   FrC   rB   rG   rv      r           re  r   rg        )r  rD  r   r"   r   rK   rL   rR  rl  r]  rP   r^  r  r   r  r  period_process_group_dictget_process_group_ranksnparrayr  rW  tolistr  r   ro  rp  rq  r`  )rS   r   r+  rG  rT  r   rs  r   rk  subgroup_size1subgroup_avg_period1subgroup_size2subgroup_avg_period2global_avg_periodr  ru  	subgroup1	subgroup2real_group_ranks_res1real_group_ranks_res2expect_group_ranks_res1expect_group_ranks_res2$expected_avg_tensor_within_subgroup1$expected_avg_tensor_within_subgroup2expected_global_avg_tensorrv  rV  s                              rU   (test_3_level_hierarchical_model_averagerzFDistributedTest._DistTestBase.test_3_level_hierarchical_model_averager  s&    ==?D,,.J.z7CK#D)!,IIIa/44Y?E))+,E__UZZ047F LN#$ N#$  !%0)>:)>:&
3&" '@@'=LH !::;OPI ::;OPI$($@$@$K!$($@$@$K!&7((4n 5678fh $
 &7((4n 5678fh $ 24KL24KL 

++,- ! 1 

++,- ! 1 

+c%
2C.DDzQ ' a!]]62
#..0F"'//%**"=FK 1 ++E,<,<,>?2:$$UZZ1KLRZ42:$$UZZ1UVRZ42:trz$$UZZ1UV $$UZZ8% %rV   r  zFCoalescing manager currently tests with NCCL only; internal test flakyc                    | j                          t        j                         }t        j                         }t	        |t
              }||   d   }t        j                  j                  |       d}d}t        |      D cg c]  }t        j                  ||       }}t        j                         5  t        |      D ]  }	t        j                  ||	           	 d d d        t        j                  ||z  |      }
t        j                  |
       t        |      D ]#  }	| j                  ||	   |
|	|z  |	dz   |z          % | j                          y c c}w # 1 sw Y   xY w)Nr   rG   r  r   rE   )r  r  rD  r   r"   r   rP   rR  r%  rW  r   _coalescing_manager
all_reducer  )rS   r   r+  rG  rT  	num_collssize_per_collrY  small_tensorsi
big_tensors              rU   test_coalescing_managerz5DistributedTest._DistTestBase.test_coalescing_manager  s6    MMO==?D,,.J.z7CK#D)!,IJJ!!),IMEJ9EUEU

=;EU   ))+y)AOOM!$45 * , I$=iPJOOJ'9%  !!$q=0AE]3JK & MMO# ,+s   E5)EE&c                    | j                          t        j                         }t        j                         }t	        |t
              }||   d   }t        j                  j                  |       d}d}t        |      D cg c]  }t        j                  ||       }}t        j                  d      5 }	t        |      D ]  }
t        j                  ||
           	 d d d        	j                          t        j                  ||z  |      }t        j                  |       t        |      D ]#  }
| j                  ||
   ||
|z  |
dz   |z          % | j                          y c c}w # 1 sw Y   xY w)Nr   rG   r  r   T)	async_opsrE   )r  r  rD  r   r"   r   rP   rR  r%  rW  r   r  r  waitr  )rS   r   r+  rG  rT  r  r  rY  r  cmr  r  s               rU   test_coalescing_manager_asyncz;DistributedTest._DistTestBase.test_coalescing_manager_async  sC    MMO==?D,,.J.z7CK#D)!,IJJ!!),IMEJ9EUEU

=;EU   ))D9Ry)AOOM!$45 * : GGII$=iPJOOJ'9%  !!$q=0AE]3JK & MMO% :9s   E*7)E//E8zNCCL Batch Send Recv Only)rG      r   zNeed NCCL 2.7+ for send/recvc                 &   | j                          t        j                         }t        j                         }t	        |t
              }||   d   }t        j                  j                  |       g }t        |      D cg c]  }d  }}t        |      D cg c]  }d  }}dD ]P  }	|	t        j                  d<   t        d|      D ]  }
t        |dz   |      j                  |
      }t        |
dz   d|      j                  d      ||
<   t        |
dz   d|      j                  |      ||
<   t        j                  t        j                  ||
   |
      }|j!                  |       t        j                  t        j"                  ||
      }|j!                  |        t        j$                  |      }|D ]  }|j'                           t        d|      D ]  }
| j)                  ||
   ||
           S | j                          y c c}w c c}w )Nr   )10TORCH_NCCL_BLOCKING_WAITrE   rT  r   rl   rT  )r  r  rD  r   r"   r   rP   rR  r%  rW  r)  r*  rU  rQ  P2POpirecvappendisendbatch_isend_irecvr  r  )rS   r   r+  rG  rT  p2p_op_listrY  recv_tensorsexpected_tensorsvalr"  send_tensorrecv_opsend_opreqsreqs                   rU   test_batch_isend_irecv_ncclz9DistributedTest._DistTestBase.test_batch_isend_irecv_nccl)  s    MMO==?D,,.J.z7CK#D)!,IJJ!!),K*/
*;<*;QD*;L<.3J.?@.?.?@!9<

56 J/C"/qI"N"T"T#K )6arY)eBi !% -:arY-eDk %S) #jj\#5FLG&&w/"jj[#FG&&w/ 0 --k:CHHJ   !J/C$$\#%68H8MN 0+ "0 MMO7 =@s   ?	H		Hc                 J   | j                          t        j                         }t        j                         }t	        |t
              }||   d   }t        j                  j                  |       g }t        ||      }t        |d|      }t        j                  t        j                  ||dz   |z        }t        j                  t        j                  ||dz
  |z   |z        }	t        j                  ||	g      }
|
D ]  }|j                           | j                          y )Nr   r  r   r  rE   )r  r  rD  r   r"   r   rP   rR  r%  rU  r  r  r  r  r  )rS   r   r+  rG  rT  r  r  recv_tensorr  r  r  r  s               rU   )test_batch_isend_irecv_ring_exchange_ncclzGDistributedTest._DistTestBase.test_batch_isend_irecv_ring_exchange_ncclQ  s     MMO==?D,,.J.z7CK#D)!,IJJ!!),K'
iHK'
"	RKjj[4!8z:QRGjj

K$(Z*?:)MG ))7G*<=D
  MMOrV   c                 h   | j                          t        j                          t        j                         }t	        t        j
                         t              }||   d   }g }|dk(  rt        |dz   |      }t        |dz   d|      }t        j                  t        j                  |d      }|j                  |       t        j                  t        j                  |d      }|j                  |       t        j                  |      }	|	D ]  }
|
j                           | j                          y )Nr   rE   r  r   r  )r  r  ri  rD  r"   r   r   rU  r  r  r  r  r  r  )rS   r   rG  rT  r  r  r  r  r  r  r  s              rU    test_batch_isend_irecv_self_ncclz>DistributedTest._DistTestBase.test_batch_isend_irecv_self_nccli  s     MMO LLN==?D.t/B/B/DgNK#D)!,IKqy+D1H	J+D1HB)T**TZZa@""7+**TZZa@""7+--k:CHHJ   MMOrV   c                    | j                          t        j                          t        j                         }t	        t        j
                         t              }||   d   }t        j                  j                  |       g }|dk(  rd}n|dk(  rd}|dv rt        |dz   |      }t        dz   d|      }t        j                  t        j                  ||      }|j                  |       t        j                  t        j                  ||      }	|j                  |	       t        j                  |      }
|
D ]  }|j!                           | j                          y )Nr   rE   rG   )rE   rG   r  r   r  )r  r  ri  rD  r"   r   r   rP   rR  r%  rU  r  r  r  r  r  r  )rS   r   rG  rT  r  peerr  r  r  r  r  r  s               rU   (test_batch_isend_irecv_no_rank_zero_ncclzFDistributedTest._DistTestBase.test_batch_isend_irecv_no_rank_zero_nccl  s   
 MMO LLN==?D.t/B/B/DgNK#D)!,IJJ!!),Kqyv~+D1H	J+D1HB)T**TZZdC""7+**TZZdC""7+--k:CHHJ   MMOrV   zGLOO Batch Send Recv CPUc                 .   | j                          t        j                         }g }t        dt        j                               D ]  }||k(  r	t        |dz         }t        |dz   d      }t        j                  t        j                  ||      }|j                  |       t        j                  t        j                  ||      }|j                  |        t        j                  |      }|D ]  }	|	j                           | j                          y )Nr   rE   r   r   r  r  rD  rW  r   rU  r  r  r  r  r  r  
rS   r   r  r"  r  r  r  r  r  r  s
             rU   test_batch_isend_irecv_glooz9DistributedTest._DistTestBase.test_batch_isend_irecv_gloo  s    MMO==?DKQ 3 3 56$;+D1H5+C!G2>**TZZcB""7+**TZZcB""7+ 7 ))+6D
  MMOrV   c                 6   | j                          t        j                         }g }t        dt        j                               D ]  }||k(  r	t        |dz         }t        |dz   d      }t        j                  t        j                  |||      }|j                  |       t        j                  t        j                  |||      }|j                  |        t        j                  |      }|D ]  }	|	j                           | j                          y )Nr   rE   r   r   tagr  r  s
             rU    test_batch_isend_irecv_gloo_tagsz>DistributedTest._DistTestBase.test_batch_isend_irecv_gloo_tags  s    MMO==?DKQ 3 3 56$;+D1H5+C!G2>**TZZcsK""7+**TZZctL""7+ 7 ))+6D
  MMOrV   c                    | j                          t        j                         }|dk(  rt        t        j                         t
              }||   d   }| j                  t        d      5  t        |dz   |      }t        j                  t        j                  |d      }t        j                  |g       d d d        y y # 1 sw Y   y xY w)Nr   z^Invalid ``op``rE   r  )r  r  rD  r"   r   r   r  r  rU  r  r&  r  )rS   r   rG  rT  r  r  s         rU   test_batch_isend_irecv_op_errz;DistributedTest._DistTestBase.test_batch_isend_irecv_op_err  s     MMO==?Dqy243F3F3H'R'-a0	++J8IJ"/qI"NK"jjaHG**G95 KJ  KJs   *AC  C	c                     | j                          t        j                         }|dk(  r7| j                  t        d      5  t        j
                  ddg       d d d        y y # 1 sw Y   y xY w)Nr   z^Invalid ``p2p_op_list``rE   rG   )r  r  rD  r  r  r  )rS   r   s     rU   "test_batch_isend_irecv_op_list_errz@DistributedTest._DistTestBase.test_batch_isend_irecv_op_list_err  sW     MMO==?Dqy++J8RS**Aq62 TS SSs    A""A+c                 H   | j                          t        j                         }t        t        j                         t
              }||   d   }t        j                  ddgd      }t        j                  ddgd      }|dk(  r| j                  t        d      5  t        |dz         }t        j                  t        j                  |d|      }t        j                  t        j                  |d|      }t        j                  ||g       d d d        y y # 1 sw Y   y xY w)Nr   rE   r  rX  r  z"All ops need to use the same group)r  r  rD  r"   r   r   r  r  r  rU  r  r  r  )	rS   r   rG  rT  
group_gloora  r  send_op_gloosend_op_nccls	            rU   (test_batch_isend_irecv_mixed_backend_errzFDistributedTest._DistTestBase.test_batch_isend_irecv_mixed_backend_err  s     MMO==?D.t/B/B/DgNK#D)!,Iq!ffEJq!ffEJqy++ D #0q"9K#'::djj+q*#UL#'::djj+q*#UL**L,+GH   s   A2DD!zNCCL Send Recv OnlyNc                    t        j                         }t        j                         }t        |t              }||   d   }t
        j                  j                  |       t        |dz   |      }||n	t               }|5 }t        d|      D ]|  }	|	|k(  r.t        d|      D ]  }
|
|k(  r	t        j                  ||
         6t        |	dz         }t        |	dz   d|      }t        j                  ||	       | j                  ||       ~ | j                          d d d        |t        j                         }|t         v r| d| dfD ]u  }t#        |      }| j%                  |       t        t        j                               D cg c]  }|dz   gdz  g }}|D ]  }| j%                  |j&                  |v        ! w y y y # 1 sw Y   xY wc c}w )	Nr   rE   r  r   r  :send:recvru   )r  rD  r   r"   r   rP   rR  r%  rU  r   rW  sendrecvr  r  r  &SEND_RECV_PROFILING_SUPPORTED_BACKENDSr   r  input_shapes)rS   profiler_ctxr   r+  rG  rT  r   profiler_clsprofr"  dstexpected_tensoroutput_tensorr  r   r   expected_shapesr   s                     rU   _test_send_recv_ncclz2DistributedTest._DistTestBase._test_send_recv_nccl   s    ==?D,,.J.z7CK#D)!,IJJ!!),"4!8yAF+7+C<L J/Cd{#(J#7C"d{ ( IIfc2 $8 +8a*@(5!G2) 		-5((H 0  # & '**,DD*1%'8WIU:K&L
!4Z!F/ :?t?R?R?T9U+9UdQhZ!^,9U ( + &,E OOE,>,>/,QR &, 'M E (' 6+s    BGGGc                 $    | j                          y rf   )r  r  s    rU   test_send_recv_ncclz1DistributedTest._DistTestBase.test_send_recv_nccl/  s     %%'rV   c                 z    t         j                  j                  j                  d      }| j	                  |       y r\  )rP   r^  r   r   r  rS   r  s     rU   %test_send_recv_nccl_autograd_profilerzCDistributedTest._DistTestBase.test_send_recv_nccl_autograd_profiler5  s/     !>>22:::NL%%l3rV   zKineto in fbcode causes hangzYtorch.profiler not enabled for mac/windows: https://github.com/pytorch/pytorch/pull/56124c                     t         j                  j                  t         j                  j                  j                  t         j                  j                  j
                  gd      }| j                  |       y ra  )rP   r   r   rc  rd  CUDAr  r  s     rU   "test_send_recv_nccl_torch_profilerz@DistributedTest._DistTestBase.test_send_recv_nccl_torch_profiler<  s]     !>>11NN3377NN3388 # 2 L %%l3rV   c                    t        j                         }|dz   }t        |      }||n	t               }|5 }t	        dt        j
                               D ]  }||k(  r@t	        dt        j
                               D ]  }||k(  r	t        j                  ||         H|dz   }	t        |	      }
t        |	d      }t        j                  ||       | j                  ||
        	 d d d        |t        j                         }|t        v r| d| dfD ]  }t        |      }t        d |D              }t        j
                         dz
  }| j                  ||       t	        t        j
                               D cg c]  }|dz   gdz  g }}|D ]:  }| j                  |j                         | j                  |j                  |v        <  y y y # 1 sw Y   xY wc c}w )	NrE   r   r   r   r  r  c              3   4   K   | ]  }|j                     y wrf   count.0r  s     rU   	<genexpr>z@DistributedTest._DistTestBase._test_send_recv.<locals>.<genexpr>j       )B6a!''6   ru   )r  rD  rU  r   rW  r   r  r  r  r  r  r   r   r  is_asyncr  )rS   r  r   	send_sizer   ctxr  r"  r  	recv_sizer  r  r  r   r   event_countexpected_event_countr  r   s                      rU   _test_send_recvz-DistributedTest._DistTestBase._test_send_recvO  s   ==?DqI"9-F".":,C D$7$7$9:Cd{#(D,?,?,A#BC"d{ ( IIfc2 $C %(!G	*7	*B(5ir(J		-5((H ;   '**,DD*1%'8WIU:K&L
!4Z!F&))B6)B&B/3/B/B/Dq/H,((6JK :?t?R?R?T9U+9UdQhZ!^,9U ( + &,E OOENN; OOE,>,>/,QR &, 'M E (! 6+s   B.G=G Gz,Nccl send/recv tested by test_send_recv_ncclc                 (    | j                  d        y Nr  )r  r  s    rU   test_send_recvz,DistributedTest._DistTestBase.test_send_recvv  s       d 3rV   z,NCCL send/recv tested by test_send_recv_ncclc                 <    t               }| j                  |       y r  )r_  r  rS   autograd_profiler_ctxs     rU    test_send_recv_autograd_profilerz>DistributedTest._DistTestBase.test_send_recv_autograd_profiler|  s     %>$?!  .C DrV   c                 :    t               }| j                  |      S r  )re  r  rS   torch_profiler_ctxs     rU   test_send_recv_torch_profilerz;DistributedTest._DistTestBase.test_send_recv_torch_profiler  s"     "8!9''5G'HHrV   c           	         t        j                         }d}t        ||      }t               }t               }||n	t	               }|5 }t        dt        j                               D ]  }	|	|k(  rt        dt        j                               D ]  }	|	|k(  r	dD ]  }
t        |d      }|
dk(  r't        j                  |      }|j                  |       nK|
dk(  rFt        j                  |      }|j                          |j                         }|j                  |       | j                  |j                        j                                  t        j                  ||	       t        j                  ||	        	 d d d        |!t        j                          }|t"        v r| d| d	fD ]  }t%        |      }| j'                  t)        d
 |D              dt        j                         dz
  z         |D ]>  }| j                  |j*                         | j'                  |j,                  |gdz  g       @  t/        j0                  t/        j2                  |      t/        j2                  |      fd      }t        t        j                               D cg c]  }t/        j4                  |       }}t        j6                  ||       g }|D ]  }||j9                         z  } ddlm} |j?                           ||      D cg c]  \  }}tA        t        |             }}}| j'                  t        j                         tA        |             | j'                  dt        j                         dz
  z  gt        j                         z  |       | jC                          y y # 1 sw Y   /xY wc c}w c c}}w )NrB   r   r   )r  r  r   r  r  r  z:recvAnySourcec              3   4   K   | ]  }|j                     y wrf   r  )r  r   s     rU   r  zKDistributedTest._DistTestBase._test_send_recv_any_source.<locals>.<genexpr>  s     @r  rG   rE   ru   )groupby)"r  rD  rU  r  r   rW  r   r  r  r  r  _source_rankr  rn   allr  r  r  r   r  r   r  r  rP   catr   r  r  r  	itertoolsr  sortr  r  )rS   r  r   send_recv_sizer   
recv_ranksirecv_ranksr  r  r  r  r  senderworkr  r   r   r   recv_ranks_tensorrY  global_recv_ranksglobal_recv_ranks_listr  keyr  	frequencys                             rU   _test_send_recv_any_sourcez8DistributedTest._DistTestBase._test_send_recv_any_source  sM   ==?DN">>FJ&K".":,C D$7$7$9:Cd{#(D,?,?,A#BC"d{ ((90=nTV0W#'6>-1YY}-EF$.$5$5f$=%)W_+/::m+DD$(IIK-1->->-@F$/$6$6v$>
 !%0@0@0H0L0L0N O ):	 $C, 		&#.		&#.5 ; : '**,DD*1%'8WI^:T&U
!4Z!F((@@@!4!4!6!:; &,E OOENN; ,,U-?-?>BRUVBVAWX &, 'V %*II\\*-u||K/HI1%!
 #4#6#6#89%9 $$%679 " %  13DE)+&/F*fmmo=* 0 .&++-7>?U7V7VeCU$7V     !4!4!6IG  $--/!3458K8K8MMy M (; `%s   D:N0N=O0N:zsendrecv anysourcez+ does not support send/recv from any sourcec                 (    | j                  d        y r  )r$  r  s    rU   test_send_recv_any_sourcez7DistributedTest._DistTestBase.test_send_recv_any_source  s    
 +++>rV   c                 <    t               }| j                  |       y r  )r_  r$  r  s     rU   +test_send_recv_any_source_autograd_profilerzIDistributedTest._DistTestBase.test_send_recv_any_source_autograd_profiler  s    
 %>$?!++9N+OrV   z!Kineto in fbcode code causes hangc                 :    t               }| j                  |      S r  )re  r$  r  s     rU   (test_send_recv_any_source_torch_profilerzFDistributedTest._DistTestBase.test_send_recv_any_source_torch_profiler  s"     "8!922@R2SSrV   c                    t        j                         }t        j                         }d}t        ||      }||n	t	               }|5 }t        d|      D ]  }||k(  rkt        d|      D ][  }	|	|k(  r	t        |d      }
t        j                  |
|	|	       | j                  |
j                  |	      j                                ] st        j                  |||        	 d d d        |t        j                         }|t        v r| d| dfD ]  }t        |      }t        d |D              }t        j                         d	z
  }| j                  ||       |D ]Z  }| j                  |j                          | j                  |j"                  |       | j                  |j$                  |gd
z  g       \  y y y # 1 sw Y   xY w)NrB   r   r   r   r  r  r  c              3   4   K   | ]  }|j                     y wrf   r  r  s     rU   r  zIDistributedTest._DistTestBase._test_send_recv_with_tag.<locals>.<genexpr>  r  r  rE   ru   )r  rD  r   rU  r   rW  r  r  rn   r  r  r  r  r   r   r  r  r   r  )rS   r  r   r+  r  r   r  r  r  r"  r  r  r   r   r  r  r   s                    rU   _test_send_recv_with_tagz6DistributedTest._DistTestBase._test_send_recv_with_tag  s   ==?D,,.JN">>F".":,C J/Cd{#(J#7C"d{ (,9.PR,SM IImScB OOM,<,<S,A,E,E,GH $8 		&#48 0  '**,DD*1%'8WIU:K&L
!4Z!F&))B6)B&B/3/B/B/Dq/H,((6JK%+E OOENN; ,,UZZD ,,U-?-?>BRUVBVAWX &, 'M E ( s   BGG
c                 (    | j                  d        y r  )r-  r  s    rU   test_send_recv_with_tagz5DistributedTest._DistTestBase.test_send_recv_with_tag  s     ))t)<rV   c                 :    t               }| j                  |      S r  )r_  r-  r  s     rU   )test_send_recv_with_tag_autograd_profilerzGDistributedTest._DistTestBase.test_send_recv_with_tag_autograd_profiler"  s"     %>$?!00>S0TTrV   c                 :    t               }| j                  |      S r  )re  r-  r  s     rU   &test_send_recv_with_tag_torch_profilerzDDistributedTest._DistTestBase.test_send_recv_with_tag_torch_profiler)  s"     "8!900>P0QQrV   c           
         t        j                         }t        j                         }||n	t               }|5 }|dk(  rnt	        d|      D cg c]"  }t        j
                  t        |d      |      $ }}|D ]1  }|j                          | j                  |j                                3 n>t        |d      }	t        j                  |	d       | j                  |	t        |d             | j                          d d d        |;t        j                         }
|
t        v r|dk(  r|
 dn|
 d}t        |      }t!        d |D              }|dk(  rt        j                         dz
  nd}| j                  ||       t	        dt        j                               D ci c]
  }||gdz  g }}|D ]  }| j                  |j"                         | j                  |j$                  |       |dk(  r,| j                  |j&                  |j)                         v        k| j                  |j&                  ||           y y y c c}w # 1 sw Y   NxY wc c}w )	Nr   rE   rB   r   r  r  c              3   4   K   | ]  }|j                     y wrf   r  r  s     rU   r  z<DistributedTest._DistTestBase._test_isend.<locals>.<genexpr>Q  s     %>v!aggvr  ru   )r  rD  r   r   rW  r  rU  r  r  is_completedr  r  r  r  r  r   r   r  r   r  values)rS   r  r   r+  r  r  destrequestsrequestr   r  expected_event_namer   r  expected_countrr  r   s                     rU   _test_isendz)DistributedTest._DistTestBase._test_isend6  s3   ==?D,,.J".":,C19 %*!Z$8 $8D 

=r#:DA$8    $,(<(<(>? $, +44FIIfa($$V]4-DE   '**,DD-1QY7)5)wiu<M ( 11DdKF"%%>v%>">KBF!)T%8%8%:Q%>QRN$$^[A /4At7J7J7L.M'.MQC!G9.M $ ' "(7((5HI19 OO % 2 2o6L6L6N N !,,U-?-?QUAVW "( E (  8's$   I'I4BI IIIzNccl does not support isendc                 (    | j                  d        y r  )r>  r  s    rU   
test_isendz(DistributedTest._DistTestBase.test_isendc  s     $/rV   c                 <    t               }| j                  |       y r  )r_  r>  r  s     rU   test_isend_autograd_profilerz:DistributedTest._DistTestBase.test_isend_autograd_profileri  s     %>$?!*?@rV   c                 <    t               }| j                  |       y r  )re  r>  r  s     rU   test_isend_torch_profilerz7DistributedTest._DistTestBase.test_isend_torch_profilerp  s     "8!9*<=rV   zNccl does not support irecvc                 `   t        j                         }t        j                         }|dk(  rt        d|      D cg c]  }t	        |d       }}t        d|      D cg c]  }t        j
                  ||dz
     |        }}t        d|      D ]_  }||dz
     j                          | j                  ||dz
     j                                | j                  ||dz
     t	        |d             a n"t	        |d      }t        j                  |d       | j                          y c c}w c c}w )Nr   rE   r   rB   )r  rD  r   rW  rU  r  r  r  r6  r  r  r  )rS   r   r+  r"  r  r9  r   s          rU   
test_irecvz(DistributedTest._DistTestBase.test_irecv}  s$    ==?D,,.Jqy6;Az6J$6JsM#r*6J ! $
  %Q
33 JJ/a8#>3  
 !J/CS1W%**,OOHS1W$5$B$B$DE$$%5cAg%>cSU@VW 0
 'tR0		&!$MMO!$s   D&#D+Fc           
         t         j                  ddft         j                  ddft         j                  ddft         j                  ddft         j
                  ddft         j                  ddft         j                  d	dffD ]  \  }}}	|	r|s|D ]  }
t        |
d
z   ||      }|r|j                  ||   d         }||
k(  rj|rCt        j                         }d|_        |
|_        | j                  dd|j                  |g|       w| j                  ddt        j                  ||
|       t        |
d
z   d|      }|r|j                  ||   d         }|rCt        j                         }d|_        |
|_        | j                  dd|j                  |g|       n$| j                  ddt        j                  ||
|       | j!                  |j#                         |j#                                | j!                  |j%                  |      j'                         t        j(                  d               | j+                          y )Ng|۽Fg0.+gTrM     g     jg  4&krE   r   
:broadcastr   )rP   floatdoublehalfint8uint8r   r   rU  rR  r  BroadcastOptions
rootTensorrootRankcall_dist_opr&  r  rS  nemaxr   r  )rS   r  r  r   rR  rG  with_optionsrO  rl   requires_cudar"  r  optsr   s                 rU   _test_broadcast_helperz4DistributedTest._DistTestBase._test_broadcast_helper  s!    fe,w.T4(R'c5)D%(UE*0+um ! C&3C!GUE&JO*9*>*>{4?PQR?S*Ts{'#'#8#8#:D./DO,/DM -- , $ ( 2 2!0 1 $ !-- , % $ / # ( "/sQwE!B%+[[T1B11E%FF'#'#8#8#:D./DO,/DM -- ,dH4F4FRV !-- , % $ & # ( ((8L8L8NO(("IIo6::<ell5>Q] !0z MMOrV   z!Nccl does not support CPU tensorsc                 R    | j                         \  }}}| j                  |||       y rf   )r  rX  r  s       rU   test_broadcastz,DistributedTest._DistTestBase.test_broadcast  s+     %)$:$:$<!E8T''x>rV   z2Only Gloo and Nccl backend supports CUDA allReducec                     | j                         \  }}}t        t        j                         t              }||   d   }t
        j                  j                  |       | j                  |||d|       y )Nr   T)	r  r"   r  r   r   rP   rR  r%  rX  rS   r  r  r   rG  rT  s         rU   test_broadcast_cudaz1DistributedTest._DistTestBase.test_broadcast_cuda  se     %)$:$:$<!E8T.t/B/B/DgNK#D)!,IJJ!!),''xt[QrV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  rX  r  s       rU   test_broadcast_groupz2DistributedTest._DistTestBase.test_broadcast_group  s+    
 %)$9$9$;!E8T''x>rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  rX  r  s       rU   test_broadcast_full_groupz7DistributedTest._DistTestBase.test_broadcast_full_group  s+     %)$>$>$@!E8T''x>rV   z/Only NCCL backend supports high priority streamc                 J   | j                         \  }}}t        t        j                         t              }||   d   }t
        j                  j                  |       t        t        dz         }|t        j                  d<   t        j                  d|t        j                               }t        |      \  }}}	t        j                  ||      }t        j                  j!                         }
d|
_        t        j                  |||	|
      }| j%                  |||d|d       y )Nr   rE   r  r   FT)r  r"   r  r   r   rP   rR  r%  r   r  r)  r*  
rendezvousrl  PrefixStoreProcessGroupNCCLOptionsis_high_priority_streamrX  )rS   r  rY  r   rG  rT  new_portgen_iteratorstorerS  rW  r  s               rU   test_nccl_high_priority_streamz<DistributedTest._DistTestBase.test_nccl_high_priority_stream  s     "335NE1d.t/B/B/DgNK#D)!,IJJ!!),;?+H(0BJJ}%??8T4;N;N;PQL $\ 2E4$$Xu5E((002D+0D(,,UD$EH''xt[RVWrV   c
                 Z   |D ]  }
t        |
dz         j                  ||
k(  r|n|      }|r|j                  |	|   d         }| j                  ddt        j
                  ||
|||j                  g       ||
k(  sx| j                  |t        |
dz   |              | j                          y )NrE   r   :reduceFtensor_shapes)	rU  rQ  rR  rR  r  r	   shaper  r  )rS   r  r  r   opmaster_valueworker_valueexpected_valuerR  rG  r"  r   s               rU   _test_reduce_helperz1DistributedTest._DistTestBase._test_reduce_helper  s     &sQw/55$(CKL\ #[[T):1)=>F!!KK#)<<. " 	 3;$$V]37N-ST# & MMOrV   r	   z does not support reducec                     | j                         \  }}}| j                  |||t        j                  j                  ddddt        |      dz
  z  z          y NrG   rB   rE   )r  ru  r  ReduceOpSUMr  r  s       rU   test_reduce_sumz-DistributedTest._DistTestBase.test_reduce_sum;  sX     %)$:$:$<!E8T$$!!R3u:>*+rV   zOnly Nccl supports CUDA reducec                 D   | j                         \  }}}t        t        j                         t              }||   d   }t
        j                  j                  |       | j                  |||t        j                  j                  ddddt        |      dz
  z  z   d|	       y Nr   rG   rB   rE   T)r  r"   r  r   r   rP   rR  r%  ru  rx  ry  r  r\  s         rU   test_reduce_sum_cudaz2DistributedTest._DistTestBase.test_reduce_sum_cudaN  s     %)$:$:$<!E8T.t/B/B/DgNK#D)!,IJJ!!),$$!!B#e*q.))
rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddt        d dgt        |      dz
  z  d             y )NrG   rB   c                     | |z  S rf   rt   r[   ys     rU   r"  zCDistributedTest._DistTestBase.test_reduce_product.<locals>.<lambda>w      QUrV   rE   )r  ru  r  rx  PRODUCTr	   r  r  s       rU   test_reduce_productz1DistributedTest._DistTestBase.test_reduce_productg  s_     %)$:$:$<!E8T$$%%*bTSZ!^-DaHrV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y Ni  rE   )r  ru  r  rx  MINr  s       rU   test_reduce_minz-DistributedTest._DistTestBase.test_reduce_minz  s@     %)$:$:$<!E8T$$xt}}'8'8$1rV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y Nr   rB   )r  ru  r  rx  MAXr  s       rU   test_reduce_maxz-DistributedTest._DistTestBase.test_reduce_max  s@     %)$:$:$<!E8T$$xt}}'8'8"b"rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddddt        |      dz
  z  z          y rw  )r  ru  r  rx  ry  r  r  s       rU   test_reduce_group_sumz3DistributedTest._DistTestBase.test_reduce_group_sum  sX     %)$9$9$;!E8T$$!!R3u:>*+rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddt        d dgt        |      dz
  z  d             y )NrG   rB   c                     | |z  S rf   rt   r  s     rU   r"  zIDistributedTest._DistTestBase.test_reduce_group_product.<locals>.<lambda>  r  rV   rE   )r  ru  r  rx  r  r	   r  r  s       rU   test_reduce_group_productz7DistributedTest._DistTestBase.test_reduce_group_product  s_     %)$9$9$;!E8T$$%%*bTSZ!^-DaHrV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  ru  r  rx  r  r  s       rU   test_reduce_group_minz3DistributedTest._DistTestBase.test_reduce_group_min  s@     %)$9$9$;!E8T$$xt}}'8'8$1rV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  ru  r  rx  r  r  s       rU   test_reduce_group_maxz3DistributedTest._DistTestBase.test_reduce_group_max  s@     %)$9$9$;!E8T$$xt}}'8'8"b"rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddddt        |      dz
  z  z          y rw  )r  ru  r  rx  ry  r  r  s       rU   test_reduce_full_group_sumz8DistributedTest._DistTestBase.test_reduce_full_group_sum  sX     %)$>$>$@!E8T$$!!R3u:>*+rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddt        d dgt        |      dz
  z  d             y )NrG   rB   c                     | |z  S rf   rt   r  s     rU   r"  zNDistributedTest._DistTestBase.test_reduce_full_group_product.<locals>.<lambda>  r  rV   rE   )r  ru  r  rx  r  r	   r  r  s       rU   test_reduce_full_group_productz<DistributedTest._DistTestBase.test_reduce_full_group_product  s_     %)$>$>$@!E8T$$%%*bTSZ!^-DaHrV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  ru  r  rx  r  r  s       rU   test_reduce_full_group_minz8DistributedTest._DistTestBase.test_reduce_full_group_min  s@     %)$>$>$@!E8T$$xt}}'8'8$1rV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  ru  r  rx  r  r  s       rU   test_reduce_full_group_maxz8DistributedTest._DistTestBase.test_reduce_full_group_max	  s@     %)$>$>$@!E8T$$xt}}'8'8"b"rV   c
                    |D ]  t        d      D 
cg c]&  }
t        dz         j                  |k(  r|n|      ( c}
|r-t        d      D ]  }
|
   j                  |	|   d         |
<   ! | j	                  ddt
        j                  d   fdd   j                  g	       |k(  sD ]!  }| j                  |t        dz   |             #  | j                          y c c}
w )NrG   rE   r   rm  Fc                  :    t        j                  d          S rX   )r  r	   )r  rq  r"  tensorss   rU   r"  zIDistributedTest._DistTestBase._test_reduce_twice_helper.<locals>.<lambda>7	  s    dkk
CX/rV   )secondary_op_callro  )
rW  rU  rQ  rR  rR  r  r	   rp  r  r  )rS   r  r  r   rq  rr  rs  rt  rR  rG  r  r   r"  r  s     ` `       @@rU   _test_reduce_twice_helperz7DistributedTest._DistTestBase._test_reduce_twice_helper	  s    
 #1X	 & "#'*00(, &	 "1X%,QZ__[5Fq5I%J
 &!!KKAJ' $+1:#3#3"4 "  3;")((sQw1WX #*1 6 MMO5s   +C2c                     | j                         \  }}}| j                  |||t        j                  j                  ddddt        |      dz
  z  z          y rw  )r  r  r  rx  ry  r  r  s       rU   test_reduce_sum_twicez3DistributedTest._DistTestBase.test_reduce_sum_twiceB	  sX     %)$:$:$<!E8T**!!R3u:>*+rV   c                 D   | j                         \  }}}t        t        j                         t              }||   d   }t
        j                  j                  |       | j                  |||t        j                  j                  ddddt        |      dz
  z  z   d|	       y r|  )r  r"   r  r   r   rP   rR  r%  r  rx  ry  r  r\  s         rU   test_reduce_sum_cuda_twicez8DistributedTest._DistTestBase.test_reduce_sum_cuda_twiceU	  s     %)$:$:$<!E8T.t/B/B/DgNK#D)!,IJJ!!),**!!B#e*q.))
rV   z#Only Nccl supports reduce_scatter_vc           
         | j                          | j                         \  }}}t        t        j                         t
              }||   d   }g }|D ]  }|j                  |dz           t        |d |       }|||   z   }	t        |      }
d}d}dD ]=  }t        |
||      }|||	 j                  |       t        j                  ||   |
|
t        j                        j                  d      j                  |      }t        j                  |t        t        j                   ||            t        j"                  j$                  ||      }|r|j'                          ddt)        |      dz
  z  z   }t        j                  ||   |
|
t        j                        }|j                  |      j                  |      }| j+                  ||       @ | j                          y )	Nr   rE   rG   rB   TFr  rN  r   )r  r  r"   r  r   r   r  r   rU  rQ  rP   rP  rJ  rR  reduce_scatterr  r  rx  ry  r  r  r  )rS   r  r  r   rG  rT  input_split_sizesr"  	start_lenend_lensum_lenrr  rs  	async_valr   
out_tensorr  rt  r  s                      rU   test_reduce_scatter_v_cudaz8DistributedTest._DistTestBase.test_reduce_scatter_v_cudan	  s    MMO$($:$:$<!E8T.t/B/B/DgNK#D)!,I "!((q1 -et45I"3D"99G+,GLL*	&w	Ry)//=KK)$/' U2YT)_  ))V->?@MM%% HHJ!"bCJN&;!<"'++%d+WgU[[# #2"7"7"G"L"LY"W  _=7 +8 MMOrV   Tc                    |r.|j                  ||   d         }|j                  ||   d         }|j                  g}| j                  ddt        j                  ||t        j
                  j                  |dd|
       |S )Nr   z:reduce_scatter_tensorFexpect_eventro  )rR  rp  rR  r  reduce_scatter_tensorrx  ry  rS   
tensor_out	tensor_inr  r   rR  rG  ro  s           rU   _reduce_scatter_tensor_helperz;DistributedTest._DistTestBase._reduce_scatter_tensor_helper	  s     %NN;t+<Q+?@	'__[->q-AB
'--.M(**!!"+   rV   z-Only Nccl supports CUDA reduce_scatter_tensorc                 x   | j                         \  }}}t        t        j                         t              }d}t        j                  |t
        j                        }t        j                  t        |      |z        }| j                  ||||d|      }t        j                  ||z  |dz   |z        t        |      z  }| j                  ||       | j                          t        j                  |t        |      |f      }| j                  ||||d|      }| j                  ||       | j                          y )NrG   rN  TrE   )r  r"   r  r   r   rP   zerosint64aranger  r  r  r  r   )	rS   r  r  r   rG  rS  r  r  r  s	            rU   test_reduce_scatter_tensor_cudaz=DistributedTest._DistTestBase.test_reduce_scatter_tensor_cuda	  s   
 %)$:$:$<!E8T.t/B/B/DgNKDT=J SZ$%67I;;Ixt[J $ll4$;T8IJSQVZWOZ9MMO i#e*d1CDI;;Ixt[J
 Z9MMOrV   c                    | j                         \  }}}t        t        j                         t              }|D ]W  }||k(  rt        |dz   d      }nt        |dz   d      }|j                  ||   d         }t               }t        j                  j                  |_
        |t        j                  k(  rt               j                  |g|      }n|j                  |g|      }t        dk(  rQ	 | j                  t         d      5  |j#                          d d d        |j+                          |j#                         }	n |j#                         }	|j+                          ddt-        |      dz
  z  z   }
| j/                  |	t        |dz   |
      g       Z | j1                          y # 1 sw Y   xY w# t$        $ r" | j'                  |j)                                Y w xY w)NrE   rG   rB   r   r  z0Work needs to be completed before calling result)r  r"   r  r   r   rU  rR  r   rx  ry  reduceOpr   r  r   	allreducer  rw  resultAssertionErrorr  r6  r  r  r  r  )rS   r  r  r   rG  r"  r   rW  r  r  rt  s              rU   test_all_reduce_result_cudaz9DistributedTest._DistTestBase.test_all_reduce_result_cuda	  s    %)$:$:$<!E8T.t/B/B/DgNK3;*37A6F*37B7F[%6q%9:') $ 1 1{000-/996(DID#--vh=Df$=!33(N !KKM	 IIK![[]F "[[]FIIK!"bCJN&;!<  -a*P)QRM N MMO' 
 * =(9(9(;<=s*   -F1F%F1%F.	*F11(GG)r  r  profile_cudaro  c          	        	 	fdg}
||
j                  |       t        j                  j                  j	                  |d      }|5 }|
D cg c]	  } |        }}|r|D ]  }|j                           d d d        |rKt        j                         t        v r/t        t        j                         |z   |      }t        j                         t        j                  j                  k7  r$| j                  t        |      t        |
             |D ]  }| j                  |j                          | j                  |j"                  d       | j%                  |j&                  d       |Yt        j                         t        j                  j                  k7  s| j                  |j(                  |d|j(                   d|         y y y c c}w # 1 sw Y   ^xY w)Nc                        i S rf   rt   )r  r  rq  s   rU   r"  z<DistributedTest._DistTestBase.call_dist_op.<locals>.<lambda>
  s    D 3F 3rV   T)use_cudar]  rE   r   zevent shape: z vs tensor )r  rP   r^  r   r   r  r  r  PROFILING_SUPPORTED_BACKENDSr   r  r  r  r  r  r  r  r  assertGreaterEqualcpu_timer  )rS   profiling_title_postfixr  rq  r  r  r  ro  r  r  op_callsr  r  op_callworksr  r   r  s      `    ``        rU   rR  z*DistributedTest._DistTestBase.call_dist_op
  s    44H , 12$)NN$;$;$C$C%T %D %! '$2:;(w(; %		 !& '  0 0 26R R,$$&)@@BW
 '')T__-C-CC$$S[#h-@AOOAJJ/$$QWWa0++AJJ:
 &1 002doo6L6LL((NN)+ANN+;;}oV   !S| < '&s   GGGGG c                 4   |D ]  }||k(  r|n|}t        |dz   |
      j                  |      }|r|j                  |	|   d         }|j                  t        j
                  k(  r!t	        j                  |      j                  g}n|j                  g}| j                  d|t        j                  |||||       |dk(  s|st        j                         t        v s| j                  d|t        j                  ||||d|	        | j                          y )NrE   rN  r   :all_reduce)async_opro  T)r  r  ro  )rU  rQ  rR  rO  rP   	complex64view_as_realrp  rR  r  r  r  !CUDA_PROFILING_SUPPORTED_BACKENDSr  )rS   r  r  r   rq  rr  rs  rt  rR  rG  rO  r  r"  
curr_valuer   ro  s                   rU   _test_all_reduce_helperz5DistributedTest._DistTestBase._test_all_reduce_helper<
  s    -1S[\l
&sQwe<BB:N#[[T):1)=>F<<5??2%*%7%7%?%E%E$FM%+\\NM!!!OO%"/ " 	 1H((*.OO%%%  !)%)&3 & 
9 P MMOrV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddddt        |      dz
  z  z          y rw  r  r  r  rx  ry  r  r  s       rU   test_all_reduce_sumz1DistributedTest._DistTestBase.test_all_reduce_sumt
  sX     %)$:$:$<!E8T((!!R3u:>*+rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddddt        |      dz
  z  z   d       y NrG   rB   rE   T)r  r  r  s       rU   test_all_reduce_sum_asyncz7DistributedTest._DistTestBase.test_all_reduce_sum_async
  s`     %)$:$:$<!E8T((!!R3u:>*+ ) 	rV   z;Only Gloo and NCCL backends will have CUDA allReduce testedc                 H   t         j                  j                  | j                         | j	                         \  }}}t        t        j                         t              }| j                  |||t        j                  j                  ddddt        |      dz
  z  z   d|	       y )NrG   rB   rE   TrP   rR  r%  r   r  r"   r  r   r   r  rx  ry  r  rS   r  r  r   rG  s        rU   test_all_reduce_sum_cudaz6DistributedTest._DistTestBase.test_all_reduce_sum_cuda
  s     JJ!!$)),$($:$:$<!E8T.t/B/B/DgNK((!!R3u:>*+
rV   c                 L   t         j                  j                  | j                         | j	                         \  }}}t        t        j                         t              }| j                  |||t        j                  j                  ddddt        |      dz
  z  z   d|d
       y r  r  r  s        rU   test_all_reduce_sum_cuda_asyncz<DistributedTest._DistTestBase.test_all_reduce_sum_cuda_async
  s     JJ!!$)),$($:$:$<!E8T.t/B/B/DgNK((!!R3u:>*+ ) rV   c                    | j                         \  }}}| j                  |||t        j                  j                  t        dd      t        dd      t        dd      t        dd      t        |      dz
  z  z   t        j                         y )NrG   ru   rB      rE   rN  )	r  r  r  rx  ry  complexr  rP   cfloatr  s       rU   test_all_reduce_sum_complexz9DistributedTest._DistTestBase.test_all_reduce_sum_complex
  s|     %)$:$:$<!E8T((!!1B1RCJN!CDll ) 	rV   c                     t         j                  j                  t         j                  j                  t         j                  j                  t         j                  j
                  t         j                  j                  t         j                  j                  g}| j                         \  }}}|D ]Q  }| j                  t        d      5  t        j                  t        dt        j                        ||       d d d        S y # 1 sw Y   ^xY wNzall_reduce does not supportrE   rN  )r  rx  r  r  r  BANDBORBXORr  r  r  r  rU  rP   r  )rS   unsupported_opsr  r  r   unsupported_ops         rU   'test_all_reduce_complex_unsupported_opszEDistributedTest._DistTestBase.test_all_reduce_complex_unsupported_ops
  s    
 !!!!%%""!!""O %)$:$:$<!E8T"1++ = OO%au||<nh  #2 s   1DD	c                    t         j                  j                  | j                         | j	                         \  }}}t        t        j                         t              }| j                  |||t        j                  j                  t        dd      t        dd      t        dd      t        dd      t        |      dz
  z  z   d|t         j                  
       y )NrG   ru   rB   r  rE   TrN  )rP   rR  r%  r   r  r"   r  r   r   r  rx  ry  r  r  r  r  s        rU    test_all_reduce_sum_cuda_complexz>DistributedTest._DistTestBase.test_all_reduce_sum_cuda_complex
  s     JJ!!$)),$($:$:$<!E8T.t/B/B/DgNK((!!1B1RCJN!CDll ) rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddt        d dgt        |      dz
  z  d             y )NrG   rB   c                     | |z  S rf   rt   r  s     rU   r"  zGDistributedTest._DistTestBase.test_all_reduce_product.<locals>.<lambda>  r  rV   rE   )r  r  r  rx  r  r	   r  r  s       rU   test_all_reduce_productz5DistributedTest._DistTestBase.test_all_reduce_product
  s_     %)$:$:$<!E8T((%%*bTSZ!^-DaHrV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  r  r  rx  r  r  s       rU   test_all_reduce_minz1DistributedTest._DistTestBase.test_all_reduce_min  s@     %)$:$:$<!E8T((xt}}'8'8$1rV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  r  r  rx  r  r  s       rU   test_all_reduce_maxz1DistributedTest._DistTestBase.test_all_reduce_max  s@     %)$:$:$<!E8T((xt}}'8'8"b"rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddddt        |      dz
  z  z          y rw  )r  r  r  rx  ry  r  r  s       rU   test_all_reduce_group_sumz7DistributedTest._DistTestBase.test_all_reduce_group_sum  sX    
 %)$9$9$;!E8T((!!R3u:>*+rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddt        d dgt        |      dz
  z  d             y )NrG   rB   c                     | |z  S rf   rt   r  s     rU   r"  zMDistributedTest._DistTestBase.test_all_reduce_group_product.<locals>.<lambda>7  r  rV   rE   )r  r  r  rx  r  r	   r  r  s       rU   test_all_reduce_group_productz;DistributedTest._DistTestBase.test_all_reduce_group_product*  s_    
 %)$9$9$;!E8T((%%*bTSZ!^-DaHrV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  r  r  rx  r  r  s       rU   test_all_reduce_group_minz7DistributedTest._DistTestBase.test_all_reduce_group_min:  s@    
 %)$9$9$;!E8T((xt}}'8'8$1rV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  r  r  rx  r  r  s       rU   test_all_reduce_group_maxz7DistributedTest._DistTestBase.test_all_reduce_group_maxD  s@    
 %)$9$9$;!E8T((xt}}'8'8"b"rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddddt        |      dz
  z  z          y rw  )r  r  r  rx  ry  r  r  s       rU   test_all_reduce_full_group_sumz<DistributedTest._DistTestBase.test_all_reduce_full_group_sumN  sX     %)$>$>$@!E8T((!!R3u:>*+rV   c                     | j                         \  }}}| j                  |||t        j                  j                  ddt        d dgt        |      dz
  z  d             y )NrG   rB   c                     | |z  S rf   rt   r  s     rU   r"  zRDistributedTest._DistTestBase.test_all_reduce_full_group_product.<locals>.<lambda>i  r  rV   rE   )r  r  r  rx  r  r	   r  r  s       rU   "test_all_reduce_full_group_productz@DistributedTest._DistTestBase.test_all_reduce_full_group_product]  s_     %)$>$>$@!E8T((%%*bTSZ!^-DaHrV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  r  r  rx  r  r  s       rU   test_all_reduce_full_group_minz<DistributedTest._DistTestBase.test_all_reduce_full_group_minl  s@     %)$>$>$@!E8T((xt}}'8'8$1rV   c           	          | j                         \  }}}| j                  |||t        j                  j                  ddd       y r  )r  r  r  rx  r  r  s       rU   test_all_reduce_full_group_maxz<DistributedTest._DistTestBase.test_all_reduce_full_group_maxu  s@     %)$>$>$@!E8T((xt}}'8'8"b"rV   c                 H   | j                         \  }}}t        |t        j                         d      }|D ]d  \  }}|D cg c]
  } ||       }	}t        j                  |	d   t        j
                  j                  |       | j                  |	d   |d          f y c c}w )NrE   )
num_inputsr   )r  r%   r  r   r  rx  ry  r  )
rS   fnr  r  r   testsinputsoutputsinputr  s
             rU   _test_sparse_all_reduce_sumz9DistributedTest._DistTestBase._test_sparse_all_reduce_sum  s    $($:$:$<!E8T.d))+E &+!289&2e9&9
DMM,=,=xH  WQZ8 &+9s   Bz+Only Gloo backend support sparse all reducec                 (    | j                  d        y )Nc                     | S rf   rt   ts    rU   r"  zJDistributedTest._DistTestBase.test_sparse_all_reduce_sum.<locals>.<lambda>  s    qrV   r  r  s    rU   test_sparse_all_reduce_sumz8DistributedTest._DistTestBase.test_sparse_all_reduce_sum  s     ,,[9rV   c                 (    | j                  d        y )Nc                 >    | j                         j                         S rf   )clonerR  r  s    rU   r"  zODistributedTest._DistTestBase.test_sparse_all_reduce_sum_cuda.<locals>.<lambda>  s    qwwy~~7GrV   r  r  s    rU   test_sparse_all_reduce_sum_cudaz=DistributedTest._DistTestBase.test_sparse_all_reduce_sum_cuda  s    
 ,,-GHrV   c           	          ddt        dd      gddt        dd      gdd| dz
  z  z   dd| dz
  z  z   t        dd      t        dd      | dz
  z  z   gt        j                  t        j                  t        j                  gfS )NrG   ru   rB   r  rE   )r  rP   rJ  r  
group_sizes    rU   $_all_reduce_coalesced_sum_test_caseszBDistributedTest._DistTestBase._all_reduce_coalesced_sum_test_cases  s     Awq!}%RR)j1n--j1n--AqMGBOzA~$FF
 ekk5<<8	 	rV   c                 z    ddgddgdd| dz
  z  z  dd| dz
  z  z  gt         j                  t         j                  gfS )NrE   rG   ru   rv   rP   rJ  r  s    rU   (_all_reduce_coalesced_product_test_caseszFDistributedTest._DistTestBase._all_reduce_coalesced_product_test_cases  sP     AAQ:>**Aj1n0E,EFekk*	 rV   c                 V    ddgddgddgt         j                  t         j                  gfS NrE   rv   rG   ru   r#  r  s    rU   $_all_reduce_coalesced_min_test_caseszBDistributedTest._DistTestBase._all_reduce_coalesced_min_test_cases  3     AAAekk*	 rV   c                 V    ddgddgddgt         j                  t         j                  gfS r&  r#  r  s    rU   $_all_reduce_coalesced_max_test_caseszBDistributedTest._DistTestBase._all_reduce_coalesced_max_test_cases  r(  rV   c                    | j                         \  }}}| j                  t        d      5  t        j                  t        dt        j                        gt        j                  j                  |       d d d        y # 1 sw Y   y xY wr  )
r  r  r  r  all_reduce_coalescedrU  rP   r  rx  r  r  s       rU   1test_all_reduce_coalesced_max_complex_unsupportedzODistributedTest._DistTestBase.test_all_reduce_coalesced_max_complex_unsupported  sd     %)$:$:$<!E8T''
4QR))"1ELL9:DMM<M<Mx SRRs   A
A>>Bc           
         t         j                  j                  | j                  t         j                  j                  | j
                  t         j                  j                  | j                  t         j                  j                  | j                  i|   } |t        |            \  }}	}
}|D ]4  }||k(  r|n|	}t        ||      D cg c]  \  }}t        |dz   ||       }}}|r$|D cg c]  }|j                  ||   d          }}g }|D ]i  }|j                  t        j                   k(  r/|j#                  t        j$                  |      j&                         O|j#                  |j&                         k | j)                  ddt         j*                  ||||       t        ||
      D cg c]  \  }}t        |dz   ||       }}}| j-                  ||       7 | j/                          y c c}}w c c}w c c}}w )NrE   rN  r   r  Frn  )r  rx  ry  r!  r  r$  r  r'  r  r*  r  r  rU  rR  rO  rP   r  r  r  rp  rR  r,  r  r  )rS   r  r  r   rq  rR  rG  test_case_funcmaster_valuesworker_valuesexpected_valuesdtypesr"  curr_valuesrO  r  r  r  ro  r   rt  r  s                         rU   !_test_all_reduce_coalesced_helperz?DistributedTest._DistTestBase._test_all_reduce_coalesced_helper  s    !!4#L#L%%t'T'T!!4#L#L!!4#L#L	
 N ESE
EAM=/6 /3s{m '*&+&>&>
s "#'3e<&>   ELMWqvvk$&7&:;WGM "%F||u6%,,U-?-?-G-M-MN%,,V\\:	 &
 !!!--"/ "  25V_1M$1M-~ "#'>G1M ! $   *:;7 : MMO7
 N $s   
G7.G=3Hc                     | j                         \  }}}| j                  |||t        j                  j                  dd        y NF)rR  rG  )r  r5  r  rx  ry  r  s       rU   test_all_reduce_coalesced_sumz;DistributedTest._DistTestBase.test_all_reduce_coalesced_sum  F    $($:$:$<!E8T22!!  3 rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  r  r  s       rU   !test_all_reduce_coalesced_productz?DistributedTest._DistTestBase.test_all_reduce_coalesced_product	  sF    $($:$:$<!E8T22%%  3 rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  r  r  s       rU   test_all_reduce_coalesced_minz;DistributedTest._DistTestBase.test_all_reduce_coalesced_min  r9  rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  r  r  s       rU   test_all_reduce_coalesced_maxz;DistributedTest._DistTestBase.test_all_reduce_coalesced_max!  sB    $($:$:$<!E8T22xt}}'8'8uRV 3 rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  ry  r  s       rU   #test_all_reduce_coalesced_group_sumzADistributedTest._DistTestBase.test_all_reduce_coalesced_group_sum(  D     %)$9$9$;!E8T22xt}}'8'8uRV 3 rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  r  r  s       rU   'test_all_reduce_coalesced_group_productzEDistributedTest._DistTestBase.test_all_reduce_coalesced_group_product0  sH     %)$9$9$;!E8T22%%  3 rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  r  r  s       rU   #test_all_reduce_coalesced_group_minzADistributedTest._DistTestBase.test_all_reduce_coalesced_group_min=  rB  rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  r  r  s       rU   #test_all_reduce_coalesced_group_maxzADistributedTest._DistTestBase.test_all_reduce_coalesced_group_maxE  rB  rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  ry  r  s       rU   (test_all_reduce_coalesced_full_group_sumzFDistributedTest._DistTestBase.test_all_reduce_coalesced_full_group_sumM  B    $($>$>$@!E8T22xt}}'8'8uRV 3 rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  r  r  s       rU   ,test_all_reduce_coalesced_full_group_productzJDistributedTest._DistTestBase.test_all_reduce_coalesced_full_group_productT  sF    $($>$>$@!E8T22%%  3 rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  r  r  s       rU   (test_all_reduce_coalesced_full_group_minzFDistributedTest._DistTestBase.test_all_reduce_coalesced_full_group_min`  sF    $($>$>$@!E8T22!!  3 rV   c                     | j                         \  }}}| j                  |||t        j                  j                  dd        y r7  )r  r5  r  rx  r  r  s       rU   (test_all_reduce_coalesced_full_group_maxzFDistributedTest._DistTestBase.test_all_reduce_coalesced_full_group_maxl  rK  rV   c                    |D ]  }t        |dz   d|      }t        |dz   ||      }	||k(  r|D 
cg c]  }
t        |dz   |
|       c}
ng }|r;|j                  ||   d         }|D cg c]  }|j                  ||   d          }}|t        j                  k(  r-|D cg c]!  }t        j                  |      j
                  # }}n|D cg c]  }|j
                   }}| j                  ddt        j                  ||||d|	       | j                  ||	        | j                          y c c}
w c c}w c c}w c c}w )NrE   r   rN  r   z:scatterF)r"  scatter_listr  r  ro  )rU  rR  rP   r  r  rp  rR  r  scatterr  r  )rS   r  r  r   rR  rG  rO  r8  r   r  r  r  r  ro  s                 rU   _test_scatter_helperz2DistributedTest._DistTestBase._test_scatter_helpert  sT    &tax5A"/q$e"L t| GLLe]4!8Qe<eL 
 #[[T):1)=>FELMWqvvk$&7&:;WGMEOO+JQ$R'QU%7%7%:%@%@'M$R6=$>gQWWgM$>!!LL!("!&"/ " 
   95 8 MMO1 M N$R$>s   D3,D8#&D=Ez&CPU tensor ops not supported by UCP TLc                 Z   | j                         \  }}}t        j                  dg      }|j                         dz  }|dk(  r9|D cg c]  }|j                         |z   }}t	        j
                  |d|       nt	        j
                  |d       | j                  |||z         |j                         dz  }|dk(  r8|D cg c]  }|j                         |z   }}t	        j
                  ||       nt	        j
                  |       | j                  |||z         y c c}w c c}w )NrE   r   r   )r"  rS  r"  )rS  )r  rP   r   r  r  rT  r  )rS   r  r  r   oneoutputr  rS  s           rU   test_scatter_checksz1DistributedTest._DistTestBase.test_scatter_checks  s    %)$:$:$<!E8T**aS/C YY[2%Fqy9>?A		a?VFV+VS4Z0 YY[2%Fqy9>?A		a?V,?V$VS4Z0  @  @s   D#D(c                 R    | j                         \  }}}| j                  |||       y rf   )r  rU  r  s       rU   test_scatterz*DistributedTest._DistTestBase.test_scatter  s+     %)$:$:$<!E8T%%eXt<rV   zOnly Nccl supports CUDA gatherc                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y NT)r  r"   r  r   r   rU  r  s        rU   test_scatter_cudaz/DistributedTest._DistTestBase.test_scatter_cuda  sD    
 %)$:$:$<!E8T.t/B/B/DgNK%%eXtT;OrV   c                 r    | j                         \  }}}| j                  |||t        j                         y rM  )r  rU  rP   r  r  s       rU   test_scatter_complexz2DistributedTest._DistTestBase.test_scatter_complex  s4     %)$:$:$<!E8T%%eXt5<<%PrV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|t        j                         y NTrN  )r  r"   r  r   r   rU  rP   r  r  s        rU   test_scatter_cuda_complexz7DistributedTest._DistTestBase.test_scatter_cuda_complex  sR    
 %)$:$:$<!E8T.t/B/B/DgNK%%xt[ & rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  rU  r  s       rU   test_scatter_groupz0DistributedTest._DistTestBase.test_scatter_group  s+     %)$9$9$;!E8T%%eXt<rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  rU  r  s       rU   test_scatter_full_groupz5DistributedTest._DistTestBase.test_scatter_full_group  s+     %)$>$>$@!E8T%%eXt<rV   c                 P   |D ]  }t        |dz   |      }||k(  r|D cg c]  }t        |dz   d       c}ng }	|r;|j                  ||   d         }|	D 
cg c]  }
|
j                  ||   d          }	}
| j                  ddt        j                  |||	|dt        |	      dkD  r|	d   j                  gnd 	       ||k(  s|D cg c]  }t        |dz   |       }}t        |	|      D ]  \  }}| j                  ||         | j                          y c c}w c c}
w c c}w )NrE   r   r   z:gatherF)r  gather_listr  r  ro  )
rU  rR  rR  r  gatherr  rp  r  r  r  )rS   r  r  r   rR  rG  r8  r   r  r  r  r  t1t2s                 rU   _test_gather_helperz1DistributedTest._DistTestBase._test_gather_helper  sF    &tax6DHDL%@%Q]4!8R0%@VX  #[[T):1)=>FELMWqvvk$&7&:;WGM!!KK '"!&8;Gq8H71:#3#3"4d " 
 4<LQ'REqdQh(BE$'R"%g/?"@B((R0 #A+ 0 MMO+ A N (Ss   DDD#c                 B   | j                         \  }}}t        j                  dg      }|dk(  rX|D cg c]  }|j                          }}t	        j
                  ||z  d|       |D ]  }| j                  ||   ||z          nt	        j
                  ||z  d       |dk(  rW|D cg c]  }|j                          }}t	        j
                  ||z  |       |D ]  }| j                  ||   ||z          y t	        j
                  ||z         y c c}w c c}w )NrE   r   )r  rj  r  )rj  )r  rP   r   r  r  rk  r  )rS   r  r  r   rX  rY  rj  r  s           rU   test_gather_checksz0DistributedTest._DistTestBase.test_gather_checks  s    %)$:$:$<!E8T**aS/C qy49:Eqsyy{E:C$JA;GA$$[^S1W=  C$JA. qy49:Eqsyy{E:C$JK@A$$[^S1W=  C$J' ; ;s   D+Dc                 R    | j                         \  }}}| j                  |||       y rf   )r  rn  r  s       rU   test_gatherz)DistributedTest._DistTestBase.test_gather(  s+     %)$:$:$<!E8T$$UHd;rV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   rn  r  s        rU   test_gather_cudaz.DistributedTest._DistTestBase.test_gather_cuda2  sD    
 %)$:$:$<!E8T.t/B/B/DgNK$$UHdD+NrV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  rn  r  s       rU   test_gather_groupz/DistributedTest._DistTestBase.test_gather_group;  s+     %)$9$9$;!E8T$$UHd;rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  rn  r  s       rU   test_gather_full_groupz4DistributedTest._DistTestBase.test_gather_full_groupF  s+     %)$>$>$@!E8T$$UHd;rV   c                    |D ]0  }t        |dz   ||      }|D 	cg c]  }	t        |dz   d|       }
}	t        j                  }|r;|j                  ||   d         }|
D cg c]  }|j                  ||   d          }
}|
d   j                  t
        j                  k(  r$t        j                  |
d         j                  g}n|
d   j                  g}| j                  dd||
||d|       |D 	cg c]  }	t        |dz   |	|       }}	t        |
|      D ]  \  }}| j                  ||        3 | j                          y c c}	w c c}w c c}	w )NrE   rN  r   r   :all_gatherFrn  )rU  r  r  rR  rO  rP   r  r  rp  rR  r  r  r  )rS   r  r  r   rR  rG  rO  r8  r   r  r  	allgatherr  ro  r  rl  rm  s                    rU   _test_all_gather_helperz5DistributedTest._DistTestBase._test_all_gather_helperQ  sm    &taxUCMRSU=2UCUS OO	#[[T):1)=>FELMWqvvk$&7&:;WGM1:##u6%*%7%7
%C%I%I$JM%,QZ%5%5$6M!!!"/ " 	 FK$EJM$(AU;U ! $ "'+;<FB$$R, =3 8 MMO5 T N $s   E#E4Ec                 R    | j                         \  }}}| j                  |||       y rf   )r  r}  r  s       rU   test_all_gatherz-DistributedTest._DistTestBase.test_all_gatherr  +     %)$:$:$<!E8T(($?rV   z"Only Nccl supports CUDA all gatherc                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r}  r  s        rU   test_all_gather_cudaz2DistributedTest._DistTestBase.test_all_gather_cuday  D    
 %)$:$:$<!E8T.t/B/B/DgNK(($kRrV   c                 r    | j                         \  }}}| j                  |||t        j                         y rM  )r  r}  rP   r  r  s       rU   test_all_gather_complexz5DistributedTest._DistTestBase.test_all_gather_complex  4     %)$:$:$<!E8T(($ell(SrV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|t        j                         y rc  )r  r"   r  r   r   r}  rP   r  r  s        rU   test_all_gather_cuda_complexz:DistributedTest._DistTestBase.test_all_gather_cuda_complex  R    
 %)$:$:$<!E8T.t/B/B/DgNK((xt[ ) rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r}  r  s       rU   test_all_gather_groupz3DistributedTest._DistTestBase.test_all_gather_group  +    
 %)$9$9$;!E8T(($?rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r}  r  s       rU   test_all_gather_full_groupz8DistributedTest._DistTestBase.test_all_gather_full_group  +     %)$>$>$@!E8T(($?rV   zOnly Nccl supports all_gather_vc           	         | j                          | j                         \  }}}t        t        j                         t
              }||   d   }g }|D ]  }|j                  |dz           t        |      }d}	dD ]  }
t        j                  ||   ||t        j                        j                  |	      j                  |      }t        |d|      }t        j                  t        t        j                   ||            |||
      }|
r|j#                          |	}t        |||      }| j%                  ||        | j                          y )Nr   rE   rG   r  rN  r   r  )r  r  r"   r  r   r   r  r   rP   rP  rJ  rQ  rR  rU  r  r  r  r  r  )rS   r  r  r   rG  rT  output_split_sizesr  r  rl   r  r   r  r  rt  r  s                   rU   test_all_gather_v_cudaz4DistributedTest._DistTestBase.test_all_gather_v_cuda  s5   
 MMO$($:$:$<!E8T.t/B/B/DgNK#D)!,I!#"))#'2 ,-GE*	KK*40'7%++ U5\T)_  +7B)L
ooZ1CDE	 HHJ!&"/^y#   _=1 +2 MMOrV   c                 L   |r.|j                  ||   d         }|j                  ||   d         }|j                  t        j                  k(  r!t        j                  |      j
                  g}n|j
                  g}| j                  ddt        j                  |||dd|	       |S )Nr   z:all_gather_into_tensorFr  )	rR  rO  rP   r  r  rp  rR  r  all_gather_into_tensorr  s           rU   _all_gather_into_tensor_helperz<DistributedTest._DistTestBase._all_gather_into_tensor_helper  s     %NN;t+<Q+?@	'__[->q-AB
5??2!&!3!3I!>!D!D E!* 1)++"+  
 rV   z.Only Nccl supports CUDA all_gather_into_tensorc           	         | j                         \  }}}t        t        j                         t              }d}t        j                  ||g      |z  }t        j                  t        |      |z  |g      dz  }| j                  ||||d|      }t        j                  |D cg c]  }t        j                  ||g      |z   c}      }	| j                  ||	       | j                          y c c}w NrG   r   T)r  r"   r  r   r   rP   r   r  r  r  r  r  
rS   r  r  r   rG  rS  r  r  r  r  s
             rU   $test_all_gather_into_cat_tensor_cudazBDistributedTest._DistTestBase.test_all_gather_into_cat_tensor_cuda  s    
 %)$:$:$<!E8T.t/B/B/DgNKD

D$<047ISZ$%6$=>"EJ<<Ixt[J $iiu(Uu!T4L)AA)Eu(UVOZ9MMO )Vs   "!C-c           	         | j                         \  }}}t        t        j                         t              }d}t        j                  ||g      |z  }t        j                  t        |      ||g      dz  }| j                  ||||d|      }t        j                  |D cg c]  }t        j                  ||g      |z   c}      }	| j                  ||	       | j                          y c c}w r  )r  r"   r  r   r   rP   r   r  r  stackr  r  r  s
             rU   &test_all_gather_into_stack_tensor_cudazDDistributedTest._DistTestBase.test_all_gather_into_stack_tensor_cuda  s    
 %)$:$:$<!E8T.t/B/B/DgNKD

D$<047ISZt$<=DJ<<Ixt[J $kkQV*WQVA5::tTl+Ca+GQV*WXOZ9MMO +Xs    !C+c           	         g }|D ]i  }|j                   t        j                  k(  r/|j                  t        j                  |      j
                         O|j                  |j
                         k | j                  ddt        j                  ||||       t        ||      D ]2  \  }}t        ||      D ]  \  }	}
t        j                  |	|
      r  y 4 y)z|
            Helper that runs all_gather_coalesced and returns true if output
            matches expectations.
            r{  Frn  T)rO  rP   r  r  r  rp  rR  r  all_gather_coalescedr  rk   )rS   output_tensor_listsinput_tensorsr  r  ro  input_tensorl1l2rl  rm  s              rU   $_run_all_gather_coalesced_and_verifyzBDistributedTest._DistTestBase._run_all_gather_coalesced_and_verify  s     M -%%8!((););L)I)O)OP!((););<	 !.
 ))#+   13CDB!"bkFB ;;r2.$ * E rV   c                    |t        dd      D ]  }t        d|      D cg c]  }t        ||||z   |       }}|D cg c])  }t        d|      D cg c]  }t        ||d|       c}+ }	}}|D 
cg c],  }
t        d|      D cg c]  }t        |||
|z   |       c}. }}
}| j                  |	|||      rJ d        | j                          y c c}w c c}w c c}}w c c}w c c}}
w )NrG   r}   rE   rN  r   z+output tensors do not match expected ouputs)rW  rZ  r  r  )rS   r  r  r   rO  test_case_id	tensor_idr  rY  r  	rank_iterr  s               rU   !_test_all_gather_coalesced_helperz?DistributedTest._DistTestBase._test_all_gather_coalesced_helper-  sf   
 #$)!QKL */q,)?	% *@I /%y$2B% *@	 " % "'+ "'A .31l-C	 .D	 3 )9b .D	 "' ( +  */( */I .31l-C	 .D	 3 )9i)6KSX .D	 */ % (  DD+]<Lh EDE 7 %0> MMO9%+(s5   C C"C)C"6C-	C("C-C"(C-allgather_coalescedz& does not support all_gather_coalescedc                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU    test_all_gather_coalesced_simplez>DistributedTest._DistTestBase.test_all_gather_coalesced_simpleT  s+    
 %)$:$:$<!E8T225(DIrV   c                 r    | j                         \  }}}| j                  |||t        j                         y rM  )r  r  rP   r  r  s       rU   !test_all_gather_coalesced_complexz?DistributedTest._DistTestBase.test_all_gather_coalesced_complex\  s9    
 %)$:$:$<!E8T22xU\\ 3 rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   test_all_gather_coalesced_groupz=DistributedTest._DistTestBase.test_all_gather_coalesced_groupf  s+     %)$9$9$;!E8T225(DIrV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   $test_all_gather_coalesced_full_groupzBDistributedTest._DistTestBase.test_all_gather_coalesced_full_groupo  s+    
 %)$>$>$@!E8T225(DIrV   c                    | j                         \  }}}|t        j                  ddg      z  t        j                  dg      |dz   t        j                  ddg      z  t        j                  dg      t        j                  dg      g}|D cg c]~  }dt        j                  ddg      z  dt        j                  dg      z  dt        j                  ddg      z  dt        j                  dg      z  dt        j                  dg      z  g }}|D cg c]x  }|t        j                  ddg      z  t        j                  dg      |dz   t        j                  ddg      z  t        j                  dg      t        j                  dg      gz }}| j                  ||||      sJ | j	                          y c c}w c c}w )NrG   r   rE   ru   r   )r  rP   r   r  r  )	rS   r  r  r   r  rY  output_tensors_listsr=  r  s	            rU   $test_all_gather_coalesced_with_emptyzBDistributedTest._DistTestBase.test_all_gather_coalesced_with_emptyw  s   
 %)$:$:$<!E8Tuzz1a&))

A3UZZA//

A3

A3M 	$ A QF++QC(QF++QC(QC(  ! 	$$ 	  A 

Aq6**JJsOUejj!Q00JJsOJJsO   	  <<$m5Ex   MMO/	$	 s   BF>A=Gc           	         |@t        |      }t        j                  ||g|      |z  }t        j                  |D 	cg c]  }	t        j                  d|g|      |	z    c}	      }
t        j                  ||g|      dz  }|rE|j	                  ||   d         }|
j	                  ||   d         }
|j	                  ||   d         }|t        j
                  k(  r!t        j                  |      j                  g}n|j                  g}| j                  ddt        j                  ||||       | j                  ||
       | j                          y c c}	w )NrN  rE   r   r   z:all_to_allF)r  ro  )r  rP   r   r  rR  r  r  rp  rR  r  all_to_all_singler  r  )rS   r  r  r   rR  rG  rO  rS  	in_tensorr  r  r  ro  s                rU   *_test_all_to_all_single_equal_split_helperzHDistributedTest._DistTestBase._test_all_to_all_single_equal_split_helper  sH    #5z!JJd|5ADH	"'))EJKUUZZD	7!;UK# #ZZtEBRG
 ){4/@/C DI&5&:&:;t;LQ;O&PO!+T1B11E!FJEOO+%*%7%7	%B%H%H$IM%.__$5M!!!**""/ "    _=MMO+ Ls   #Ec           	         |&t        |      }|D cg c]  }|dz   	 }	}|D 
cg c]  }
|dz   	 }}
t        j                  t        |	      |g|      |z  }t        j                  |dz   |z  |g|      }t        j                  |D cg c]!  }t        j                  |dz   |g|      |z  # c}      }|rE|j                  ||   d         }|j                  ||   d         }|j                  ||   d         }t        j                  ||||	|       | j                  ||       | j                          y c c}w c c}
w c c}w NrE   rN  r   r  )
r  rP   r   r   r  rR  r  r  r  r  )rS   r  r  r   rR  rG  rO  rS  r  	in_splitsrY  
out_splitsr  r  r  s                  rU   ,_test_all_to_all_single_unequal_split_helperzJDistributedTest._DistTestBase._test_all_to_all_single_unequal_split_helper  sI    #5z,12EqQUE	20561dQh
6!JJI'=UKdR	"ZZ$(d):D(AO
"'))LQREqUZZ4 0>BER#  ){4/@/C DI&5&:&:;t;LQ;O&PO!+T1B11E!FJ&&	:y   _=MMO 36 Ss   D;E 
&Ec                 "   |Xt        |      }|D cg c]  }|dz   	 }	}t        |      D 
cg c]$  \  }}
t        j                  |	|   |g|      |z  & }}}
|D 
cg c]  }
t        j                  |dz   |g|        }}
|D cg c]!  }t        j                  |dz   |g|      |z  # }}|rl|D cg c]  }|j	                  ||   d          }}|D cg c]  }|j	                  ||   d          }}|D cg c]  }|j	                  ||   d          }}t        j                  |||       t        ||      D ]  \  }}| j                  ||        | j                          y c c}w c c}
}w c c}
w c c}w c c}w c c}w c c}w r  )
r  	enumeraterP   r   rR  r  
all_to_allr  r  r  )rS   r  r  r   rR  rG  rO  rS  r  r  rY  
in_tensorsout_tensorsr  r  rl  rm  s                    rU   _test_all_to_all_helperz5DistributedTest._DistTestBase._test_all_to_all_helper  s    #5z,12EqQUE	2 !*% 0 01 JJ	!d35ADH 0  
 JOINAEJJD1?   LQ$KPaEJJq$/u=A5 ! $ HR!S
1!&&T):1)=">
J!S>N(>N{4034>N % ( JU"UA166+d*;A*>#?K"UZxH!+/?@FB$$R, AMMO) 3$ "T( #Vs(   E-)E2#E8&E=6FF>Fmpiz'Only MPI supports CPU all_to_all_singlec                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   "test_all_to_all_single_equal_splitz@DistributedTest._DistTestBase.test_all_to_all_single_equal_split  s+     %)$:$:$<!E8T;;E8TRrV   z)Only Nccl supports CUDA all_to_all_singlec                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   'test_all_to_all_single_equal_split_cudazEDistributedTest._DistTestBase.test_all_to_all_single_equal_split_cuda  sK    
 %)$:$:$<!E8T.t/B/B/DgNK;;rV   c                 r    | j                         \  }}}| j                  |||t        j                         y rM  )r  r  rP   r  r  s       rU   *test_all_to_all_single_equal_split_complexzHDistributedTest._DistTestBase.test_all_to_all_single_equal_split_complex	  s9     %)$:$:$<!E8T;;xU\\ < rV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|t        j                         y rc  )r  r"   r  r   r   r  rP   r  r  s        rU   /test_all_to_all_single_equal_split_cuda_complexzMDistributedTest._DistTestBase.test_all_to_all_single_equal_split_cuda_complex  sR    
 %)$:$:$<!E8T.t/B/B/DgNK;;xt[ < rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   $test_all_to_all_single_unequal_splitzBDistributedTest._DistTestBase.test_all_to_all_single_unequal_split  s+     %)$:$:$<!E8T==eXtTrV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  r  r"   r  r   r   r  r  s        rU   )test_all_to_all_single_unequal_split_cudazGDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_cuda$  sK    
 %)$:$:$<!E8T.t/B/B/DgNK==rV   c                 r    | j                         \  }}}| j                  |||t        j                         y rM  )r  r  rP   r  r  s       rU   ,test_all_to_all_single_unequal_split_complexzJDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_complex3  s9     %)$:$:$<!E8T==xU\\ > rV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|t        j                         y rc  )r  r"   r  r   r   r  rP   r  r  s        rU   1test_all_to_all_single_unequal_split_cuda_complexzODistributedTest._DistTestBase.test_all_to_all_single_unequal_split_cuda_complex<  sW    
 %)$:$:$<!E8T.t/B/B/DgNK==ll > rV   zOnly MPI supports all_to_allc                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   test_all_to_allz-DistributedTest._DistTestBase.test_all_to_allL  r  rV   z"Only NCCL supports CUDA all_to_allc                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   test_all_to_all_cudaz2DistributedTest._DistTestBase.test_all_to_all_cudaS  r  rV   c                 r    | j                         \  }}}| j                  |||t        j                         y rM  )r  r  rP   r  r  s       rU   test_all_to_all_complexz5DistributedTest._DistTestBase.test_all_to_all_complex\  r  rV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|t        j                         y rc  )r  r"   r  r   r   r  rP   r  r  s        rU   test_all_to_all_cuda_complexz:DistributedTest._DistTestBase.test_all_to_all_cuda_complexc  r  rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   (test_all_to_all_single_equal_split_groupzFDistributedTest._DistTestBase.test_all_to_all_single_equal_split_groupn  s+    
 %)$9$9$;!E8T;;E8TRrV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   -test_all_to_all_single_equal_split_group_cudazKDistributedTest._DistTestBase.test_all_to_all_single_equal_split_group_cudav  sK     %)$9$9$;!E8T.t/B/B/DgNK;;rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   *test_all_to_all_single_unequal_split_groupzHDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_group  s+    
 %)$9$9$;!E8T==eXtTrV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  r  r  s        rU   /test_all_to_all_single_unequal_split_group_cudazMDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_group_cuda  sK     %)$:$:$<!E8T.t/B/B/DgNK==rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   test_all_to_all_groupz3DistributedTest._DistTestBase.test_all_to_all_group  r  rV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   test_all_to_all_group_cudaz8DistributedTest._DistTestBase.test_all_to_all_group_cuda  sD     %)$9$9$;!E8T.t/B/B/DgNK(($kRrV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   -test_all_to_all_single_equal_split_full_groupzKDistributedTest._DistTestBase.test_all_to_all_single_equal_split_full_group  s+     %)$>$>$@!E8T;;E8TRrV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   2test_all_to_all_single_equal_split_full_group_cudazPDistributedTest._DistTestBase.test_all_to_all_single_equal_split_full_group_cuda  sK    
 %)$>$>$@!E8T.t/B/B/DgNK;;rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   /test_all_to_all_single_unequal_split_full_groupzMDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_full_group  s+     %)$>$>$@!E8T==eXtTrV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   4test_all_to_all_single_unequal_split_full_group_cudazRDistributedTest._DistTestBase.test_all_to_all_single_unequal_split_full_group_cuda  sK    
 %)$>$>$@!E8T.t/B/B/DgNK==rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   test_all_to_all_full_groupz8DistributedTest._DistTestBase.test_all_to_all_full_group  r  rV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   test_all_to_all_full_group_cudaz=DistributedTest._DistTestBase.test_all_to_all_full_group_cuda  sD    
 %)$>$>$@!E8T.t/B/B/DgNK(($kRrV   c                 t   d}|D ]  }t        j                  d      j                  d      }|r|j                  ||   d         }||k(  rk|j                  t	        j                         |z          t        j                  |||       t	        j                  |dz          t        j                  |       t        j                  |||       t        j                  |       | j                  t        t	        j                               t        |d         d||fz  dz          ! | j                  d	       y )
N333333?rE           r   rr  z!destination rank: %d, my rank: %dz3 (if you see this failure, please report in #14554)r   r  )rP   DoubleTensorrQ  rR  ru  r  r&  rx  ri  r  rJ  r  )	rS   r  r  r   rR  rG  	WAIT_TIMEr8  r  s	            rU   _test_barrier_helperz2DistributedTest._DistTestBase._test_barrier_helper  s    I % 2 21 5 ; ;C @$1$6$6{47H7K$LM4<!''		i(?@NN=$AJJy3/LL*NN=$ALL*11diik*mA./;tTlJOP 0 MM"M%rV   z MPI doesn't supports GPU barrierzSkipped internallyc                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   test_barrier_cudaz/DistributedTest._DistTestBase.test_barrier_cuda  sD     %)$:$:$<!E8T.t/B/B/DgNK%%eXtT;OrV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   test_barrier_group_cudaz5DistributedTest._DistTestBase.test_barrier_group_cuda  sD     %)$9$9$;!E8T.t/B/B/DgNK%%eXtT;OrV   c                     | j                         \  }}}t        t        j                         t              }| j                  |||d|       y r^  )r  r"   r  r   r   r  r  s        rU   test_barrier_full_group_cudaz:DistributedTest._DistTestBase.test_barrier_full_group_cuda"  sD     %)$>$>$@!E8T.t/B/B/DgNK%%eXtT;OrV   zcpu barrierz does not support CPU barrierc                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   test_barrierz*DistributedTest._DistTestBase.test_barrier,  s+    
 %)$:$:$<!E8T%%eXt<rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   test_barrier_groupz0DistributedTest._DistTestBase.test_barrier_group4  s+     %)$9$9$;!E8T%%eXt<rV   c                 R    | j                         \  }}}| j                  |||       y rf   )r  r  r  s       rU   test_barrier_full_groupz5DistributedTest._DistTestBase.test_barrier_full_group=  s+    
 %)$>$>$@!E8T%%eXt<rV   c                     |j                         D ]B  }|j                  t        j                         5  ||j                  z  }d d d        d |_        D y # 1 sw Y   xY wrf   )r]  rq  rP   no_gradrS   r   rs  s      rU   _model_stepz)DistributedTest._DistTestBase._model_stepE  sH    ))+::)+ )!%EJ	 ,(s   AA 	c                 "   |j                         D ]p  }|j                  t        j                         5  ||j                  z  }d d d        |j                  j	                  d       |j                  j                          r y # 1 sw Y   AxY wNF)r]  rq  rP   r  requires_grad_zero_r  s      rU   _model_step_with_zero_gradz8DistributedTest._DistTestBase._model_step_with_zero_gradL  sc    ))+::)+ )JJ--e4JJ$$& ,(s   BB	c                     t        t        j                  d         }||z  }t        j                  |d      }t        j                  |d      }t        j                         }||||fS )Nr'  rG   rv   )r   r)  r*  rP   rQ   rK   MSELoss)rS   local_bsr+  	global_bs	input_cputargetr   s          rU   _prepare_dummy_dataz1DistributedTest._DistTestBase._prepare_dummy_dataT  sY    RZZ56J"X-IIq1I[[A.F::<Di55rV   c                     |j                           ||      } |||      |z  }|j                          |"| j                  |j                  |             y y )Nmemory_format)trainbackwardr  is_contiguous)	rS   r   	input_varr  r   scale_factorr  rY  ls	            rU   _test_DDP_helperz.DistributedTest._DistTestBase._test_DDP_helper^  sV     KKM9%FVV$|3AJJL( 4 4= 4 QR )rV   c                     | j                  t        |      t        |             t        ||      D ]  \  }}| j                  ||        y rf   )r  r  r  )rS   	param_gpu	param_DDPp_gpup_DDPs        rU   _assert_equal_paramz1DistributedTest._DistTestBase._assert_equal_paramh  s?    S^S^< #Iy 9u  . !:rV   c           	      $   t        |      D ]  }| j                  |||||       |
||z  }
| j                  |||
|
|z    ||
|
|z    ||dk7  r||z  |z  nd|       |r#| j                  |       | j                  |       n"| j                  |       | j                  |       | j	                  t        |j                               t        |j                  j                                      |t        j                  |         }|	s|dk(  st        j                  d      st        j                         5 }t        j                  dk(  r=t        j                   ||       |j#                  d       t        j$                  |      }n?t        j                   ||j&                         t        j$                  |j&                        }d d d         t        j(                         5 }t        j                   ||       |j#                  d       t        j$                  |      }d d d        |j+                         D ]6  }| j-                  |j+                         |   j+                         |          8 y # 1 sw Y   wxY w# 1 sw Y   `xY w)Nr  r   rE   rG   r  r=   )rW  r#  r  r  r)  r  r]  r   rP   randpermr   r   rE  NamedTemporaryFiler4  r5  saveseekloadr   TemporaryFile
state_dictr  )rS   
model_base	model_DDPr  r  r   r  r   
batch_size	test_saveoffsetr+  	zero_gradr  n_iteridxtmptmp_filesaved_modelks                       rU   _test_DDP_niterz-DistributedTest._DistTestBase._test_DDP_niterm  s/   " V}%%vt= &  >!H_F %%&6H#456FX$56:D/J)J6q"/ &  33J?33I>$$Z0$$Y/((..014	8H8H8S8S8U3V
 ennZ89 k.D.DY.O!446#<<72!JJy#6HHQK(-

3I!JJy#((;(-

388(<I 76G %X '')X

9h/a #jj2 * ))+  !5!5!7!:K<R<R<TUV<WX , 76 *)s   ;BI9*=J9J	Jc                 j   t         }t        j                  |      }|j                  |d          t        j                  |      }	|	j                  |d          t        j
                  j                  |	|||      }	|r|	j                          t        j                         5 }
t        j                  dk(  r=t        j                  |	|
       |
j                  d       t        j                  |
      }	n?t        j                  |	|
j                          t        j                  |
j                         }	d d d        t#        |      }| j%                  |      \  }}}}| j'                  ||	|j                  |d         |j                  |d         ||||d	       | j)                          y # 1 sw Y   xxY w)Nr   
device_idsgradient_as_bucket_viewstatic_graphr=   T)DDP_NETro  rp  rR  rK   parallelr   _set_static_graphrE  r,  r4  r5  rP   r-  r.  r/  r   r  r  r>  r  )rS   
gpu_subsetr   output_devicerB  rC  set_static_graph_twicer   	model_gpur3  r:  r  r  r  r  r   s                   rU   _test_DistributedDataParallelz;DistributedTest._DistTestBase._test_DistributedDataParallel  sg    E e,INN:a=) e,INN:a=);;%(?)	 < I &++- ,,.#<<7*JJy#.HHQK %

3IJJy#((3 %

388 4I / :H151I1I(1S.Iy&$   z!}-JqM*
 MMO3 /.s   "BF))F2c                 ,   | j                         \  }}}t        }t        j                  |      }t        j
                  j                  ||      }d}| j                  |      \  }}	}
}| j                  |||	|
||||dd
       | j                          |S )NrB  rG   FT)r7  )
r  rD  ro  rp  rK   rE  r   r  r>  r  )rS   rB  r  r  r   r2  r3  r  r  r  r  r   s               rU    _test_DistributedDataParallelCPUz>DistributedTest._DistTestBase._test_DistributedDataParallelCPU  s     %)$:$:$<!E8T !J j1I;;3J < I
 H151I1I(1S.Iy&$    !  MMOrV   z'nccl does not support DDP on CPU modelsc                 $    | j                          y rf   rN  r  s    rU   test_DistributedDataParallelCPUz=DistributedTest._DistTestBase.test_DistributedDataParallelCPU  s     113rV   c                 (    | j                  d       y )NTrM  rP  r  s    rU   ,test_DistributedDataParallelCPU_grad_is_viewzJDistributedTest._DistTestBase.test_DistributedDataParallelCPU_grad_is_view  s     11$1OrV   ddpz1 backend does not support DistributedDataParallelc                 R    | j                  t        d        | j                          y )Nc                  d    t         j                  j                  t        j                               S rf   )rK   rE  r   r   rt   rV   rU   r"  zZDistributedTest._DistTestBase.test_DistributedDataParallel_requires_grad.<locals>.<lambda>#  s    bkk&I&I"))+&VrV   )r  rw  r  r  s    rU   *test_DistributedDataParallel_requires_gradzHDistributedTest._DistTestBase.test_DistributedDataParallel_requires_grad  s#     V MMOrV   r'  c                      G d dt         j                        } |       j                  | j                        }t         j                  j                  || j                  g      }y )Nc                        e Zd Z fdZ xZS )MDistributedTest._DistTestBase.test_ddp_zero_output_features.<locals>.ToyModelc                     t         |           t        j                  dd      | _        t        j
                         | _        t        j                  dd      | _        y )NrB   r   )rI   rJ   rK   rL   net1r   r   net2rR   s    rU   rJ   zVDistributedTest._DistTestBase.test_ddp_zero_output_features.<locals>.ToyModel.__init__.  s?    G$& "		"b 1DI "	DI "		"a 0DIrV   )r^   r_   r`   rJ   ra   rb   s   @rU   ToyModelrZ  -  s    1 1rV   r^  rA  )rK   r   r'  r   rE  r   )rS   r^  r   	ddp_models       rU   test_ddp_zero_output_featuresz;DistributedTest._DistTestBase.test_ddp_zero_output_features'  sL    1299 1 JMM$)),E;;499+ < IrV   zGloo-only testc                 B    G d dt         j                        } |       }t        j                   j                  j	                  |      }t        d      D ]H  } |       j                  d       | j                  t        d |j                         D                     J y )Nc                   $     e Zd Z fdZd Z xZS )BDistributedTest._DistTestBase.test_ddp_create_graph.<locals>.Modelc                 |    t         |           t        j                  t	        j
                  d            | _        y )N      ?)rI   rJ   rK   r   rP   r   r   rR   s    rU   rJ   zKDistributedTest._DistTestBase.test_ddp_create_graph.<locals>.Model.__init__<  s'    G$&\\%,,s*;<DFrV   c                 8    | j                   j                  d      S r   )r   powr  s    rU   r\   zJDistributedTest._DistTestBase.test_ddp_create_graph.<locals>.Model.forward@  s    66::a=(rV   r]   rb   s   @rU   Modelrd  ;  s    =)rV   ri     T)create_graphc              3   4   K   | ]  }|j                     y wrf   r   )r  rs  s     rU   r  zFDistributedTest._DistTestBase.test_ddp_create_graph.<locals>.<genexpr>L  s     P9O++9Or  )
rK   r   rP   rE  r   rW  r  r  r  r]  )rS   ri  r   r`  rY  s        rU   test_ddp_create_graphz3DistributedTest._DistTestBase.test_ddp_create_graph9  s{    )		 ) GE))AA%HI1X $$$$7P9M9M9OPP rV   c                 ^   t         j                  j                  | j                        }| j                  }t         j                  j	                  |      5  t         j
                  j                  j                  t         j
                  j                  ddd      j                  |      |g      }t        d      D ]Y  }|j                  j                  j                  }|!|j                  d       |j                          t        j                  |g      j!                         j                  |      } ||      j#                         }|j%                          |j                  j                  j                  }|j'                         }t)        j*                  |       t-        t.        j0                  d         }	|j3                  |	       t#        d t        |	      D              |	z  }
| j5                  |d   |
d	|
 d
| d| j                          \ 	 d d d        y # 1 sw Y   y xY w)NrE   FrC   r_  r   r'  c              3       K   | ]  }|  y wrf   rt   r  r  s     rU   r  z`DistributedTest._DistTestBase.test_DistributedDataParallel_non_default_stream.<locals>.<genexpr>n       'E3Da3D   r   r   zExpected gradient of 	 but got z	 on rank msg)rP   rR  Streamr   streamrK   rE  r   rL   rW  r   weightrq  r  r  r   rJ  r   r  r  r  r  r   r)  r*  div_r  )rS   rx  r   netr  rq  batchr   avgr+  expected_grads              rU   /test_DistributedDataParallel_non_default_streamzMDistributedTest._DistTestBase.test_DistributedDataParallel_non_default_streamO  s    ZZ&&tyy1F99D""6*hh''??HHOOAquO5::4@dV @  tA::,,11D'++E2

!LL$0668==dCEu:>>+DMMO::,,11D**,C OOC(!$RZZ%=!>JHHZ($''E53D'E$E
$RM$$D	%3M?)C5PYZ^ZcZcYde % ) %	 +**s   GH##H,rR  z@ backend does not support DDP communication hook on CUDA devicesc                 D   t         j                  t         j                  t        j                  t        j
                  t        j                  t        j                  g}t        j                  j                  t        j                  j                  g}|D ]  }t        j                  j                  j!                  t        j                  j#                  ddd      j%                  | j&                        | j&                  g      }|j)                         }| j+                  |j-                  d      d        |j/                  d |       |j)                         }| j+                  |j-                  d      |j0                          |D ]  }t        j                  j                  j!                  t        j                  j#                  ddd      j%                  | j&                        | j&                  g      }|j)                         }| j+                  |j-                  d      d        |j3                  |       |j)                         }| j+                  |j-                  d      t5        |              t        j                  j                  j!                  t        j                  j#                  ddd      j%                  | j&                        | j&                  g      }|j)                         }| j+                  |j-                  d      d        t7        d      D ]J  }t        j8                  dd| j&                        } ||      j;                         }|j=                          L |j)                         }| j+                  |j-                  dd      d       y )	NrE   FrC   r_  	comm_hookrG   r    )defaultallreduce_hookfp16_compress_hookpowerSGDr   batched_powerSGD_hookquantization_hooksquantization_pertensor_hookquantization_perchannel_hookr  BuiltinCommHookType	ALLREDUCEFP16_COMPRESSrP   rK   rE  r   rL   rR  r   _get_ddp_logging_datar  r  register_comm_hookr`   _register_builtin_comm_hookr   rW  r   r   r  )	rS   hookscpp_builtin_hooksr   r`  ddp_logging_datar  r   r   s	            rU   test_ddp_comm_hook_loggingz8DistributedTest._DistTestBase.test_ddp_comm_hook_loggingu  s    &&**&&..">>"??E ((22((66!
 !HH--EEHHOOAquO5::499E $		{ F 	 $-#B#B#D   !1!5!5k!BDI,,T48#,#B#B#D   !1!5!5k!BDDUDUV  *!HH--EEHHOOAquO5::499E $		{ F 	 $-#B#B#D   !1!5!5k!BDI55d;#,#B#B#D   !1!5!5k!BCIN * ))AA15166tyyA II; B I  )>>@-11+>E1XjjAdii8 ~))+ 
  )>>@ -11+rBBGrV   c                 B	   | j                   }t        j                  j                  |       t        j                  |       t        j                  j	                  |       t               t        j                  dd      j                         fg}t        rS|j                  t        j                  j                         t        j                  dddd      j                         f       |D ]  \  }	}
t        j                  j                  j                  ddd      5  t        j                  j                   j#                  t%        j&                  |	      j                         | j                   g||      }t        j                  j                   j#                  t%        j&                  |	      j                         | j                   g||      }|j)                         }|j)                         }|r=t+        |      }t+        |      }| j-                  t/        |      d       |d   g}|d   g}|r |j0                  |g|d	|i| n |j0                  |g|i |  ||g|i |}t3        |j)                         |j)                               D ]  \  }}| j5                  ||        t%        j&                  t+        |j)                                     }t7        d
      D ]:  }|j9                           ||
      }|j;                         }|j=                          < t?        j@                          t7        d
      D ]J  }|j9                           ||
      }|j;                         }|j=                          |jC                          L t?        j@                          t3        |j)                         |j)                               D ]  \  }}| j5                  ||        |re| jE                  |d   tG        tI        |j)                                            | j5                  |dd  t+        |j)                               dd         n)| jE                  |t+        |j)                                      t?        j@                          d d d         y # 1 sw Y   xY w)NrE   r   ru   TFenableddeterministic	benchmarkr@  r   optim_paramsrj  )%r   rP   rR  r%  manual_seedr   rQ   HAS_TORCHVISIONr  torchvisionmodelsresnet50r$  cudnnflagsrK   rE  r   ro  rp  r]  r  assertGreaterr  _register_fused_optimr  r  rW  r7  r   r  r  ri  rv  assertNotEqualrl  iter)rS   grad_as_bucket_viewrC  	optim_clsoptimize_subsetfunctional_optim_argsfunctional_optim_kwargsr   models_to_testr   r   ddp_model_with_optimizer_hookddp_model_with_no_hookhook_paramsno_hook_paramsoptimizer_no_hook
hook_paramallreduce_paramopt_hook_init_paramsr  r  r   s                         rU   $_test_ddp_hook_with_optimizer_parityzBDistributedTest._DistTestBase._test_ddp_hook_with_optimizer_parity  s2    99DJJ!!$'d#JJ""4(U[[D16689N %% ''002EKK1a4N4S4S4UV !/^^))//  0 
 ))AA MM%0557(,		{4G)5	 B  2 .3XX->->-V-Ve,113$(II;0C%1	 .W .* #@"J"J"LK%;%F%F%HN&&*;&7)-n)=**3{+;Q?'21~&6*8*;)<
 'K5KK%2 *5 6	 L5KK%2 6 )2&).) 2)% 8;5@@B.99;83
O ((_E	8 ,0==:EEGH,(
 #1X5??A;C@"wwy	 & LLN #1X.88:4S9"wwy)..0 & LLN 8;5@@B.99;83
O ((_E	8 '++03 &C&N&N&P!QR
 ((04 !>!I!I!KLQRP
 ++0 !>!I!I!KL LLN]  !/ s   M+RR	c                    t         j                  j                  | j                         t	               j                         }dg}t         j
                  j                  j                  j                  ||       t         j
                  j                  j                  || j                  g      }t         j
                  j                  j                  j                  |d      }|D ]  \  }}| j                  d|d    |        t        t        |j                                     dz
  }d}t         j
                  j                  j                  j                  |d      }|D ]  }|dz  }	 | j                  ||       y )	Na.weightr_  T)named_paramsmodule.r   rE   F)rP   rR  r%  r   r   rK   rE  r   +_set_params_and_buffers_to_ignore_for_model_get_data_parallel_paramsr  r  r  r]  r  )	rS   r   params_to_ignorer`  	dp_paramsr   rY  num_ddp_paramsr  s	            rU   test_get_data_parallel_paramsz;DistributedTest._DistTestBase.test_get_data_parallel_params  sK   JJ!!$)),"$))+E *|HH55aa' ))AA499+ B I ))AA[[D \ I %a##g.>q.A-B$CTJ %
 !e&6&6&8!9:Q>NE))AA[[\apu[vI
 UN3rV   c           
         t        j                  | j                         t         j                  j                  | j                         t         j                  j	                  | j                         t        j                  t        j                  dd      t        j                  dd      t        j                  dd            j                         g}t        r;|j                  t        j                  j                         j                                t        |      D ][  \  }}t        j                  |      }t
        j                   j#                  || j                  g|      } ||j%                         fi |}	|rt'        ||j%                         |       t
        j                   j#                  || j                  g|      }|st'        ||j%                         |       t)        |j%                         |j%                               D ]  \  }
}| j+                  |
|d        t         j,                  j.                  j1                  ddd      5  t3        d      D ]  }|d	k(  rt        j4                  d	dd
d
d      nt        j4                  ddd      } ||      j7                         j9                          |	j;                           ||      j7                         j9                          t)        |j%                         |j%                               D ]<  \  }
}| j+                  |
|d|        | j=                  |j>                  d u d|        > |	jA                  d        	 d d d        ^ y # 1 sw Y   jxY w)Nru   rA  rB  optimizer_classrV  optimizer_kwargszParameters not initially equal!TFr  r  rE   r   rR  r   rB   zParams not equal at iteration z&Optim in backward grad is not None at set_to_none)!rP   r  r   rR  r%  rK   r[  rL   r  r  r  r  r  r  ro  rp  rE  r   r]  r   r  r  r$  r  r  rW  rQ   r   r  rv  r  rq  r7  )rS   r  optim_kwargsinit_beforerB  r  jr   model_optim_in_bwdr{  p1p2r  r   s                 rU   !_test_ddp_apply_optim_in_backwardz?DistributedTest._DistTestBase._test_ddp_apply_optim_in_backward  s    dii(JJ""499-JJ!!$)), bii1oryyA		!QPUUWN %%k&8&8&A&A&C&H&H&JK%n55%)]]5%9";; $		{,C < 
 "%"2"2"4EE0(11<<>)5
 &([[%H%H& $		{,C &I &"
 #0(11<<>)5 "%"2"2"46H6S6S6UVFB$$R-NO W ^^))//  0  #1X  !Av "KK1dDH!&R6!B 
 c
(113

*#%
&)!,,.0B0M0M0O'FB !,, "B*H(L !OO "4"H L' D91 & ; 6: s   DN  N
	c                 "   t        j                  t        j                  j                  t        j                  j
                  gddg      D ]6  \  }}| j                  |      5  | j                  |ddi|       d d d        8 y # 1 sw Y   CxY w)NTF)r  rz  Q?)r  r  r  )r  productrP   r{  r|  AdamsubTestr  )rS   r  r  s      rU    test_ddp_apply_optim_in_backwardz>DistributedTest._DistTestBase.test_ddp_apply_optim_in_backward  s}    *3*;*;%++"2"23dE]+&	; \\I\6::"+&*D\$/ ;  76+ 76s   #BB	c                 p    dD ]1  }| j                  t        j                  j                  ddi|d       3 y )Nr  rz  r  F)r  r  r  rB  )r  rP   r{  r|  )rS   r  s     rU   :test_ddp_apply_optim_in_backward_grad_as_bucket_view_falsezXDistributedTest._DistTestBase.test_ddp_apply_optim_in_backward_grad_as_bucket_view_false  s7    ,66#kkoo"& +,1	 7   -rV   c           
      *   t         j                  j                  | j                         dD ]  }| j	                  |      5  t        j
                  | j                         t         j                  j                  | j                         t               }dg}t         j                  j                  j                  j                  ||       |r6t        t         j                  j                  |j                         ddi       t         j                  j                  j                  |j                  | j                        | j                  g      }|s6t        t         j                  j                  |j                         ddi       t        j                  dd	      } ||      \  }}|j!                  d
d      |z  j#                         j%                          t'        t)        j*                               D cg c]  }d  }	}t)        j,                  |	|       |	d
   |	dd  }}
|D ]  }| j/                  |
j0                  j2                  |j0                  j2                         | j5                  t7        |
j8                  j                               t7        |j8                  j                                      | j5                  |
j0                  j:                  |j0                  j:                          	 d d d         y c c}w # 1 sw Y   xY w)Nr  )r  r  rz  r  r  r_  rE   rB   r   )rP   rR  r%  r   r  r  r   rK   rE  r   r  r   r{  r|  r]  rQ   	transposer   r  rW  r  r   all_gather_objectr  rM   ry  r  r  rN   rD   )rS   r  r   r  r{  r   rM   rN   rY  r  rank0_model	remainderms                rU   /test_ddp_apply_optim_in_backward_ignored_paramszMDistributedTest._DistTestBase.test_ddp_apply_optim_in_backward_ignored_params  sH   JJ!!$)),,\\k\:%%dii0JJ**4995*,E(2|$HH%%==ii/ #4,1KKOO#(#3#3#5.2D\
  ((++CC

499-$(II; D C '4,1KKOO#(#3#3#5.2D\
  ++a,Cs8DAq[[A&*//1::< -2$2E2E2G,HI,Hqd,HFI**659-3AYqr
K&++KMM,@,@!##**M(( !9!9!;<d133>>CS>T ((););QSSXXF 'E ;:  -@ J? ;:s    F:L<	LC1LLL	returnc                 r    t        t        j                  t        j                  t        j                        S )N)param_dtypereduce_dtypebuffer_dtype)r   rP   float16r  s    rU   _get_fp16_configz.DistributedTest._DistTestBase._get_fp16_config4  s$    "!MM"]]"]] rV   c                 R   | j                   }t        j                  |       t        j                  j                  |       t        j                  j	                  |       t               }|j                  dt        j                  d             ddg}t        j                  j                  j                  j                  ||       | j                         }t        j                  j                  j                  |j                  |      |g|d      }|D cg c]  }d| 	 }}t        |      }d}t        j                   |j#                         |j%                               D ]  \  }	}
|	|v r<|dz  }| j'                  t)        |
d	             | j'                  t)        |
d
             F| j+                  |j,                  |
j.                  j0                         | j+                  t        j2                  |
j4                  j0                          | j+                  ||       y c c}w )NrF   r}   r  TrA  mixed_precisionrB  r  r   rE   	_mp_param	_fp_param)r   rP   r  rR  r%  r   rO   r   rK   rE  r   r  r  r'  r  r  chainnamed_parametersr  r5  hasattrr  r  r  rO  float32r  )rS   r   r   	to_ignore	mp_configr{  r   expected_ignored	n_ignorednr   s              rU   .test_ddp_native_mixed_precision_ignored_paramszLDistributedTest._DistTestBase.test_ddp_native_mixed_precision_ignored_params;  s   99Dd#JJ""4(JJ!!$'"$E!!(EJJqM:#X.IHH55aay --/I((##;; 6 )(,	 < C 7@@id74&)iI@"9~I#//#*>*>*@#BSBSBUVA	>NI$$WQ%<=$$WQ%<=$$Y%:%:AKK<M<MN$$U]]AKK4E4EF W -y9 As   H$c                      j                   }t        j                  |       t        j                  j                  |       t        j                  j	                  |       t        j
                  dd      } j                          G  fddt        j                  j                        } |       }t        j                  j                  j                  |j                  |      |g|      } j                  |j                  j                  j                  j                          |j#                         D ]f  } j                  j$                  |j&                  j                          j                  t        j(                  |j*                  j                         h t-        d      D ]  }	 ||      j/                         }
|
j1                          |j3                         D ]a  \  }} j                  |j                  t        j(                         |j4                  |dk(  rCJ  j                  |j4                  j                  t        j(                         t-        t7        j8                  |j:                              D cg c]!  }t        j<                  |j4                        # }}t7        j>                  ||j4                         |d   |dd  }} j                  |j                  t        j(                         |D ]>  } j                  |j                  t        j(                          j                  ||       @ d |jA                  |	        y c c}w )
NrB   rE   c                   ,     e Zd Z fdZfdZ xZS )ODistributedTest._DistTestBase._test_ddp_native_mixed_precision.<locals>.MyModelc                 ,   t         |           t        j                  j	                  dd      | _        | j                  dt        j                  dd             t        j                  j                  t        j                  dd      d      | _	        y )NrE   r}   rF   rG   rB   Fr   )
rI   rJ   rP   rK   rL   r  rO   rQ   r   r   rR   s    rU   rJ   zXDistributedTest._DistTestBase._test_ddp_native_mixed_precision.<locals>.MyModel.__init__i  sj    G$&"XX__Q2DF((5;;q!3DE"XX//B*% 0 DFrV   c                 z   | j                   j                         }|D ](  }j                  j                  |j                         * j                  | j
                  j                  j                         j                  j                  |j                         | j                  |      | j                  z   S rf   )r  r]  r  r  rO  rF   r  r   )self_r[   rV  r   r  rS   s       rU   r\   zWDistributedTest._DistTestBase._test_ddp_native_mixed_precision.<locals>.MyModel.forwardq  s    "WW//1F#(()>)>H $ $$U\\%7%79O9OP$$Y%:%:AGGD 771://rV   r]   )rT   r  rS   s   @rU   MyModelr  h  s    0 0rV   r  r  rj  zmodule.pr   r  )!r   rP   r  rR  r%  rQ   r  rK   r   rE  r   r'  r  r   rF   rO  r  r]  r  r  r  r  rW  r   r  r  rq  r  r   rW  
zeros_liker  r7  )rS   rB  set_grad_to_noner   r   r  r  r{  r   r  r   r  rs  rY  tensor_listgrestg_r  s   `                 @rU    _test_ddp_native_mixed_precisionz>DistributedTest._DistTestBase._test_ddp_native_mixed_precision^  s    99Dd#JJ""4(JJ!!$'++b!$C--/I0%((// 0& 	A((##;;T
 6 )(?	 < C SZZ..44i6L6LM ^^%  !6!68I8IJ  0A0AB & 1X3x||~ # 4 4 6HAu$$U[[%--@zz) J.(()9)95==I &+4+>+>s?P?P+Q%R'%R ",,UZZ8%R $ ' UZZ@"-a.+ab/4((%--@"&B ,,RXXu}}E ,,Q3 #' !7  *:;) 's   &M%c                 *    | j                  dd       y )NFrB  r  r  r  s    rU   Gtest_ddp_native_mixed_precision_no_grad_as_bucket_view_no_set_grad_nonezeDistributedTest._DistTestBase.test_ddp_native_mixed_precision_no_grad_as_bucket_view_no_set_grad_none  s    11(-!& 2 rV   c                 *    | j                  dd       y )NTFr  r  r  s    rU   Dtest_ddp_native_mixed_precision_grad_as_bucket_view_no_set_grad_nonezbDistributedTest._DistTestBase.test_ddp_native_mixed_precision_grad_as_bucket_view_no_set_grad_none  s    11(,!& 2 rV   c                 *    | j                  dd       y NTr  r  r  s    rU   Dtest_ddp_native_mixed_precision_grad_as_bucket_view_set_grad_to_nonezbDistributedTest._DistTestBase.test_ddp_native_mixed_precision_grad_as_bucket_view_set_grad_to_none      11(,t 2 rV   c                 *    | j                  dd       y r  r  r  s    rU   Gtest_ddp_native_mixed_precision_no_grad_as_bucket_view_set_grad_to_nonezeDistributedTest._DistTestBase.test_ddp_native_mixed_precision_no_grad_as_bucket_view_set_grad_to_none  r	  rV   c           
         | j                   }t        j                  j                  dd      }	 |j                  }t        j                  j                  j                  t        j                  |      j                  |      |g|      }|j                  ||       t        j                  j                  j                  t        j                  |      j                  |      |g|      }t        d      D ]  }	|j                  j                  j                  |j                  j                  j                  fD ]&  }
|
|
j!                  d       |
j#                          ( t        j$                  |g      j'                         j)                  |      } ||      j+                         }|j-                          |j                  j                  j                  }|j/                         }t+        d t        t1        j2                               D              t1        j2                         z  } ||      j+                         }|j-                          |j                  j                  j                  }|j/                         }|	|k  s| j5                  |d   j7                         |d	| d
|d           | j5                  |d   |d   d|d    d|d            y # t
        $ r |}Y w xY w)NrE   r}   rA  rW  r   r   r<  Fc              3       K   | ]  }|  y wrf   rt   rp  s     rU   r  zFDistributedTest._DistTestBase._test_ddp_hook_parity.<locals>.<genexpr>  s     @#?a#?rr  rs  zExpected hook grad of rt  ru  z,Expected hook grad to be close to allreduce z
, but got )r   rP   rK   rL   rW  AttributeErrorrE  r   ro  rp  r'  r  rW  r   ry  rq  r  r  r   rJ  rR  r   r  r  r  r   r  item)rS   r   r   num_validated_itersr   r  rW  net_with_hooknet_without_hookr  r  r|  r   rq  r}  r~  	loss_hook	grad_hookavg_hooks                      rU   _test_ddp_hook_parityz3DistributedTest._DistTestBase._test_ddp_hook_parity  s   99D1%A& % 3 3 "HH--EEa ##D) 6+ F M
 ,,5t,D$xx00HHa ##D) 6+  I  
 3Z %++2277!((//44A }((/	 dV,22499$?'.224'..55::jjl@5)<)<)>#?@@4CVCVCXX  *%0446	""$)0077<<	$??,**$$ ++-%4]O9XVZ^L\] %  $$ D	J3t9+U_`him`n_op % A   " & %&s   K# #K21K2c                 F    | j                  d t        j                         y )Nr  )r  r  r  r  s    rU   test_ddp_hook_parity_allreducez<DistributedTest._DistTestBase.test_ddp_hook_parity_allreduce  s     &&T8N8N&OrV   c                 $   t        j                         }t        |t              }t	        |      D cg c]  }|t        |         d    }}t        j                  j                  |      }| j                  |t        j                         y c c}w )Nr   r  )r  r   r"   r   rW  r   rP   distributedr  r  r  r  )rS   r+  rG  r=  gpusrW  s         rU   ,test_ddp_hook_parity_allreduce_process_groupzJDistributedTest._DistTestBase.test_ddp_hook_parity_allreduce_process_group  s|     ,,.J.z7CK49*4EF4EqKA'*4EDF!--77=M&&]AWAW&X Gs   Bc                     dD ]<  }t        j                  d dd|      }| j                  |t         j                         > y )Nr  rE   rG   )rW  matrix_approximation_rankstart_powerSGD_iter
warm_startr  )r  PowerSGDStater  r   )rS   r"  powersgd_states      rU   test_ddp_hook_parity_powerSGDz;DistributedTest._DistTestBase.test_ddp_hook_parity_powerSGD	  sJ     ,
!)!7!7"&./())	" **(x/E/E +  ,rV   zmDisabled for environments that                          don't support multiprocessing with spawn start methodc                 v   t        j                  d t        j                  j                  d      }| j                  |t         j                         d}t        j                  d t        j                  j                  |d      }| j                  |t         j                  |       | j                  t        j                         k(  r9t        j                  d d d      }| j                  |t         j                         t        j                  d d d      }| j                  |t         j                         y )NrB   )rW  r2  start_localSGD_iterr  F)rW  r2  r'  post_local_gradient_allreduce)r   r   r  r   )	post_localSGDPostLocalSGDStater  r  r  r  r   r+  r   )rS   r   r'  s      rU   "test_ddp_hook_parity_post_localSGDz@DistributedTest._DistTestBase.test_ddp_hook_parity_post_localSGD  s    "33"TZZ-=-=SUE &&-"B"B '  #%!33"))$7.3	E &&"55$7 '  $"5"5"77%77"&2 **m&F&F +  "33"TtE &&-"B"B ' rV   c                 z   t               }|r|d   nt        j                  d|z        }t        t	        j
                  |      j                  |      ||d|      }	|j                  |       t        j                  |d      j                  |      }
t        j                  |d      j                  |      }||	|
|fS )Nr   cuda:%dMbP?)rA  rW  bucket_cap_mbrB  rG   rv   )r   rP   r   r   ro  rp  r'  rQ   )rS   r   rW  devicesrA  global_batch_sizerB  r   r   r`  r  r  s               rU   _prepare_single_device_modulez;DistributedTest._DistTestBase._prepare_single_device_moduleR  s     EE#*WQZY=M0NF/e$''/%+#(?I HHVKK 11588@E[[!2A699&AF)UF22rV   c                     t               }t        t        j                  |      |d|      }t	        j
                  |d      }t	        j
                  |d      }||||fS )Nr.  )rW  r/  rB  rG   rv   )r   r   ro  rp  rP   rQ   )rS   rW  r1  rB  r   r`  r  r  s           rU   _prepare_cpu_modulez1DistributedTest._DistTestBase._prepare_cpu_modulel  s]     EE/e$+#(?	I KK 115E[[!2A6F)UF22rV   c           	      .   | j                         \  }}}t               }t        dk(  s	t        dk(  r|}d}	| j                  |||      \  }
}}}t        dk(  rt	        t        j                         t              }||   dd }|D cg c]#  }t        j                  dt        |      z         % }}|}t        |      }	| j                  ||||||      \  }
}}}|j                  ||       d }t        j                         5  j                         5  |j                           |       ddd       ddd       t        |      D ]!  } |
       ||	z  |dz   |	z   }|||	z  |dz   |	z   }|dz  d	k(  r$j                         5   ||||       ddd       n
 |||       t!        |
j#                         |j#                               D ]g  \  }}|j$                  s|dz  d	k(  r'| j'                  |j(                  |j(                         B| j+                  |j(                  |j(                         i t        j,                  d
|z          |t        j.                           }$ yc c}w # 1 sw Y   HxY w# 1 sw Y   MxY w# 1 sw Y   xY w)a#  
            This is the recommended way to implement accumulate grads.
            If ``ddp_comm_hook`` input was specified, it will also register that hook
            to the ``ddp_model``. The hook fed into this function should not change
            the resulting gradients.
            r  r  rE   r  Ncuda:c                     | j                           | |      }t        j                  ||j                  |j                              }|j                          y rf   )r  r   mse_lossr'  r   r  )r   r  r  rY  r   s        rU   
step_modelzTDistributedTest._DistTestBase._test_accumulate_gradients_no_sync.<locals>.step_model  s:    uzz&&))FMM*BCrV   rG   r   9  )r  r   r   r4  r"   r  rP   r   r   r  r2  r  r  no_syncr  rW  r  r]  r   r  rq  r  r  r+  )rS   	num_itersddp_comm_hookrB  r  r  r   r+  r1  local_batch_sizer   r`  r  r  rG  int_devicesr  r0  r9  	iteration	ddp_input
ddp_targetr  s                          rU   "_test_accumulate_gradients_no_syncz@DistributedTest._DistTestBase._test_accumulate_gradients_no_sync}  s    %)$:$:$<!E8T')J %7f#4$.!#$ 262J2J/1H3/y% & 243F3F3H'R)$/3CNO;a5<<#a&(89;O$.!#&w< 262T2T%+3/y% (,,X}E  &&(OO%e$ ) ! #9-	5%0!++tax;K.K	 $++tax;K.K
 q=A%"**,"9iD -, y)Z@ 0 0 2I4H4H4JKDAq??  1})++AFFAFF;((8 L !!$"23enn->?@9 .9 P. )( !$ -,s6   (I,I>I1+I>J1I;	6I>>JJ	z2get_future is only supported on mpi, nccl and glooc                 $    | j                          y)V
            Runs _test_accumulate_gradients_no_sync using default inputs
            NrC  r  s    rU   !test_accumulate_gradients_no_syncz?DistributedTest._DistTestBase.test_accumulate_gradients_no_sync  s     335rV   c                 (    | j                  d       y)rE  TrM  NrF  r  s    rU   .test_accumulate_gradients_no_sync_grad_is_viewzLDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_grad_is_view  s     33D3QrV   c                     t               dt        dt        j                  dt        j
                  j                  t        j                     ffd}| j                  d|       y)z
            Runs multiple iterations on _test_accumulate_gradients_no_sync
            using allreduce hook and validates whether future result was properly
            passed as gradients in reducer.
            r  bucketr  c                     |j                         z  g}| j                  |      j                         j                  d       S )Nc                 (    | j                         d   S rC  r   futs    rU   r"  zxDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_hook.<locals>.allreduce_hook.<locals>.<lambda>  s    ciik!nrV   )rF   r  
get_futurethen)r  rK  r  r+  s      rU   r  zfDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_hook.<locals>.allreduce_hook  s=     "==?Z78&&w/Z\T45rV   rv   r<  r=  N	r   objectr  
GradBucketrP   futuresFuturerj   rC  )rS   r  r+  s     @rU   0test_accumulate_gradients_no_sync_allreduce_hookzNDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_hook  s[     ()J *.//%%ell3 33> 4 rV   c                     t               dt        dt        j                  dt        j
                  j                  t        j                     ffd}| j                  d|       y)ao  
            Runs multiple iterations on _test_accumulate_gradients_no_sync using allreduce
            hook that also uses then callbacks. In first then callback result is multiplied
            by 2, and the second callback divides the result by 2 * world_size. It validates
            whether final result was properly passed as gradients in reducer.
            r  rK  r  c                     | j                  |j                         g      j                         }d }fd}|j                  |      j                  |      S )Nc                 .    d| j                         d   z  S )NrG   r   r  rN  s    rU   multzDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_with_then_hook.<locals>.allreduce_with_then_hook.<locals>.mult  s    sxxz!},,rV   c                 0    | j                         dz  z  S r   r\  )rO  r+  s    rU   divzDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_with_then_hook.<locals>.allreduce_with_then_hook.<locals>.div  s    88:Z88rV   )r  rF   rP  rQ  )r  rK  rO  r]  r_  r+  s        rU   allreduce_with_then_hookzzDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_with_then_hook.<locals>.allreduce_with_then_hook  sL     ((&--/):;FFH-9 xx~**3//rV   rv   rR  NrS  )rS   r`  r+  s     @rU   :test_accumulate_gradients_no_sync_allreduce_with_then_hookzXDistributedTest._DistTestBase.test_accumulate_gradients_no_sync_allreduce_with_then_hook   s\     ()J0 0*.//0%%ell30 33+C 4 rV   c                    d }d }| j                         \  }}}t        dd      }t        dk(  r;t        t	        j
                         t              }||   d   }|j                  |      }|j                  |g      j                         }	|	j                  |      j                  |      j                         }
t        ddt        |      z  dz  dz         }| j                  |
d   |       y )Nc                 N    | j                         D cg c]  }|dz  	 c}S c c}w )Nru   r\  rO  r  s     rU   r]  z;DistributedTest._DistTestBase.test_get_future.<locals>.mult(  $    '*xxz2z!Az222   "c                 N    | j                         D cg c]  }|dz   	 c}S c c}w rX   r\  rd  s     rU   r  z:DistributedTest._DistTestBase.test_get_future.<locals>.add+  re  rf  ru   rG   r  r   rE   )r  rU  r   r"   r  r   r'  r  rP  rQ  r  r  r  )rS   r]  r  r  r  r   r  rG  rT  rO  resexpecteds               rU   test_get_futurez-DistributedTest._DistTestBase.test_get_future"  s    33 %)$:$:$<!E8T!!Q'E& 243F3F3H'R'-a0	+$$eW-88:C((4.%%c*//1C$QCJ(:Q(>?HSVX.rV   c           
      (   | j                         \  }}}t        t        j                         t              }t        ||         }t        j                  dd      D ]  \  }}| j                  ||||       | j                  ||||d       | j                  ||t        j                  d      ||       |D cg c]#  }t        j                  dt        |      z         % }	}| j                  |	|t        j                  d      ||        y c c}w )N)FT)rG  r   rB  rC  T)rG  r   rB  rC  rI  rR  )rG  r   rH  rB  rC  r6  )r  r"   r  r   r   r  r  r  rK  rP   r   r   )
rS   r  r  r   rG  r  use_bucket_viewrC  r  	gpus_lists
             rU   r   z:DistributedTest._DistTestBase.test_DistributedDataParallel:  s)    %)$:$:$<!E8T.t/B/B/DgNKD)*D1:1B1B}2- 22#,;!-	 3  22#,;!-+/ 3  22#"',,v"6,;!- 3  FJJTU\\'CF*:;T	J22("',,v"6,;!- 3 =2: Ks   9(Dc                    t        j                  d       t        j                  t              j                         }t         j                  j                  |j                         d      }t               }t        j                  j                  || j                  g|      }t        j                  t        j                          dz  d      j                         }t        j                  t        j                          dz  d      j                         }t        j"                         }|j                         D ]2  }	| j%                  |	d u       | j%                  |	j&                  d u        4 t)        d      D ]i  }
|j+                          t-               5   ||      } |||      }d d d        |j/                        j1                          |j                         D ]  }	|	j2                  s| j%                  |	j&                  d u       | j5                  |	j&                  j7                         j9                                | j5                  |	j&                  j;                         j9                                 |j=                  |       |j?                          t        j                  d|
z          |t        j@                  t        j                          dz           }l |S # 1 sw Y   AxY w)	Nz  r  ry  r  rG   rv   r   r:  )!rP   r  ro  rp  rD  rR  r{  r|  r]  r   rK   rE  r   r   rQ   r  r   r  r  rq  rW  r7  r   scaler  r   r5  isnananyisinfrv  updater+  )rS   grad_is_viewr   	optimizerscalerr`  r  r  loss_fnr   r9  rY  r   s                rU   &_test_DistributedDataParallel_with_ampzDDistributedTest._DistTestBase._test_DistributedDataParallel_with_ampj  s-   e$MM'*//1E(8(8(:tDI  \F;;499+| < I KK 3 3 5 91=BBDE[[!4!4!6!:A>CCEFjjlG ))+.$/ , Ry##%Z&u-F"662D   T"++- #--/Ad(:;((););)=>((););)=>	 0 I&  !!$*-ennT-@-@-BQ-FGH; !> 9  Zs   K((K2	c                 "   t         j                  j                  | j                         | j	                  d      }| j	                  d      }t        |j                         |j                               D ]  \  }}| j                  ||        y )NFru  T)rP   rR  r%  r   ry  r  r]  r  )rS   ddp_model_grad_not_viewddp_model_grad_is_viewr  r  s        rU   6test_DistributedDataParallel_with_amp_and_grad_is_viewzTDistributedTest._DistTestBase.test_DistributedDataParallel_with_amp_and_grad_is_view  s     JJ!!$)),&*&Q&Q" 'R '# &*%P%P! &Q &" '224&1131   A&	rV   c                    |rt         nt        }t        j                  |      }	|	j	                  |d          t
        j                  j                  t        j                  |            }
|
j	                  |d          t
        j                  j                  |
|      }
t        j                         5 }t        j                  dk(  r=t        j                  |
|       |j!                  d       t        j"                  |      }
n?t        j                  |
|j$                         t        j"                  |j$                        }
d d d        t        j&                  |d      }t        j&                  |d      }t        j(                         }| j+                  |	|
|j	                  |d         |j	                  |d         ||||d|t-        j.                         |rdnd       | j1                          y # 1 sw Y   xY w)Nr   r_  r=   rG   rv   Tr}   )BN_NETBN_NET_NO_AFFINEro  rp  rR  rK   SyncBatchNormconvert_sync_batchnormrE  r   rE  r,  r4  r5  rP   r-  r.  r/  r   rQ   r  r>  r  r   r  )rS   rG  r   r  r  r6  rH  r   r   rJ  r3  r:  r  r  r   s                  rU   +_test_DistributedDataParallel_SyncBatchNormzIDistributedTest._DistTestBase._test_DistributedDataParallel_SyncBatchNorm  s    %F*:E e,INN:a=) ((??e@TUINN:a=);;j < I
 ,,.#<<7*JJy#.HHQK %

3IJJy#((3 %

388 4I / Iq1I[[A.F::<D   z!}-JqM*##% MMO; /.s   3BG22G;c                    d}t         j                  j                  j                  t	        j
                  t              j                         | j                  g|      } |       }t         j                  j                  |j                         |      }t         j                  j                  j                  t	        j
                  t              j                         | j                  g|      } |       }| j                  |||      }	t        j                  t        j                         dz  d      j                         }
t        j                  t        j                         dz  d      j                         }t        j                          }t#        d      D ]  }| j%                  ||||
|       |j'                  |j                                | j%                  |	|||
|       t)        |j                         |j                               D ]+  \  }}| j+                  |j,                  |j,                         -  | j+                  |j.                  |j.                         y )Nr  r  ry  rG   rv   r   )rP   rK   rE  r   ro  rp  rD  rR  r   r{  r|  r]  _create_post_localSGD_optimizerrQ   r  r   r  rW  _perform_a_train_stepr`  r  r  r  rv  )rS   create_averagerru  learning_rater{  ru  r~  net_using_post_localSGD_opt	averager2post_localSGD_optr  r  rx  rY  r  r  s                   rU   $_test_post_localSGD_optimizer_parityzBDistributedTest._DistTestBase._test_post_localSGD_optimizer_parity  s    M((##;;g&++- II;(4 < C
 '(H++//#.."2}/EC*/((*;*;*S*Sg&++- II;(4 +T +' ()I $ D D+]I! KK 3 3 5 91=BBDE[[!4!4!6!:A>CCEFjjlG2Y**3WeVL++CNN,<=**%/ "NN$&A&L&L&NFB $$RWWbgg6 " X]]INN;rV   c                 0    t        j                  dd      S )Nrv   rB   ri  )rm  rn  r  s    rU   _create_periodic_model_averagerz=DistributedTest._DistTestBase._create_periodic_model_averager  s    22!"MMrV   c                     t        j                  t        j                  j	                  |j                         |      |      S )Nry  )r{  ru  )post_localSGD_optimizerPostLocalSGDOptimizerrP   r{  r|  r]  )rS   r{  r  ru  s       rU   r  z=DistributedTest._DistTestBase._create_post_localSGD_optimizer!  s3    *@@kkoocnn&6=oI! rV   c                     |j                           ||      } |||      }|j                          |j                          y rf   )r7  r  rv  )rS   rv  r{  rx  r  r  rY  r   s           rU   r  z3DistributedTest._DistTestBase._perform_a_train_step'  s6    !ZF66*DMMONNrV   c                    d}t         j                  j                  j                  t	        j
                  t              j                         | j                  g      } |       }| j                  |||      } |       }| j                  |||      }t        j                  t        j                         dz  d      j                         }	t        j                  t        j                         dz  d      j                         }
t        j                         }t        d      D ]  }| j                  ||||	|
        | j                  dk(  r&t        j                   d|j#                         i|       t        j$                          ddz  d| j                  z  i}t        j&                  ||	      }|j)                  |d          | j+                  |j,                  d       | j/                  |j,                  |j,                         |d   d
= | j1                  d
|d          | j3                  t4        d      5  |j)                  |d          d d d        | j/                  |j,                  d       y # 1 sw Y   &xY w)Nr  r_  rG   rv   r   r   optimizer_state_dictr-  map_locationrv  z]Loaded state dict does not contain a step counter for an averager. Setting step counter to 0.)expected_warningexpected_regex)rP   rK   rE  r   ro  rp  rD  rR  r   r  rQ   r  r   r  rW  r  r-  r1  ri  r/  load_state_dictr  rv  r  r  assertWarnsRegexUserWarning)rS   r  
chkpt_filer  r  ru  r  r  dummy_post_localSGD_optr  r  rx  rY  r  
checkpoints                  rU   )_test_post_localSGD_optimizer_step_reloadzGDistributedTest._DistTestBase._test_post_localSGD_optimizer_step_reload.  s4    !M*/((*;*;*S*Sg&++-499+ +T +' '(H $ D D+]H! ()I&*&J&J+]I'# KK 3 3 5 91=BBDE[[!4!4!6!:A>CCEFjjlG2Y**%/  yyA~

+->-I-I-KLj LLN%M9tyy+@ALJ\JJ#33J?U4VW 	2X]]INN; 126:VZ0F%GH &&!, - ' 
 (7756 Y^^Q/ s   9I33I<c                     t         j                  j                  | j                         | j	                  | j
                  d       y NFr{  rP   rR  r%  r   r  r  r  s    rU   #test_post_localSGD_optimizer_parityzADistributedTest._DistTestBase.test_post_localSGD_optimizer_parityn  s9     JJ!!$)),5544" 6 rV   c                     t         j                  j                  | j                         | j	                  | j
                  d       y NTr{  r  r  s    rU   0test_post_localSGD_optimizer_parity_grad_is_viewzNDistributedTest._DistTestBase.test_post_localSGD_optimizer_parity_grad_is_viewz  s9     JJ!!$)),5544! 6 rV   c                 r    t        ddt        j                         fg      }t        j                  |d      S )N)rG   rG   rv   r  )r   r  r   r  r  )rS   r  s     rU   #_create_hierarchical_model_averagerzADistributedTest._DistTestBase._create_hierarchical_model_averager  s8    %0&1d>Q>Q>S:T1U%V""<<'=A rV   c                     t         j                  j                  | j                         | j	                  | j
                  d       y r  rP   rR  r%  r   r  r  r  s    rU   9test_post_localSGD_optimizer_parity_with_hierarchical_sgdzWDistributedTest._DistTestBase.test_post_localSGD_optimizer_parity_with_hierarchical_sgd  s9     JJ!!$)),5588" 6 rV   c                     t         j                  j                  | j                         | j	                  | j
                  d       y r  r  r  s    rU   Ftest_post_localSGD_optimizer_parity_with_hierarchical_sgd_grad_is_viewzdDistributedTest._DistTestBase.test_post_localSGD_optimizer_parity_with_hierarchical_sgd_grad_is_view  s9     JJ!!$)),5588! 6 rV   c                     t         j                  j                  | j                         t	               5 }| j                  | j                  |       d d d        y # 1 sw Y   y xY wrf   )rP   rR  r%  r   rK  r  r  )rS   r;  s     rU   (test_post_localSGD_optimizer_step_reloadzFDistributedTest._DistTestBase.test_post_localSGD_optimizer_step_reload  sG     JJ!!$)), "h>>88( #""s   AA#c                     | j                  t        j                         | j                  t        j                         y rf   )>_test_DistributedDataParallel_SyncBatchNorm_with_memory_formatrP   channels_lastchannels_last_3dr  s    rU   8test_DistributedDataParallel_SyncBatchNorm_Channels_LastzVDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_Channels_Last  s4     OO## OO&&rV   c                    | j                         \  }}}t        j                         }d}t        |dz        }t        |dz        }t        }	t        j                  |	      j                  |      }
t        j                  j                  |
|g      }|dddg|t        j                  u rg ndgz   }t        j                  |dt        j                  ij                  |      j                  |      }t        j                  |dt        j                  ij                  |      j                  |      }t        j                          }| j#                  |
|||||||d|t        j                         |       | j%                          y )NrG   r_  rv   rO  r  T)r  r  r   r   ONLY_SBN_NETro  rp  rR  rK   rE  r   rP   r  rQ   rJ  r'  r  r>  r  )rS   r  r  r  r   r  r  	bs_offsetr  r   rJ  r3  shapes	input_gpu
target_gpur   s                   rU   r  z\DistributedTest._DistTestBase._test_DistributedDataParallel_SyncBatchNorm_with_memory_format  sk    %)$:$:$<!E8T //1MHD1HIMA-.I Ee,11$7I;;tf < I  Aq)#u':'::F
 V75;;7d-0  V75;;7d-0 
 ::<D   ##%+ !  MMOrV   c           
         | j                         \  }}}t        j                         }|g}d}t        |dz        }t        |dz        }| j	                  |||||       | j	                  |||||t        j                  d             |D 	cg c]#  }	t        j                  dt        |	      z         % }}	| j	                  |||||t        j                  d             y c c}	w )NrG   rG  r   r  r  r6  rR  )rG  r   r  r  r6  rH  r6  )r  r  r   r   r  rP   r   r   )
rS   r  r  r   r+  r  r  r  r  r  s
             rU   *test_DistributedDataParallel_SyncBatchNormzHDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm  s     %)$:$:$<!E8T,,.J 6DHD1HIJN+I<<!#  =  <<!# #ll62 =  =AADqELL3q6!12DDA<<!# #ll62 =  Bs   (C#c                     | j                         \  }}}t        j                         }|g}d}t        |dz        }t        |dz        }| j	                  |||||d       y )NrG   F)rG  r   r  r  r6  r   )r  r  r   r   r  )	rS   r  r  r   r+  r  r  r  r  s	            rU   4test_DistributedDataParallel_SyncBatchNorm_No_AffinezRDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_No_Affine  su     %)$:$:$<!E8T,,.J 6DHD1HIJN+I<<!#  = rV   c                 \   | j                         \  }}}|g}t        j                  d      }t        j                  |      }|j                  |d          t        j                  j                  t        j                  |            }|j                  |d          t        j                  j                  ||      }t        |      dz  }t        j                         |z  }	t        j                  |	d      }
t        j                  |	d      }t        j                         }t        j                   j"                  j%                  d      5  | j'                  |||
j                  |d         |j                  |d         ||||	d	       | j)                          d d d        y # 1 sw Y   y xY w)NrG   r   r_  FT)r  rK   r   ro  rp  rR  r  r  rE  r   r  r  r   rP   rQ   r  r$  r  r  r>  r  rS   r  r  r   r  r   rJ  r3  r  r  r  r  r   s                rU   3test_DistributedDataParallel_SyncBatchNorm_2D_InputzQDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_2D_Input7  sZ    %)$:$:$<!E8T 6DNN1%E e,INN47# ((??e@TUINN47#;;IRV;WI4y1}H++-8IIq1I[[A.F::<D
 %%++E2$$NN47+KKQ(
  322s   AF""F+c                 >   | j                         \  }}}|g}t        j                  d      }t        j                  |      }|j                  |d          t        j                  j                  t        j                  |            }|j                  |d          t        j                  j                  ||      }d}t        j                         }	t        j                  |	d      }
t        j                  |	d      }t        j                         }t        j                  j                   j#                  d      5  | j%                  |||
j                  |d         |j                  |d         ||||	d	       | j'                          d d d        y # 1 sw Y   y xY w)NrG   r   r_  rE   FT)r  rK   r   ro  rp  rR  r  r  rE  r   r  r   rP   rQ   r  r$  r  r  r>  r  r  s                rU   Ctest_DistributedDataParallel_SyncBatchNorm_Single_Input_Per_ProcesszaDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_Single_Input_Per_Processe  sM    %)$:$:$<!E8T 6DNN1%E e,INN47# ((??e@TUINN47#;;IRV;WIH++-IIq1I[[A.F::<D
 %%++E2$$NN47+KKQ(
  322s   <AFFc                 R   | j                         \  }}}t        j                  j                  t        j                  |      |g      }g }t        t        j                               D ]t  }t        j                  t        j                  ddd|dz   z        d|dz
  z  z  t        j                  ddd|dz   z        d|dz
  z  z  gd      }|j                  |       v t        j                  |D cg c]A  }|j                  ddd      j                         j                  t        j                   d	      C c}d      j                  |      }	t        d
      D ]:  } |||   j                  |            }
|
j#                         j%                          < |j&                  j(                  |j&                  j*                  }}t        j,                  j/                  ||	j#                  d             t        j,                  j/                  ||	j1                  d             y c c}w )Nr_  rG   rE   rB   rr  r  r   r   r   r<  )r  rK   rE  r   r  rR  rW  r  r   rP   r  r   r  permute
contiguousviewnum_featuresmeanr  r   running_meanrunning_vartestingassert_closer  )rS   r  r  r   r   r   r  input_var_rankr[   all_input_varr  r  r  s                rU   Itest_DistributedDataParallel_SyncBatchNorm_Diff_Input_Sizes_Running_ValuezgDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_Diff_Input_Sizes_Running_Value  s    %)$:$:$<!E8TKK77!!$'TF 8 E I4..01!&

1aA731q5>J

1aA731q5>J "   0 2 "II '& IIaA&113889R9RTVW&  d4j  3Z)D/..t45!!#  
 ))(( &L MM&&|]5G5G5JKMM&&{M4E4Ea4HI!s   3AH$c                     | j                         \  }}}|g}t        }t        j                         }|dz   }t	        |dz   |z  dz        }t	        |dz   |z  dz        }	| j                  ||||	|       y )NrG   ru   r  )r  r  r  r   r   r  )
rS   r  r  r   r  r   r  r  r  r  s
             rU   Dtest_DistributedDataParallel_SyncBatchNorm_Diff_Input_Sizes_gradientzbDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_Diff_Input_Sizes_gradient  s     %)$:$:$<!E8T6D E //1MaxHTAX-12I]Q.-?!CDI<<!#  = rV   c                    | j                         \  }}}t        j                  t              }|j	                         }t
        j                  j                  j                  |      }t        j                  j                  |j                  |      |g      }t        j                  ddt
        j                  t        j                  |            } ||      }| j                  |j                   t
        j                         |j#                         j%                          |j'                         D ]6  }| j                  |j(                  j                   t
        j                         8 y )Nr_  rG   )rO  r   )r  ro  rp  r  rL  rP   rK   r  r  rE  r   rR  rQ   r  r   r  rO  r   r  r]  rq  )rS   r  r  r   r   r   r  rs  s           rU   /test_DistributedDataParallel_SyncBatchNorm_halfzMDistributedTest._DistTestBase.test_DistributedDataParallel_SyncBatchNorm_half  s     %)$:$:$<!E8TMM&)EJJLEHH**AA%HEKK77

48HVZU[7\E++a%--T@RSC*CSYY6GGI ))+  !1!15==A ,rV   c           	         t        j                         }t        j                  t              }|r2t
        j                  j                  |j                  |      |g      }nt
        j                  j                  |      }d}| j                  |      \  }}}}|r"|j                  |      }|j                  |      }|j                  d       t        d      D ]  }	||z  }
| j                  |||
|
|z    ||
|
|z    |d       | j                  |       |j                         }|	dkD  r|	dk  s|	dz  dk(  r| j                  |j!                  d      d       | j                  |j!                  d      d       | j                  |j!                  d	      d       | j                  |j!                  d      |j!                  d
             | j                  |j!                  d	      |j!                  d
             | j#                  |j!                  d      |	       n&|	dkD  r!| j%                  |j!                  d      |	       |t'        j(                  |         } |S )Nr_  rG   r   rE   r   rB   forward_compute_timebackward_compute_timebackward_comm_time"backward_compute_comm_overlap_timer@  )r  rD  ro  rp  rD  rK   rE  r   rR  r  $_set_ddp_runtime_logging_sample_raterW  r#  r  r  r  r  r  r  rP   r+  )rS   is_gpur   r3  r  r4  r  r  r   r9  r6  r  s               rU   _test_ddp_logging_dataz4DistributedTest._DistTestBase._test_ddp_logging_data  sP   ==?Dg.IKK??NN4(dV @ 	 KK??	J	 H.2.F.Fx.P+Jvt

4(T*::1=Ry %%&6H#456FX$56 //	: $-#B#B#D 7bC!GqL++(,,-CDa ++(,,-DEq ++(,,-ABA ++(,,-DE(,,-QR ++(,,-AB(,,-QR $$%5%9%9+%FL1W ''(8(<(<[(I3O ennZ89] !` rV   c           
      2   d }t        j                  t         j                  j                         | j	                         \  }}}| j                  d      }|j                         }| j                  |j                  d      t        j                                | j                  |j                  d      t        j                                | j                  |j                  d      d       | j                  |j                  d      d	       | j                  |j                  d
      d       | j                  |j                  d      d       | j                  |j                  d      d       | j                  |j                  d      d       | j                  |j                  d      d       | j                  |j                  d      t        j                  |             | j                  |j                  d      d       t        |j                               }d}d}	t        t        d |            }|D ]+  }
|dz  }|	|
j                         |
j!                         z  z  }	- | j                  |j                  d      d       | j                  |j                  d      |	       | j                  |j                  d      |       | j                  |j                  d      t#        |	             | j                  |j                  d       |d             | j                  |j                  d       |d             | j                  |j                  d        |d!             | j                  |j                  d"       |d#             |j                  d      d$k(  rq| j                  |j                  d%       |d&             | j                  |j                  d'       |d(             d)}| j                  |j                  d*      |       | j                  |j                  d+      d        | j                  |j                  d,      d        | j                  |j                  d-      d        | j                  |j                  d.      d        | j                  |j                  d/      d        | j                  |j                  d0      d        | j                  |j                  d1d      d       | j                  |j                  d2      d       | j                  |j                  d3      t#        |	             |j                  d4      }t        t%        t'        d5      D cg c]  }t#        |       c}            }| j                  |d6j)                  |             |j                  d7      }| j                  |d8j)                  |             | j+                  |j                  d9      d       | j+                  |j                  d:      d       | j+                  |j                  d;      d       | j+                  |j                  d:      |j                  d<             | j+                  |j                  d;      |j                  d<             |j                  d=      }|j                  d>      }|j                  d?      }|j                  d@      }|j                  dA      }| j+                  ||       | j+                  ||       | j+                  ||       | j+                  ||       t-               }|j/                          |j0                  j3                          t4        j6                  j9                  |dBC      }|j                         }t        |j                               }| j                  |j                  d      t;        dD             |d   j                         |d   j!                         z  |d   j                         |d   j!                         z  g}| j                  |j                  d      d6j)                  dE |D                     | j                  |j                  d      dF       y c c}w )GNc                 P    | t         j                  v rt         j                  |    S dS )Nr  r  r  s    rU   	parse_envzJDistributedTest._DistTestBase.test_ddp_logging_data_cpu.<locals>.parse_env6  s     *-*;rzz#FFrV   Fr  r+  r   module_namer   rA  r  rH  r   broadcast_buffersrE   bucket_cap_bytesi  find_unused_parametersr   rB  backend_namer@  r  c                     | j                   S rf   r   )	parameters    rU   r"  zIDistributedTest._DistTestBase.test_ddp_logging_data_cpu.<locals>.<lambda>P  s
    93J3JrV   r3  rJ  total_parameter_size_bytesnum_parameter_tensorsbucket_sizesmaster_portr  master_addrr  torch_distributed_debugTORCH_DISTRIBUTED_DEBUGcuda_visible_devicesCUDA_VISIBLE_DEVICESr  gloo_socket_ifnameGLOO_SOCKET_IFNAMEgloo_device_transportGLOO_DEVICE_TRANSPORTrG   gloo_num_threadsnccl_socket_ifnamenccl_blocking_waitnccl_async_error_handling
nccl_debugnccl_nthreadsnccl_ib_timeoutunused_parameter_sizehas_rebuilt_bucketsrebuilt_bucket_sizes'prev_iteration_grad_ready_order_indicesru   ,  rebuilt_per_bucket_param_indices avg_forward_compute_timeavg_backward_compute_timeavg_backward_comm_time&avg_backward_compute_comm_overlap_timeforward_compute_time_startbackward_compute_time_startbackward_compute_time_endbackward_comm_time_startbackward_comm_time_endg      ?)r/  g      8Ac              3   2   K   | ]  }t        |        y wrf   )r   )r  r[   s     rU   r  zJDistributedTest._DistTestBase.test_ddp_logging_data_cpu.<locals>.<genexpr>  s     7,Q#a&,s   zdouble, float)r  set_debug_levelr  INFOr  r  r  r  r  r   rD  r  r  r]  filternumelelement_sizer   reversedrW  r2  r  r   rJ  r   rK  rK   rE  r   r   )rS   r  r  r  r   r3  r  rV  
num_params
param_sizer   default_gloo_threadsgrad_ready_orderr[   expected_orderbucket_indicesfwd_host_side_timebwd_comp_start_host_side_timebwd_comp_end_host_side_timebwd_comm_start_host_side_timebwd_comm_end_host_side_timer   r  s                          rU   test_ddp_logging_data_cpuz7DistributedTest._DistTestBase.test_ddp_logging_data_cpu2  s   G   !5!56$($:$:$<!E8T3353AI(>>@-11,?ATATAVW-11&94==?K-11-@%H-11,?D -11/BBG-112EFJ-112DEGWX-112JKQO-112KLaP $$^4d6F6Fx6P -11+>C)..01FJJ&!JFSTFa
aggi!..*:::
  -11(;WE $$%ABJ -112IJJW-11.A3z?S $$]3Y}5M  $$]3Y}5M  $$%>?34  $$%;<01  ##N3v=  $(()=>23   $(()@A56 ()$  $(();<(
 -112FGN-112FGN-112MNPTU-11,?F-11/BDI-112CDdK
 -112I1MqQ-112GH!L $$%;<c*o  0339  "(E!H+EHqCFH+E"FGN-tyy/HI-112TUN^SXXn-EF ##$4$8$89S$TVWX## $$%@A1 ##$4$8$89Q$RTUV## $$%@A $$%MN ## $$%=> $$%MN
 "2!5!56R!S,<,@,@--) +;*>*>++' -=,@,@*-) +;*>*>?W*X'##+-J ##-/L ##+-J ##$ACUV JEKKMII;;EQT;UI(>>@)..01F $$%78#>O:P q	!F1I$:$:$<<q	!F1I$:$:$<<L  $$^4		7,77 -11(;_My ,Fs   5bc           
         | j                         \  }}}| j                  d      }|j                         }| j                  |j	                  d      t        |             | j                  |j	                  d      |       |j	                  d      }t        t        t        d      D cg c]  }t        |       c}            }| j                  |dj                  |             |j	                  d      }	| j                  |	d	j                  |             | j                  |j	                  d
      d       | j                  |j	                  d      d       | j                  |j	                  d      |j	                  d             | j                  |j	                  d      |j	                  d             |j	                  d      }
|j	                  d      }|j	                  d      }|j	                  d      }|j	                  d      }| j                  ||       | j                  ||       | j                  ||       | j                  ||
       y c c}w )NTr  rA  rH  r  ru   r  r  r  r  rE   r  r  r  r	  r
  r  r  r  )r  r  r  r  r  r   r  r  rW  r2  r  )rS   r  r  r   r3  r  r  r[   r  r  r  r  r  r  r  s                  rU   test_ddp_logging_data_gpuz7DistributedTest._DistTestBase.test_ddp_logging_data_gpu  sV    %)$:$:$<!E8T3343@I(>>@-11,?TK-11/BDI/339  "(E!H+EHqCFH+E"FGN-tyy/HI-112TUN^SXXn-EF ##$4$8$89S$TVWX## $$%MNPQ ## $$%@A $$%MN ## $$%=> $$%MN
 "2!5!56R!S,<,@,@--) +;*>*>++' -=,@,@*-) +;*>*>?W*X'##+-J ##-/L ##+-J ##$ACUVQ ,Fs   *Ic           	      |   t         j                  j                  t              }d}| j	                  t
        |      5  d}| j                  |      \  }}}}t        j                         |z  }| j                  |||||z    ||||z    |d       |j                          d d d        t        ||       y # 1 sw Y   xY w)Nz,should be called before training loop startsrG   rE   )rK   rE  r   rD  r  rw  r  r  rD  r#  rF  r1   )	rS   r3  expected_errr  r4  r  r  r   r6  s	            rU   test_static_graph_api_cpuz7DistributedTest._DistTestBase.test_static_graph_api_cpu  s     ;;GDIIL''lC262J2J82T/
E6483 %%&6H#456FX$56 ++- D  $I|<! DCs   A!B22B;c                 T   d}t         j                  j                  |g      }t        j                  j                         }t        j                  j                  t        j                  |      |      }|j                  d   j                  j                  }| j                  ||       y rC  )rP   r  r  r  r  r  rK   r  r  ro  rp  layer1bn1rW  r  )rS   process_idsrW  res50_modelres50_model_syncprocess_group_syncs         rU    test_SyncBatchNorm_process_groupz>DistributedTest._DistTestBase.test_SyncBatchNorm_process_group  s     K!--77FM%,,557K!//FFk*M  "2!8!8!;!?!?!M!M/?rV   c                     |t         j                  k7  r|t        d| d      |5 ||||       t        j                         |k(  r| j	                  ||       y y  |||       | j	                  ||       y )NzReduction fn z must specify dst!)r  r  r  rD  r  )rS   r   r  rq  reduction_fnr  s         rU   _run_reduction_testz1DistributedTest._DistTestBase._run_reduction_test&  sy     t.3; =>P!QRRVS"-==?c)$$V_= * VR(  9rV   c                    t         j                  j                  | j                         | j                  dz  dk(  }t        j
                  j                  t        j
                  j                  fD ]  }t        j                  ||g      j                  | j                        }| j                  |t        j                  ddg      j                  | j                        |       t        j                  ddg      j                  | j                        }|j                         }| j                  |||        t        j
                  j                  t        j
                  j                  fD ]s  }t        j                  ||g      j                  | j                        }| j                  |t        j                  ddg      j                  | j                        |       u y )NrG   r   FT)rP   rR  r%  r   r  rx  r  r  r   r'  r0  r  ry  r  )rS   elementrq  r  r  s        rU    test_nccl_backend_bool_allreducez>DistributedTest._DistTestBase.test_nccl_backend_bool_allreduce4  s^    JJ!!$)),ii!mq(G}},,dmm.?.?@$||Wg,>?BB499M(( %,,u~">"A"A$))"Lb
  %||T4L9<<TYYG"."4"4"6((K A }}(($--*;*;<$||Wg,>?BB499M(( %,,d|"<"?"?		"JB =rV   c                    t         j                  j                  | j                         ddgddgd}t        j                  || j                  dz           j                  | j                        }|j                         }t        t        j                               D cg c]2  }t        j                  ddg      j                  | j                        4 }}t        j                  ||       | j                  t        |      t        j                                t        |      D ]K  \  }}t        j                  ||dz           j                  | j                        }| j                  ||       M | j                  ||       y c c}w )NTFr   rE   rG   )rP   rR  r%  r   r   r'  r  rW  r  r   r  r  r  r  )	rS   r   r  input_tensor_copyrY  r  r  r  ri  s	            rU    test_nccl_backend_bool_allgatherz>DistributedTest._DistTestBase.test_nccl_backend_bool_allgatherO  s?    JJ!!$)),Tlt}5C <<DIIM(:;>>tyyIL , 2 2 4 t22455A eU^,//		:5   OOK6S-t/B/B/DE!+.1 <<AE
366tyyA  H- /
 .=s    7Fc                    t         j                  j                  | j                         ddgddgd}t        j
                  j                  t        j
                  j                  fD ]  }t        j                  || j                  dz           j                  | j                        }t        j                  ddg      j                  | j                        }| j                  |||t        j                  d       t        j                  ddg      j                  | j                        }|j                         }| j                  |||t        j                  d        t        j
                  j                  t        j
                  j                  fD ]  }t        j                  || j                  dz           j                  | j                        }| j                  dk(  r0t        j                  ddg      j                  | j                        n|j                         }| j                  |||t        j                  d        y )NTFr5  rG   r   rp  )rP   rR  r%  r   r  rx  r  r  r   r'  r0  r	   r  ry  r  )rS   r   rq  r  ri  r  s         rU   test_nccl_backend_bool_reducez;DistributedTest._DistTestBase.test_nccl_backend_bool_reducee  s    JJ!!$)),Tlu~6C}},,dmm.?.?@$||C		A,>?BB499M <<7::499E((xT[[VW(X  %||T4L9<<TYYG"."4"4"6(( /2t{{ )  A }}(($--*;*;<$||C		A,>?BB499M yyA~ LL$.11$))<%++- 
 ((xT[[VW(X =rV   c           	      r   d}t        j                  t        |      D cg c]*  }| j                  dk(  rt	        j                         dk  nd, c}      j                  | j                        }t        j                  |d       t        t        j                               D cg c]G  }t        j                  t        |      D cg c]  }d c}      j                  | j                        I }}t        j                  ||       |d   }|dd  D ]  }| j                  ||        y c c}w c c}w c c}w )NrB   r   rZ  FrW  rE   )rP   r   rW  r   randomr'  r  r&  r   r  r  )rS   tensor_sizerY  bcast_tensorr  ri  r   s          rU    test_nccl_backend_bool_broadcastz>DistributedTest._DistTestBase.test_nccl_backend_bool_broadcast  s    K << #;// /3ii1nV]]_s*%G/
 bm  NN<Q/ t22455A U;-?@-?e-?@ADDTYYO5   OOK6"1~H%ab/  2 * As   /D*$!D4	D/"D4/D4c                 V    t        j                         dz   dz   }t        |      D cg c]3  }t        j                  d      j                   j                        |z  5 }}t        dz  dz
  d      }t        |      D cg c]3  }t        j                  d      j                   j                        |z  5 }}t        |d      }|j                  |j                  }}|z  dk7  rt        j                  |z
  z        n|z  }	 j                  ||	        j                  ||z         t        t        |            }
 j                  t!        |
      |        fd} ||       t        |      }|j                  |j                  }} j                  |t        j                  |z                j                  ||z         t        t        |            }
 j                  t!        |
      |        ||       t        |      }|j                  |j                  }} j                  |t        j                  |z                j                  ||z         t        t        |            }
 j                  t!        |
      |        ||       y c c}w c c}w )	Nr<  rE   rG   T)dataset	drop_lastr   c                    t              D cg c]1  }t        j                  dg      j                  j                        3 }}t        j                  |t        j                  | g      j                  j                               |D cg c]  }|j                          }}j                  t        t        |            d       y c c}w c c}w Nr   rE   )rW  rP   
LongTensorr'  r   r  r  r   r  r  r  r  )local_num_samplesrY  world_samplessamplerS   r+  s       rU   validate_global_samplesz^DistributedTest._DistTestBase.test_DistributedSampler_padding.<locals>.validate_global_samples  s     BGzAR!ARAE$$aS),,TYY7AR  ! !5<<1B0C#D#G#G		#R >K K]6] K  S%7!8!<! !Ls   6CC)r@  )r  r   rW  rP   r   r'  r   rT  r<   num_samples
total_sizemathceilr  r  r  r  )rS   dataset_sizer  r@  dataset_tiny_sizedataset_tinydist_samplerrE  local_dataset_sizeeffective_dataset_sizeindices_listrH  dist_sampler_added_samplesdist_sampler_added_samples_tinyr+  s   `             @rU   test_DistributedSampler_paddingz=DistributedTest._DistTestBase.test_DistributedSampler_padding  s    ,,.J +a/L@El@ST@S1uzz!}''		2Q6@SGT !$J!Oa$7 ;9>?P9Q9QA

1  +a/9Q  
 .gNL((''  2  *,1 		<*4
BC!J. #
 .0FG/1BZ1OP\ 23LS.0AB	= $$56 *<G)L&*66*55  2 .		,:S0TU/1BZ1OP%? @ALS.0AB $$56 /A.V+/;;/::  2 !499->-K#L /1BZ1OP%D EFLS.0AB#$56M Us   8J!8J&c           	      z   t         j                         }t        j                  d   }|dk(  rD| j                  dz   t        | j                        z  }t        j                  j                  |       |dk(  r0|j                  t        t        j                  ddd                   t        t        j                               D cg c]  }d  }}t        j                   ||| j                  t#        |      z     |       t%        |      D ](  \  }}||t#        |      z     }	| j'                  ||	       * y c c}w )Nr   r  rE   ru   r   r   r  )COLLECTIVES_OBJECT_TEST_LISTro  r)  r*  r   r   r+  rP   rR  r%  r  rd   rQ   rW  r  r   r  r  r  r  )
rS   r2  gather_objectsr  	next_rankrY  output_gatheredr  r  ri  s
             rU   _test_allgather_objectz4DistributedTest._DistTestBase._test_allgather_object  s    :>>@Njj+G& !YY]c$//.BB	

%%i0 & %%c%++a1*E&FG-243F3F3H-IJ-It-IOJ""tyy3~+>>? $O43)!c..A*AB  h/ 5 Ks   	D8r   )OFFr  r  )levelsc                 "    | j                         S rf   )r\  r  s    rU   !test_all_gather_object_default_pgz?DistributedTest._DistTestBase.test_all_gather_object_default_pg  s     ..00rV   )r  r]  r  c                     t               }t        j                  |      }t        j                  |      }| j	                  |      S )Nr  )r2  )r   r  r  r  r\  rS   r  r  r2  s       rU   test_all_gather_object_subgroupz=DistributedTest._DistTestBase.test_all_gather_object_subgroup
  s>     )*G&&w/G~~g6H...AArV   c           	         t         j                         }t        j                  |      }t        j
                  d   }|dk(  rD| j                  dz   t        | j                        z  }t        j                  j                  |       |dk(  r0|j                  t        t        j                  dd|                   t        t        j                   |            D cg c]  }d  }}d}t        j"                  || j                  t%        |      z     ||k(  r|nd ||       ||k7  r<| j'                  |t        t        j                                D cg c]  }d  c}       n6t)        |      D ](  \  }	}
||	t%        |      z     }| j'                  |
|       *  G d d	      } |       }t        t        j                                D cg c]  }| }}| j+                  t,        d
      5  t        j.                  t        t        j                                D cg c]  }d  c}|| j                     |       d d d        y c c}w c c}w c c}w c c}w # 1 sw Y   y xY w)Nr   r  rE   ru   r   r   )object_gather_listr  r  c                       e Zd Zy)>DistributedTest._DistTestBase._test_gather_object.<locals>.BarN)r^   r_   r`   rt   rV   rU   Barrg  8  s    rV   rh  zCan't pickle local objectr  )rX  ro  r  rD  r)  r*  r   r   r+  rP   rR  r%  r  rd   rQ   rW  r   gather_objectr  r  r  r  r  r  )rS   pgrY  my_rankr  rZ  rY  r[  gather_on_rankr  r  ri  rh  rN   s                 rU   _test_gather_objectz1DistributedTest._DistTestBase._test_gather_object  s   9>>@NmmB'Gjj+G& !YY]c$//.BB	

%%i0 & %%c%++a7*K&LM-243F3Fr3J-KL-Kt-KOLNtyy3~+>>?n, $3" .(  #E$:M:M:O4P%Q4Pqd4P%Q (8FAs-a#n2E.EFH$$S(3 9
  A).t/B/B/D)EF)EAa)ENF''8ST&&#()<)<)>#?@#?aT#?@"499- UT3 M &R G A UTs0   	I	I
	I)/I	I
!IIIc                 "    | j                         S rf   )rm  r  s    rU   test_gather_objectz0DistributedTest._DistTestBase.test_gather_objectD  s     ++--rV   c                     t               }t        j                  |      }t        j                  |      }| j	                  |      S Nr  )r   r  r  r  rm  rb  s       rU   test_gather_object_subgroupz9DistributedTest._DistTestBase.test_gather_object_subgroupL  s;     )*G&&w/G~~g6H++H55rV   c                 V   t        |j                  j                         j                               }|D ]n  }t	        t        j                               D cg c]  }t        j                  |       }}t        j                  ||       |D ]  }| j                  ||        p y c c}w rf   )r  r   r1  r7  rW  r  r   rP   r  r  r  )rS   r{  net_module_statesr  rY  r  r   s          rU   validate_net_equivalencez6DistributedTest._DistTestBase.validate_net_equivalenceW  s     $SZZ%:%:%<%C%C%E F&16t7J7J7L1M1MAE$$Q'1M   Q/)F$$VQ/ * 's   B&c                 L   d}| j                   }d}t        j                  |       t        j                  ||d      }t        j                  j
                  j                  |j                  |      | j                   gd      }t        j                  ||d      j                  |      }t        j                  |      |_
        t        |j                  j                         j                               }|D ]  }t        t        j                                D 	cg c]  }	t        j"                  |       }
}	t        j$                  |
|       t'        |
      D ]/  \  }}||k(  r| j)                  ||       | j+                  ||       1  t-        |j                  |j.                  |j0                  ||j2                         | j5                  |       ||k(  rE|j                         j                         }t7        ||      D ]  \  }}| j)                  ||        y y c c}	w )NrG   rE   FrC   rA  r/  )r   rW  broadcast_bucket_sizer"  params_and_buffers_to_ignore)r   rP   r  rK   rL   rE  r   rR  ro  rp  r   r  r1  r7  rW  r  r   r  r  r  r  r  r   rW  rx  parameters_to_ignoreru  r  )rS   r   r   rank_to_broadcastr   r{  	new_modelrt  r  rY  r  r  r   expected_statesri  s                  rU   test_ddp_sync_module_statesz9DistributedTest._DistTestBase.test_ddp_sync_module_statesc  s    C99D !d#IIc3U3E((##;;

4 dii[ < C 		#s7<<TBIy1CJ $SZZ%:%:%<%C%C%E F&16t7J7J7L1M1MAE$$Q'1M   Q/!*;!7IAvDy((F3 ++Av6 "8 '  zz!//&)&?&?%-0-E-E ))#. (("+"6"6"8"?"?"A#&'8/#JKAx$$Q1 $K )-s   H!c                    d}d}d}| j                   }t        j                  ||d      }t        j                  ||| j                         |z  }t        j                  j
                  j                  |j                  |      | j                   gd      }d}| j                   d	kD  r|d
z  }|j                  d      5  t        |      D ]  }	 ||      j                         }
|
j                          t        j                  ||| j                         |z  }t        t        |j                                     }| j                  ||j                          |j#                          t        j                  j%                  | j                           	 d d d        |j                  d      5  t        |      D ]  } ||      j                         }
|
j                          t'        j(                         }|dk\  r|dz  }t        j                  ||| j                         |z  |z  t'        j(                         z  }t        t        |j                                     }| j                  ||j                          |j#                          t        j                  j%                  | j                           	 d d d        y # 1 sw Y   0xY w# 1 sw Y   y xY w)Nr}   rE   r   FrC   r   rw  ru   r   rG   )divide_by_initial_world_sizeT)r   rK   rL   rP   r   rE  r   rR  r2  rW  r   r  rl  r  r]  r  rq  r7  synchronizer  r   )rS   r   r|  
grad_scaler   r   r   r{  n_itersrY  r   r~  rs  r  effective_wss                  rU   test_ddp_grad_div_uneven_inputsz=DistributedTest._DistTestBase.test_ddp_grad_div_uneven_inputs  s1    CEJ99DIIc3U3E**UC		:ZGC((##;;

4 dii[ < C Gyy1}1u=wAs8<<>DMMO
 %*JJsC		$JZ$WM cnn&6!78E$$]EJJ?MMOJJ**$))*< ( >" t<wAs8<<>DMMO#'#6#6#8LAv$)

3DII>$%&' ++-	%.M
 !cnn&6!78E$$]EJJ?MMOJJ**$))*< ( =<# >=" =<s   >CK-D
KKKc                    d}d}d}t         j                  j                  | j                         t	        j
                  ||d      }t        j                  ||| j                        }t         j                  j                  j                  |j                  | j                        | j                  g      }t        j                  |      }|5 }	t        |      D ](  }
 ||      j                         }|j                          * 	 d d d        t        j                          d}t!        |	      }t        d	 |D              }| j#                  ||       |D ]9  }| j%                  |j&                         | j#                  |j(                  |       ; t        j                          d
}t!        ||	      }t        d |D              }| j+                  |d       |D ]  }| j#                  |j(                  |         t         j                  j                  j                  |j                  | j                        | j                  gd      }t        d      D ](  }
 ||      j                         }|j                          * |5 }	 ||      j                         }|j                          d d d        t!        ||	      }| j+                  t-        |      d       | j+                  |d   j.                  d       | j#                  |d   j(                  |       |D ]  }| j%                  |j&                          t!        d|	      }| j#                  t-        |      d       y # 1 sw Y   _xY w# 1 sw Y   xY w)Nru   rB   rj  FrC   r   r_  r  c              3   4   K   | ]  }|j                     y wrf   r  r  s     rU   r  zDDistributedTest._DistTestBase._test_ddp_profiling.<locals>.<genexpr>  s     6v!aggvr  rI  c              3   4   K   | ]  }|j                     y wrf   r  r  s     rU   r  zDDistributedTest._DistTestBase._test_ddp_profiling.<locals>.<genexpr>  s     @/?!agg/?r  rE   TrA  r  r   search_unused_parameters)rP   rR  r%  r   rK   rL   randrE  r   ro  rp  rW  r   r  r  r  r   r  r  r  r   r  r  r  )rS   r  r|  r   r<  r   r   r{  profiler_ctx_copyr  r  r   all_reduce_event_namer   r  r   broadcast_event_namebroadcast_eventss                     rU   _test_ddp_profilingz1DistributedTest._DistTestBase._test_ddp_profiling  s   ECIJJ!!$)),IIc3U3E**UC		:C((##;;

499% II; < C !%l ;y)As8<<>DMMO * 
 (,'7'7'9&:+$F!()>EF6v66K[)4/  -BC   '+&6&6&8%9#D 23GN@/?@@K##K3)  -AB * ((##;;

499% II;'+ < C
 1X3x||~  #d3x||~ # ))>EF##CK3##F1IOOQ7VAY^^-BC/   ))CTJFS[!,[ B #"s   7M%'M2%M/2M;c                 v    t         j                  j                  j                         }| j	                  |      S r  )rP   r^  r   r   r  r  s     rU   $test_ddp_profiling_autograd_profilerzBDistributedTest._DistTestBase.test_ddp_profiling_autograd_profiler	  s2     %*NN$;$;$C$C$E!++9N+OOrV   c                     t         j                  j                  j                  }t         j                  j                  j                  }t         j                  j                  ||g      }| j                  |       y )N)rb  r  )rP   r   rc  rd  r  r   r  )rS   cpu_actcuda_actr  s       rU   !test_ddp_profiling_torch_profilerz?DistributedTest._DistTestBase.test_ddp_profiling_torch_profiler  s]     nn5599G~~66;;H!&!7!7GXCV!7!W$$2D$ErV   c                     d}d}d}t        j                  ||d      }t        j                  ||| j                        }t        j                  |      }|j                  | j                        }t        t        j                               D ci c]  }|d|dz   z   }}t        |j                               }	t        j                  j                  |j                         |	      }
t        |	      D ]J  }|
j!                           ||      }|j                         }|j#                          |
j%                          L || j                     }t        j                   j&                  j)                  |j                  | j                        | j                  g
      }t        j                  j                  |j                         |t        j                         z  	      }|j+                         5  t        |      D ]t  }|j!                           ||      }|j                         }|j#                          t        j                  j-                  | j                         |j%                          v 	 d d d        t/        |j1                         j3                         |j4                  j1                         j3                               D ]  \  \  }}\  }}| j7                  ||        y c c}w # 1 sw Y   }xY w)Nru   rB   r  FrC   r   rG   rE   ry  r_  )rK   rL   rP   r  r   ro  rp  rR  rW  r  r   r   r7  r{  r|  r]  r7  r  rv  rE  r   r2  r  r  r1  rp   r   r  )rS   r|  r   r  r   r   local_modelr   rank_to_iter_mappinglocal_iterslocal_optimrY  r  r   r<  r{  	ddp_optimr  local_tensordist_tensors                       rU   test_ddp_join_model_equivalencez=DistributedTest._DistTestBase.test_ddp_join_model_equivalence  sb    EC MIIc3U3E**UC		:C--.K%**4995K16t7J7J7L1M$1Ma4!8n$1M ! $ 299;<K++//+*@*@*B}/UK;'%%'!#&wwy  " ( -TYY7I((##;;

499%499+ < C   "}t7J7J7L'L ( I y)A'')c(C779DMMOJJ**$))*<NN$ *  8;&&(..0#**2G2G2I2O2O2Q83!L#3A{   {;8?$* s   K/4BK44K=c           	         |j                   }|j                  }| j                  }|j                  }t        j
                  j                  |       t        j                          t        j                  j                  j                  |j                  |      |gd|      }|j                  >|j                  |j                  |j                         t        d|j                          ||   }	t	        j                   |	gt        j
                  j#                               }
t        j$                  |
t        j&                  j(                         |
j+                         }d}|j,                  rA||	k(  r%| j/                  t0        d| j                   d      }n!| j/                  t0        d	      }n
t3               }|5  |j5                  |j,                  
      5  t7        |	      D ]  }||z  dk7  r|j9                         }n
t3               }|5  t;        |t<              r || j?                         }n ||      j?                         }|jA                          | jC                  |       t        j
                  jE                  |       d d d        |dz  } 	 d d d        d d d        |j,                  r| jG                  ||       n| jI                  ||       t        j
                  jE                  |       |j,                  s6| jK                  |jL                         t	        j                   |jL                  g| j                        }t7        t        jN                               D cg c]  }t	        jP                  |       }}t        jR                  ||       t        jN                         dz
  }| jU                  |h|D ch c]  }|j+                          c}       | jW                  |       |jY                         }| jK                  |j[                  d             t        j                          y y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY wc c}w c c}w )NrE   )rA  r/  r  zregistered hook r   )rq  r   Rank  exhausted all inputs1Detected at least one rank that exhausted inputs.r   join_uneven_inputs).r   r   r   r   rP   rR  r%  r  ri  rK   rE  r   r   r  r   printr   current_devicer  rx  r  r  r   r  rw  r   r2  rW  r;  ri   r   r   r  r  r  r  r  r  _authoritative_rankr   r  r  assertSetEqualru  r  r  )rS   	test_caseiteration_mappingfind_unused_paramsr   r   r   r   r{  r<  num_iters_tensormin_num_iterstotal_itersr  r  contextr   final_rank_tensorrY  r  max_rankr   r  s                          rU   _run_uneven_inputs_testz5DistributedTest._DistTestBase._run_uneven_inputs_testO  s    OOE--C99D%33MJJ!!$'LLN ((##;;

4  6'9	 < C ~~)&&y	G((89: *$/I  %||EJJ$=$=$?  OO,1B1BC,113MK33 I-$($:$:$dii[8M&N%M
 %)$:$:$K%M
 !,XX/8/S/S   #9- },1&)kkmG&1mG$)#u5'*Cy}}'*3x||~ MMO ,,S1 "JJ22$2? % $q(% . . 33  m< '']C JJ""$"/ 77 7 78$)LL,,-dii%!
 #4#6#6#899 $$%679   ->?..014##J[ I[6[ I --c2#&#<#<#>  0 4 45I JK) 81 %W  R !JsI   >Q4QA<Q 	QQ/Q'Q,
 Q
QQ	QQ$c                     G d dt         j                  j                        }t         j                  j	                  | j
                         t        }t        j                  j                  t        j                  |            j                  | j
                        } |       j                  | j
                        }t        j                  dd      j                  t         j                  j                               }||fD ]$  }t         j                  j                  j                  || j
                  g      }d}| j
                  dk7  r'|}| j                  t         d| j
                   d	      }n|dz  }| j                  t         d
      }d}	|5  |j#                  d      5  t%        |      D ]>  }
 ||      j'                         }|j)                          | j+                  |       |	dz  }	@ 	 d d d        d d d        | j-                  |	|       | j/                  |       ' y # 1 sw Y   8xY w# 1 sw Y   <xY w)Nc                   $     e Zd Z fdZd Z xZS )bDistributedTest._DistTestBase.test_ddp_uneven_inputs_stop_iteration_sync_bn.<locals>.ModelWithCommc                 \    t         |           t        j                  ddd      | _        y )NrG   r   FrC   rI   rJ   rK   rL   r   rR   s    rU   rJ   zkDistributedTest._DistTestBase.test_ddp_uneven_inputs_stop_iteration_sync_bn.<locals>.ModelWithComm.__init__  s"    G$&!yyBU;DHrV   c                 R    | j                  |      }t        j                  |       |S rf   )r   r  r  rZ   s     rU   r\   zjDistributedTest._DistTestBase.test_ddp_uneven_inputs_stop_iteration_sync_bn.<locals>.ModelWithComm.forward  s     AOOA&HrV   r]   rb   s   @rU   ModelWithCommr    s    <rV   r  rB   rG   r_  r}   r   r  r  r  Tr  rE   )rP   rK   r   rR  r%  r   r  r  r  ro  rp  rQ   r  rE  r   r  rw  r2  rW  r   r  r  r  ru  )rS   r  model_bn
comm_modelmodel_inputr   r  r<  r  r  r  r   s               rU   -test_ddp_uneven_inputs_stop_iteration_sync_bnzKDistributedTest._DistTestBase.test_ddp_uneven_inputs_stop_iteration_sync_bn  s     JJ!!$)),H''>>h'd499o  '--dii8J++b!,11%**2K2K2MNK"J/))AA $		{ B  !"99> -I$($:$:$dii[8M&N%M
 !. 1I$($:$:$K%M "tD!&y!1A#(#5#9#9#;D MMO ,,U3FA	 "2 E #   M2--e4= 0* ED #]s%   H8'AH,5H8,H51H88I	c                 x	   d}d}t        j                  t        j                  ddd      t        j                         t        j                  ddd      t        j                         t        j                  ddd      t        j                               }t        j                  ||d      }t               } G d	 d
t         j                        } |d      } |d      }| j                  }	t        d|t        j                  |d|	      d      t        d|t        j                  |||||	      d      t        d|t        j                  |||	      d      t        d|t        j                  |d|	      |	fd      t        d|t        j                  |d|	      |	fd      g}
t        d|t        j                  d t        j                  |||	      d      t        d|t        j                  t        j                  d dddd      t        j                  |||	      d      g}|
j!                  |       t"        rRt$        j&                  j)                         }|
j+                  t        d|t        j                  dddd      d             g }t-        |
      D ]D  \  }}|j+                  t        |j.                  |j0                  |j2                  |dz                F g }|
D ]I  }|j+                  t        |j.                  |j0                  |j2                  |j4                  d             K |
j!                  |       |
j!                  |       ddg}g d}dg}t7        j8                         dkD  r|j+                  d       g }|D ]  }|D ]  }|D ]  }t;        d|      D 	ci c]  }	|	| }}	|dkD  r+|j=                         D ]  }	|	dkD  s	||	xx   |dz  z  cc<    |j?                  t;        |t7        j8                               D 	ci c]  }	|	||z   
 c}	       |j+                  |          tA        jB                  |
|      D ]\  \  }}| j                  dk(  r(tE        d|j.                   d|j4                   d|        | jG                  ||d |j.                  v !       ^ y c c}	w c c}	w )"Nr   rE   r   r}          FrC   c                   *     e Zd Z fdZd Zd Z xZS )ODistributedTest._DistTestBase.test_ddp_uneven_inputs.<locals>.UnusedParamModulec                 l    t         |           t               | _        t               | _        || _        y rf   )rI   rJ   r   t0rl  unused_params_rank)rS   r  rT   s     rU   rJ   zXDistributedTest._DistTestBase.test_ddp_uneven_inputs.<locals>.UnusedParamModule.__init__
  s(    G$&"fDG"fDG.@D+rV   c                 Z    | j                   j                  | j                  j                  fS rf   )r  r   rl  r  s    rU   task_parametersz_DistributedTest._DistTestBase.test_ddp_uneven_inputs.<locals>.UnusedParamModule.task_parameters  s     GGIItwwyy11rV   c                     || j                   k7  r | j                  | j                  |            S | j                  |      S rf   )r  rl  r  )rS   r[   r   s      rU   r\   zWDistributedTest._DistTestBase.test_ddp_uneven_inputs.<locals>.UnusedParamModule.forward  s?      4#:#:: 
+ "WWQZrV   )r^   r_   r`   rJ   r  r\   ra   rb   s   @rU   UnusedParamModuler  	  s    A2rV   r  r   batch_norm_netrG   r   )r   r   r   r   large_conv_modelsmall_model&unjoined_rank_with_unused_params_model$joined_rank_with_unused_params_modelsmall_model_allreduce_hook)r   r   r   r   r   r   small_model_power_sgd_hook)rW  r   r!  r"  use_error_feedbackresnet_modelru   T)r   r   r   r   r   )rG   ru   rB   zRunning test: z' sync interval
                        z0 with iteration mapping
                        unused_params_model)r  )$rK   r[  r\  r   rL   r   r   r   r   rP   r   r  r  r  r   r#  extendr  r  r  r  r  r  r   r   r   r   r  r   rW  keysrt  r  r  r  r  )rS   r   r|  large_modelr  bn_netr  r  r  r   r  models_with_hookr  models_with_syncr  
test_inputthrow_on_early_term_testsbaseline_num_itersiteration_offsetsnum_uneven_ranksiteration_mappingsnum_early_join_ranksbaseline_iterr6  mappingr  r  s                              rU   r  z4DistributedTest._DistTestBase.test_ddp_uneven_inputs  s    CE--		!R#			"b!$			"c1%	K ))C59K!^FBII " 6Gq5I23DQ3G099D #) 

5!D9"#	 #+%

5%c$G"#	 #&%

5#d;"#	 #A@E1T:DA"#	 #?>E1T:DA"#	9"NL #5% //

5#d;"# #5%!//"00&*23 -.#(+0 

5#d;"# 4 !!"23 *11::<%%&+*!JJq!T48&'	  "!*>!:: ''&'__(..&NN&'!e	 "; )+%,
)00&'__(..&NN&0&>&>37 - !!"23!!";< #$Q * !s""$q( ''*!#
 )9$%7M"3 ).a1E(F#(F !-/(F   # 0!3(/#'!8$+DMVq[$@M )7   -2$8$:M:M:O-"-"D !%mf&< <-" +11':) #4 &8 )90 3<2C2C 23.- 99>*9>>*: ;"001 2*+/
 ,,%(=(O - 3+#s   *
R2
R7c                    t        j                  | j                         t         j                  j                  j                  t         j                  j                  dd      j                  | j                        | j                  g      }t        j                  d      | j                  z  }d}t        j                         }|j                  d      5  t        |      D ]  }|j                  j                  j                  }|!|j!                  d       |j#                           ||      }|j%                         }|j'                          t%        d t        |      D              |z  }	| j)                  |j                  j                  j                  j+                         |	        	 d d d        |j,                  }
| j/                  |
j0                         | j3                  |       y # 1 sw Y   BxY w)NrE   r_  r}   F)enablec              3       K   | ]  }|  y wrf   rt   rp  s     rU   r  zSDistributedTest._DistTestBase.test_ddp_uneven_input_join_disable.<locals>.<genexpr>  rq  rr  )rP   r  r   rK   rE  r   rL   rR  r   r  r   r2  rW  r   ry  rq  r  r  r   r  r  r  _join_configr5  r  ru  )rS   r{  r   r  r+  rY  rq  r  r   r~  join_configs              rU   "test_ddp_uneven_input_join_disablez@DistributedTest._DistTestBase.test_ddp_uneven_input_join_disable  sv    dii(((##;;1%**4995499+ < C **Q-$))+CG,,.J'wA::,,11D'++E2

c(C779DMMO %('E53D'E$E
$RM$$SZZ%6%6%;%;%@%@%BMR ( ( **K[//0))#.# ('s   CG11G:c                    d G fddt         j                        } |       }t        j                   j                  j	                  |j                  | j                        | j                  g      }t        j                  d      }| j                  t              5  |j                         5   ||      }|j                         }|j                          d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)NzIntentional errorc                   *     e Zd Z fdZfdZ xZS )VDistributedTest._DistTestBase.test_ddp_uneven_input_exception.<locals>.ExceptionModulec                     t         |           t        j                  t	        j
                  dd            | _        y )NrE   Tr   )rI   rJ   rK   r   rP   r   rs  rR   s    rU   rJ   z_DistributedTest._DistTestBase.test_ddp_uneven_input_exception.<locals>.ExceptionModule.__init__  s)    G$&!#ejj$.O!PDJrV   c                     t              rf   )r  )rS   rY  	error_strs     rU   r\   z^DistributedTest._DistTestBase.test_ddp_uneven_input_exception.<locals>.ExceptionModule.forward  s    $Y//rV   r]   )rT   r  s   @rU   ExceptionModuler    s    Q0 0rV   r  r_  rE   )rK   r   rP   rE  r   rR  r   r   r  r  r2  r   r  )rS   r  exception_moduler{  r   r  r   r  s          @rU   test_ddp_uneven_input_exceptionz=DistributedTest._DistTestBase.test_ddp_uneven_input_exception  s     ,I0")) 0  /0((##;; %%dii0dii[ < C **Q-C''
I>XXZc(C779DMMO   ?>Z ?>s$    C71)C+C7+C4	0C77D c           	         t         j                         }| j                  dz   t        | j                        z  }t
        j                  d   }|dk(  rt        j                  j                  |       d}|dk(  r0|j                  t        t        j                  ddd                   t        r.|j                  t        t        j                  dd                   | j                  |k(  r|n|D cg c]  }d  c}}|dk7  rq|d   g}| j                  |k7  r| j                  |d   |d          t        j                   |d|t        j"                  d      	       | j%                  |d   |d          |dk7  rt        j                  j'                         t        | j                        k(  rq|d   g}| j                  |k7  r| j                  |d   |d          t        j                   |d|t        j"                  |      	       | j%                  |d   |d          |dk(  rt        j                  j'                         t        | j                        k(  rq|d   g}| j                  |k7  r| j                  |d   |d          t        j                   |d|t        j"                  |      	       | j%                  |d   |d          |d   g}| j                  |k7  r| j                  |d   |d          t        j                   |d|
       | j%                  |d   |d          | j                  |k7  r| j                  ||       t        j                   |d|
       | j%                  ||       y c c}w )NrE   r   r  r   ru   r   i
r#  )r"  r  r   r!  )rX  ro  r   r   r+  r)  r*  rP   rR  r%  r  rd   rQ   r7   r  r  rG  r   r  r  )	rS   r  rY  rZ  r  src_rankrY  objectssingle_obj_lists	            rU   _test_broadcast_object_listz9DistributedTest._DistTestBase._test_broadcast_object_list  s   9>>@N Q#doo*>>Ijj+G& 

%%i0H& %%c%++a1*E&FG %%c%++a*C&DE 99( $23NqdN3  & #*1:,99(''(:N1<MN**#%U@S   !3^A5FG
 & UZZ%<%<%>#dooBV%V#*1:,99(''(:N1<MN**#%Y@W   !3^A5FG & UZZ%<%<%>#dooBV%V#*1:,99(''(:N1<MN**#%Y@W   !3^A5FG  'qzlOyyH$##OA$6q8IJ&&AUK_Q/1BC yyH$##G^<&&wAUCWn5] 4s   1	M0r  zCTest is failing, see https://github.com/pytorch/pytorch/pull/113620c                 "    | j                         S rf   )r  r  s    rU   test_broadcast_object_listz8DistributedTest._DistTestBase.test_broadcast_object_list>  s     3355rV   c                     t               }t        j                  |      }t        j                  |      }| j	                  |      S rq  )r   r  r  r  r  rb  s       rU   $_test_broadcast_object_list_subgroupzBDistributedTest._DistTestBase._test_broadcast_object_list_subgroupG  s;     )*G&&w/G~~g6H33H==rV   c                     G d dt         j                        }| j                  }t        j                  ddgddg      D ]  \  }} || j                        j                         j                  |      j                  j                  dt        j                  d| j                  z   | j                               t        j                  j                               }t        j                  j                               }t        fdj                         D              }j                  j!                         D 	
cg c]  \  }	}
| d	|	  }}	}
j                  j#                         D 
cg c]  \  }}
| d	|  }}}
t        j                   j$                  j&                  j)                  ||z          t        j                   j$                  j'                  |g|||
      }t        j*                  ddd      j                  |      |j,                  _        t/        j0                  |j,                        j3                  | j                        }t        j4                  dt        j
                        j                  |      | j                  dz   z  }t7        d      D ]Y  } ||      j9                         j;                           ||      j9                         j;                          t=        |j,                  j                  j                         |j                  j                               D ]+  \  }}| j?                  |j@                  |j@                         - t=        |j,                  jB                  j                         |jB                  j                               D ]-  \  }}| jE                  |j@                  |j@                  k(         / |D ]  }| jG                  |j@                  d u        ! \ t        j2                  jI                  | j                          y c c}
}	w c c}
}w )Nc                   $     e Zd Z fdZd Z xZS )LDistributedTest._DistTestBase._test_ddp_ignore_params_arg.<locals>.TestModelc                     || _         t        | 	          t        j                  ddd      | _        | j                   dk(  rt        j                  ddd      | _        y t        j                  ddd      | _        y )NrE   FrC   r   rB   )r   rI   rJ   rK   rL   r   r   rS   r   rT   s     rU   rJ   zUDistributedTest._DistTestBase._test_ddp_ignore_params_arg.<locals>.TestModel.__init__T  s^     $DIG$&!yyAE:DH yyA~#%99Q#?#%99R%#@rV   c                 J    | j                  |      }| j                  |      }|S rf   r   rZ   s     rU   r\   zTDistributedTest._DistTestBase._test_ddp_ignore_params_arg.<locals>.TestModel.forward_  s!    AAHrV   r]   rb   s   @rU   	TestModelr  S  s    	ArV   r  FTignore_bufferr}   r   c              3   D   K   | ]  \  }}|j                   u r|  y wrf   )r   )r  r  r   r   s      rU   r  zLDistributedTest._DistTestBase._test_ddp_ignore_params_arg.<locals>.<genexpr>q  s*      &/D+V*  /Ds    r  )rA  r  r  rC  rE   rC   rN  rj  )%rK   r   r   r  r  rJ  r'  r   rO   rP   r  r  r]  r  rl  named_modulesr  r  rE  r   r  rL   r   ro  rp  rR  r   rW  r   r  r  r  rq  r   r5  r  r  )rS   rC  r  rT  find_unusedr  proxy_paramsproxy_buffersmodel_fc2_name
param_namerY  proxy_param_namesbuf_nameproxy_buffer_namesrT  r  r   r  materialized_paramlocal_paramsynced_paramproxy_paramr   s                         @rU   _test_ddp_ignore_params_argz9DistributedTest._DistTestBase._test_ddp_ignore_params_argR  sv   BII " 		I4=4E4Et}50/ "$)),22477	B 		))#U[[TYYtyy%Q  $EII$8$8$:; $UYY%6%6%8 9!% &/4/B/B/D& " */)C)C)E%)E
A &&a
|4)E " % (-yy'>'>'@&'@! &&az2'@ # & !!99ee,/AA hh''?? ){+6&7!- @  "$1ae!<!?!?	!J

 #mmCJJ7<<TYYGjj%++699)D		TUVqAHLLN++-$((*335 <?

113[__5O5O5Q<7*K (();)@)@+BRBRS< 69

113[__5O5O5Q61k (():):k>N>N)NO6 (4(8(8D(@A (4% ". 

&&dii&8G5 %&s   P75P=c                 L    | j                  d       | j                  d       y )NFrC  T)r  r  s    rU   test_ddp_ignore_params_argz8DistributedTest._DistTestBase.test_ddp_ignore_params_arg  s&     ,,%,@,,$,?rV   c           
          G d dt         j                        }t        j                   j                  j	                   |       j                  | j                        | j                  g      }t        d      D ]y  }t        j                  dd      }|dkD  r8	  ||      j                         j                          | j                  dd	       V ||      j                         j                          { t%        j6                          y # t        $ r}t        |      }t        ||       t        t         t"        g}t%        j&                         t$        j(                  j*                  k(  r|j-                  t.               n3d
j1                  dg      }|j-                  d| j                   d|        |D ]  }	| j3                  |	|v d|	 d|         | j                  t4        |v        Y d }~xd }~ww xY w)Nc                   $     e Zd Z fdZd Z xZS )`DistributedTest._DistTestBase.test_ddp_unused_params_rebuild_buckets_exception.<locals>.ToyModelc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        y r   rI   rJ   rK   rL   r\  r]  rR   s    rU   rJ   ziDistributedTest._DistTestBase.test_ddp_unused_params_rebuild_buckets_exception.<locals>.ToyModel.__init__  s6    G$& "		"bu =DI "		"bu =DIrV   c                 $    | j                  |      S rf   )r\  rZ   s     rU   r\   zhDistributedTest._DistTestBase.test_ddp_unused_params_rebuild_buckets_exception.<locals>.ToyModel.forward  s    99Q<'rV   r]   rb   s   @rU   r^  r    s    >
(rV   r^  r_  rG   rE   rB   r   Tz'DDP unused parameters error not raised.r  znet2.weightdid not receive grad for rank : 	Expected 
 to be in )rK   r   rP   rE  r   rR  r   rW  r  r   r  r5  rw  r   r1   !ddp_prev_reduction_unfinished_str$ddp_recommend_find_unused_params_str ddp_outputs_not_used_in_loss_strr  r  r  r]  r  ddp_suggest_debug_mode_strr2  r  "ddp_find_unused_params_enabled_strri  )
rS   r^  rT  r  r   r  rv  expected_strsunreduced_paramsss
             rU   0test_ddp_unused_params_rebuild_buckets_exceptionzNDistributedTest._DistTestBase.test_ddp_unused_params_rebuild_buckets_exception  s   (299 ( ((##;;
		*		{ < C 1XjjB'q5C//1, (( "K HLLN++-C F LLN7 ( T!!f/S9=@<)  //1T__5H5HH)001KL/3yy-/I,)00"@2N^M_ ` "/A OOAH	!Jse6TU "/(()Ks)RSS'Ts   $D

	G6CG11G6c                 .    G d dt         j                        }t        j                  j	                  | j
                          |       j                  t        j                  j                               }dD ]  }t        j                   j                  j                  t        j                  |      | j
                  gd|      }t        j                  dd| j
                        }t        d	      D ]  } ||      }|dz  }|j                          !  y )
Nc                   $     e Zd Z fdZd Z xZS )VDistributedTest._DistTestBase.test_ddp_shared_grad_acc_unused_params.<locals>.ToyModelc                 "   t         |           t        j                  ddd      | _        t        j
                  t        j                  d            | _        | j                  | j                  _        t        j                  dd      | _	        y )NrB   r}   FrC   )
rI   rJ   rK   rL   r\  r   rP   r  rD   r]  rR   s    rU   rJ   z_DistributedTest._DistTestBase.test_ddp_shared_grad_acc_unused_params.<locals>.ToyModel.__init__  s]    G$& "		"ae <DI "U[[^ <DI &*YYDIIN "		"a 0DIrV   c                 @    | j                  |      j                         S rf   )r]  r   rZ   s     rU   r\   z^DistributedTest._DistTestBase.test_ddp_shared_grad_acc_unused_params.<locals>.ToyModel.forward  s    99Q<++--rV   r]   rb   s   @rU   r^  r+    s    	1.rV   r^  r  T)rA  r  rC  r   rB   r   rj  )rK   r   rP   rR  r%  r   r'  r  rE  r   ro  rp  rQ   rW  r  )rS   r^  r   staticr`  r   r  r   s           rU   &test_ddp_shared_grad_acc_unused_paramszDDistributedTest._DistTestBase.test_ddp_shared_grad_acc_unused_params  s    
.299 . JJ!!$)),JMM%**";";"=>E'!HH--EEMM%( $		{+/!'	 F 	 kk"b;qA$S>DBJDMMO	 " (rV   c                 (    t        j                  dd      j                   j                        }d G d d      } fd} fd} fd} fd}||t        |t
        |t        |t        |t        |i G  fd	d
t        j                   j                        }t        j                   j                  j                   |       j                   j                         j                  g      fd}t	        d t              D              }	 ||	t               t              D 
cg c]  }
t        j                  dd       }	}
 ||	t
                |t        j                  dd            }	 ||	|       d}d}t        j                  ||      }t        j                  ||      }t        ||      }	 ||	t!        |	             t        ||      }	 ||	t!        |	             t"        d   |t"        d   |i}	 ||	t!        |	             y c c}
w )NrB   rG   c                       e Zd ZddgZd Zy)DDistributedTest._DistTestBase.test_ddp_device.<locals>.TensorWrapperr  moved_to_gpuc                      || _         d| _        y r  )r  r3  )rS   r  s     rU   rJ   zMDistributedTest._DistTestBase.test_ddp_device.<locals>.TensorWrapper.__init__  s    DF(-D%rV   N)r^   r_   r`   	__slots__rJ   rt   rV   rU   TensorWrapperr2    s     .1	.rV   r6  c           	      &   j                  t        |              j                  dt        | D ch c]  }|j                   c}             j                  | d   j                  j                  j
                         | d   | d   z   S c c}w )NrE   r   )r  r  r  r   indexr   )r[   r  expected_lenrS   s     rU   tuple_and_list_validatorzODistributedTest._DistTestBase.test_ddp_device.<locals>.tuple_and_list_validator  sv    A5  C1(=1a1(=$>?  1!2!2DII>tad{" )>s   Bc                    j                  | j                  t               j                  | j                  j                  j
                  | j                  j                  j
                         j                  | j                  j                  j
                  j                         | j                  | j                  z   S rf   )r  _fieldsEXPECTED_FIELDSrM   r   r8  rN   r   r[   rS   s    rU   namedtuple_validatorzKDistributedTest._DistTestBase.test_ddp_device.<locals>.namedtuple_validator   sw      O<  !1!1133::3C3CD  !1!1499=ssQSSy rV   c                     j                  | j                  xs" t        | j                  j                        dk(         | j                  j                  j                        | _        d| _        | j                  S )Nr#  T)r  r3  r   r  r   r'  r   r>  s    rU   custom_type_validatorzLDistributedTest._DistTestBase.test_ddp_device.<locals>.custom_type_validator&  sQ     L3qsszz?e3KMccffTYY'!%ss
rV   c           	         j                  t        d   | j                         v        j                  t        d   | j                         v        j                  dt	        | j                         D ch c]  }|j                   c}             j                  | t        d      j                  j                  j                         | t        d      | t        d      z   S c c}w rC  )	r  r=  r  r  r  r7  r   r8  r   )r[   r  rS   s     rU   dict_validatorzEDistributedTest._DistTestBase.test_ddp_device.<locals>.dict_validator,  s     2affh >? 2affh >?  C188:(F:a:(F$GH  ?1#5!6!=!=!C!CTYYO+,q1C/DDD )Gs   5C&c                   ,     e Zd Z fdZfdZ xZS )?DistributedTest._DistTestBase.test_ddp_device.<locals>.ToyModelc                 \    t         |           t        j                  ddd      | _        y r   r  _selfrT   s    rU   rJ   zHDistributedTest._DistTestBase.test_ddp_device.<locals>.ToyModel.__init__=  s"    G$& "		"bu =EIrV   c                 r    j                  t        ||              |   |      }| j                  |      S rf   )r  ri   r   )rH  r[   expected_type
fwd_tensorrS   
validatorss       rU   r\   zGDistributedTest._DistTestBase.test_ddp_device.<locals>.ToyModel.forwardA  s8     OOJq-$@A!:M!:1!=J 99Z00rV   r]   )rT   rS   rL  s   @rU   r^  rE  <  s    >1 1rV   r^  r_  c                 t    t        d      D ])  } | |      }|j                         j                          + y )Nrv   )rW  r   r  )r   
input_typerY  r  r   s       rU   
train_iterzADistributedTest._DistTestBase.test_ddp_device.<locals>.train_iterL  s/    qAZ0CGGI&&( "rV   c              3   H   K   | ]  }t        j                  d d         yw)rB   N)rP   rQ   )r  rY  s     rU   r  z@DistributedTest._DistTestBase.test_ddp_device.<locals>.<genexpr>S  s     I5HB+5Hs    "r}   r   rE   )rK   rL   r'  r   r   r  TestNamedTupleInput_0r   r  rP   r   rE  r   rW  rQ   r  typer=  )rS   r  r6  r:  r?  rA  rC  r^  rO  r   rY  r|  r   rM   rN   r9  r   rL  s   `              @@@rU   test_ddp_devicez-DistributedTest._DistTestBase.test_ddp_device
  s    		"b!$$TYY/AL. .#!E 4/.%';%';nJ
1588?? 
1 HH%%==
dii(dii[ > E) IU<5HIICsE" 16l0CD0C15;;r2&0CCDsD!  B 34CsM* EC

5#&A

5#&A'1-CsDI&'1-CsDI&  "A"AC sDI&1 Es   +Hc                     d}d}t        j                  || j                        t        j                  || j                         G  fddt         j                  j                        }t         j                  j
                  j                   |       j                   j                         j                  g      }t              } ||t        |             t              } ||t        |             y )Nr}   rB   r   c                   .     e Zd Z fdZfdZ xZS )KDistributedTest._DistTestBase.test_ddp_namedtuple.<locals>.NamedTupleModulec                 X    t         |           t        j                  dd      | _        y )NrB   rE   r  rG  s    rU   rJ   zTDistributedTest._DistTestBase.test_ddp_namedtuple.<locals>.NamedTupleModule.__init__|  s     G$& "		"a 0EIrV   c           	      |   j                  t        ||      d| dt        |              j                  |j                  t
               j                  |j                         j                  |j                         | j                  t        j                  |j                  |j                              S )NzExpected type rt  )r  ri   rR  r  r<  r=  rM   rN   r   rP   mul)rH  r  rJ  rM   rN   rS   s      rU   r\   zSDistributedTest._DistTestBase.test_ddp_namedtuple.<locals>.NamedTupleModule.forward  s    OO"5-8(yeN $$U]]OD$$Q0$$Q0 99UYYuww%@AArV   r]   )rT   rM   rN   rS   s   @rU   NamedTupleModulerV  {  s    1	B 	BrV   rZ  r_  )rP   r  r   rK   r   rE  r   rR  rQ  rR  r   )rS   r|  r   rZ  r   r   rM   rN   s   `     @@rU   test_ddp_namedtuplez1DistributedTest._DistTestBase.test_ddp_namedtupler  s     EC

5#dii8A

5#dii8AB B588?? B  HH%%== "''		2		{ > E (1-C#tCy!'1-C#tCy!rV   c           
      L   d}d}t        j                         }t        j                  j	                  | j
                         t        j                  j                  j                  t               j                  | j
                        | j
                  gd      }t        j                  ||| j
                        }t        j                  ||| j
                        }t        d      D ]  }|dz  dk(  r	 ||      }n ||      }|j                         }	|	j                          |j                  j!                         }
|dz  dk(  r3t        j"                  |dg| j
                  t        j$                  	      }n2t        j"                  ||g| j
                  t        j$                  	      }|
}| j'                  ||        t        j                  j                  j                  t               j                  | j
                        | j
                  gd
      }t        d      D ]g  }|dk(  r' ||      j                         }	|	j                          /	  ||      j                         }	|	j                          | j)                  dd       i t        jF                          y # t*        $ r}t-        |      }t/        ||       d}t0        t2        t4        d| j
                   d| g}t        j6                         t         j8                  j:                  k(  r|j=                  t>               n3djA                  dg      }|j=                  d| j
                   d|        |D ]  }| jC                  ||v d| d|         | j)                  tD        |v        Y d }~yd }~ww xY w)Nr   rB   Tr  r   rj  rG   r   r   rO  FDDP error not raisedrE   6Parameter indices which did not receive grad for rank r  r  lin2.weightr  r  r  )$r  r   rP   rR  r%  r   rK   rE  r   r   rQ   r   rW  r   r  reducer_get_local_used_mapr   int32r  r5  rw  r   r1   r   r!  r"  r  r  r]  r  r#  r2  r  r$  ri  )rS   r|  r   r+  r   random_input
ones_inputr  r  r   local_used_mapri  variable_usage_tensorr  rv  unused_param_indexr%  r&  r'  s                      rU   'test_ddp_control_flow_same_across_rankszEDistributedTest._DistTestBase.test_ddp_control_flow_same_across_ranks  s   
 EC,,.JJJ!!$)),HH%%==#%**4995 II;'+ > E
 !;;uc$))DLE3tyyAJ1Xq5A:-C
+Cwwy "'!B!B!Dq5A:$||#Q		 H  %||#Z0%++ H
 )7%  !6A+ 2 HH%%==#%**4995 II;', > E
 1X6 .224DMMOG$\26682 ((/EFA D LLN5 ( T!!f/s;-.*=@<TUYU^U^T__abtauv	)  //1T__5H5HH)001KL/3yy-/I,)00"@2N^M_ ` "/A OOAH	!Jse6TU "/(()Ks)RSS-Ts   &J$$	N#-C+NN#c                    t        j                         }t        j                  j	                  | j
                         t        j                  j                  j                  t               j                  | j
                        | j
                  gd      }t        j                  dd| j
                        }t        j                  dd| j
                        }d}| j                  t        |      5  t        d      D ];  }|dz  dk(  r	 ||      }n ||      }|j                         }|j!                          = 	 d d d        t#        ||       | j                  t        d	      5  t        d      D ];  }|dz  dk7  r	 ||      }n ||      }|j                         }|j!                          = 	 d d d        t#        |d
       y # 1 sw Y   xY w# 1 sw Y   "xY w)NTrA  rC  r   rB   r   z1Your training graph has changed in this iterationrG   r   af  Expected to have finished reduction in the prior iteration before starting a new one. This error indicates that your training graph has changed in this iteration, e.g., one parameter is used in first iteration, but then got unused in the second iteration. this is not compatible with static_graph set to True.
Parameter indices which did not receive grad forz#Expected to have finished reduction)r  r   rP   rR  r%  r   rK   rE  r   r   rQ   r   r  rw  rW  r   r  r1   )	rS   r+  r   rd  re  r$  r  r  r   s	            rU   test_invalid_static_graphz7DistributedTest._DistTestBase.test_invalid_static_graph  s    ,,.JJJ!!$)),HH%%==#%**4995 II;! > E
 !;;r2dii@LB499=J OL''lCqA1uz#L1#J/779DMMO " D $E<8 ''C	 qA1uz#L1#J/779DMMO "	$ $E+PQ? DC	 	s   2A
G'A
GGGc           
         dd G fddt         j                        }t        j                         }t        j
                  j                  | j                         t        j                   j                  j                   || j                        j                  | j                        | j                  gd      }t	        j                  | j                        }t	        j                  | j                        }t        d      D ]  }|d	z  d
k(  r	 ||      }n ||      }|j                         }|j                          |j                  j!                         }	|d	z  d
k(  r3t	        j"                  |d
g| j                  t        j$                        }
n2t	        j"                  |dg| j                  t        j$                        }
|	}| j'                  ||
        t        j                   j                  j                   || j                        j                  | j                        | j                  gd      }t        d	      D ]g  }|d
k(  r' ||      j                         }|j                          /	  ||      j                         }|j                          | j)                  dd       i t        jF                          y # t*        $ r}t-        |      }t/        ||       d}t0        t2        t4        d| j                   d| g}t        j6                         t        j8                  j:                  k(  r|j=                  t>               n3djA                  dg      }|j=                  d| j                   d|        |D ]  }| jC                  ||v d| d|         | j)                  tD        |v        Y d }~yd }~ww xY w)Nr   rB   c                   ,     e Zd Z fdZfdZ xZS )\DistributedTest._DistTestBase.test_ddp_control_flow_different_across_ranks.<locals>.ToyModelc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        || _        y r   rI   rJ   rK   rL   r   r   r   r   s     rU   rJ   zeDistributedTest._DistTestBase.test_ddp_control_flow_different_across_ranks.<locals>.ToyModel.__init__!  =    G$& "		"bu =DI "		"bu =DI $DIrV   c                 D   t        j                  |t        j                  |j                              xr | j                  dk(  }|r3| j                  t        j                  | j                  |                  S t        j                  | j                  |            S )Nr   rE   )	rP   rk   r   r   r   r   r   r   r   )rS   r[   r   r|  r   s      rU   r\   zdDistributedTest._DistTestBase.test_ddp_control_flow_different_across_ranks.<locals>.ToyModel.forward'  su     Auzz%QXX'NO + IIN %
 (#yy		!)=>> vvdiil33rV   r]   )rT   r|  r   s   @rU   r^  ro     s    %4 4rV   r^  Tr  r   rj  rG   r   r]  rE   Fr^  r_  r  r  r`  r  r  r  )$rK   r   r  r   rP   rR  r%  r   rE  r   rQ   r   rW  r   r  ra  rb  r   rc  r  r5  rw  r   r1   r   r!  r"  r  r  r]  r  r#  r2  r  r$  ri  )rS   r^  r+  r   rd  re  r  r  r   rf  ri  rg  r  rv  rh  r%  r&  r'  r|  r   s                     @@rU   ,test_ddp_control_flow_different_across_rankszJDistributedTest._DistTestBase.test_ddp_control_flow_different_across_ranks  s/   
 EC4299 4( ,,.JJJ!!$)),HH%%==#((3 II;'+ > E
 !;;uc$))DLE3tyyAJ1Xq5A:-C
+Cwwy "'!B!B!Dq5A:$||#Q		 H  %||#Q		 H )7%   !6A/ 6 HH%%==#((3 II;', > E
 1X6 .224DMMOG$\26680 ((/EF? B LLN3 ( T!!f/s;-.*=@<TUYU^U^T__abtauv	)  //1T__5H5HH)001KL/3yy-/I,)00"@2N^M_ ` "/A OOAH	!Jse6TU "/(()Ks)RSS+Ts   &K	OC+OOc                 .   d}| j                   |k(  rt        nt        D cg c]  }d  c}}t        j                         }|d | }d}t	        |      |k  r(|j                  ||          |dz  }t	        |      |k  r(d g}t        j                  |||       | j                  |d   t        | j                   t	        t              z            | j                  t        d      5  t        j                  g ||       d d d        y c c}w # 1 sw Y   y xY w)Nr   rE   rW  zMExpected argument scatter_object_output_list to be a list of size at least 1.)
r   rX  r  r   r  r  scatter_object_listr  r  r  )rS   r  rY  rS  r+  r  output_obj_lists          rU   test_scatter_object_listz6DistributedTest._DistTestBase.test_scatter_object_list  s   H 99( -$@A$@qd$@A 
 ,,.J'4LAl#j0##LO4Q l#j0  $fO$$_lQ",II$@ AA ''_ ((\xH	 % B$ s   	D$DDc           	      
   t        j                  dgt         j                        t        j                  dgt         j                        t        j                  dgt         j                        t        j                  dgt         j                        t        j                  dgt         j                        t        j                  dgt         j                        g}|D cg c]  }|j	                          }}|S c c}w )Nr   rN  r  )rP   rP  rJ  rK  	to_sparse)rS   r  r  tensors_sparses       rU   3_generate_sparse_tensors_for_bucket_assignment_testzQDistributedTest._DistTestBase._generate_sparse_tensors_for_bucket_assignment_test  s    RD4RD5RD4RD5RD4RD5G 6==WakkmWN=!! >s   %D c                 D   t        j                  t        d      t         j                  j                        }dt
        j                  d<   t        j                  t        j                         t        d            }t        j                  j                  | j                         t        d      }t        j                  j                  j                  |j!                  | j                        | j                  g|	      }d
}| j#                  t$        |      5  | j'                         }|r$t        j(                  |dg|j*                        }nt        j(                  |dg      }d d d        |rt-        ||       t        j.                  |       y # 1 sw Y   -xY w)Nr
  r  rz  r  r  r  r}   r  rz  r   r  zNo support for sparse tensors.i  )logger)r  r  r   r  r  r)  r*  r  rP   rR  r%  r   r   rK   rE  r   r'  r  rw  r|  "_compute_bucket_assignment_by_sizer  r1   ri  )rS   
use_loggerr  group_to_user{  r$  r{  r  s           rU   '_test_compute_bucket_assignment_by_sizezEDistributedTest._DistTestBase._test_compute_bucket_assignment_by_size  sE   !"-t||7H7HJ
 69BJJ12>>((*Ia4HL JJ!!$)), .a0C((##;;tyy! II;* < C <L''lCLLN  !DD&cjjF "DD&F D '\: LL$# DCs   AFFc                 (    | j                  d       y NFr  r  r  s    rU   Btest_compute_bucket_assignment_by_size_sparse_error_without_loggerz`DistributedTest._DistTestBase.test_compute_bucket_assignment_by_size_sparse_error_without_logger  s     88E8JrV   c                 (    | j                  d       y NTr  r  r  s    rU   ?test_compute_bucket_assignment_by_size_sparse_error_with_loggerz]DistributedTest._DistTestBase.test_compute_bucket_assignment_by_size_sparse_error_with_logger  s     88D8IrV   c                    |rd}| j                  t        |      }||fS t        j                         t        j                  j
                  k(  }| j                  dk(  rit        j                  |      t        j                  j                  k(  r|sd}| j                  t        |      }||fS d }| j                  t              }||fS d}| j                  t        |      }||fS )Nz'DDP expects same model across all ranksr   #caught collective operation timeoutzappears not to match)r  rw  r  r  r  r  r   r  r  r  r  )rS   r  r   r$  r  is_detail_dbg_modes         rU   2_determine_expected_error_verify_model_across_rankzPDistributedTest._DistTestBase._determine_expected_error_verify_model_across_rank  s     H,,\<HL((!%!5!5!74??;Q;Q!QyyA~$$\2dll6G6GG.#HL00|LC $$ $(L++L9C $$  6,,\<H$$rV   c                 *   t        j                  t        d      t         j                  j                        }dt
        j                  d<   t        j                  t        j                         t        d            }t        j                  j                  | j                         | j                  |      \  }}t        d      }t        j                  j                  j!                  |j#                  | j                        | j                  g|	      }t        j$                  | j                  dk(  rd
ndd      |j&                  _        |5  |r9t+        |j,                  t/        |j1                               |j2                         n-t+        |j,                  t/        |j1                                      t        j4                  |       d d d        |r| j                  dk7  rt7        ||       t        j4                  |       y # 1 sw Y   <xY w)Nr
  r  r~  r  r  r}   r  r   r  r<  rB   rE   )r  r  r   r  r  r)  r*  r  rP   rR  r%  r   r  r   rK   rE  r   r'  rL   r   r   r   rW  r  r]  r  ri  r1   )rS   r  r  r  r  r$  r{  s          rU   _test_verify_model_across_rankz<DistributedTest._DistTestBase._test_verify_model_across_rank  s   !"-t||7H7HJ
 69BJJ12>>((*Ia4HL JJ!!$)), $ W W!C
 .a0C((##;;tyy! II;* < C  YYdii1ns"aHCJJN 8))40@+A3:: 9))40@+A
 \*  dii1n'\: LL$) s   A>H		Hc                 (    | j                  d       y r  r  r  s    rU   )test_verify_model_across_rank_with_loggerzGDistributedTest._DistTestBase.test_verify_model_across_rank_with_logger2   s     //4/@rV   c                 (    | j                  d       y r  r  r  s    rU   ,test_verify_model_across_rank_without_loggerzJDistributedTest._DistTestBase.test_verify_model_across_rank_without_logger:   s     //5/ArV   c                 &   |5  t         j                  j                  j                  |j	                  | j
                        | j
                  g|      }t        j                  |       d d d        t        j                  |       y # 1 sw Y   xY w)Nr  )rP   rK   rE  r   r'  r   r  ri  )rS   r  r{  	ddp_groupr  s        rU   $_run_test_ddp_model_with_diff_paramszBDistributedTest._DistTestBase._run_test_ddp_model_with_diff_paramsB   sh    hh''??FF499%499+Y @ 
 Y'  LL$ s   A&BBc                    t        j                  t        d      t         j                  j                        }dt
        j                  d<   t        j                  t        j                         t        d            }t        j                  j                  | j                         | j                  |      \  }}t        | j                        }| j                  ||||       y )Nr
  r  r~  r  r  rB   r  r  r  r   r  r  r)  r*  r  rP   rR  r%  r   r  r   r  rS   r  r  r  r$  r{  s         rU   &test_ddp_model_diff_shape_across_rankszDDistributedTest._DistTestBase.test_ddp_model_diff_shape_across_ranksQ   s     !"-t||7H7HJ
 69BJJ12>>((*Ib4IL JJ!!$)), $ W W!C
 .dii8C55S,
rV   c                    t        j                  t        d      t         j                  j                        }dt
        j                  d<   t        j                  t        j                         t        d            }t        j                  j                  | j                         | j                  |d	      \  }}t        | j                  | j                  d
k(  	      }| j                  ||||       y )Nr
  r  r~  r  r  rB   r  T)r   rE   r  r  s         rU   +test_ddp_model_diff_num_params_across_rankszIDistributedTest._DistTestBase.test_ddp_model_diff_num_params_across_ranksk   s     !"-t||7H7HJ
 69BJJ12>>((*Ib4IL JJ!!$)), $ W Wd !X !C .		DIINC 55	rV   c                 
    |       }t        j                  |      }t        j                  j                  j                  t        j                  |      j                  | j                        | j                  gd      }t        j                  dd      }|t        k(  r ||      d   \  }} ||      d   \  }	}
n ||      \  }} ||      \  }	}
|
j                         }|j                          |t        k(  r| j                  |j                  j                  j                  j                  j                   d u        | j#                  |j                  j                  j                  j                  j                   |j                  j                  j                  j                          n| j                  |j                  j                  j                  j                   d u        | j#                  |j                  j                  j                  j                   |j                  j                  j                          d }d }|j%                          |j%                          t'        d      D ]  }|t        k(  r ||      d   \  }} ||      d   \  }	}
n ||      \  }} ||      \  }	}
|dk  r+||z  }|	|
z  }|j                         }|j                         }n |j                         }|
j                         }|j                          |j                          |dk(  r|t        k(  r_|j                  j                  j                  j                   }|j                  j                  j                  j                  j                   }nJ|j                  j                  j                   }|j                  j                  j                  j                   }| j#                  ||       n|dk\  r|t        k(  r| j#                  |j                  j                  j                  j                  j                   |       | j#                  |j                  j                  j                  j                   |       nj| j#                  |j                  j                  j                  j                   |       | j#                  |j                  j                  j                   |       t)        |j+                         |j+                               D ]/  \  }}|j                   }|j                   }| j#                  ||       1  t-        j.                          y )NTr  rB   r   rj  rG   rE   )ro  rp  rP   rK   rE  r   rR  r   rQ   r   r   r  r  r   rM   ry  rq  r  r7  rW  r  r]  r  ri  )rS   
module_clsrB  r   	local_netr{  r   rM   rN   a_distb_dist	loss_distsaved_a_local_gradsaved_a_dist_gradr  r  t_distr   r  
dist_param
local_grad	dist_grads                         rU   _test_output_unused_in_lossz9DistributedTest._DistTestBase._test_output_unused_in_loss   s#   LEe,I((##;;e$))$))4 II;'+ < C ++b"%C
 -- ~m41!$S-!8 ~1!$S

I  --

 1 1 3 3 : : ? ?4 GH  JJ%%''..33Y5E5E5G5G5N5N5S5S 

 3 3 8 8D @A  !4!4!9!99;;;M;M;R;RS!% $MMO!1X!11$S>-8DAq%(Xm%<NFF$S>DAq%(XNFFq5 AA#f_F557D &

I 557D &

I""$6!%55-6-=-=-?-?-F-F-K-K*,/JJ,=,=,?,?,F,F,K,K)-6[[-?-?-D-D*,/JJLL,?,?,D,D)$$%79JK!V!%55((JJ--//66;;=N ((%,,..55::<N (()<)<)A)ACTU((););)@)@BTU 25((*CNN,<2-[* "-!1!1J *I$$Z;2Y f LLNrV   c                 B    t         }dD ]  }| j                  ||        y Nr  )r   r  rS   r  r  s      rU   'test_output_unused_in_loss_tuple_modulezEDistributedTest._DistTestBase.test_output_unused_in_loss_tuple_module   s%     3J'4#00=PQ (5rV   c                 B    t         }dD ]  }| j                  ||        y r  )r   r  r  s      rU   &test_output_unused_in_loss_dict_modulezDDistributedTest._DistTestBase.test_output_unused_in_loss_dict_module   s%     *J'4#00=PQ (5rV   c                 ^   t        j                  dd      j                  | j                        }t	               j                  | j                        }t        j                  |      }t         j                  j                  j                  || j                  gd      } ||      j                         } ||      j                         }t        j                  j                  j                         |      j                          t        j                  j                  j                         |      j                          t        |j!                         |j!                               D ]B  \  \  }}\  }}	|j"                  }
|	j"                  }| j%                  |
|d| d|
 d| d|        D y )	NrE   rG   Tr  z
DDP param z with grad z0
                    does not match local param z with grad
                    )rP   r   r'  r   r   ro  rp  rK   rE  r   r   _C
_functionsUndefinedGradr  r  r  rq  r  )rS   r[   r{  r  r  	local_outdist_param_namer  local_param_namer  r  r  s               rU   ,test_undefined_grad_parity_unused_parameterszJDistributedTest._DistTestBase.test_undefined_grad_parity_unused_parameters !  sj    

1a ##DII.A%((499%Cc*I((##;; II;'+ < C
 a&**,C!!((*IHH--/4==?HH--/	:CCERU$$&	(B(B(DSN-*/N0@+ 'OO	(--
  "?"3;yk J00@/A BL$SrV   c                 X    G d dt         j                        }t        j                  d       t	        j
                         }t        j                  j                  | j                          || j                        j                  | j                        }t        j                   j                  j                  || j                  g|d|      }t        j                  dd| j                        }t        d      D ]*  } ||      }	|	j                         }
|
j                          , |S )	Nc                   $     e Zd Z fdZd Z xZS )RDistributedTest._DistTestBase._test_different_graph_across_ranks.<locals>.ToyModelc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        || _        y r   rq  r   s     rU   rJ   z[DistributedTest._DistTestBase._test_different_graph_across_ranks.<locals>.ToyModel.__init__&!  rr  rV   c                     | j                   dk(  r3| j                  t        j                  | j	                  |                  S t        j                  | j	                  |            S rC  )r   r   r   r   r   rZ   s     rU   r\   zZDistributedTest._DistTestBase._test_different_graph_across_ranks.<locals>.ToyModel.forward,!  sD    yyA~#yy		!)=>> vvdiil33rV   r]   rb   s   @rU   r^  r  %!  s    %4rV   r^  ro  T)rA  r  rB  rC  r   rB   r   )rK   r   rP   r  r  r   rR  r%  r   rE  r   rQ   rW  r   r  )rS   r  rC  r^  r+  r   r`  rd  r  r  r   s              rU   "_test_different_graph_across_ranksz@DistributedTest._DistTestBase._test_different_graph_across_ranks"!  s    4299 4 e$,,.JJJ!!$)),TYY',,TYY7E))AA II;'=(,) B I !;;r2dii@L2Y-wwy  rV   c                    | j                  d      }| j                  |j                         j                  dd             | j                  d      }| j	                  |j                         j                  dd             t        |j                         |j                               D ]  \  }}| j                  ||        y )NT)r  r  r   r  )r  r5  r  r  r  r  r]  r  )rS   
base_modelstatic_modelr  r  s        rU   !test_different_graph_across_ranksz?DistributedTest._DistTestBase.test_different_graph_across_ranksD!  s     @@'+ A J 002667LaP  BBPTBULOO224889NPQR J113\5L5L5NO1  A& PrV   zUMacOS uses uv transport which does not have as robust error handling as tcp transportc                 6   t        j                  d      | j                  z  g}t        d      D ]*  }t	        j
                  t        j                  |             , t        d      }t	        j                  |       t        d      D ]*  }t	        j
                  t        j                  |             , t	        j                  |d       d}d}| j                  |k(  r:| j                  t        d	| d
      5  t	        j                  |       d d d        nV| j                  |k7  rGd	| j                   d| }| j                  t        |      5  t	        j                  |       d d d        | j                  d       y # 1 sw Y   xY w# 1 sw Y   (xY w)NrB   rG   r  r  Trz  wait_all_ranksrE   r   r    failed to pass monitoredBarrierb successfully reached monitoredBarrier, but received errors while waiting for send/recv from rank    )rP   r   r   rW  r  r  r  r   monitored_barrierr  rw  r  )rS   r  rY  rz  failed_rankr  	err_regexs          rU   test_monitored_barrier_glooz9DistributedTest._DistTestBase.test_monitored_barrier_glooT!  sV    zz"~		12G2Y		' 23   *G""732Y		' 23 ""74H KHyyH$++ E+6V"W **7;  k) DII; ' z# 
 ++L)D**7; E
 MM"M%!  EDs   :FFFFc                 2   d}d}t        j                  ddg      }| j                  |k(  ry | j                  dk(  r:| j                  t        d| d      5  t        j
                  ||       d d d        y t        j
                  ||       y # 1 sw Y   y xY w)NrE   rr  r   rY  r  r  )r  r  r   r  rw  r  )rS   r  rz  r2  s       rU   $test_monitored_barrier_gloo_subgroupzBDistributedTest._DistTestBase.test_monitored_barrier_gloo_subgroup|!  s     KG~~QF3HyyK'yyA~++ E+6V"W **8W=  &&x9 s   BBc           	         t        j                  t        t        t	        | j
                                    t        d      t         j                  j                        }t        j                  t        t        t	        | j
                                    t         j                  j                        }t        j                  d| j                        | j                  z  g}|j                  |      j                  t        d             | j                  dk7  r~t        j                         t         j                   j"                  k(  rd	}nd
}| j%                  t&        |      5  |j                  |      j                  t        d             d d d        n|rLdj)                  t        dt	        | j
                              D cg c]  }t+        |       c}      }d| d}nd}d| d}t        d      }	| j%                  t&        |      5  |j-                  |	|       d d d        | j/                  d       y # 1 sw Y   xY wc c}w # 1 sw Y   -xY w)Nrh  r  )rY  rz  r  rX  rB   r   r}   r   zTimed out waitingr  rr  r  rE   Ranks r  r  r  r  r  )r  r  r  rW  r   r+  r   r  r  r  rP   r   r   r  r  r  r  r  r  rw  r2  r   r  r  )
rS   r  nccl_pggloo_pgr  r  r  rank_strexpected_first_fail_rank!monitored_barrier_timeout_secondss
             rU   &_test_monitored_barrier_allreduce_hangzDDistributedTest._DistTestBase._test_monitored_barrier_allreduce_hang!  s   nn5T__!567 ""-))G nn5T__!567))G zz"TYY7$))CDG g&++Ia,@A yyA~
 '')T__-C-CC 3I EI++L)D%%g.33Ic4JK ED
 "#yy).q#doo2F)GH)GAQ)GH H #)
2R SI/0,"'(@'AAa bI4=c4J1++L)D--9. .  E
 MM"M%' ED I EDs   &+IIII
Ic                 (    | j                  d       y )NFr  r  r  s    rU   %test_monitored_barrier_allreduce_hangzCDistributedTest._DistTestBase.test_monitored_barrier_allreduce_hang!  s     77u7MrV   c                 (    | j                  d       y )NTr  r  r  s    rU   4test_monitored_barrier_allreduce_hang_wait_all_rankszRDistributedTest._DistTestBase.test_monitored_barrier_allreduce_hang_wait_all_ranks!  s     77t7LrV   c           	      F   t        j                  t        t        t	        | j
                                          }t        d      }| j                  dk(  r?| j                  t        d| j                   d      5  |j                  |       d d d        y y # 1 sw Y   y xY w)Nr  r   r  r  z timed out in monitoredBarrier)r  r  r  rW  r   r+  r   r   r  rw  r  )rS   rW  rz  s      rU   *test_monitored_barrier_gloo_rank_0_timeoutzHDistributedTest._DistTestBase.test_monitored_barrier_gloo_rank_0_timeout!  s     !NNeC<P6Q1RSM*GyyA~++ E$))4R"S "33G<   s   ;BB c                    d}t        d      }d}| j                  |k(  r9| j                  t        d|       5  t	        j
                  |       d d d        y | j                  dk(  rHd| j                   d| }| j                  t        |      5  t	        j
                  |       d d d        y y # 1 sw Y   y xY w# 1 sw Y   y xY w)NrG   r  r   r  r  rE   r  )r   r   r  rw  r  r  )rS   expected_first_failed_rankrz  r  r  s        rU   $test_monitored_barrier_failure_orderzBDistributedTest._DistTestBase.test_monitored_barrier_failure_order!  s     *+&*GHyyH$++ E*D)E"F **7;  aDII; ' z# 
 ++L)D**7; ED  	  EDs   B1B=1B:=Cc                 `   | j                   dk(  rt        d      }dj                  t        dt	        | j
                              D cg c]  }t        |       c}      }d| d}| j                  t        |      5  t        j                  |d	       d d d        y y c c}w # 1 sw Y   y xY w)
Nr   rr  r  r  rE   r  r  Tr  )r   r   r2  rW  r   r+  r   r  rw  r  r  )rS   rz  r  r  r  s        rU   %test_monitored_barrier_wait_all_rankszCDistributedTest._DistTestBase.test_monitored_barrier_wait_all_ranks!  s    
 yyA~#C099eAs4???S6T%U6Tc!f6T%UV$XJ.NO	++L)D**74P ED	 %UDDs   B=B$$B-r  c                    t               }t        j                  j                  j	                  |j                  | j                        | j                  g      }ddd}|j                         \  }}|j                  |      }| j                  ||       t               }dg}t        j                  j                  j                  j                  ||       t        j                  j                  j	                  |j                  | j                        | j                  g      }ddi}|j                         \  }}|j                  |      }| j                  ||       t               }t        j                  j                  j	                  |j                  | j                        | j                  g      }|j                         \  }}| j                  dk(  rt        t        |d                |j                  t        j                  j                  t        j                  d            t        j                  j                  t        j                  d            g       | j!                  t"        d      5  |j                  |       d d d        |d d }| j!                  t"        d	      5  |j                  |       d d d        |j                  t        j                  j                  t        j                  d            t        j                  j                  t        j                  d            g       y # 1 sw Y   xY w# 1 sw Y   xY w)
Nr_  r  zb.weightr5  r   rE   zExpected param to name mappingrN  zParam with name)r   rP   rK   rE  r   rR  r   _build_params_for_reducer"_build_debug_param_to_name_mappingassertDictEqualr  r  rR  r  r   r   r  r  )rS   r   r{  expected_mapping
net_paramsrY  param_to_name_mappingr  s           rU   *test_ddp_build_debug_param_to_name_mappingzHDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mapping	"  s    #$E((##;;

499% II; < C $.*=99;MJ$'$J$J:$V!  !13HI #$E *|HH55aa' ((##;;

499% II; < C !":99;MJ$'$J$J:$V!  !13HI
 #$E((##;;

499% II; < C  99;MJyyA~d:a=)*HH&&uzz!}5HH&&uzz!}5 ''
4TU66zB V $CRJ''
4EF66zB G HH&&uzz!}5HH&&uzz!}5 VU GFs   :L=/M	=M	Mc                 V    G d dt         j                        } |       }t        j                   j                  j	                  |j                  | j                        | j                  g      }ddi}|j                         \  }}|j                  |      }| j                  ||       y )Nc                   $     e Zd Z fdZd Z xZS )cDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mapping_requires_grad.<locals>.Netc                     t         |           t        j                  dd      | _        | j                  j
                  j                  d       y )NrB   F)rI   rJ   rK   rL   r   rD   r  rR   s    rU   rJ   zlDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mapping_requires_grad.<locals>.Net.__init__P"  s8    G$&!yyR0DH HHMM007rV   c                 $    | j                  |      S rf   r   rZ   s     rU   r\   zkDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mapping_requires_grad.<locals>.Net.forwardW"  s    88A;&rV   r]   rb   s   @rU   r   r  O"  s    8'rV   r   r_  r   z
lin.weight)
rK   r   rP   rE  r   rR  r   r  r  r  )rS   r   r   r{  r  r  rY  r  s           rU   8test_ddp_build_debug_param_to_name_mapping_requires_gradzVDistributedTest._DistTestBase.test_ddp_build_debug_param_to_name_mapping_requires_gradH"  s    	'bii 	' EE((##;;

499%499+ < C <   99;MJ$'$J$J:$V!24DErV   c           
         t        j                         t         j                  j                  k(  } G d dt        j
                         G fddt        j
                        } |       }g }|r|j                         D ][  \  }}||j                  j                  j                  k(  s*|j                  d      D ]  \  }}	| d| }
|j                  |
        ] t        j                  j                  j                  j                  ||       |j                  j                  j                   |j                  j                   j"                  g}nOt%        |j                  j                  j'                               |j                  j                   j"                  gz   }g }g }i }d}|j                         D ]  \  }}|j                  d      D ]g  \  }}	| d| }
|||
<   |
|vr|d	z  }||v r|j                  |
       1|r$||j                  j                  j                  k7  sW|j                  |
       i  t        j                  j                  j                  |j)                  | j*                        | j*                  g
      }d\  }}t        j,                  ||      }t/        d      D ]k  }|dk(  r) ||      }|j1                         }|j3                          1	  ||      }|j1                         }|j3                          | j5                  dd       m y # t6        $ r}t9        |      }||j;                  d      d  }|D ]B  }| j5                  ||v xs |       | j5                  t9        ||         |v d||    d|        D |D ]  }| j=                  ||v         |D ]  }| j=                  ||v         Y d }~d }~ww xY w)Nc                   $     e Zd Z fdZd Z xZS )^DistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error.<locals>.SubModulec                     t         |           t        d      | _        t	               | _        t               | _        t        j                  ddd      | _
        y )Nr   rv   rB   FrC   )rI   rJ   r   embedding_netr   r   r   r   rK   rL   	lin_layerrR   s    rU   rJ   zgDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error.<locals>.SubModule.__init__i"  sB    G$&)DQ)GD&-/DH*nDG%'YYq"5%ADNrV   c                     | j                  |      }| j                  |      }| j                  j                  |      }|S rf   )r   r  r   rM   rZ   s     rU   r\   zfDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error.<locals>.SubModule.forwardp"  s5    
Aq)A

1A HrV   r]   rb   s   @rU   	SubModuler  h"  s    BrV   r  c                   (     e Zd Z fdZd Z xZS )\DistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error.<locals>.MyModelc                 :    t         |                   | _        y rf   )rI   rJ   
sub_module)rS   r  rT   s    rU   rJ   zeDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error.<locals>.MyModel.__init__y"  s    G$&&/kDOrV   c                 $    | j                  |      S rf   )r	  rZ   s     rU   r\   zdDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_error.<locals>.MyModel.forward}"  s    ??1--rV   r]   )rT   r  s   @rU   r  r 	  x"  s    2.rV   r  F)recurser  r   rE   r_  )rB   rG   rG   zExpected error was not raised!zdid not receive gradzDid not find index z for )r  r  r  r]  rK   r   r  r	  r  r   r  r  rP   rE  r   r  r   rN   r  modulesrR  r   r   rW  r   r  r  rw  r   findr5  )rS   ignore_sparsedebug_mode_offr  r   sparse_embedding_fqnsr  r   parameter_namers  fqnunused_modulesexpected_unused_param_fqnsused_param_fqnsfqn_to_param_indexr8  r{  r|  r   r   r  r  r   r  unused_param_substrunused_param_fqnused_param_fqnsparse_param_fqnr  s                               @rU   -_test_ddp_multiple_nested_unused_params_errorzKDistributedTest._DistTestBase._test_ddp_multiple_nested_unused_params_errore"  s   !113t7J7JJNBII  .")) . IE$&!+0+>+>+@'K!1!1!?!?!I!II5;5L5L$) 6M 61NE &1M>2B"CC188=	6 ,A !!99ee0 $$2266$$((**"
 "&e&6&6&D&D&L&L&N!O$$((**S " *,& O!#E',':':'<#V-3-D-DU-D-S)NE(M>*:;C.3&s+"77
/299#> !.%)9)9)G)G)Q)QQ+2237 .T (= ((##;;

499% II; < C JE3**UC(C1X6c(C779DMMO!Q!#h"wwy< /OPO  ( VF./7M0N0P.Q+ 1K, OO 04G G !2#1 !OO #$67G$H I#6!7"56HIY6Z5[[`aq`r s 1K />N ,,^?R-RS /> 1F, ,,-=AT-TU 1F5Vs   %(L""	O+BOOc                 (    | j                  d       y )NFr	  r	  r  s    rU   ,test_ddp_multiple_nested_unused_params_errorzJDistributedTest._DistTestBase.test_ddp_multiple_nested_unused_params_error"  s     >>U>SrV   c                 (    | j                  d       y )NTr	  r	  r  s    rU   8test_ddp_multiple_nested_unused_params_err_ignore_paramszVDistributedTest._DistTestBase.test_ddp_multiple_nested_unused_params_err_ignore_params"  s     >>T>RrV   c                 L   | j                   }t        j                  j                  |       t	               j                         }t        j                  |      }t        j                  j                  j                  ||g      }t        j                  ddd      j                         }t        j                  |      }t        j                  j                  j                  ||g      }t        j                  dd|      }t        j                  dddd|      }|||f|||fg}|D ]f  }	|	\  }
}}| j                   d	k(  s|
j                          |j                          t        d
      D ]   }| j                   |
|       ||             " h | j                  d       y )Nr_  rG   r   Fr  track_running_statsrB   r   rv   r   rj  r  r  )r   rP   rR  r%  r   ro  rp  rK   rE  r   r  rQ   evalrW  r  r  )rS   r   r   r  syncbn_modellocal_syncbn_modelr   
inp_syncbnr  test
test_modeltest_local_modeltest_inprY  s                 rU   test_ddp_inferencez0DistributedTest._DistTestBase.test_ddp_inference"  sx    99DJJ!!$'EJJLE--.KHH%%== 6 > E ++Dedf  "&|!< 88,,DD$ E L ++b!D1CRAq>JS)1:>E 9=6
,h99>OO%$))+"1X((&x02B82L &  MM"M%rV   c                    | j                   }t        j                  j                  |       t	        j
                  ddd      j                  |      }t        j                  j                  j                  ||g      }t        j                  j                  j                         5 }t        d      D ]Q  }t        j                  dddd      j                  |      } ||      }|j                         }|j                          S 	 d d d        t        d	k(  rt!        d
      }nt!        d      }| j#                  g |       |j$                  }	| j                   dk(  r|	j'                          t        j                  j                  j                         5 }t        d      D ]Q  }t        j                  dddd      j                  |      } |	|      }|j                         }|j                          S 	 d d d        t        d	k(  rt!        d
|      }nt!        d|      }| j)                  g |       y y # 1 sw Y   0xY w# 1 sw Y   LxY w)NrG   r   Fr	  r_  rj  rB   rv   r  _all_gather_baser  r   )r   rP   rR  r%  rK   r  rE  r   r^  r   r   rW  rQ   r   r  r   r   r  r   r	  r  )
rS   r   r   r  r  r   r  r   all_gather_callsmodel_inferences
             rU   !test_ddp_sync_bn_training_vs_evalz?DistributedTest._DistTestBase.test_ddp_sync_bn_training_vs_eval#  s    99DJJ!!$' $$Q5QVVE HH%%==eQUPV=WE((002dqA++b!Q277=C*C779DMMO	 " 3 & #67I4#P #6|T#J $45 $llOyyA~$$&^^,,446$"1X#kk"aA6;;DA-c2"wwy	 & 7 f$':;Mt'T$':<'N$  %56 % 32( 76s   'A H6A I6I Ic                 L   t               j                  | j                        }t        j                  j
                  j                  || j                  g      }d}| j                  t        |      5  |j                  i i        d d d        t        ||       y # 1 sw Y   xY w)Nr_  zmust be callable)r   rR  r   rP   rK   rE  r   r  	TypeErrorr  r1   )rS   r   r$  s      rU   test_ddp_python_error_loggedz:DistributedTest._DistTestBase.test_ddp_python_error_loggedD#  s     #$))$))4EHH%%== II; > E .L''	<@((R0 A $E<8 A@s   2BB#c           	         | j                   }t        j                  j                  |        G d dt        j                  j
                        }fd |       j                  |      }t        j                  |      }t        j                  j                  j                  ||g      }t        j                  j                  j                  ||gd      }t        j                  dd      }t        t        t        d	}|j                         D ]  }t        d
      D ]  } |||      }	 |	      }
|
j!                          | j#                  |        |||      }| j%                  t'        |||                 |      }|j!                          | j#                  |       t)        |j+                         |j+                               D ]  \  }}| j-                  ||          y )Nc                   $     e Zd Z fdZd Z xZS )\DistributedTest._DistTestBase.test_ddp_static_graph_nested_types.<locals>.NestedOutputModulec                 \    t         |           t        j                  ddd      | _        y )Nr<  rE   FrC   r  rR   s    rU   rJ   zeDistributedTest._DistTestBase.test_ddp_static_graph_nested_types.<locals>.NestedOutputModule.__init__e#  s"    G$&!yyae<DHrV   c                 8   |dk(  r3| j                  |      | j                  |      | j                  |      ffS |dk(  r3| j                  |      | j                  |      | j                  |      ggS |dk(  r%| j                  |      d| j                  |      idS y )Nr   r  r  r   r~   r  )rS   r   output_types      rU   r\   zdDistributedTest._DistTestBase.test_ddp_static_graph_nested_types.<locals>.NestedOutputModule.forwardi#  s    "g- HHSM $ $   %. HHSM $ $   %.!%# #TXXc]"   /rV   r]   rb   s   @rU   NestedOutputModuler1	  d#  s    =rV   r5	  c                 B   d}t        | t        j                        r| j                         S t        | t              r"| j                         D ]  }| |      z  } |S t        | t        t        f      r| D ]  }| |      z  } |S t        dt        |              )Nr  zUnknown model output type )
ri   rP   rj   r   r  r7  r   r  r  rR  )model_outputr   rl   r[   get_losss       rU   r8	  zRDistributedTest._DistTestBase.test_ddp_static_graph_nested_types.<locals>.get_loss#  s    lELL9'++--d3!-!4!4!6/ "7   udm<)+ *  %'A$|BTAU%VWWrV   r_  Trk  rB   r<  )r  r   r  rj  )r4	  )r   rP   rR  r%  rK   r   ro  rp  rE  r   rQ   r  r   r  r  rW  r  r  r  ri   r  r]  r  )rS   r   r5	  r   model_static_graphr   type_mappingr4	  r  r  r   
out_staticloss_staticr   p_staticr8	  s                  @rU   "test_ddp_static_graph_nested_typesz@DistributedTest._DistTestBase.test_ddp_static_graph_nested_typesY#  s    99DJJ!!$'UXX__ < '(--d3E!%u!5HH%%== 6 > E "'!2!2!J!J 6! "K "
 ++b#&CL
  ,002qA=C#C=DMMO$$U+!3C[!QJOOJz<;T$UV"*:"6K((*$$%78),((*,>,I,I,K*H ((H5* "  3rV   c                 X   t         j                  j                  | j                          G d dt        j
                        } |       j                  | j                        }t        j                  dd| j                        }t        j                  ddgddg      D ]  \  }}t        || j                  g| j                  ||      }t        d	      D ]Q  } ||      }| j                  |d
   j                         |d
   |d   z   j                         }	|	j                          S  y )Nc                   $     e Zd Z fdZd Z xZS )SDistributedTest._DistTestBase.test_ddp_returns_tensor_with_no_grad.<locals>.MyModelc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        y r   r   rR   s    rU   rJ   z\DistributedTest._DistTestBase.test_ddp_returns_tensor_with_no_grad.<locals>.MyModel.__init__#  s6    G$&!yyRe<DH!yyRe<DHrV   c                     | j                  t        j                  | j                  |                  }|j	                         }|j                         }|j                  rJ ||fS rf   )r   r   r   r   r  detachr   )rS   r[   r  s      rU   r\   z[DistributedTest._DistTestBase.test_ddp_returns_tensor_with_no_grad.<locals>.MyModel.forward#  sM    !45A	A
A ..q6MrV   r]   rb   s   @rU   r  rA	  #  s    =
"rV   r  rE   rB   r   TF)rA  rH  r  rC  rj  r   )rP   rR  r%  r   rK   r   r'  rQ   r  r  r   rW  r5  r   r   r  )
rS   r  r   r   r  rC  rT  r  r  os
             rU   $test_ddp_returns_tensor_with_no_gradzBDistributedTest._DistTestBase.test_ddp_returns_tensor_with_no_grad#  s     JJ!!$)),"")) " ILL+E++aDII6C/8/@/@ue}0+l . $		{"&))+6!- qAc(C$$SV%9%9:Q#a&--/AJJL	 "0rV   c                     G d dt         j                        }t        j                  j	                  | j
                          |       j                         }dD ]  }t        j                   j                  j                  || j
                  g|      }t        j                  ddd      }t        d	      D ]V  } |||d
      }|j                         }|j                          | j                  |j                  j                                X  t        j                   j                  j                  || j
                  gd      }t        j                  ddd      }t        d	      D ]3  }	 ||d|	dz  dk(        }|j                         }|j                          5 | j                  |j                  j                                y )Nc                   $     e Zd Z fdZd Z xZS )RDistributedTest._DistTestBase.test_detect_ddp_is_actually_static.<locals>.ToyModelc                     t         |           t        j                  ddd      | _        t        j                  dd      | _        y r   r  rR   s    rU   rJ   z[DistributedTest._DistTestBase.test_detect_ddp_is_actually_static.<locals>.ToyModel.__init__#  s4    G$& "		"bu =DI "		"b 1DIrV   c                     |r3|r | j                  | j                  |            S | j                  |      S | j                  | j                  |            S rf   )r]  r\  )rS   r[   r  dynamics       rU   r\   zZDistributedTest._DistTestBase.test_detect_ddp_is_actually_static.<locals>.ToyModel.forward#  sD    ""#'99TYYq\#::#'99Q</#yy166rV   r]   rb   s   @rU   r^  rI	  #  s    2
7rV   r^  r  r  rE   rB   rR  r   rj  F)r  rL	  TrG   r   )rK   r   rP   rR  r%  r   rE  r   rQ   rW  r   r  r  ra  _ddp_graph_staticr5  )
rS   r^  r   r  rT  r   rY  r  r   r  s
             rU   "test_detect_ddp_is_actually_staticz@DistributedTest._DistTestBase.test_detect_ddp_is_actually_static#  sg   7299 7  JJ!!$)),JOO%E,hh''?? $		{+6 @ 
 kk!R7qAc{EJC779DMMOOOCKK$A$A$CD	 "  - ((##;; II;'+ < C
 ++aF3C1X#4Q!Dwwy  S[[::<=rV   c           	          G d dt         j                        } |       j                  | j                        }dD ]7  }t	        || j                  g| j                  d||      }t        d      D cg c]  }d  }}t        d      D ]  }|j                          t        j                  dd	| j                  
      }	 ||	|d   |d   |d         \  }
|d<   |d<   |d<   t        t        |            D ]\  }t        j                  ||         r | j                  ||   j                  d        ;| j                  ||   d   j                  d        ^ |
j                         j                           : y c c}w )Nc                   *     e Zd Z fdZd Zd Z xZS )JDistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd.<locals>.MyModelc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        | j                  j                  j                  | _        y r   )rI   rJ   rK   rL   r   r   ry  r   rR   s    rU   rJ   zSDistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd.<locals>.MyModel.__init__$  sK    G$&!yyRe<DH!yyRe<DH"&((//"8"8DKrV   c                 J    t        j                  dd| j                        }|S )NrE   rB   r   )rP   rQ   r   )rS   r~  s     rU   
__init_optzUDistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd.<locals>.MyModel.__init_opt$  s    ++aDKK@CJrV   c                    t        j                  | j                  |            }| j                  |      }|| j	                         }|| j	                         }|t        j                  |      s| j	                         }|||d|ifS )Nr   )r   r   r   r   _MyModel__init_optrP   	is_tensor)rS   r[   opt_1opt_2
opt_nesteds        rU   r\   zRDistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd.<locals>.MyModel.forward$  sz    txx{+AA} $ 1} $ 1!)1L%)__%6
 eUXz,BBBrV   )r^   r_   r`   rJ   rV	  r\   ra   rb   s   @rU   r  rQ	  $  s    9CrV   r  r  F)rA  rH  r  r  rC  ru   rG   rE   rB   r   r   )rX	  rY	  rZ	  r   )rK   r   r'  r   r   rW  r7  rP   rQ   r  rW	  r  grad_fnr  r  )rS   rC  r  r   r  rT  rY  r~  r  r[   r  s              rU   _test_ddp_new_tensor_in_fwdz9DistributedTest._DistTestBase._test_ddp_new_tensor_in_fwd$  sI   C")) C0 ILL+E,- $		{"&))&++6!- &+1X.XtX.qAMMOAr$))<A25Qs1v#a&3/CQQQ #3s8_ ??3q62 ,,SV^^TB ,,SVH-=-E-EtL	 -
 HHJ'') "  - /s   3	E8c                 &    | j                  d      S )NFr  r\	  r  s    rU   test_ddp_new_tensor_in_fwdz8DistributedTest._DistTestBase.test_ddp_new_tensor_in_fwd>$  s     333GGrV   c                 &    | j                  d      S )NTr  r^	  r  s    rU   'test_ddp_new_tensor_in_fwd_static_graphzEDistributedTest._DistTestBase.test_ddp_new_tensor_in_fwd_static_graphF$  s     333FFrV   c                    | j                   }t        j                  j                  |       t        j                  |       t        j                  j	                  |       fd}t        j
                  j                  j                  j                  j                  }t        j
                  j                  j                  j                  j                  }||fD ]  }t               j                  |      }t        j
                  j                  j                  || j                   g      }|j                  |||       t        j
                  j                  j                  t        j                  |      | j                   gd      }	t        j                   dd|      }
t#        d      D ]&  } ||
      j%                         }||k(  r?t'        |	j(                  j+                               }|D ]  }t-        j.                  |         |	|
      j%                         }||k(  r?t'        |	j(                  j+                               }|D ]  }t-        j.                  |        t        j                  j1                          s| j3                  ||	       |j5                          |j5                          s||k(  s| j3                  ||	       ) t-        j6                           y )Nc                 <   |j                         D cg c]  \  }}|	 }}}|D cg c]2  }t        j                  || j                  d      j	                         4 }}r|S t
        j                  j                  |      j                          y c c}}w c c}w )NT)r  r  )	rp   r  r  rW  rP  rP   rV  collect_allr  )rT  r  rY  rF   r  futsreturn_futuress         rU   buffer_comm_hookzWDistributedTest._DistTestBase._test_ddp_buffer_hook_allreduce.<locals>.buffer_comm_hookT$  s    5B5H5H5JK5Jkq&65JK
 #*	 #* OOc&7&7$ jl# #*	   "KMM--d388: Ls
   B7Br_  F)rA  r  rG   rB   r   )r   rP   rR  r%  r  rK   rE  r  _BufferCommHookLocationPRE_FORWARDPOST_FORWARDr?   r   _register_buffer_comm_hookro  rp  rQ   rW  r   r  r   r  r  r  r  r  r  ri  )rS   rf	  r   rg	  hook_pre_fwdhook_post_fwdhook_run_locationr   	model_ddpmodel_ddp_no_hookr   r  r  model_no_hook_buffersr   loss_no_hooks    `              rU   _test_ddp_buffer_hook_allreducez=DistributedTest._DistTestBase._test_ddp_buffer_hook_allreduceN$  s_   99DJJ!!$'d#JJ""4(; !!--EEQQ  !!--EERR  &! '(--d3!HH--EE $		{ F 	 44/1B %*HH$5$5$M$MMM%( $		{&+ %N %!
 kk!R5qA )# 2 2 4I )L8045F5M5M5U5U5W0X-&;F OOF3 '< $5S#9#=#=#?L(M9045F5M5M5U5U5W0X-&;F OOF3 '<JJ**, *229>OP&&( ))+ &*;}*L229>OP? "@ e&rV   c                 (    | j                  d       y )NTrf	  rs	  r  s    rU   ,test_ddp_buffer_hook_allreduce_return_futurezJDistributedTest._DistTestBase.test_ddp_buffer_hook_allreduce_return_future$  s     000ErV   c                 (    | j                  d       y )NFru	  rv	  r  s    rU   test_ddp_buffer_hook_allreducez<DistributedTest._DistTestBase.test_ddp_buffer_hook_allreduce$  s     000FrV   c                 (   | j                   }t        j                  j                  |       t        j                  |       t        j                  j	                  |       d }t               j                  |      }t        j                  j                  j                  || j                   g      }|j                  ||       t        j                  j                  j                  t        j                  |      | j                   g      }t        j                  dd|      }t        d      D ]`  } ||      j                         } ||      j                         }	| j                  ||       |j!                          |	j!                          b y )Nc                 v    |j                         D cg c]  \  }}|	 }}}| j                  |       y c c}}w rf   )rp   _default_broadcast_coalesced)rT  r  rY  rF   r  s        rU   rg	  zZDistributedTest._DistTestBase.test_ddp_broadcast_buffer_via_hook.<locals>.buffer_comm_hook$  s<     6C5H5H5JK5Jkq&65JK009 Ls   5r_  rG   rB   r   )r   rP   rR  r%  r  r?   rK   rE  r   rk	  ro  rp  rQ   rW  r   r  r  )
rS   r   rg	  r   ro	  rp	  r   r  r  rr	  s
             rU   "test_ddp_broadcast_buffer_via_hookz@DistributedTest._DistTestBase.test_ddp_broadcast_buffer_via_hook$  s>    99DJJ!!$'d#JJ""4(: #$))$/E))AA II; B I 00<LM % 1 1 I Ie$ II; !J ! ++aD1C1X%cN..0	0599;**96GH""$%%' rV   c                     G d dt         j                  j                         G fddt        j                        } || j
                        }t        j                  ddd      j                  | j
                        }t         j                  j                  j                  || j
                  g      }| j                  t              5   ||      j                         j                          d d d        |j                          d	|_        t         j                  j                  j                  || j
                  g      } ||      j                         j                          y # 1 sw Y   {xY w)
Nc                   ,    e Zd Zed        Zed        Zy)SDistributedTest._DistTestBase.test_ddp_remove_autograd_hooks.<locals>.SimulateErrorc                     |S rf   rt   r  r  s     rU   r\   z[DistributedTest._DistTestBase.test_ddp_remove_autograd_hooks.<locals>.SimulateError.forward$       LrV   c                     t               rf   rw  r  grad_outputs     rU   r  z\DistributedTest._DistTestBase.test_ddp_remove_autograd_hooks.<locals>.SimulateError.backward$  
    &.(rV   Nr^   r_   r`   staticmethodr\   r  rt   rV   rU   SimulateErrorr	  $  (    ! ! ) )rV   r	  c                   *     e Zd Z fdZfdZ xZS )MDistributedTest._DistTestBase.test_ddp_remove_autograd_hooks.<locals>.MyModelc                     t         |           d| _        t        j                  dd      j                  |      | _        y )NTrB   )rI   rJ   errorrK   rL   rR  r   rS   r   rT   s     rU   rJ   zVDistributedTest._DistTestBase.test_ddp_remove_autograd_hooks.<locals>.MyModel.__init__$  s2    G$&!%DJ!yyR055f=DHrV   c                 ~    | j                   r | j                  j                  |            S | j                  |      S rf   )r	  r   apply)rS   r   r	  s     rU   r\   zUDistributedTest._DistTestBase.test_ddp_remove_autograd_hooks.<locals>.MyModel.forward$  s2    zz#xx(;(;C(@AA#xx},rV   r]   rT   r	  s   @rU   r  r	  $  s    >
- -rV   r  rB   Tr   r_  F)rP   r^  FunctionrK   r   r   r  rR  rE  r   r  rw  r   r  _remove_autograd_hooksr	  )rS   r  r   r  
model_ddp1
model_ddp2r	  s         @rU   test_ddp_remove_autograd_hooksz<DistributedTest._DistTestBase.test_ddp_remove_autograd_hooks$  s   ) 7 7 )
-")) 
-  DII&EJJr2T:??		JE**BB II; C J
 ""<05!%%'002 1 --/  EK**BB II; C J u!!#,,. 10s   %E--E6zSTest is failing, tracking issue at https://github.com/pytorch/pytorch/issues/102751c                 j   t          G d d              G fddt        j                        } || j                        }t	        j
                  ddd      j                  | j                        }t        j                  j                  j                  || j                  gdd	
      }| j                  dk(  r* ||      \  }}|j                         j                          n: ||      \  }}|j                         |j                         z   j                          | j                  dk(  rW| j                  t        d      5  |j                          d d d        | j                  t        d      5   ||       d d d        y |j                           ||       y # 1 sw Y   JxY w# 1 sw Y   y xY w)Nc                   ,    e Zd ZU ej                  ed<   y)EDistributedTest._DistTestBase.test_ddp_has_finalized.<locals>.MyClassobjN)r^   r_   r`   rP   rj   r   rt   rV   rU   MyClassr	  %  s    \\!rV   r	  c                   *     e Zd Z fdZfdZ xZS )EDistributedTest._DistTestBase.test_ddp_has_finalized.<locals>.MyModelc                     t         |           || _        t        j                  dd      j                  |      | _        t        j                  dd      j                  |      | _        y )N   i   )rI   rJ   r   rK   rL   rR  r   r   r   s     rU   rJ   zNDistributedTest._DistTestBase.test_ddp_has_finalized.<locals>.MyModel.__init__%  sO    G$& $DI!yyt499$?DH!yyx8==dCDHrV   c                     | j                   dk(  r(| j                  |       | j                  |            fS | j                  |      | j                  |      fS rC  )r   r   r   )rS   r   r	  s     rU   r\   zMDistributedTest._DistTestBase.test_ddp_has_finalized.<locals>.MyModel.forward%  sI    yyA~#xx}gdhhsm.DDD#xx}dhhsm;;rV   r]   )rT   r	  s   @rU   r  r	  %  s    D< <rV   r  rB   r	  Tr   g      p?rA  r  r/  r   r   )r   rK   r   r   rP   r  rR  rE  r   r   r  r  rw  _check_reducer_finalized)	rS   r  r   r  rT  out1rY  out2r	  s	           @rU   r  z4DistributedTest._DistTestBase.test_ddp_has_finalized%  s^    " " "<")) < DII&EJJr4t<AA$))LE((##;; II;'+5	 < C yyA~e*a
##% Z
ddhhj(224yyA~++L:vw002 x ++L:vwJ xw ,,.E
 xw xws   F2	F)F&)F2z4TORCH_NCCL_USE_COMM_NONBLOCKING only applies to NCCLc                 >   dt         j                  d<   t        j                          t	        d      }t        j
                  t        t        t        t         j                  d         | j                  |       dfd}| j                  dk7  rd	d
l
}|j                  || j                  f      }|j                          | j                  t              5  t        j                   j#                          d
d
d
       d|j%                          y
y
# 1 sw Y   xY w)z
            Tests that we can abort a NCCL communicator during initialization and
            recover appropriately.
            r  TORCH_NCCL_USE_COMM_NONBLOCKINGrE   r  r'  r  Tc                     t               }rK|j                  t        j                  |             j	                          t        j                  d       rJy y rX   )r   _get_backendrP   r   	_shutdownru  rx  )r   rj  runnings     rU   abortzADistributedTest._DistTestBase.test_nccl_init_abort.<locals>.abortQ%  s<    ')OOELL$89CCEJJqM rV   r   N)r  r  F)r)  r*  r  r  r   r  r   r   r   r   	threadingThreadstartr  rw  rP   r  ri  r2  )rS   rz  r	  r	  r  r	  s        @rU   test_nccl_init_abortz2DistributedTest._DistTestBase.test_nccl_init_abort8%  s     =@BJJ89&&(*G##'rzz,78YY G" yyA~ $$E$E	&&|4%%--/ 5    54s   DDc           	         	
 d } G d dt         j                  j                         G fddt         j                  j                        }t        j
                  ddd	      j                   j                        	t         j                  j                  j                   | j                         j                  gdd
      }t        j                  |      
	
 fd} |        d |       k(  sJ |rt        j                  dd
g      }|j                  |        j                  dv r |        t        j                  g d      }|j                  |        j                  dv r |        |j                  t                       |        nt        j                           j                  dv rwt        j                    j"                  t$        d j                  d
z
  t'        t(                     |j                  t                       |        t        j                           j+                  d       t        j                    j"                  t$        d j                  t'        t(                     d |       k(  sJ y )Nc                      t         j                  j                  j                  } | D cg c]  }t	        | |          }}t	        |      dk(  rdS t        |      S c c}w rC  )rP   _dynamoutilsguard_failuresr  rT  )r	  codenum_recompiless      rU   get_num_torch_recompilesz]DistributedTest._DistTestBase._run_ddp_update_process_group.<locals>.get_num_torch_recompilesb%  sW    !&!4!4!C!CHV!W#nT&:";!W/14qM#n:MM "Xs   Ac                   ,    e Zd Zed        Zed        Zy)RDistributedTest._DistTestBase._run_ddp_update_process_group.<locals>.SimulateErrorc                     |S rf   rt   r	  s     rU   r\   zZDistributedTest._DistTestBase._run_ddp_update_process_group.<locals>.SimulateError.forwardh%  r	  rV   c                     t               rf   r	  r	  s     rU   r  z[DistributedTest._DistTestBase._run_ddp_update_process_group.<locals>.SimulateError.backwardl%  r	  rV   Nr	  rt   rV   rU   r	  r	  g%  r	  rV   r	  c                   *     e Zd Z fdZfdZ xZS )LDistributedTest._DistTestBase._run_ddp_update_process_group.<locals>.MyModelc                 Z   t         |           t        j                  j	                  dd      j                  |      | _        t        j                  j	                  dd      j                  |      | _        t        j                  j	                  dd      j                  |      | _        y )Nr	  )	rI   rJ   rP   rK   rL   rR  r   r   r   r	  s     rU   rJ   zUDistributedTest._DistTestBase._run_ddp_update_process_group.<locals>.MyModel.__init__q%  sq    G$&$xxtT:??GDH$xxtT:??GDH$xxtT:??GDHrV   c           	          |r>| j                  | j                  | j                  j                  |                        S | j                  | j                  | j                  |                  S rf   )r   r   r   r	  )rS   r   r	  r	  s      rU   r\   zTDistributedTest._DistTestBase._run_ddp_update_process_group.<locals>.MyModel.forwardx%  sR    #xx-:M:Mc:R1S(TUU#xx#(?@@rV   r]   r	  s   @rU   r  r	  p%  s    HA ArV   r  rB   r	  Tr   rE   r	  c                  r    d      } | j                         j                          t        j                  j	                          j                  t              5   d      } | j                         j                          d d d        t        j                  j	                          y # 1 sw Y   (xY w)NF)r	  T)r   r  rP   rR  r  r  rw  )r  r  r   rS   s    rU   run_iterationzRDistributedTest._DistTestBase._run_ddp_update_process_group.<locals>.run_iteration%  s    E/	""$

&&( &&|4T2CGGI&&( 5 

&&( 54s   )B--B6r   r  r5  )rE   rG   ru   ru   r  r  rv   r  )rP   r^  r	  rK   r   r  rR  r   rE  r   compiler  r  _update_process_groupr   r  r  r  r   r   r  r  )rS   new_pgr	  r  rT  r	  group_size_2group_size_3r	  r  r   s   `       @@@rU   _run_ddp_update_process_groupz;DistributedTest._DistTestBase._run_ddp_update_process_groupa%  s   N
) 7 7 )A%((// A JJr4t<AA$))LE((##;;		" II;'+	 < C MM#&E
) O02222#~~QF;)),799&!O  $~~I>)),799	)!O ))*<*>? **,99	)++$($4$4 '#$!YY] )2D E --.@.BC!O..0 q) '' $ 0 0# %.@A 02222rV   c                 (    | j                  d       y )NTr	  r	  r  s    rU   'test_ddp_update_process_group_new_groupzEDistributedTest._DistTestBase.test_ddp_update_process_group_new_group%  s     ..d.;rV   c                 (    | j                  d       y )NFr	  r	  r  s    rU   +test_ddp_update_process_group_default_groupzIDistributedTest._DistTestBase.test_ddp_update_process_group_default_group%  s     ..e.<rV   c                    | j                   }t        j                  j                  |       t        j                  |       t        j                  j	                  |        G d dt
        j                        } |       j                  |      }t        j
                  j                  j                  || j                   g      }t        j                  dd|      }t        d      D ]  }|dk(  r(|j                  j                  dz   |j                  _         ||      j                         }|j                          t        t        j                                D cg c]+  }t        j"                  |j                  j                        - }	}t        j$                  |	|j                  j                         |	d   }
|	dd  D ]  }| j'                  |
|         y c c}w )	Nc                   $     e Zd Z fdZd Z xZS )ODistributedTest._DistTestBase.test_ddp_broadcast_buffer.<locals>.NetWithBuffersc                     t         |           t        j                  ddd      | _        t        j                  ddd      | _        | j                  dt        j                  dd             y rA   rH   rR   s    rU   rJ   zXDistributedTest._DistTestBase.test_ddp_broadcast_buffer.<locals>.NetWithBuffers.__init__%  sQ    G$&YYr2E:DFYYr159DF((5;;q!3DErV   c                 B    | j                  | j                  |            S rf   )rN   rM   rZ   s     rU   r\   zWDistributedTest._DistTestBase.test_ddp_broadcast_buffer.<locals>.NetWithBuffers.forward%  s    66$&&),,rV   r]   rb   s   @rU   r?   r	  %  s    F-rV   r?   r_  rG   rB   r   r   rE   )r   rP   rR  r%  r  rK   r   rE  r   rQ   rW  r   rF   r   r  r  r   r  r  r  )rS   r   r?   r   ro	  r   r  r   rY  bufs
rank_0_bufr  s               rU   test_ddp_broadcast_bufferz7DistributedTest._DistTestBase.test_ddp_broadcast_buffer%  s    99DJJ!!$'d#JJ""4(- - #$))$/E))AA II; B I ++aD1C1X19.7.>.>.E.E.II$$+ ~))+ #4#6#6#899 $$Y%5%5%<%<=9   i&6&6&=&=>!!W
8C$$Z5 $ s   0Gz8Only Nccl & Gloo backend support DistributedDataParallelc                     G d dt         j                        }t        j                  j	                  | j
                         t        j                  dd| j
                  dz   z  z          |       j                  | j
                        }t        j                  |      }t        j                   j                  j                  || j
                  gd      }t        j                  dd	d
      }t        d      D ]  }|j                          |j                           ||      } ||      }|j                         |j                         z   }|j                          | j
                  dk(  su|j!                         }	t        d      D ]C  } ||	      } ||	      }|j                         |j                         z   }|j                          E t#        j$                         }
|j'                         D ]2  }|j(                  t#        j$                         z  |j(                  _        4 t-        |j'                         |j'                               D ]X  \  }}| j/                  t        j0                  |j(                  |j(                        |j(                   d|j(                          Z  t#        j2                          y )Nc                   $     e Zd Z fdZd Z xZS )JDistributedTest._DistTestBase.test_static_graph_multi_forward.<locals>.Netc                     t         |           t        j                  dd      | _        t        j
                         | _        y r  )rI   rJ   rK   rL   r   r   r   rR   s    rU   rJ   zSDistributedTest._DistTestBase.test_static_graph_multi_forward.<locals>.Net.__init__	&  s-    G$&!yyR0DH "	DIrV   c                 B    | j                  | j                  |            S rf   )r   r   rZ   s     rU   r\   zRDistributedTest._DistTestBase.test_static_graph_multi_forward.<locals>.Net.forward&  s    99TXXa[11rV   r]   rb   s   @rU   r   r	  &  s    *
2rV   r   *   r:  rE   Trk  rG   rB   rR  r   ru   r   z vs )rK   r   rP   rR  r%  r   r  ro  rp  rE  r   r   rW  r7  r   r  r  r  r   r]  rq  r  r  r  allcloseri  )rS   r   r   r  r   rY  rM   rN   r   	inp_clonewsr   p_ddpp_locals                 rU   test_static_graph_multi_forwardz=DistributedTest._DistTestBase.test_static_graph_multi_forward&  s   2bii 2 JJ!!$)),bDDIIM$::;EJJtyy)E--.KHH%%==499+D > E **Q62C1X!%%'#J#Juuw(99> #		I"1X'	2'	2 uuw0	 & ,,.B(335&'fft/B/B/D&D 6 +.((*#..0+w !NN %

GLL  %zzl$w||n=		+) > LLNrV   c                 R   t         }| j                  }|j                  |      }t        j                  j
                  j                  t        j                  |      | j                  g      }|j                         }|j                  dd      }| j                  |       t        j                  j                  |      }t        j                  j
                  j                  || j                  g      }|j                         }|j                  dd      }| j                  |       y )Nr_  has_sync_bnTF)r  r   rR  rP   rK   rE  r   ro  rp  r  r  r5  r  r  r  )rS   r   r   rJ  
no_sync_bnr  sync_bn_loggedr3  s           rU   test_sync_bn_loggedz1DistributedTest._DistTestBase.test_sync_bn_logged:&  s     E99D

4(I**BBi( II; C J  *??A-11-FN^,((??	JI))AA II; B I  )>>@-11-GNOON+rV   c                     G d dt         j                  j                        }| j                  } |       j	                  |      }t         j                  j
                  j                  ||g      }t        j                  d      j	                  |      }t        j                  dgg|d      }t        j                  dg|d      }t        j                  dg|	      }|||d
}|j                  j                  j                  j                         }	|j                  j                  j                         }
t         j                  j                  |||      }| j!                  ||       |j                  j                  j                  }|j                  j                  }| j!                  ||	       | j!                  ||
       |j#                          | j%                  |j&                         | j%                  |j&                         | j)                  |j&                         | j)                  |j                  j                  j                  j&                         | j)                  |j                  j                  j*                  j&                         | j)                  |j                  j                  j&                         y )Nc                   $     e Zd Z fdZd Z xZS )MDistributedTest._DistTestBase.test_stateless_api_with_ddp.<locals>.MockModulec                     t         |           t        j                  j	                  dd      | _        t        j                  d      }| j                  d|       y )NrE   rF   )rI   rJ   rP   rK   rL   r  r   rO   )rS   rF   rT   s     rU   rJ   zVDistributedTest._DistTestBase.test_stateless_api_with_ddp.<locals>.MockModule.__init__[&  sA    G$&#hhooa3DG"ZZ]F((6:rV   c                 >    | j                  |      | j                  z   S rf   )r  rF   rZ   s     rU   r\   zUDistributedTest._DistTestBase.test_stateless_api_with_ddp.<locals>.MockModule.forwarda&  s    771:33rV   r]   rb   s   @rU   
MockModuler	  Z&  s    ;4rV   r	  r_  )rE   rE   rf  T)r   r   r  r   )zmodule.l1.weightzmodule.l1.biaszmodule.buffer)rP   rK   r   r   r'  rE  r   r  r   r   r  ry  r  rF   r!  functional_callr  r  assertIsNotNonerq  rF  rD   )rS   r	  r   r   r[   ry  rD   rF   r]  prev_weightprev_bufferrh  
cur_weight
cur_buffers                 rU   test_stateless_api_with_ddpz9DistributedTest._DistTestBase.test_stateless_api_with_ddpT&  s   4UXX__ 4 YYF\__V,FXX&&>>F8 ? F 

6"%%f-A\\C5'&MF<<fDID\\3%7F$*"&!'J
 !--**11779K --..446K**,,VZCCQ$))00J--JZ5Z5LLN  -  +fkk*fmm..55::;fmm..33889fmm22778rV   c                 H    G d dt         j                        }d }d }d } |       } |       }|j                  j                  |       |j                  j	                  |       |j                  j                  |       |j                  j	                  |       |j                  j                  |       |j                  j                  |       t        |j                  | j                        | j                  g      }t        j                  dd      } ||      } ||j                  | j                              }	| j                  ||	       |j                         j                          |	j                         j                          |j                         D 
cg c]  }
|
j                   }}
| j                  |d	   |j                  j                   j                         | j                  |d
   |j                  j"                  j                         y c c}
w )Nc                   $     e Zd Z fdZd Z xZS )TDistributedTest._DistTestBase.test_ddp_forward_backward_hook.<locals>.DummyTestModelc                     t         |           t        j                  d       t	        j
                  dd      | _        y )Nr   rG   )rI   rJ   rP   r  rK   rL   r   rR   s    rU   rJ   z]DistributedTest._DistTestBase.test_ddp_forward_backward_hook.<locals>.DummyTestModel.__init__&  s-    G$&%%a( ii1oDGrV   c                 $    | j                  |      S rf   r   rZ   s     rU   r\   z\DistributedTest._DistTestBase.test_ddp_forward_backward_hook.<locals>.DummyTestModel.forward&  s    771:%rV   r]   rb   s   @rU   DummyTestModelr	  &  s    .
&rV   r	  c                 F    t         j                  j                  |d         S rC  )rK   
functionalr   )r   r  s     rU   	relu_hookzODistributedTest._DistTestBase.test_ddp_forward_backward_hook.<locals>.relu_hook&  s    }}))%(33rV   c                 @    t         j                  j                  |      S rf   )rK   r	  gelur   _inputrY  s      rU   	gelu_hookzODistributedTest._DistTestBase.test_ddp_forward_backward_hook.<locals>.gelu_hook&  s    }}))&11rV   c                 H    t         j                  j                  |d         fS rC  )rK   r	  celur
  s      rU   	celu_hookzODistributedTest._DistTestBase.test_ddp_forward_backward_hook.<locals>.celu_hook&  s    **6!9577rV   r_  r}   rG   r   rE   )rK   r   r   register_forward_pre_hookregister_forward_hookregister_backward_hookr   r'  r   rP   r  r  r   r  r]  rq  ry  rD   )rS   r	  r	  r
  r
  r  r`  
input_dataoutput_local
output_ddpr   	ddp_gradss               rU   test_ddp_forward_backward_hookz<DistributedTest._DistTestBase.test_ddp_forward_backward_hook&  s   & &428 )*K&(INN44Y?NN00;LL229=LL..y9NN11)<LL//	:/TYY'TYYKI Aq)J&z2L":==#;<J\:6'')NN%%')2)=)=)?@)?A)?I@Yq\;>>+@+@+E+EFYq\;>>+>+>+C+CD As   #Hc                 2   t        j                  d       d}t        j                         dz   }| j                  }t        j
                  dd|      }t        j
                  dd|      }t         j                  j                  dd      j                  |      }t        t        j                  |      |g      }	t        t        j                  |      |g      }
t         j                  j                  |	j                         |	      }|	j                  ||       |	j!                          t#        d
      D ]P  }|j%                           |	|      }t'        j(                  ||      }|j+                          |j-                          R |	j/                         ||d}|dk(  r| j1                  d      5 }t        j2                  ||       d d d        | j5                  t7        j8                        d       | j5                  |j8                  d   j;                         d       t=        j>                          ddz  d|z  i}| j1                  d      5 }t        j@                  ||      }d d d        | j5                  t7        j8                        d       | j5                  |j8                  d   j;                         d       |
jC                  d          |d   }|d   }t         j                  j                  |
j                         |	      }| j5                  |jD                  |jD                         | j5                  |jF                  |jF                         |jF                  D ]4  }|dk7  s	|dk7  s| j5                  tI        ||      tI        ||             6 | j5                  |jJ                  tM                      tO        |jP                  jS                         |jP                  jS                               D ]%  \  }}tT        jV                  jY                  ||       ' |
j                  ||       |
j!                          t#        d
      D ]  }|j%                          |j%                           |	|      } |
|      }t'        j(                  ||      }t'        j(                  ||      }|j+                          |j+                          |j-                          |j-                           tO        |	j                         |
j                               D ]+  \  }}| j5                  |jZ                  |jZ                         - t=        j>                          |dk(  rt]        j^                  |       y y # 1 sw Y   xY w# 1 sw Y   !xY w)Nr   g{Gz?z/checkpoint.ptr  rE   r   r}   r_  ry  rB   )r1  r  comm_hook_stateztorch.distributedzHNOTE: Process group is not serializable and excluded from a saved state.r-  r  zNOTE: Process group will be set to a default group (i.e. the world size).                If a different group is desired, please set `self.process_group` after PowerSGD state is loaded.r1  r  r
  rW  rng)0rP   r  rE  
gettempdirr   rQ   rK   rL   r'  r   ro  rp  r{  r|  r]  r  r  rW  r7  r   r8  r  rv  r1  
assertLogsr-  r  r  records
getMessager  ri  r/  r  r`   r5  getattrrW  r   r  r
  	get_stater  r  assert_array_equalrq  r)  rH  )rS   r   
hook_stater  r  r   r  r  r{  r`  dummy_ddp_modelrv  rY  r  r   r   capturedr  r  
dummy_hookdummy_hook_statedummy_optimizerentryentry1entry2
out_origin	out_dummyloss_origin
loss_dummy
orig_paramdummy_params                                  rU   _test_hook_picklingz1DistributedTest._DistTestBase._test_hook_pickling&  s   a  M!,,.1AAJ99DKK1T2E[[Ad3F((//!Q'**40C/c0BPTvVI5c"vO 	(<(<(>=QI((T:OO2Y##%&zz#v.   (224!#-E qy__%89XJJuj1 :   X%5%5!6:  $$Q'224^
 LLN%M9t+;<L!45"ZZ
N
 6 S!1!12A6  #..0r ++J|,DE#K0J)*;<#kkoo**, . O
 Z44d6G6GH Z113C3M3MN *33O+$$ 0%8'*e:T 4 -;;=O=QR #&((*,<,@,@,J,J,L# 

--ff=#
 ../?L!!#2Y##%))+&u-
+E2	jjV<ZZ	6:
$$&##% $$&  ,/$$&(B(B(D,'
K   +2B2BC,
 LLNqy		*% k :9 65s   5U?V?V	Vc                 x    t         j                  }t        j                  d dd      }| j                  ||       y )NrE   rv   )rW  r   r!  )r  r   r#  r)
  )rS   r   r$  s      rU   test_ddp_hook_pickling_powerSGDz=DistributedTest._DistTestBase.test_ddp_hook_pickling_powerSGD''  s:     ))D%33"*+$%N
 $$T>:rV   c                 N   t        t        j                  d         }ddlm}  |d|f      }t               }t        j                  j                  | j                         t               j                         }t        j                  j                  j                  ||      }| j                  |j                  |       | j                  |j                  j!                  d      |       | j#                  t$        d      5  t        j                  j                  j                  |||      }d	d	d	       | j#                  t$        d
      5   |dd|dz  f      }t        j                  j                  j                  ||      }d	d	d	       y	# 1 sw Y   bxY w# 1 sw Y   y	xY w)zC
            Test DDP with device_mesh initialization.
            r'  r   )init_device_meshrR  )device_mesh)mesh_dimz<Cannot specify both process_group and device_mesh arguments.)rW  r.
  Nz!Only 1D device mesh is supported,rG   )r   r)  r*  torch.distributed.device_meshr-
  r   rP   rR  r%  r   r   rK   rE  r   r  r.
  	get_groupr  rw  )rS   r+  r-
  r.
  rj  r   r`  s          rU   #test_ddp_device_mesh_initializationzADistributedTest._DistTestBase.test_ddp_device_mesh_initialization9'  se    RZZ56JF*6J=AK#%BJJ!!$)),"$))+E))AA%U`AaIY22K@Y22<<a<H"M''\ "HH--EE F 	 ''A /v:?7KL!HH--EE{ F 		   s   -F:FFF$c                    t         j                  j                  dd      j                  | j                        }t        j                  |      }t         j                  j                  j                  || j                  g      }t         j                  j                  j                  || j                  gd      }t        j                  |      }t        j                  |      }t        j                  dd      j                  | j                        }t        d      D ]  } ||      j                         } ||      j                         }| j                  ||       |j                          |j                          t        |j!                         |j!                               D ]+  \  }	}
| j                  |	j"                  |
j"                         -  y)z>Tests that DDP works with torch compile when static_graph=TruerB   r_  Trk  rj  N)rP   rK   rL   rR  r   ro  rp  rE  r   r	  r  rW  r   r  r  r  r]  rq  )rS   r   model_clonerT  
ddp_staticr  rY  out_ddpout_ddp_staticr  r  s              rU   test_ddp_compile_static_graphz;DistributedTest._DistTestBase.test_ddp_compile_static_graph['  se    HHOOB+00;E--.K((##;; II; < C **BB II;! C J
 --$Cz2JJJr2&++DII6E1Xe*..*!+E!2!6!6!8  .9  "'')!#.."2J4I4I4KLFB$$RWWbgg6 M rV   rf   )FNF)FN)TN)rf  N)Nr   FNr}   )NFFFr   r   )r<  )rG   NFr^  )FF(  r^   r_   r`   r  r  r  r  r  r  r  r  r  r   r  r  r  r  r;   r   r   r   r  r'   r  r  r+  r%  r2   r  r,  r)   r.  unittestskipIfr1  r:  r+   r>  rA  rH  rK  rP  rR  rb  rw  r  r  r  r7   r9   r  r  r-   r  r  r  r  r  r  r  r  r  r  r  r  r4   r5   r  r  r	  r  r  r$  skip_collectiver&  r(  r*  r-  r/  r1  r3  r>  r@  rB  rD  rF  rX  rZ  r]  r_  ra  rk  ru  rz  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rR  rP   rJ  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r	  r  r  r  r  r	  r!  r$  r'  r*  r-  r5  r8  r;  r=  r?  rA  rD  rF  rH  rJ  rM  rO  rQ  rU  rZ  r\  r_  ra  rd  rf  rh  rn  rq  rs  ru  rw  ry  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r&   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r  r  r  r#  r)  r>  rK  rN  rQ  rS  rW  r   r)  r*  ra  rm  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r%  r8   r+  r2  r4  rC  r*   rG  rI  rX  ra  rj  r   ry  r~  r  r  r  r  r  r  r  r  r  r(   r  r  r  r  r  r  r  r  r  r  r  r  r  r   r"  r%  skipIfNoTorchVisionr-  r  r  r0  r3  r7  r9  r>  rV  r\  r,   r0   r`  rc  rm  ro  rr  ru  r~  r  r  r  r  r  r  r  r  r  r  r  skipr  r  r  r  r(  r/  rS  r[  ri  rl  rt  rx  r|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r/   r  r  r  r  r  r  r  r	  r	  r	  r&	  r+	  r.	  r>	  rF	  rN	  r\	  r_	  ra	  rs	  rw	  ry	  r}	  r	  r  r	  r	  r	  r	  r	  r	  r	  r	  r
  r)
  r+
  r2
  r8
  rt   rV   rU   _DistTestBaser  `  sxB   	*	+	+	+	..	.<	.	/ 
	'	1	>	1	G
	, 
(vD

 
(&&y110

	B



	B$ 
!	'vD

	>

 
!	> 
(vD

	>

	>	J: 
&m&C&CE&J	K	A		!		E 
 
 
L	E 
&m&C&CE&J	K	!		E)+g	h	J 
i 
 
L	J 
(=88DD7)WX

 
A		!	
	5 
 
	


	5 
(=88DD7)WX

 
	( 
	


	( 
(=88DD7)WX

 
A		!		& 
 
	

	& 
(=88DD7)WX

 
A		!		5 
 
	

	5( 
(=88DD7)WX

 
A		!		 
 
	

	 
(=88DD7)WX

 
		 
	


		 
(=88DD7)WX

 
A		!		 
 
	

	 
(=88DD7)WX

 
!	"	M 
	


"	MH 
(=88DD7)WX

 
!		= 
	


	=< 
!	*	" 
*	"X 
(=88DD7)WX

 
!	%	= 
	


%	=N 
(=88DD7)WX

 
A		!	N	9 
 
	

N	9b 
	'v;;mT

		

 

	: 
	'v;;mT

		

 

	< 
	'6(9;V	W	y*H	I#	 
J 
X 
#	J 
	'6(9;V	W	y*H	I	 
J 
X 
	* 
	'6(9;V	W	y*H	I	 
J 
X 
	0 
	 	'6(9;V	W	y*H	I	 
J 
X 
! 
	> 
(6(9;U	V	 
W	, 
(6(9;U	V	 
W	, 
(6(9;V	W	y*H	I		6 
J 
X		6 
(6(9;V	W	y*H	I	3 
J 
X	3 
(6(9;V	W	y*H	I	I 
J 
X	I" 
	'6(9;P	Q	y*H	I*	S 
J 
R 
*	SX 
	'6(9;P	Q	y*H	I	( 
J 
R 
	( 
	'6(9;P	Q	y*H	I	4 
J 
R 
	4 
	'6(9;P	Q	y*H	I	'	3Q	R	'"
g

	4	

 
S 
J 
R 
	4%	SN 
(vM

	4

	4 
(vM

	E

	E 
(vM

 
(	3Q	R	'"
g

	I	

 
S

	I
K	 Z 
(}445IJJiBC

	?	

	? 
(}445IJJiBC

	P	

	P 
(}445IJJiBC

 
(	3V	W	'"
g

	T	

 
X	

	T
 	YD 
(vM

	=

	= 
(vM

	U

	U 
(vM

 
(	3V	W	'"
g

	R	

 
X

	R
+	XZ 
(v<

	0

	0 
(v<

	A

	A 
(v<

 
(	3V	W	'"
g

	>	

 
X

	>
 
(v<

	

	: F	P 
(vB

	?

	? 
(v3'V"3@

 
	R 
	


	R 
!	'vB

	?

 
!	? 
(vB

	?

	? 
(v=

 
	X 
	


	X8 	B 
(vB

 
(}44X>>i/0


		




	 
(v?

 
(}44X>>i/0

 
	 
	



	" 
(vB

 
(}44X>>i/0


		




	 
(vB

 
(}44X>>i/0

		



	 
(vB

 
(}44X>>i/0

		



	 
(vB

 
(}44X>>i/0

 
!
	 
!	




	 
(vB

 
(}44X>>i/0

 
!
	 
!	




	 
(vB

 
(}44X>>i/0

 
!	 
!	



	 
(vB

 
(}44X>>i/0

 
!	 
!	



	 
(vB

 
(}44X>>i/0


		




	 
(vB

 
(}44X>>i/0


		




	 
(vB

 
(}44X>>i/0

		



	 
(vB

 
(}44X>>i/0

		



	  '	R 
(vB

 
(}44X>>i/0


		




	 
(v?

 
(}44X>>i/0

 
	 
	



	" 
(vD

 
(}44X>>i/0

 
+	 
	



+	^ QU	* 
(vN

 
	 


	4 
	%m&C&CE&J	K*	 
L 
*	d "4	@ ++6	p 
(vB


	


	 
(vB

	

	 
(v3'V"3I

 
	 
	


	  
(v3'V"3I

 
	 
	


	" 
(vB

	

	 
(vB

	

	$ 
(v3'V"3I

 
	 
	


	" 
(vB


	


	 
(vB

	

	 
(vB

	

	 
!	'vB


	

 
!
	 
!	'vB


	

 
!
	 
!	'vB

	

 
!	 
!	'vB

	

 
!	 
(vB


	


	 
(vB


	


	 
(vB

	

	 
(vB

	

			9 
(vL

	:

	: 
(vL

 
	I 


	I 

	 

	 
	 
	 
	 
	 
	 
	 
(vB

	

	 1	f 
&vh	/		 
0		 
&vh	/		 
0		 
&vh	/		 
0		 
&vh	/	 
0	 
!	%vh	/	 
0 
!	 
!	%vh	/		 
0 
!		 
!	%vh	/	 
0 
!	 
!	%vh	/	 
0 
!	 
&vh	/	 
0	 
&vh	/		 
0		 
&vh	/		 
0		 
&vh	/	 
0	 /4U[[	B 
(vB

 
(uF

	1



	1, 
(vB

 
(uF

	=



	= 
(v?

 
	P 


	P
 
(vB

 
(uF

	Q



	Q 
(v?

 
	 


	 
(vB

 
(uF

 
!	= 
!



	= 
(vB

 
(uF

	=



	= BF	: 
(vB

 
(uF

	(



	(, 
(vB

 
(uF

	<



	< 
(v?

 
	O 


	O
 
(vB

 
(uF

 
!	< 
!



	< 
(vB

 
(uF

	<



	< /4U[[	B 
(vB

	@

	@ 
(vC

 
	S 


	S
 
(vB

	T

	T 
(vC

 
	 


	 
!	'vB

	@

 
!	@ 
(vB

	@

	@ 
(v@

 
%	 


%	R QU	. 
(vO

 
	 


	" 
(vO

 
	 


	"	< 05{{%	N 
(}445JKKi=>

	J	

	J 
(}445JKKi=>

		

	 
!	'}445JKKi=>

	J	

 
!
	J 
(}445JKKi=>

	J	

	J 
(}445JKKi=>

 		

 	H /4U[[	> /4U[[	6 ++	B 
(uG

	S

	S 
(vJ

 
		 


		 
(uG

	

	 
(vJ

 
	 


	 
(uG

	U

	U 
(vJ

 
		 


		 
(uG

	

	 
(vJ

 

	 



	 
(u<

	@

	@ 
(vC

 
	S 


	S
 
(u<

	T

	T 
(vC

 
	 


	 
(uG

 
!	S 
!

	S 
(vJ

 
	 		 
! 



		 
(uG

 
!	U 
!

	U 
(vJ

 
	 		 
! 



		 
(u<

 
!	@ 
!

	@ 
(vJ

 
!		S 
 
!


	S
 
(uG

	S

	S 
(vJ

 
		 


		 
(uG

	U

	U 
(vJ

 
		 


		 
(u<

	@

	@ 
(vC

 
	S 


	S BF	&> 
	'u@

 
(u.0D

	P



 
	P
 
!		'u@

	P

 
 
!
	P
 
!		'u@

	P

 
 
!
	P
 
(}44]CCi45

	=	

	= 
!	'}44]CCi45

	=	

 
!
	= 
(}44]CCi45

	=	

	=	&	'	6 SW	S	/  B	YP $)#(:	x!	F 
(vH

	4

	4 
(vH

	P

	P 
(=88??7)LM

		

	 
(=88??7)LM

 
#bjj67	8	 
9	


	 
(6(9;K	L	 
M	* 
(=88??7)LM

 
#bjj67	8	 
9	


	B 
(=88@@7)[\

 
#bjj67	89	H 
9	


9	HvD	#LK	Z 
!		4 
	4> %)L	:\ 
!			 
		 
!		 
	 
!	*	G 
*	GX	o 	 
!	 	: 
 	:DA	<F 
!		 
	 
!		 
	 
!		 
	
 
!		 
	
7	r 
(=88@@7)[\

 
#bjj67	8	P 
9	


	P 
(=88@@7)[\

 
#bjj67	8	Y 
9	


	Y 
(=88@@7)[\

 
#bjj67	8
	 
9	



	 
(=88@@7)[\

 
($P


 
#bjj67	8,	 
9

	

,	j %*	3< %*		3$ LQO	Ab 
(uHF!2Hw&7H@

 
w	*	6 
+	


	6 
(uHF!2Hw&7H@

 
w	*	R 
+	


	R 
(uHF!2Hw&7H@

 
w	*	 
+	


	. 
(uHF!2Hw&7H@

 
w	*	 
+	


	: 
(uHF!2Hw&7H@

 
w	*	/ 
+	


	/& 
(=88??7)LM

 
)	 
	


)	V5	n 
(=88??7)LM

 
	' 
	


	'* 9	v-	<^	N		>	0@ 
!		'=88??7)LM

		

 

	 
!		'=88??7)LM

		

 

		 
!			'=88??7)LM

		

 
 
	 
!			'=88??7)LM

		

 
 
	 
!		'=88??7)LM

		

 

	 
(=88??7)LM

 
	 
	


	.	` 
(=88??7)LM

 
&	 
	


&	P 
(=88??7)LM

 
	 
	


	( 
(=88??7)LM

 
'	  
	


'	 R 
(=88??7)LM

 
	A	'	  
 
	

'	 R 
(=88??7)LM

 
$	J 
	


$	JL 
(=88??7)LM

 
	 
	


	* 
(=88??7)LM

 
	B 
	


	BC	J 
(vH

P	N

P	Nd 
(=88??7)LM

 
1	W 
	


1	Wf 
(vH

	=

	=* 
	@ 
	@ =AOOQU	: 
&vh	/	!		 
 
0	2 
&vh	/	!		> 
 
0	>( 
&vh	/	#bjj67	8	Y 
9 
0	Y2 
&vh	/	!		3 
 
0	3& 
(=88??7)LM

 
#bjj67	8L	7 
9	


L	7\	04 
&m&C&CE&J	K	(

<()2::i+@

 
 '@	A	1 
B

 
L
	1 
&m&C&CE&J	K	(

<()2::i+@

 
 '@	A	B 
B

 
L
	B-	^ 
(uF

 
&m&C&CE&J	K	'@	A	. 
B 
L


	. 
(uF

 
&m&C&CE&J	K	'@	A	6 
B 
L


	6
	0 
!		'=88??7)LM

*	2	

 

*	2X 
!		'=88??7)LM

2	=	

 

2	=h:	-x 
&m&C&CE&J	K	!		P 
 
L	P 
&m&C&CE&J	K	!		'	3V	W	'"
g

	F	

 
X 
 
L	F 
!		'=88??7)LM

,	<	

 

,	<\i	V 
!		'=88??7)LM

4	5	

 

4	5l 
!		'=88??7)LM

}		

 

}	~ 
!		'=88??7)LM

	/	

 

	/: 
!		'=88??7)LM

	$	

 

	$0D	6L 
&m&C&CE&J	K	(

<()2::i+@

 
 z	2	\	]	6 
^ 
3

 
L	6 
&m&C&CE&J	K	(

<()2::i+@

 
 z	2	> 
3

 
L
	>W	9r 
&m&C&CE&J	K	!		@ 
 
L	@ 
 '@	A	%m&C&CE&J	K	!	0	 
 
L 
B0	d 
&m&C&CE&J	K	!	 	$ 
 
L 	$D 
&m&C&CE&J	K	!	d	' 
 
Ld	'L 
&m&C&CE&J	K	!		" 
 
L	"B 
 '@	A	%m&C&CE&J	K	!	N	 
 
L 
BN	` 
&m&C&CE&J	K	!	,	R 
 
L,	R\ 
 '@	A	%m&C&CE&J	K	!	c	 
 
L 
Bc	J 
&vh	/	I 
0	I:	"(	%T 
&m&C&CE&J	K	!		K 
 
L	K 
&m&C&CE&J	K	!		J 
 
L	J 16	%<3	%j 
&m&C&CE&J	K	'u.0D

 
!		A 


 
L
	A 
&m&C&CE&J	K	'u.0D

 
!		B 


 
L
	B	% 
&m&C&CE&J	K	'u.0D

 
!		 


 
L
	* 
&m&C&CE&J	K	'u.0D

 
!		 


 
L
	8^	@ 
(=88??7)LM

 
!		R 
	


	R
 
(=88??7)LM

 
!		R 
	


	R
 
(=88??7)LM

 
!		 
	


	< >C 	D 
&m&C&CE&J	K	!		' 
 
L	' 
&vh	/	'"
c

!	&	

 
0
!	&F 
&vh	/	: 
0	:*3	&j 
!	%m&C&CE&J	K	#bjj67	8	N 
9 
L 
!	N
 
!	%m&C&CE&J	K	#bjj67	8	M 
9 
L 
!	M
 
&vh	/	= 
0	= 
&vh	/	 	'"
c

	<	

 
! 
0	<* 
&vh	/	 	Q 
! 
0	Q 
&m&C&CE&J	K	x	0	!	:	 
 
1 
L:	x 
(=88??7)LM

 
 x	0	!		F 
 
1	

	F.r	Qh 
 '@	A	%m&C&CE&J	K	!		T 
 
L 
B	T 
 '@	A	%m&C&CE&J	K	!		S 
 
L 
B	S
 
(=88??7)LM

 
!	$	& 
	


$	&L 
(=88??7)LM

 
!		\	]*	7 
^ 
	

*	7X 
!		'=88??7)LM

	9	

 

	9  
!		'=88??7)LM

Q	6	

 

Q	6f 
!		'=88??7)LM

!	!	

 

!	!F 
!		'=88??7)LM

+	>	

 

+	>Z1	*f 
!		'=88??7)LM

	H	

 

	H 
!		'=88??7)LM

	G	

 

	GK	Z 
!		'=88??7)LM

	F	

 

	F 
!		'=88??7)LM

	G	

 

	G 
!		'=88??7)LM

	(	

 

	(@ 
!		'=88??7)LM

.	/	

 

.	/` 
!		'=88??7)LM

 
l	m+	 
n	

 
+	Z 
!		'vB

"		

 

"	Hc	3J 
!		A		'=88??7)LM

	<	

 
 
	< 
!		A		'=88??7)LM

	=	

 
 
	= 
!		'=88??7)LM

#	6	

 

#	6J 
!		'v3'V"3F

1		

 

1	f 
!		'v3'V"3F

	,	

 

	,* 
!		'=88??7)LM

+	9	

 

+	9Z 
&m&C&CE&J	K	!	&	E 
 
L&	EPu	&n 
(=88@@7)[\

 
#bjj67	8	'u.0D

	;

 
9	

	; 
&m&C&CE&J	K	!		 
 
L	@ 
!		A		'=88??7)LM

	7	

 
 
	7rV   r>
  N)r^   r_   r`   r>
  rt   rV   rU   r  r  _  s    YT7 YT7rV   r  )ro  r  rK  r)  r;  r4  rE  ru  collectionsr   r   
contextlibr   r   dataclassesr   datetimer   	functoolsr	   typingr
   r   r   r   r9
  numpyr  rP   
torch.cudatorch.distributedr  r  6torch.distributed.algorithms.model_averaging.averagers
algorithmsmodel_averagingrm  Htorch.distributed.algorithms.model_averaging.hierarchical_model_averagerhierarchical_model_averagerr  2torch.distributed.algorithms.model_averaging.utilsr	  r_  torch.nnrK   torch.nn.functionalr	  r   torch._utils_internalr   r  r   r  torch.cuda.ampr   r   +torch.distributed.algorithms.ddp_comm_hooksr   r)  r   r  r   r  r   r  torch.distributed.optimr   "torch.distributed.distributed_c10dr   r   r   r   torch.distributed.utilsr   r   torch.nn.parallelr   torch.nn.parallel.distributedr   r   *torch.testing._internal.common_distributedr    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   $torch.testing._internal.common_utilsr3   r4   r5   r6   r7   r8   r9   r:   r;   /torch.distributed.optim.post_localSGD_optimizerr{  r  torch.utils.data.distributedr<   r  r  ImportErrorr5  r6  r:  r   r?   rd   r}  barrQ   foo_cpu_tensorrX  r  r  r  r  r  r  r  r  r=  rQ  r   r<
  r*  r   getenvr   r  r  r   r   r!  r$  r"  r#  r   r   r   r   r   r   r   r   r   r   r   rD  r  r  r  r  r	  r  r  r%  r,  rA  rK  rJ  rU  rZ  r_  re  rg  r  r  r>
  rt   rV   rU   <module>r`
     sh      	  
   / 2 !   3 3       J J b b R R    A A /  A 
 6 V     *
 
 
 R Q ;O <<7	!RYY 	! $ G	[U[[A&' Hd#34	4+,   	LLLLLLLL	   	LLLLLLLL	% ! 	LLLLLLLL	* & "<A J 
 5)  **Y
biix04c: 
 A "
 A % &V " A !
 T 
 299 #")) #$	ryy 	299 #299 #
		 

ryy 
	RYY 	")) &(")) ( %	u- rD1  
 " )*  
 4  &    #%++ W 15EKK V?$ $NC(* C(LZT7 ZT7zh << =Cy  Os   <R, ,R76R7