
    PhE                        d dl mZmZmZmZmZ d dlZd dlmZm	Z	 d dl
mZ d dlmZ d dlmZmZmZmZmZmZ ej*                  j,                  Z eej.                  j0                  ej2                  j0                  ej4                  j0                  g      dede	fd	       Z eej8                  j                  ej:                  j                  ej<                  j                  g      dede	fd
       Z eej8                  j@                  ej:                  j@                  ej<                  j@                  ejB                  j@                  g      dede	fd       Z" eejF                  j@                  ejH                  j@                  g      defd       Z% eejL                  jN                  g      defd       Z( eejR                  j0                  g      defd       Z* eejV                  j0                        dede	fd       Z, eejZ                  j0                        dede	fd       Z. eej^                  j0                        dede	fd       Z0 eejb                  jd                        dede	fd       Z3 eejh                  j0                        dede	fd       Z5 eejl                  j0                        dede	fd       Z7dedee2   dee   fdZ8 eejr                  j0                        dede	fd       Z:y)    )castListOptionalSequenceTupleN)OpSchemaOutputSharding)pointwise_rule)register_prop_rule)_PartialDTensorSpec	Placement	ReplicateShard
TensorMeta	op_schemareturnc                     | j                   d   }t        |t              rt        d |D              sJ t	        |      S )Nr   c              3   <   K   | ]  }t        |t                y wN
isinstancer   .0ss     sC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/_spmd/experimental_ops.py	<genexpr>z'_prop__foreach_unaop.<locals>.<genexpr>        )Sd*Q*Dd   output_specargs_schemar   listallr	   )r   selfs     r   _prop__foreach_unaopr'      s>       #DdD!c)Sd)S&SSS d++    c           	      R   | j                   d d \  }}t        | j                         dk  rd n| j                   d   }t        |t              rt	        d |D              s
J d|        t        |t              rt	        d |D              s
J d|        t        |      t        |      k(  sJ dt        |       dt        |              t        d t        ||      D              r7t        d t        | j                  |r|||fn||f| j                  	      g
      S t        |      S )N      c              3   <   K   | ]  }t        |t                y wr   r   r   s     r   r   z,_prop__foreach_binop_list.<locals>.<genexpr>.   s      *,0q
1k"Dr   z#Expect a List[DTensorSpec] but got c              3   <   K   | ]  }t        |t                y wr   r   )r   os     r   r   z,_prop__foreach_binop_list.<locals>.<genexpr>1   s      +,1q
1k"Er   z/Two tensor lists must match in length, but got z and c              3   ,   K   | ]  \  }}||k7    y wr    )r   r   r.   s      r   r   z,_prop__foreach_binop_list.<locals>.<genexpr>9   s     
/.da16.s   opr#   kwargs_schemar!   schema_suggestionsr    r#   lenr   r$   r%   anyzipr	   r   r2   r3   )r   r&   otherscalars       r   _prop__foreach_binop_listr<   $   sO    ''+KD%../!3T9N9Nq9QFdD!c *,0* ' 4	,TF34  eT"s +,1+ ( 5	,UG45  t9E
" 	t9+U3u:,	0"
 
/c$.
//  ||8>tV 4T4L"+"9"9 	
 		
 $//r(   c                     | j                   \  }}t        |t              rt        d |D              sJ t        |t              rJ t	        |      S )Nc              3   <   K   | ]  }t        |t                y wr   r   r   s     r   r   z._prop__foreach_binop_scalar.<locals>.<genexpr>U   r   r   r    r"   )r   r&   r;   s      r   _prop__foreach_binop_scalarr?   K   sK     ((LD&dD!c)Sd)S&SSS&$'''d++r(   c           
         | j                   d d \  }}}t        | j                         dk  rd n| j                   d   }t        |t              rt	        d |D              sJ t        |t              rt	        d |D              sJ t        |t              rt	        d |D              sJ t        d t        |||      D              r9t        d t        | j                  |r||||fn|||f| j                        g      S t        |	      S )
Nr+      c              3   <   K   | ]  }t        |t                y wr   r   r   s     r   r   z/_prop__foreach_addcop_scalar.<locals>.<genexpr>c   r   r   c              3   <   K   | ]  }t        |t                y wr   r   r   s     r   r   z/_prop__foreach_addcop_scalar.<locals>.<genexpr>d        ,VQUAZ;-GQUr   c              3   <   K   | ]  }t        |t                y wr   r   r   s     r   r   z/_prop__foreach_addcop_scalar.<locals>.<genexpr>e   rD   r   c              3   <   K   | ]  \  }}}||k7  xs ||k7    y wr   r0   )r   r   t1t2s       r   r   z/_prop__foreach_addcop_scalar.<locals>.<genexpr>f   s)     
K/J)!R17a2g/Jr   r1   r4   r    r6   )r   r&   tensor1tensor2r;   s        r   _prop__foreach_addcop_scalarrK   Z   s    '222A6D'7../!3T9N9Nq9QFdD!c)Sd)S&SSSgt$,VQU,V)VVVgt$,VQU,V)VVV

Ks4'/J
KK  || "&tT6 :d+"+"9"9 
 	
 $//r(   c                     | j                   \  }}t        |t              rt        d |D              sJ t	        |      S )Nc              3   <   K   | ]  }t        |t                y wr   r   r   s     r   r   z7_prop__foreach_pow_scalar_and_tensor.<locals>.<genexpr>}   s      .,4q
1k"Hr   r    r"   )r   scalaexponents      r   $_prop__foreach_pow_scalar_and_tensorrP   z   sF    ++OE8h%# .,4. +   h//r(   c           
      >    d} j                   d | }t        d |D              sJ t        d |D              sJ |D cg c]  }t        |      s| c}t        fdD              s"J dD cg c]  }t        |       c} d       t        d t	         D              rRt         fd|D              }t        d t         j                  | j                   |d  z    j                  	      g
      S t         j                   d   f|z        S c c}w c c}w )N   c              3   <   K   | ]  }t        |t                y wr   )r   r$   )r   schemas     r   r   z$_prop__fused_adam.<locals>.<genexpr>   s     G6FFz&$'6Fr   c              3   J   K   | ]  }|D ]  }t        |t                 y wr   r   )r   rT   r   s      r   r   z$_prop__fused_adam.<locals>.<genexpr>   s%      1Av6a
1k"6"1As   !#c              3   R   K   | ]  }t        |      t        d          k(     ywr   N)r7   )r   r   tensor_schemass     r   r   z$_prop__fused_adam.<locals>.<genexpr>   s%     HAs1v^A.//s   $'z8expect the same number of gradients and states, but got .c              3   F   K   | ]  t        fd D                yw)c              3   .   K   | ]  }|d    k7    ywrW   r0   )r   ttss     r   r   z._prop__fused_adam.<locals>.<genexpr>.<genexpr>   s     &2aqBqEz2s   N)r8   )r   r]   s    @r   r   z$_prop__fused_adam.<locals>.<genexpr>   s     
F1E23&2&&1Es   !c              3   V   K   | ]   }t        |      rj                  d    n| " ywrW   )r7   r#   )r   r   r   s     r   r   z$_prop__fused_adam.<locals>.<genexpr>   s,      6
?O!AI!!!$A5?O   &)r1   r4   r   r    )
r#   r%   r7   r8   r9   tupler	   r   r2   r3   )r   NTtesnor_list_argsrT   r   new_schemasrX   s   `     @r   _prop__fused_adamrd      sK   	
B1:1F1Fs1KG6FGGGG 1A   
 .0-6V-0N HHH B*+NqCFN+
,A	/H
 
Fn1E
FF05 6
?O6
 1
  || +i.C.CBC.H H"+"9"9 	
 		
 9+@+@+C*E*JKK10 ,s    DD2Dc           
         | j                   d d \  }}t        |t              sJ t        |t              sJ |j                  |j                  k7  rlt        |j                  |j                  |j
                        }t        d t        | j                  ||f| j                   dd  z   | j                        g      S t        t        |j                  t               f      t        |j                  t               f      f      S )Nr*   mesh
placementstensor_metar1   r4   rg   rh   r    )r#   r   r   rh   rg   ri   r	   r   r2   r3   r   r   )r   r&   targetnew_selfs       r   _prop_nll_loss_forwardrm      s    ((!,LD&dK(((fk***&+++ ((((

  ||!)6 2Y5J5J125N N"+"9"9 	
 		
  
}E	~F
 	
r(   c                     | j                   d d \  }}t        |t              sJ t        |t              sJ t        |      S )Nr*   r    )r#   r   r   r	   )r   grad_outputr&   s      r   _prop_nll_loss_backwardrp      sD    !--bq1Kk;///dK(((d++r(   c                 B   | j                   d   t        | j                         dk(  rdnt        t        | j                   d         t	        t
              rt              dkD  sJ d       t        d D              s
J d        t        fdD              sJ d d       t        fd	D              sJ d
 d       t        fdd   j                  D              sJ d       t        t        d   j                  d   j                              S )Nr      z#expect at least one tensor to stackc              3   <   K   | ]  }t        |t                y wr   r   )r   r\   s     r   r   z_prop_stack.<locals>.<genexpr>   s      ,3q
1k"Gr   z'expect a list of DTensorSpecs, but got c              3   V   K   | ]   }|j                   d    j                   k(   " ywrW   )shaper   r\   tensorss     r   r   z_prop_stack.<locals>.<genexpr>   s(      -471:###Wr_   z3expect all tensors to have the same shape, but got rY   c              3   V   K   | ]   }|j                   d    j                   k(   " ywrW   )rh   rv   s     r   r   z_prop_stack.<locals>.<genexpr>   s(      7>!
---wr_   z8expect all tensors to have the same placements, but got c              3   B   K   | ]  }|j                           y wr   is_shardr   pdims     r   r   z_prop_stack.<locals>.<genexpr>   s!      %:AJJsO%:   z4DTensor does not support stack on sharded dimension.rj   r    )r#   r7   r   intr   r$   r%   rh   r	   r   rg   )r   r~   rw   s    @@r   _prop_stackr      sX   ##A&G9(()Q.!Di>S>STU>V4WC7D!c'lQ&6-,-6 ,3  ;	0	:;   -4  H	<WIQGH   7>  M	A'!LM   %,QZ%:%:  >=>  WQZ__AVAVW r(   c                    | j                   d d \  }t        |t              sJ t        t              sJ |j                  }t        fd|D              sJ d       g }|D ]Z  }t        |t              r7|j                  kD  r(|j                  t        |j                  dz
               J|j                  |       \ t        t        |j                  t        |                  S )Nr*   c              3   B   K   | ]  }|j                           y wr   rz   r|   s     r   r   z_prop_select.<locals>.<genexpr>   s       %/AJJsOZr   z5DTensor does not support select on sharded dimension.rr   rj   r    )r#   r   r   r   rh   r%   r   r~   appendr	   rg   r`   )r   tensorrh   new_placementsr}   r~   s        @r   _prop_selectr      s    ''+KFCfk***c3&,&7&7J %/  ?>?  ')N aAEECK!!%	"23!!!$  V[[U>=RS r(   c                 &   | j                   \  }}}}}t        |t              sJ t        |t        t        f      sJ |0t        |t              sJ t        d |j                  D              sJ |0t        |t              sJ t        d |j                  D              sJ t        |j                        t        |      z
  t        fd|j                  D              sJ t        |j                  |j                        }t        |||f      S )Nc              3   <   K   | ]  }t        |t                y wr   r   r   r   r}   s     r   r   z*_prop_native_layer_norm.<locals>.<genexpr>       G5F:a+5Fr   c              3   <   K   | ]  }t        |t                y wr   r   r   s     r   r   z*_prop_native_layer_norm.<locals>.<genexpr>       E_:a+_r   c              3      K   | ]8  }t        |t              xs" t        |t              xr |j                  k  f : y wr   )r   r   r   r~   )r   r}   
batch_ndims     r   r   z*_prop_native_layer_norm.<locals>.<genexpr>  s@      !A 	1i RZ5%9%Paeej>P$RR!s   >Arj   r    )r#   r   r   r`   r$   r%   rh   r7   ru   rg   r	   )r   inputnormalized_shapeweightbiaseps
stats_specr   s          @r   _prop_native_layer_normr   
  s   1:1F1F.EVT3e[)))&666&+...GV5F5FGGGG$,,,ET__EEEEU[[!C(8$99J !!    ZZ##J uj*&EFFr(   c                    | j                   \  }}}}}}}}t        |t              sJ t        |t        t        f      sJ |0t        |t              sJ t        d |j                  D              sJ |0t        |t              sJ t        d |j                  D              sJ t        d |j                  D              sJ d|j                          |r@t        |j                  t	        t               g|j                  j                  z              nd }	|r@t        |j                  t	        t               g|j                  j                  z              nd }
t        |d   r|nd |d   r|	nd |d   r|
f	      S d f	      S )
Nc              3   <   K   | ]  }t        |t                y wr   r   r   s     r   r   z3_prop_native_layer_norm_backward.<locals>.<genexpr>2  r   r   c              3   <   K   | ]  }t        |t                y wr   r   r   s     r   r   z3_prop_native_layer_norm_backward.<locals>.<genexpr>5  r   r   c              3   ^   K   | ]%  }t        |t              xr |j                  d k(   ' ywrW   )r   r   r~   r   s     r   r   z3_prop_native_layer_norm_backward.<locals>.<genexpr>8  s+      5D
1e+!+_s   +-zGot rj   r   rr   r*   r    )r#   r   r   r$   r`   r%   rh   r8   rg   r   ndimr	   )r   gradr   r   result1result2r   r   grad_input_maskweight_grad	bias_grads              r    _prop_native_layer_norm_backwardr   "  s    		dK(((oe}555&+...GV5F5FGGGG$,,,ET__EEEE  59__   	doo   	 	hj\FKK,<,<<=	

   	 	hj\DIINN:;	

    $A&DD*1-K4(+I
  26
 r(   
active_dimc                 $   g }| j                   dd D ]  }t        |t              r|j                  J |j	                  t        |j
                  |j                  t        |8t        j                  |j                  d| dz   |j                  |dz   d z         n|j                  |j                  j                  |j                  j                                      t        | j                  |i       } t        | d	
      }|j                   r2t        |j                   t              sJ |j                   j                  S |j"                  J |j"                  d   j                   d   }t        |t              sJ t%        |j                        S )zvConsiders 2 first inputs of op_schema as having same shape, and returns suggested placement for a pointwise operation.Nr*   r   )rr   rr   )ru   stridedtyperf   r1   F)	linearity)r#   r   r   ri   r   rg   rh   r   torchSizeru   r   r   r   r2   r
   r!   r5   r`   )r   r   r#   r   output_sharding
out_schemas         r   _refine_shardingr   V  sl    K""2A&![)amm.GGGVV<<& "-  ***-4qwwzA~?O7PP ==//----	
 '$ <<I
 %Y%@O""/55{CCC**55511===$77:FFqI
*k222Z**++r(   c                    d}| j                   |t        | j                         d  z   \  }}}}}}t        |t              sJ t        |t              sJ t        |t              sJ |dk  r||j
                  z  }|j                  |   |j                  |   k(  r|dk(  sJ ||j                  |   k\  sJ d }t        t        | |            }t        |      D ]3  \  }	}
t        |
t              s|
j                  |k(  s't               ||	<   5 t        |      }|t        |j                        k(  rM|j                  t        |j                        k(  r+t        t        |j                   |j                              S t        d t#        | j$                  t        |j                   ||j&                        t        |j                   ||j&                        f| j                   dd  z   | j(                        g      S )	N)NNr   NNrr   r   rj   r    rf   r*   r1   r4   )r#   r7   r   r   r   r   ru   r$   r   	enumerater   r~   r   r`   rh   r	   rg   r   r2   ri   r3   )r   defaultsr   srcr~   startendstepinput_suggestionir}   s              r   prop_slice_scatterr     s    .HY-B-B)C)E FF &E3UC e[)))c;'''c3
Qwuzz
 {{3399S>)zzciin$$$ ,Y<=*+1aAEESL"++Q , -.5!1!122s~~J 8 #ZZ ++
 	
  ||#!&'7(-(9(9
 $!$'7(+!  ++AB/!0 #,"9"9 
 	
r(   );typingr   r   r   r   r   r   #torch.distributed._tensor.op_schemar   r	   *torch.distributed._tensor.ops.common_rulesr
   #torch.distributed._tensor.ops.utilsr   )torch.distributed._tensor.placement_typesr   r   r   r   r   r   opsaten_foreach_negdefault_foreach_reciprocal_foreach_sqrtr'   _foreach_add_foreach_div_foreach_mulr<   Scalar_foreach_subr?   _foreach_addcdiv_foreach_addcmulrK   _foreach_powScalarAndTensorrP   _fused_adamrd   nll_loss_forwardrm   nll_loss_backwardrp   stackr   selectr   r   native_layer_normr   native_layer_norm_backwardr   r   slice_scatterr   r0   r(   r   <module>r      s   8 8  H E B  yy~~ !!  (("",H , ,, 0 0n 00@         	,8 , ,, $$$$0H 004 T&&66780H 0 90 T%%--./!L !L 0!LH D))112"
h "
> "
 3"
J D**223,x ,N , 4, DJJ&&'8   (2 DKKOO$H   %2 D**223Gx GN G 4G. D33;;<0 0^ 0 =0f(,(,%-c](,i(,V D&&../E
( E
~ E
 0E
r(   