
    Phw                     	   U d dl Z d dlZd dlmZ d dlmZmZ d dlmZ d dl	m
Z
mZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
l m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z* d dl+m,Z,m-Z- d dl.m/Z0 d dl1m2Z2m3Z3m4Z4m5Z5 da6ee jn                     e8d<   ejr                  jt                  Z: G d de      Z;e G d d             Z<e G d d             Z=de
de>fdZ?	 	 dDdej                  j                  dee
df   deeeBe
f      deeej                  eej                  eee%   ee%   f   f      de
f
dZEd  ZFd!ej,                  j                  d"e!ddfd#ZHd$eej                  e
f   d%e
de
fd&ZId'eee=eJf      d(edeeeJ   ee%   f   fd)ZKd!ej                  d*ee
df   defd+ZLd!ej                  d*ee
df   defd,ZMd!ej                  d*ee
df   deeBe
f   d-edef
d.ZNd!ej                  d*ee
df   deeBe
f   d-edef
d/ZOd!ej                  d*ee
df   deeBe
f   d-edef
d0ZPe:j                  j                  eLe:j                  j                  eLe:j                  j                  eMe:j                  j                  eLiZVeej                  j                  ef   e8d1<   e:j                  j                  eNe:j                  j                  eOe:j                  j                  eOiZZeej                  j                  ef   e8d2<   e:j                  j                  ePe:j                  j                  ePe:j                  j                  ePiZ]eej                  j                  ef   e8d3<   d4dd5d!ej                  d$eej                  e
f   d6e>d-ee   deej                     f
d7Z_d8ed$eej                  e
f   deej                  e
f   fd9Z`d:ej                  d!ej                  d$eej                  e
f   dej                  fd;Zad:ej                  d<eej,                  j                  ej,                  j                  f   ddfd=Zbd>ej                  deej                  eej                     f   fd?Zd	 	 dEd:ej                  d@eej                     dAee<   d-ee   dBe>deej                  eeBe<f   f   fdCZey)F    N)	dataclass)autoEnum)partial)	AnyCallablecastDictListOptionalSequenceTupleUnion)_get_tracer)OP)
get_logger)
DeviceMeshDTensor)OpSchema)_PartialDTensorSpec	Placement	ReplicateShard
TensorMeta)redistribute_local_tensor)make_fx
proxy_slot)_pytree)tree_flattentree_maptree_map_onlytree_unflattenloggerc                   (    e Zd Z e       Z e       Zy)TrainingPhaseN)__name__
__module____qualname__r   FORWARDBACKWARD     mC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/_spmd/distribute.pyr&   r&   %   s    fGvHr-   r&   c                   (    e Zd ZU eed<   ee   ed<   y)Schemamesh
placementsN)r'   r(   r)   r   __annotations__r   r   r,   r-   r.   r0   r0   *   s    
Yr-   r0   c                   n    e Zd ZU dZeed<   eed<   eed<   defdZe	de
j                  dedd fd	       Zy
)DSymInta;  DSymInt represents a value retrieved by a SymInt op from a DTensor.

    DSymInt helps View and Factory ops to determine the placement and shape of the
    output tensor, as those operators either do not have an input DTensor or
    the input DTensor is insufficient to determine the output tensor's placement.
    global_valuelocal_valuer1   returnc                 4    | j                   | j                  k7  S N)r7   r6   )selfs    r.   is_shardzDSymInt.is_shard=   s    4#4#444r-   nodedtensorc                    d}|j                   t        j                  k(  r^t        t        |j
                  d         } | |j                  |      |j                         j                  |      |j                        S |j                   t        j                  k(  r? | |j                         |j                         j                         |j                        S |j                   t        j                  k(  r^t        t        |j
                  d         } | |j                  |      |j                         j                  |      |j                        S t        d|j                          )Nr      )r6   r7   r1   zDSymInt does not support )targetatensym_sizer	   intargssizeto_localdevice_mesh	sym_numelnumel
sym_stridestrideNotImplementedError)clsr=   r>   dims       r.   	from_nodezDSymInt.from_node@   s"   ;;$--'sDIIaL)C$\\#.#,,.33C8(( 
 [[DNN*$]]_#,,.446(( 
 [[DOO+sDIIaL)C$^^C0#,,.55c:((  &(A$++&OPPr-   N)r'   r(   r)   __doc__rD   r3   r   boolr<   classmethodfxNoder   rP   r,   r-   r.   r5   r5   0   sY     
5$ 5 QRWW Qw Q9 Q Qr-   r5   objr8   c                 x    t        | t              syd}| j                  D ]  }t        |t              sd} |S  |S )zECheck if object is 1) DTensor and  2) with any placement of _Partial.FT)
isinstancer   r2   r   )rV   
is_partial	placements      r.   _is_partial_dtensorr[   [   sB    c7#J^^	i*J $
 r-   op
local_args.kwargsspecsc                 \    |i }i dt         dt         ffd} | t        ||      i |S )Nargr8   c                    |    \  }}}}t        || j                         | j                        }t        |t	        |      |      }t        |t	        |      |      }t        | t        j                        r| v rt        | ||      S | S )N)rL   dtype)tensor_meta)	r   rL   rc   r   tuplerX   torchTensorr   )	ra   tensor_shaper1   current_placementtarget_placementrd   current_spectarget_specr_   s	           r.   redistributez2_dispatch_with_local_tensors.<locals>.redistributey   s    BG*?d-/? ::<))

 #%)*
 "%(){
 #u||, &c<E	
 	
r-   )r   r!   )r\   r]   r^   r_   rm   s      ` r.   _dispatch_with_local_tensorsrn   i   sE     ~}
# 
# 
* xj1<V<<r-   c                 t   t        |       \  }}t        j                  |j                        }i }t	        |      D ]l  \  }}t        |t              s|rE|j                         ||   j                  |j                  ||   j                  f||j                  <   |j                  ||<   n t        ||      }	||	fS r:   )r    pytreetree_leavesargs_schema	enumeraterX   r   rF   r1   r2   _local_tensorr#   )
rE   target_schemarm   flatten_argsargs_tree_specflatten_args_schemar_   ira   unflattened_argss
             r.   _update_specs_for_redistributer{      s    #/#5 L. ,,]-F-FG 	 
 L)3c7#HHJ'*//NN'*55	,c''( &)%6%6" * &&9>J"""r-   r=   	op_schemac                    t        | j                        \  }}t        j                  |j                        }dt
        t        t        j                  j                  f   dt        fd}t        |      t        |      k(  sJ t        t        ||            D ]'  \  }\  }} ||      st        |t              s#|||<   ) t        ||      }	t        |	      D ]  \  }
}| j!                  |
|        y )Nra   r8   c                     t        | t        j                  j                        r;| j                  t
        j                  t
        j                  t
        j                  fv S t        | t              S r:   )
rX   rf   rT   rU   rA   rB   rC   rI   rK   rD   )ra   s    r.   is_sym_int_or_intz6_update_node_from_op_schema.<locals>.is_sym_int_or_int   sN    c588==)::"  
 #s##r-   )r    rE   rp   rq   rr   r   rD   rf   rT   rU   rR   lenrs   ziprX   r#   
update_arg)r=   r|   	flat_argsrw   flat_args_schemar   ry   ra   
arg_schemarE   idxs              r.   _update_node_from_op_schemar      s     ,TYY 7I~)))*?*?@$uS%((--%78 $T $ y>S!12222 )#i9I*J KCS!jS&A%IaL !L )^4DdOSS! $r-   node_to_objra   c                     t        |t        j                  j                        r=| |   }t	               r,t        t        t        t        f   |j                        t        = |S |S r:   )
rX   rf   rT   rU   r   r	   r
   r   __dict__r   )r   ra   rV   s      r.   
_remap_argr      sJ    #uxx}}%#= T#s(^S\\2:>

r-   sizesr1   c                    | D cg c]   }t        |t              r|j                  n|" }}t        |       D cg c]0  \  }}t        |t              r|j	                         rt        |      2 c}}xs t               g}t        |      |j                  k(  s!J dt        |       d|j                   d       ||fS c c}w c c}}w )Nz"The number of sharded dimensions (z2) must match number of dimensions in device mesh (z).)	rX   r5   r7   rs   r<   r   r   r   ndim)r   r1   slocal_sizesry   ar2   s          r.   unpack_sizes_and_dimsr      s     AF@E1Aw/Q6  
 e$#$DAqq'"qzz| 	a$# # +	  z?dii' 
,S_,= >66:ii[	D'
 
""#s
   %B=5CrE   c                    t        |      dk(  sJ d| j                   d|        t        |d   t              sJ d|d           t        |d   t              sJ d|d           t        |d   |d   j                        \  }}| j                  d   |f| _        t        t        j                  j                  | j                        }t        j                   ||d   j                  |      |d   j                  |d	      S )
N   zExpect two args but got op z with args r   z*Expect 1st argument to be DTensor but got r@   $Expect 2nd argument as list but got Flocal_tensorrH   r2   	run_check)r   rA   rX   r   listr   rH   rE   r	   rf   _ops
OpOverload
from_localrt   )r=   rE   r   r2   r\   s        r.   binop_sym_int_consumer_ruler      s   t9>W8[QUPVWW>Q >	3DG9=>  d1gt$V(LTRSWI&VV$ 4DGT!W=P=PQK 1{+DI	ejj##T[[	1BQ--{;G''	 r-   c           
      r   |\  }}}}}}|D cg c]   }t        |t              r|j                  n|" }	}t        j                  |	|j
                  |j                        }
t        j                  t        j                  |
|j                         ||||      |j                  |j                  d      S c c}w )N)devicerc   Fr   )rX   r5   r7   rf   zerosr   rc   r   r   slice_scatterrG   rH   r2   )r=   rE   grad_outputinput_sizesrO   startendstepr   r   input_tensors              r.   #slice_backwad_sym_int_consumer_ruler     s     7;3Kc5#t AL@K1Aw/Q6   ;;K..k6G6GL ((+..0#uc4
  ++)) s   %B4default_meshc                    t        j                  | }t        d |D              rJ d| j                   d| d       t	        |d   t
              sJ d|d           t        |d   |      \  }}|g|dd  | _        t        t        j                  j                  | j                        }t        j                   || j                  i |||d	      S )
Nc              3   <   K   | ]  }t        |t                y wr:   )rX   r   .0r   s     r.   	<genexpr>z*factory_with_sizes_rule.<locals>.<genexpr>  s     =9a:a)9s   z4Not expect DTensor argument for factory op, but got z with arguments .r   r   r@   Fr   )rp   arg_tree_leavesanyrA   rX   r   r   rE   r	   rf   r   r   r   r   )r=   rE   r^   r   r   r   r2   r\   s           r.   factory_with_sizes_ruler     s     &&-I=9== 
>t{{m Lq	"= d1gt$V(LTRSWI&VV$3DG\JK(tABx(DI	ejj##T[[	1B-f- 	 r-   c                     t        d |      | _        t        t        j                  j
                  | j                        }t        j                   || j                  i ||t               gd      S )Nc                 >    t        | t              r| j                  S | S r:   rX   r5   r7   r   s    r.   <lambda>z%factory_arange_rule.<locals>.<lambda>6  s    Jq'4J1==#QPQ#Qr-   Fr   )
r!   rE   r	   rf   r   r   rA   r   r   r   r=   rE   r^   r   r\   s        r.   factory_arange_ruler   0  s_     QSWXDI	ejj##T[[	1B-f- K=	 r-   c                     ||c| _         | _        t        t        j                  j
                  | j                        }t        j                   || j                   i | j                  |t               gd      S )NFr   )
rE   r^   r	   rf   r   r   rA   r   r   r   r   s        r.   default_factory_op_ruler   @  sc     "6DIt{	ejj##T[[	1B2dkk2 K=	 r-   VIEW_SYM_INT_CONSUMERSFACTORY_SYM_INT_CONSUMERSFACTORY_OPSF)force_make_fxr   r   c                P   t        j                         5  t        t        t        |      | j
                        }t        t        t        |      | j                        }t        t         j                  j                  | j                        }t        d t        j                  | D              r|t        v r4t        |      dk(  s
J d|        t        |   | |      || <   	 d d d        y |t         v r'|J d       t!        |   | |||      || <   	 d d d        y t#        t$        t&        j(                        sJ t$        j+                  d||       | j                  t,        j.                  j0                  k(  rt,        j2                  j0                  }t        d |      }t        d |      }|t4        v rt5        |   | |||      || <   	 d d d        y t        t6        |||      } t9        |d	
      |      }|j:                  j=                          |cd d d        S # 1 sw Y   y xY w)Nc              3   \   K   | ]$  }t        |t              r|j                          & y wr:   )rX   r5   r<   r   s     r.   r   z._get_dtensor_dispatch_graph.<locals>.<genexpr>x  s(      
2!W% JJL2s   *,r   zExpect empty kwargs, but got z%Requires default mesh for factory opszYAssuming using local_value from SymInt for %sis mathematically correct. Full args are %s.c                 >    t        | t              r| j                  S | S r:   r   r   s    r.   r   z-_get_dtensor_dispatch_graph.<locals>.<lambda>  s    :a3I!--"Pq"Pr-   c                 >    t        | t              r| j                  S | S r:   r   r   s    r.   r   z-_get_dtensor_dispatch_graph.<locals>.<lambda>  s    z!W'=ammD1Dr-   )r^   r_   F)_allow_non_fake_inputs)rf   no_gradr!   r   r   rE   r^   r	   r   r   rA   r   rp   r   r   r   r   rX   r$   loggingLoggerwarningrB   viewdefaultreshaper   rn   r   grapheliminate_dead_code)	r=   r   r   r   rE   r^   op_overloaddispatchgms	            r.   _get_dtensor_dispatch_graphr   j  s    

K8$))D'*k:DKKH5::00$++> 
++T2
 

 446{a'Q+H)QQ'$:;$Gd$SD! 
   99#/X1XX/$=k$J$%D! + 
. "&'..999C	 ;;$))+++
 ,,..K PRVWDf
 +% !,K 8dFL!K g 
j (	
 =WXe<TB 	$$&Y 
s    CH*&HB/HAHH%dtc                    dt         j                  dt         j                  dt         j                  fd}| j                  }t        j                  | j                        } t	        |      ||      }|j
                  j                  D cg c]"  }|j                  t        j                  k(  s!|$ }}|j
                  j                  D cg c]"  }|j                  t        j                  k(  s!|$ }}t        |      dk(  sJ t        |      dk(  sJ | ||d   <   t        j                  || j                  t               gd	      ||d   <   t!        |d   |d
      }	|	J |	||d      fS c c}w c c}w )zCreate a graph for a dummy add function from a partial DTensor.

    This dummy add is used for triggering all_reduce on a Partial DTensor
    during the DTensor expansion of the traced graph.
    Also returns the actual DTensor after resharding.
    gradzeror8   c                     | |z   S r:   r,   )r   r   s     r.   	dummy_addz)_build_dummy_add_graph.<locals>.dummy_add  s    d{r-   r   r@   r   Fr   T)r   )rf   rg   rt   
zeros_liker   r   nodesr\   r   PLACEHOLDERCALL_FUNCTIONr   r   r   rH   r   r   )
r   r   r   r   r   
traced_addnplaceholderscall_functionstraced_dispatchs
             r.   _build_dummy_add_graphr     sf    ELL U\\  ))D))"*:*:;D##D$/J)//55P5!9OA5LP!+!1!1!7!7T!7A1442CSCS;Sa!7NT|!!!~!####%KQ #*#5#5bnny{mu$KQ  2q;dO &&& Kq(9:::! QTs   
"E--E-"E2.E2r   c           
         g }d}|j                   d   D ]  }t        |t        j                        s|j	                  |       0||   }t        |      s|j	                  |       Rd}t        t        |      }t        ||      \  }}	|j                  j                  D 
cg c]"  }
|
j                  dk(  s|
j                  dk(  r|
$ }}
|j                  j                  D 
cg c]  }
|
j                  dk(  s|
 }}
t        |      dk(  rt        |      dk(  sJ |d   j                  |d          |j                  j                          |	||d   <   i |j                  j                  D ]~  }|j                  t         j"                  k(  r||<   '|j                  t         j$                  k(  rt        |j                         dk(  rt        |j                   d         dk(  s*J d|j                    d	t        |j                                 |j	                  |j                   d   d             ||j                   d   d      ||j                   d   d      <   |j                  t         j&                  k(  r+t)        | |j*                  t-        ||j*                               | j                  j/                  |      5  | j                  j1                  |fd
      |<   d d d          |r6| j                  j3                  |       | j                  j5                  |      S |S c c}
w c c}
w # 1 sw Y   xY w)NFr   T	wait_commwait_tensoraddr@   !Expecting single output, but got  c                     |    S r:   r,   r   value_remaps    r.   r   z!_convert_output.<locals>.<lambda>'  
    UVr-   )rE   rX   rT   rU   appendr[   r	   r   r   r   r   namer   replace_all_uses_withr   r\   r   r   OUTPUTGET_ATTRsetattrrA   getattrinserting_before	node_copy
erase_nodeoutput)r   r=   r   new_argshas_partialargumentrV   r   r   
result_objr   waitr   dtnr   s                 @r.   _convert_outputr     s   
 HKIIaL(BGG,OOH%(#"3'OOH% '3&<R&M# %**00
0vv$-(? 0 	 

 *//55I5Q5q5I4yA~#c(a-// 	A$$T!W-113  *DG.0"((..Cvv' $,C 299$MQ&3sxx{+;q+@R6sxxj#chh-QR@CHHQKN ;<
 <GsxxPQ{ST~;VKA78 66R[[(

<
 XX..t4')xx'9'9#?W'XK$ 54/ /G !x 
D!xxx((]

 JF 54s    'L*!L/6L/?#L44L>node_replacementsc                    | j                   j                  D ]v  }||vr	||   }t        j                  |j                   }di c}|j                   j                  D ]-  }|j
                  t        j                  k(  s!||   |<   |dz  }/ | j                   j                  |      5  |j                   j                  D ]  }|j
                  t        j                  k(  r"|j
                  t        j                  k(  rt        |j                        dk(  s-J d|j                   dt        |j                  d                 |j                  d   }t        |      dk(  r|d   }nd }	t        |      D ]  \  }}
|
	|
j
                  dk(  sJ |
j                  j                  dk(  sJ |
j                  j                  dk(  sJ |	|	|
j                  d   k(  sJ |
j                  d   }	|
j                  d   |k(  rJ  |	J |	}|   }|j                  |       ^| j                   j!                  |fd      |<   t#        d	 ||fD              s|j                  |           n | j                   j%                  |       d d d        y | j                   j'                          | j)                          y # 1 sw Y   xY w)
Nr   r@   r   r   call_function	_operatorgetitemc                     |    S r:   r,   r   s    r.   r   z _rebuild_graph.<locals>.<lambda>j  r   r-   c              3      K   | ]c  }t        |j                  t        j                  j                        xr/ |j                  j
                  j                  j                  d        e yw))zaten::_foreachzaten::_fused_adamN)rX   rA   rf   r   r   _schemar   
startswith)r   r   s     r.   r   z!_rebuild_graph.<locals>.<genexpr>k  s\      
 "-A	 #188UZZ-B-BC HH,,11<<C "-s   A)A+)r   r   rp   r   rE   r\   r   r   r   r   r   rs   rA   r(   r'   r   r   allr   r   	recompile)r   r   r=   r   rv   ry   r   outputsr   sourceoutnew_noder   s               @r.   _rebuild_graphr  /  s   
 ((+D1 --tyy9B;"((..Cvv'#/?C Q / XX&&t,&,,2266R^^+ VVryy(CHH*Y:388*Ac#((ST+FVEWXY*!hhqkG 7|q(!(
 "&&/&8FAs"{ (#&66_#<<#<#&::#8#8K#GG#G#&::#6#6)#CC#C#)>Vsxx{5JJ#J%(XXa[F#&88A;!#33#3 '9  &111!'*62H..x8')xx'9'9#?W'XK$ 
 #&t   22;s3CDo 3t HH%w -,! Z HH  "LLN} -,s   (EJ=<AJ=2J==K	r   c                 T   i i dt         j                  dt         j                  dd ffdt        | j                        D ]^  t         j                  j                  j                  fd       t         j                  j                  j                  fd       ` S )Narg_nodeconsumerr8   c                 \    | vr'|| <   j                  |g       j                  |        y y r:   )
setdefaultr   )r  r  last_consumer_to_nodesnode_to_last_consumers     r.   _register_final_consumerz=_get_last_consumer_to_nodes.<locals>._register_final_consumer  s7    00.6!(+"--h;BB8L 1r-   c                      |       S r:   r,   )r  r  r=   s    r.   r   z-_get_last_consumer_to_nodes.<locals>.<lambda>  s    (@4(Pr-   c                      |       S r:   r,   )
kwarg_noder  r=   s    r.   r   z-_get_last_consumer_to_nodes.<locals>.<lambda>  s    7
DIr-   )rT   rU   reversedr   r=   map_argrE   r^   )r   r  r  r=   r  s    @@@@r.   _get_last_consumer_to_nodesr    s     57;=M277 Mbgg M$ M
 %
IIP	
 	KKI	
	 & "!r-   inpsschemas_allow_partialc                    t        d      at        j                  D ch c]  }t	        t        |       }}i }i }t        | j                        }	i }
t        | j                  j                        D ]  \  }}t        J t        j                  d||j                  |j                         |j                  t        j                  k(  rq|t        |      k  sJ d|dz    dt        |       d       t        j                   ||   j#                         ||   j$                  ||   j&                  d      ||<   nt)        |j                  t*        j,                  j.                        r;t1        t        ||j2                  d	            }t4        j7                  ||      ||<   nt)        |j                  t*        j,                  j8                        rt;        |||
      }|Z|||<   nS|j                  t        j<                  k(  r|st?        | ||      }|j2                  d	   D ]`  }t)        |t@        jB                        s||   }t)        |t              s4tE        |jF                  |j&                        |
|jH                  <   b n|j                  t        jJ                  k(  r}tM        tO        tP        |      |j2                        }tM        tO        tP        |      |jR                        }tU        tW        d |tY        |j[                               z               |j                  |v rt              d	kD  rt]        fdD              sJ d       t_        t4        d |      }t_        t4        d |      }t_        t4        d |      }t_        t4        d |      }||_        ||_)        t5         |j                  |i | |j                  |i |d	   j$                        ||<   nQt              d	k(  sJ d|j                   d        |j                  |i |||<   nta        d|j                         ||	v s|	|   D ]  }||=   tc        | |       | |
fS c c}w )zTransform a graph module to a distributed graph module.

    Returns:
        - transformed graph module
        - map from output name to DTensorSpec

    spmd_expznode%s: op=%s target=%szgot more placeholder nodes (r@   z) than inputs ()Fr   r   )r   c                 "    t        | t              S r:   )rX   r5   r   s    r.   r   z)_convert_to_distributed.<locals>.<lambda>  s    Aw!7r-   c              3   V   K   | ]   }d    j                   |j                   k(   " yw)r   N)r1   )r   ddsymintss     r.   r   z*_convert_to_distributed.<locals>.<genexpr>  s(      8@1HQK$$.s   &)z&all DSymInts must have the same mesh. c                     | j                   S r:   r7   r   s    r.   r   z)_convert_to_distributed.<locals>.<lambda>  s    ammr-   c                     | j                   S r:   r(  r   s    r.   r   z)_convert_to_distributed.<locals>.<lambda>  s    r-   c                     | j                   S r:   r6   r   s    r.   r   z)_convert_to_distributed.<locals>.<lambda>  s    q~~r-   c                     | j                   S r:   r+  r   s    r.   r   z)_convert_to_distributed.<locals>.<lambda>  s    r-   )r7   r6   r1   zBSPMD expansion does not support SymInt in non-operator nodes, got r   zUnrecognized node.op type )2r   r$   operator__all__r   r  r   rs   r   infor\   rA   r   r   r   r   r   cloner1   r2   rX   rf   r   OpOverloadPacketr	   rE   r5   rP   r   r   r   r   rT   rU   r0   rH   r   r   r!   r   r   r^   r   filterre   valuesr  r"   
ValueErrorr  )r   r  r  r   r  r   	operatorsr   r   r  output_schemasry   r=   r>   replacementinp_argrV   rE   r^   r]   local_kwargsglobal_argsglobal_kwargsr  r&  s                           @r.   _convert_to_distributedr<    s    
#F5=5E5EF5ET4(5EIF&(K DF8B(*NRXX^^,4!!!-q$''4;;G77bnn$s  Q-a!eWOCI;aPQ 
 !( 2 2Q

%%!K UZZ%@%@A7K		!$=>G ' 1 1$ @KUZZ%:%:;5kK &*5!$'WW		!! 'r4=  99Q<grww/%g.C!#w/7=OOS^^8w||4	 ( WW(((GJ<diiHDgj+>LF7fmmo@V9VWH {{i'CMA,= 8@  <;<  +74KTR
,W6MvV+G5MtT -g7OQW X&	*$+ +Z H< H!,k!K]!K!!))%D! 8}) ""&++a1) %0DKK$@$@D!9$''CDD))248) 9c -h 2()~ Gs   Q#)NN)NF)fr   r-  dataclassesr   enumr   r   	functoolsr   typingr   r   r	   r
   r   r   r   r   r   rf   (torch.distributed._spmd.experimental_opstorch.fxrT   #torch.distributed._spmd.comm_tensorr   #torch.distributed._spmd.graph_utilsr   !torch.distributed._spmd.log_utilsr   torch.distributed._tensorr   r   #torch.distributed._tensor.op_schemar   )torch.distributed._tensor.placement_typesr   r   r   r   r   r   &torch.distributed._tensor.redistributer   "torch.fx.experimental.proxy_tensorr   r   torch.utilsr   rp   torch.utils._pytreer    r!   r"   r#   r$   r   r3   opsrB   r&   r0   r5   rR   r[   r   r   strrg   Sizern   r{   rU   r   r   rD   r   r   r   r   r   r   _unsafe_viewr   expandslice_backwardr   r   fullaranger   r   scalar_tensorr   r   GraphModuler   r   r   r  Graphr  r<  r,   r-   r.   <module>rX     s     !   T T T  /  ; 2 8 9 8  M B ) U U $(  'yy~~D 
      
 'Q 'Q 'QTS T " (, 	%=

%=c3h%= T#s(^$%= LL%**j(9*=x	?RRSU	
	%= 	%=V#@ehhmm  T 0	D#. 	S 	S 	#gsl#$#,6#
49d9o%&#(bgg U38_  ,
''sCx,
''
S/ cN 	
 0
''
S/ cN 	
  
''
S/ cN 	
 ( 	:KK4!DII2	A UZZ22H<=  	II.KK,KK*D 4

 5 5x ?@  	 7KK.JJ/6T%**''12   )-S
''Sbggsl#S 	S
 :&S bnnSl";";"277C<0";
2>>3";JG
G
''G bggsl#G WW	GTS
SEHHMM588+?+??@S 
Sl"88"	"''4=
 !"> *. o
o
u||
o &\o :&	o
 o 2>>4V,,-or-   