
    Pht9                        d dl mZ d dlmZmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ d dlmZ 	 d dlmZmZ eedf   Zeeef   Zeeeeee      f      Zd	efd
ZdefdZdefdZe G d d             Z G d d      Z G d de      Z G d de      Z e G d d             Z!e G d d             Z"e G d d             Z#e G d d             Z$y# e$ r d dlmZmZ Y w xY w)    )	dataclass)DictListOptionalSequenceTupleUnionN)
OpOverload)DTensorSpec)
DeviceMesh)tree_map_onlyTreeSpec.returnc                     | j                   J d       t        j                  | j                   j                  | j                   j                  | j                   j
                        S )zN "
    This is used to propagate tensor metadata, must be under fake mode
    z)DTensorSpec does not contain tensor_meta.)dtype)tensor_metatorchempty_stridedshapestrider   )args    nC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/_tensor/op_schema.py!_rebuild_tensor_from_dtensor_metar      sT     ??&S(SS&oo##     opc                 :    | j                   j                  d   dk(  S )N_)_schemanamer   s    r   _is_inplace_opr"   &   s     ::??2#%%r   c                 2    d| j                   j                  v S )Nout)r   overload_namer!   s    r   _is_out_variant_opr&   -   s     BJJ,,,,r   c                   h    e Zd ZU dZeed<   dZeee      ed<   dZ	ee
e
e         ed<   d ZdefdZy)	PlacementStrategyz
    A placement strategy describes an acceptable sharding placements of the output
    and the tensor arguments of an operation.
    output_specNinput_specsredistribute_costc                 \    dj                  |D cg c]  }t        |       c}      S c c}w )N )joinstr)self
placementsps      r   pretty_print_placementsz)PlacementStrategy.pretty_print_placementsD   s'    ww
3
1A
3443s   )r   c           	         | j                   d}nHddj                  | j                   D cg c]  }| j                  |j                         c}      z   dz   }| j                  | j                  j                        }| | S c c}w )Nr-   (, z) -> )r*   r.   r3   r1   r)   )r0   input_specs_strspecoutput_spec_strs       r   __str__zPlacementStrategy.__str__G   s    # O )) %)$4$4$4D 44T__E$4   66t7G7G7R7RS!"?"344s   "B
)__name__
__module____qualname____doc__r   __annotations__r*   r   r   r+   r   floatr3   r/   r:    r   r   r(   r(   4   sO    
 37K(;/07 6:xT%[ 12955 5r   r(   c                       e Zd ZdZy)StrategyTypezi
    Base class type for op strategy, We have two StrategyType:
        OpStrategy and TupleStrategy
    N)r;   r<   r=   r>   rA   r   r   rC   rC   Y   s    
 	r   rC   c                   j     e Zd ZdZdee   ddf fdZdefdZde	fdZ
ed        Zed	        Z xZS )

OpStrategyz[
    OpStrategy that consists of a list of placement strategies associated with the op
    
strategiesr   Nc                 0    t         |           || _        y N)super__init__rF   )r0   rF   	__class__s     r   rJ   zOpStrategy.__init__g   s    3=r   c                     dj                  | j                  D cg c]  }t        |       c}      }| j                  d   j                  j                  j
                  }d| d| S c c}w )Nr6   r   zOpStrategy:[z	] @mesh: )r.   rF   r/   r)   meshr   )r0   strategystrategy_list_str
mesh_shapes       r   r:   zOpStrategy.__str__k   sc     IIT__&U_s8}_&UV__Q'3388>>
/0	*FF 'Vs   A(c                 z    t        | j                  D cg c]  }|j                  j                   c}      S c c}w )zR
        Returns the max number of shards across all placement strategies
        )maxrF   r)   
num_shards)r0   rN   s     r   max_num_shardszOpStrategy.max_num_shardsp   s1     DOOTOH((33OTUUTs   8c                 H    | j                   d   j                  j                  S Nr   )rF   r)   r   r0   s    r   output_shapezOpStrategy.output_shapev   s    q!--333r   c                 H    | j                   d   j                  j                  S rV   )rF   r)   ndimrW   s    r   output_ndimzOpStrategy.output_ndimz   s    q!--222r   )r;   r<   r=   r>   r   r(   rJ   r/   r:   intrT   propertyrX   r[   __classcell__rK   s   @r   rE   rE   b   se    >4(9#: >t >G G
V V 4 4 3 3r   rE   c                   >     e Zd ZdZdee   ddf fdZdefdZ xZ	S )TupleStrategya  
    TupleStrategy represents the output strategy of this op is a tuple
    of strategy, i.e. If the output of this op is a tuple of tensors or list of tensors
    with possibly different placement strategies, we should return a TupleStrategy that
    contains a tuple of OpStrategy.

    NOTE: if the output of the op is a List[Tensor] and they share the same placement
    strategy, then we should return a single OpStrategy instead of a TupleStrategy
    childsr   Nc                 0    t         |           || _        y rH   )rI   rJ   rb   )r0   rb   rK   s     r   rJ   zTupleStrategy.__init__   s    .4r   c           	          dj                  t        | j                        D cg c]  \  }}t        |        c}}      }d| dS c c}}w )Nr6   zTupleStrategy())r.   	enumeraterb   r/   )r0   idxstratchild_strategies_strs       r   r:   zTupleStrategy.__str__   sS    #yy/8/EF/EeE
|_/EF 
   45Q77 Gs   A
)
r;   r<   r=   r>   r   rC   rJ   r/   r:   r^   r_   s   @r   ra   ra      s,    5x5 5$ 58 8r   ra   c                   H    e Zd ZU dZdZeed<   dZee	e
      ed<   dZeed<   y)RuntimeSchemaInfoa  
    RuntimeSchemaInfo stores the operator schema related information for runtime (eager)
    execution. This is mainly used for two ways: 1. to generate hash for args to determine
    whether to re-run sharding prop or not 2. to determine if we need pytree
    d   static_argnumNstatic_kwargkeyFneeds_pytree)r;   r<   r=   r>   rm   r\   r?   rn   r   r   r/   ro   boolrA   r   r   rk   rk      s2     M3+/OXd3i(/ L$r   rk   c                       e Zd ZU dZeed<   eed<   eed<   dZe	e
   ed<   edeedf   fd	       Zdefd
ZdefdZddZdedefdZdefdZdefdZdefdZdedefdZdefdZdefdZddZy)OpSchemaa  
    OpSchema is a data class that describes an operator input schemas, it
    includes DTensor DTensorSpecs and non-tensor args/kwargs (positional order
    preserved). It is mainly used by the dispatching logic below to run things like
    sharding propagation.

    NOTE: this should be used as a read only data class
    TODO: make this a frozen dataclass

    Args:
        op: the operator overload we are intercepting
        args_schema: contains args except that the DTensor args have been replaced
            with its DTensorSpec
        kwargs_schema: contains kwargs except that the DTensor kwargs have been replaced
            with its DTensorSpec
    r   args_schemakwargs_schemaNschema_infor   .c                 :    t        d | j                  D              S )z
        args_spec: Tuple[DTensorSpec, ...]: contains a clean list of args spec list
            with NO non-DTensor positional arguments (i.e. int/float/tuple, etc)
            mainly used by sharding propagation to propagate the output spec
        c              3   B   K   | ]  }t        |t              s|  y wrH   
isinstancer   ).0items     r   	<genexpr>z%OpSchema.args_spec.<locals>.<genexpr>   s     X&6d*T;:WT&6s   )tuplers   rW   s    r   	args_speczOpSchema.args_spec   s     Xd&6&6XXXr   c                 V    d| j                    d| j                   d| j                   dS )NzOpSchema(op=z, args_schema=z, kwargs_schema=re   )r   rs   rt   rW   s    r   __repr__zOpSchema.__repr__   s;    477) $ ,,- ."0014	
r   c                    g }d }| j                   D ]L  }t        |t              r1|j                  t	        |             |j
                  j                  }Et        |t              rdt        |j                        dk(  sJ |j                  d   j                  }|j                  t	        |             |j
                  j                  }t        |t              rj|j                  d   }t        |t              sJ |j                  d   j                  j
                  j                  }|j                  t	        |             3|j                  t	        |             O d| j                   ddj                  |       d| dS )N   r   zOp(op=z, args_sharding=r6   z@ mesh: re   )rs   ry   r   appendr/   rM   r   rE   lenrF   r)   ra   rb   r   r.   )r0   args_shardingrP   r   arg_specfirst_op_strtgys         r   r:   zOpSchema.__str__   s7   #%
##C#{+$$SX. XX^^
C,3>>*a///>>!,88$$S]3%]]00
C/"%**Q-!/:>>>,77:FFKKQQ
$$SX.$$SX. $  y 0=1I0J(S]R^^_``r   c                     d}| j                   D ]Q  }t        |t              s|j                  !t	        d |j                  j
                  D              sHd} || _        y  || _        y )NFc              3   P   K   | ]  }t        |t        j                           y wrH   )ry   r   SymInt)rz   ss     r   r|   z)OpSchema.__post_init__.<locals>.<genexpr>   s     P<Oqz!U\\2<Os   $&T)rs   ry   r   r   anyr   has_symints)r0   r   as      r   __post_init__zOpSchema.__post_init__   s\    !!A![)amm.GPAMM<O<OPP"&K& "
 'r   arg_idxc                     | j                   |   }t        |t              }|ryt        |t              syt	        d |D              S )NTFc              3   H   K   | ]  }t        |t              xs |d u   y wrH   rx   )rz   es     r   r|   z?OpSchema.arg_type_tensor_or_tensor_list_like.<locals>.<genexpr>   s$     HCq:a-:d:Cs    ")rs   ry   r   listall)r0   r   r   	is_tensors       r   #arg_type_tensor_or_tensor_list_likez,OpSchema.arg_type_tensor_or_tensor_list_like   sB    w'sK0	#t$HCHHHr   c                     | j                   j                  j                  }t        |      dkD  xr' t	        |d   j
                  t        j                        S )Nr   r   )r   r   returnsr   ry   typer   
TensorTyper0   return_typess     r   return_type_tuple_tensorsz"OpSchema.return_type_tuple_tensors   sH    ww..< 1$ 
O  %"2"2*
 	
r   c                     | j                   j                  j                  }t        |d   j                  t
        j                        S rV   )r   r   r   ry   r   r   r   r   s     r   return_type_tensorzOpSchema.return_type_tensor  s4    ww.. ,q/..0@0@AAr   c                      j                   st         j                        d }n, j                   j                   j                   j                  }t         fdt         j                        D              }|,t         fd|D              }t         j                  ||f      S t         j                  |f      S )Nc              3      K   | ]:  \  }}j                  |      s|k\  rt        |t              rt        |      n| < y wrH   )r   ry   r   r}   )rz   ir   r0   rm   s      r   r|   z$OpSchema.__hash__.<locals>.<genexpr>  sD      
3177:a=>P #1d+E!H23s   A Ac              3   V   K   | ]   }j                   j                  |d        " y wrH   )rt   get)rz   kr0   s     r   r|   z$OpSchema.__hash__.<locals>.<genexpr>  s(      #9HA""&&q$/s   &))	ru   r   rs   rm   rn   r}   rf   hashr   )r0   rn   args_to_hashkwargs_to_hashrm   s   `   @r   __hash__zOpSchema.__hash__  s     0 01M"O ,,::M"..>>O 
!$"2"23
 

 &" #9H# N ,?@@,/00r   otherc                    t        |t              sy| j                  |j                  k7  ryt        | j                        t        |j                        k7  ry| j
                  st        | j                        }d }n,| j
                  j                  }| j
                  j                  }t        t        | j                  |j                              D ],  \  }\  }}t        |t              r||k7  r y||k\  s&||k7  s, y |rB|D ]=  }| j                  j                  |d       |j                  j                  |d       k7  s= y y)NFT)ry   rr   r   r   rs   ru   rm   rn   rf   zipr   rt   r   )r0   r   rm   rn   r   self_arg	other_argkeys           r   __eq__zOpSchema.__eq__!  s(   %*77ehht C(9(9$::  0 01M"O ,,::M"..>>O(1  %"3"34)
$A$) (K0X5Jm#I(=)
 &%%))#t48K8K8O8O9  !	 ' r   c                 @    t        t        t        | j                        S )z
        gen_fake_args: generate fake args for the operator, this is mainly used
            by sharding propagation rules to generate fake args for the operator
            to run the local tensor operator and get the output spec.
        )r   r   r   rs   rW   s    r   gen_fake_argszOpSchema.gen_fake_argsF  s     :D<L<L
 	
r   c                 @    t        t        t        | j                        S )z
        gen_fake_kwargs: generate fake kwargs for the operator, this is mainly used
            by sharding propagation rules to generate fake kwargs for the operator
            to run the local tensor operator and get the output spec.
        )r   r   r   rt   rW   s    r   gen_fake_kwargszOpSchema.gen_fake_kwargsP  s     :D<N<N
 	
r   c                     | j                   }g }d}|j                  D ]=  }t        |t              r|j	                  ||          |dz  }-|j	                  |       ? t        |      | _        |j                  | _        y )Nr   r   )r~   rs   ry   r   r   r}   rt   )r0   origin_schemasuggestion_args_specnew_arg_schemaidx_of_args_specr   s         r   !_inplace_rewrap_schema_suggestionz*OpSchema._inplace_rewrap_schema_suggestionZ  s|    #~~') ,,C#{+%%&:;K&LM A% %%c* - !0*88r   )r   N)r   rr   r   N)r;   r<   r=   r>   r
   r?   ArgsType
KwargsTyperu   r   rk   r]   r   r   r~   r/   r   r:   r   r\   rp   r   r   r   r   objectr   r   r   r   rA   r   r   rr   rr      s    " 	N/3K+,3Y5c!12 Y Y
# 
a a*'	I3 	I4 	I
4 
BD B1# 1,#F #t #J
x 

 
9r   rr   c                   X    e Zd ZU dZeed<   dZeee	      ed<   dZ
ee   ed<   dZeed<   y)OutputShardinga  
    OutputSharding is a data class that is used by the sharding propagation
    rules, it could set the output_spec upon successful propagation, and if
    it failed, output_spec would become None and sharding propagation rules
    could give a list of suggestions for inputs to reshard.

    NOTE: the schema_suggestion generated by sharding propagation should be
    exactly the same as the operator OpSchema, except the DTensor DTensorSpecs
    r)   Nschema_suggestionsfailed_reasonFneeds_redistribute)r;   r<   r=   r>   OutputSpecTyper?   r   r   r   rr   r   r/   r   rp   rA   r   r   r   r   h  s<      37h07#'M8C='$$r   r   c                       e Zd ZU dZeed<   eed<   ee   ed<   e	e   ed<   e
eef   ed<   dZee   ed<   dZee   ed	<   y)
OpInfoz7
    All Runtime Op execution info are packed here
    rM   schemaflat_args_schema
local_argslocal_kwargsNargs_tree_specoutput_sharding)r;   r<   r=   r>   r   r?   rr   r   r   r   r   r/   r   r   r   r   r   rA   r   r   r   r   z  sY     6l"  sF{##)-NHX&- 15OXn-4r   r   )%dataclassesr   typingr   r   r   r   r   r	   r   
torch._opsr
   )torch.distributed._tensor.placement_typesr   torch.distributed.device_meshr   torch.utils._cxx_pytreer   r   ImportErrortorch.utils._pytreer   r   r/   r   r   r   r"   r&   r(   rC   rE   ra   rk   rr   r   r   rA   r   r   <module>r      sD   ! ? ?  ! A 4? #v+
 %Xh{6K-L LMN	f 	&z &-: - !5 !5 !5H	 	3 3:8L 8,   ( z9 z9 z9z % % %" 5 5 5a   s   C C"!C"