
    Ph"D                         d dl mZ d dlmZ d dlmZmZmZmZm	Z	m
Z
mZmZ d dlZd dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlmZ d d	lm Z  ejB                  jD                  Z"d
e#fdZ$ G d d      Z%y)    )	lru_cache)chain)CallablecastDictListOptionalSequenceTupleUnionN)
OpOverload)FakeTensorMode)
DTensorSpecOpInfoOpSchema
OpStrategyOutputShardingOutputSpecTypePlacementStrategyRuntimeSchemaInfoStrategyTypeTupleStrategy)
TensorMeta)
DeviceMeshreturnc                 @    | yt        | t              syt        |       S )Nr      )
isinstancer
   len)objs    rC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/_tensor/sharding_prop.py_lengthr"      s     
{c8$s8O    c                       e Zd ZddZ	 ddedeegef   dee	   fdZ
	 ddedeeegef   dee	   fd	Zd
ededeee   eedf   f   fdZdedededeee   eedf   f   ddfdZdeddfdZd
edefdZdedefdZy)ShardingPropagatorr   Nc                 n    i | _         i | _        i | _         t        d       | j                        | _        y N)op_to_rulesop_strategy_funcsop_to_schema_infor    propagate_op_sharding_non_cachedpropagate_op_sharding)selfs    r!   __init__zShardingPropagator.__init__#   s<    SU  	
 GI%4Yt_T5Z5Z%["r#   op_overload	rule_funcschema_infoc                 F    || j                   |<   ||| j                  |<   yy)zG
        Register a sharding propagation rule for an operator.
        N)r(   r*   )r-   r/   r0   r1   s       r!   register_sharding_prop_rulez.ShardingPropagator.register_sharding_prop_rule-   s/     )2%"2=D"";/ #r#   strategy_funcc                 F    || j                   |<   ||| j                  |<   yy)zI
        Register a sharding strategy generator for an operator.
        N)r)   r*   )r-   r/   r4   r1   s       r!   register_op_strategyz'ShardingPropagator.register_op_strategy:   s/     /<{+"2=D"";/ #r#   	op_schema.c                    |j                   t        j                  j                  k(  ryt	               5  |j                         }|j                         } |j                   |i |}ddd       t        t        j                        r0t        |j                  |j                         |j                        S t        |t        t        f      rg }|D ]\  }t        |t        j                        s|j!                  t        |j                  |j                         |j                               ^ t        |t              rt        |      S |S y# 1 sw Y   xY w)z{
        Propagate the tensor metadata, it could either return a TensorMeta
        or a list/tuple of TensorMetas
        N)shapestridedtype)opatenequaldefaultr   gen_fake_argsgen_fake_kwargsr   torchTensorr   r9   r:   r;   tuplelistappend)r-   r7   	fake_argsfake_kwargsfake_outtensor_meta_listfake_out_items          r!   _propagate_tensor_metaz)ShardingPropagator._propagate_tensor_metaG   s$    <<4::--- !//1I#335K#y||Y>+>H 
 h-nnX__->hnn  5$-0!!)mU\\:$++""/"5"5#0#7#7#9"/"5"5 "* h. &' & 9 s   3EEr<   output_specoutput_tensor_metac           
      x   t        |t              rbt        |t              sJt        |t        t        f      st        d      t        d|j                          dt        |       d      ||_        y	t        |t        t        f      rt        |t        t        f      rt        |      t        |      k7  r5t        d|j                          dt        |       dt        |       d      t        |      D ]A  \  }}t        |t              s||   }t        |t              st        d| d      ||_        C y	y	)
zP
        Wrap the output_spec with the tensor metadata from the output.
        zGShardingPropagator error: output does not have an associated TensorMetazFor the op zL, `output_spec` has 1 output which does not equal the number of op outputs: .z, `output_spec` has z7 outputs which does not equal the number of op outputs z!ShardingPropagator error: output z' does not have an associated TensorMetaN)r   r   r   rD   rE   
ValueErrornamer   tensor_metar"   	enumerate)r-   r<   rM   rN   ispecoutput_tensor_meta_is          r!   _wrap_output_spec_tensor_metaz0ShardingPropagator._wrap_output_spec_tensor_metar   sO    k;/0*=!"4udmD$a  !!"'') ---01C-D,EQH  '9K#eT]305$-@CE'(E) !!"''),@[AQ@R S,,34F,G+HK  %[14dK0+=a+@(%&:JG(?sBij  (<D$ 2 4r#   op_infoc                     |j                   j                  r#| j                  |j                         }||_        y | j                  |j                         }||_        y r'   )schemahas_symintsr+   r,   output_sharding)r-   rY   r]   s      r!   	propagatezShardingPropagator.propagate   sM    
 >>%%"CCGNNSO #2 #88HO"1r#   c           
      r  ! |j                   t        j                  j                  u rt	        d|g      S | j                  |      }dt        dt        f!fd!|j                   | j                  v rd}|j                  D ][  }t        |t              r|j                  } n=t        |t        t        f      s8t        |d   t              sL|d   j                  } n |J d|j                           |j                  D cg c]
  } !|       }}|j                  j                         D ci c]  \  }}| !|       }	}}t!        |j                   t        |      |	      }
 | j                  |j                      ||
      }t        |t"              r| j%                  |      }d}g }t'        |j(                        D ]Y  \  }}|j*                  |j,                  n|j*                  |   }|j/                  |       |j0                  |j0                  k7  sXd	}[ d}|r4t!        |j                   t        |      i       }|j3                  |       |g}|j5                         rt        t7        t9        |j                   j:                  j<                              D cg c]L  }t        |j,                  j                  |j,                  j0                  |j,                  j>                  
      N c}      }n|jA                         r|j,                  }nd}t	        |||      }nt        |tB              rg }|jD                  D ]@  }t        |t"              sJ | j%                  |      }|j/                  |j,                         B d}g }|j                  D ]  }t        |t        t        f      rt        |d   t              rtg }t'        |      D ]7  \  }}|j0                  ||   j0                  k7  rd	}|j/                  ||          9 |j/                  t        |t              rt        |      n|       t        |t              r2|d   }|j0                  |j0                  k7  rd	}|j/                  |       |j/                  |        d}|r-t!        |j                   t        |      |j                        }|g}t	        |t        |      nd||      }ntG        d      | jI                  |j                   |j,                  |       |S |j                   | jJ                  v r| jJ                  |j                      }	  ||      }|j,                  b|jR                  '|jT                  JtQ        d| d|jT                         |jR                  d   } ||      } | j,                  |_        d	|_+        | jI                  |j                   |j,                  |       |S tM        d|j                    d      c c}w c c}}w c c}w # tL        $ r}|d}~wtN        $ r}tQ        d| d|       |d}~ww xY w)zM
        Propagate the sharding for an operator given the op_schema.
        NrV   r   c                 Z   t        | t              rt        t        |       g      S t        | t        t
        f      rit        | d   t              rV| D cg c]
  } |       }}t        t        t           |      }t        t        | t
              rt        |            S |      S | S c c}w )Nr   )
r   r   r   r   rE   rD   r   r
   r   r   )rV   stuple_strategyspec_to_strategys      r!   rc   zMShardingPropagator.propagate_op_sharding_non_cached.<locals>.spec_to_strategy   s    $,!#4T#:";<<D4-0ZQ5U?C!Dt!"21"5t!D!%h|&<n!M$-7e-DE.) JX   "Es   B(r   zCannot find mesh for op )r<   args_schemakwargs_schemaFT)mesh
placementsrS   )needs_redistributezUnsupported op strategy typez"Sharding propagation failed on op z	.
Error: z!Failed reason: z	Operator z. does not have a sharding strategy registered.),r<   r=   _local_scalar_denser?   r   rL   objectr)   rd   r   r   rf   rE   rD   re   itemsr   r   _select_strategyrT   	args_specinput_specsrM   rF   rg   !_inplace_rewrap_schema_suggestionreturn_type_tuple_tensorsranger   _schemareturnsrS   return_type_tensorr   childsrQ   rX   r(   NotImplementedError	ExceptionRuntimeErrorschema_suggestionsfailed_reasonrh   )"r-   r7   out_tensor_metarf   argrU   args_op_strategykvkwargs_op_strategystrategy_schemaop_strategyoutput_strategyrh   expected_input_specsidx
input_specdesired_specsuggestion_schemareshard_schema_rM   r]   out_spec_liststrategysuggestion_argsexpected_input_spec_listarg_specexpected_input_specsharding_prop_funcesuggested_input_schemapropagation_resrc   s"                                    @r!   r+   z3ShardingPropagator.propagate_op_sharding_non_cached   s?    <<433;;;!$4455i@	6 	f 	 <<4111D ,,c;/88DdE]3
3q6;8Wq6;;D - #N'?	~%NN# >G=R=RS=R 0 3=RS 4=3J3J3P3P3R"3R41a#A&&3R  "
 )1<<!"230)O ?$00>t_UK+z2"&"7"7"D%*"')$'01D1D'EOC +66> (33,88= !
 )//=!,,0G0GG-1* (F %)!%%-!e,@&A2&N #DDYO)7(8%668 38 &+3y||/C/C/K/K+L%M	 &N (%4%@%@%E%E+:+F+F+Q+Q,;,G,G,S,S
 &N	3K 113"1"="=K"&K"0%'9#
 K7 " + 2 2H%h
;;;&*&;&;H&EO!(()D)DE !3
 &+"02$00C!#e}5*A; 460-6s^MC'22mC6H6S6SS59 24;;M#<NO .< (..)#u5 "":;!9
 $C5.;A.>+>>-@-K-KK15.'../BC'..s3) 1, %)!%%-!eO&<i>U>U&N *8(8%"0,;,GE-(T%'9# !!?@@ ..o99? #"\\T---!%!1!1),,!?"4Y"? **2"55=&44@*@ L..=.K.K-LN  .=-O-OPQ-R*&89O&PO 3B2M2MO/9=O6 ..o99? #"%ILL>)WX K  T"R	d '  "8>RSQTUs7   W8/W=AX X 	X6XX6X11X6r   c                 \   t        |j                        dk(  r|j                  d   S g }|j                  D ]N  }|j                  J d       t        t	        j
                  |j                              }|j                  |       P |j                  |j                  t        |               S )Nr   r   z)must set redistribute cost each strategy!)	r   
strategiesredistribute_costsumr   from_iterablerF   indexmin)r-   r   strategy_costsstrtgr   s        r!   rl   z#ShardingPropagator._select_strategyu  s    x""#q(&&q))&(((E''3;:;3 #E$7$78O8O$P Q!!"34 ) "">#7#7N8K#LMMr#   )r   Nr'   )__name__
__module____qualname__r.   r   r   r   r   r	   r   r3   r   r   r6   r   r   r   r   rL   r   rX   r   r^   r+   r   r   rl    r#   r!   r%   r%   "   s9   \ 48	>> XJ67> /0	>" 48	>>  X 6 DE> /0	>)!)	tZj!15S3II	J)V'<'< $'< "*d:.j#o0FF
	'< 
'<R	2 	2D 	2M( M~ M^N N8I Nr#   r%   )&	functoolsr   	itertoolsr   typingr   r   r   r   r	   r
   r   r   rB   
torch._opsr   torch._subclassesr   #torch.distributed._tensor.op_schemar   r   r   r   r   r   r   r   r   r   )torch.distributed._tensor.placement_typesr   torch.distributed.device_meshr   opsr=   intr"   r%   r   r#   r!   <module>r      s\      O O O  ! ,   A 4yy~~C aN aNr#   