
    Ph                         d dl mZmZmZmZ d dlZd dlmZ d dlm	c m
Z d dlmZ d dlmZmZmZ d dlmZmZmZ d dlmZmZ ej(                  j2                  Z G d d      Zy)	    )CallableDictListSetN)Tensor)
DeviceMesh	ReplicateShard)DimSpecInputDimops)_PartialDTensorSpecc                       e Zd ZdZddeddfdZdeddfdZdej                  deddfd	Z	dej                  defd
Z
ddej                  defdZdej                  dedefdZy)BatchDimAnalyzera  This class is used to analyze the batch dimension of each tensor/node in the graph.

    We need to know the batch dimension of each tensor/node so that we know
    exactly the sharding layout of intermediate tensors.

    We possibly should evaluate using symbolic shapes to track the batch dimension.
    We can experiment it later with dynamo integration (as dynamo have mark_dynamic
    API which allows marking batch dimension only) or try to use FakeTensorMode to
    mark the batch dimension. For now, let's just use the batch dimension of the first
    input tensor as the hint to track the batch dimension of all tensors/nodes in
    the graph.
    	batch_dimreturnNc                 \   || _         i | _        d| _        t        j                  j
                  t        j                  t        j                  j                  t        j                  t        j                  j
                  t        j                  t        j                  j
                  t        j                  t        j                  j
                  t        j                  t        j                  j
                  t        j                  t        j                  j
                  t        j                  t        j                  j
                  t        j                  t        j                  j
                  t        j                  t        j                   j"                  t        j                   i
| _        y )N)r   batch_dim_mapbatch_dim_sizeatensqueezedefaulttorchdimviewr   reshape_unsafe_view	unsqueezeexpandpermuterepeat	transposeintdim_rule_map)selfr   s     rC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/_spmd/batch_dim_utils.py__init__zBatchDimAnalyzer.__init__$   s    "13  LL  %--LLemmIIv{{LL  %--%%v{{NN""EOOKKLL  %--KKNNW
    r   c                     | j                   dk7  r+| j                   |k7  rt        d| d| j                    d      || _         y)z=Initialize batch dim size base on the first input batch size.r   z=batch dim size is already initialized! Found new batch size: z' not matching existing batch dim size: !N)r   RuntimeError)r'   r   s     r(   init_batch_dim_sizez$BatchDimAnalyzer.init_batch_dim_size8   sX    "$)<)<)N))7(8 95595H5H4IL 
 -r*   nodec                 "    || j                   |<   y N)r   )r'   r/   r   s      r(   set_batch_dimzBatchDimAnalyzer.set_batch_dimB   s    #,4 r*   c                 Z    || j                   vrt        d| d      | j                   |   S )Nz#batch dim analysis failed on node: r,   )r   r-   )r'   r/   s     r(   get_batch_dimzBatchDimAnalyzer.get_batch_dimE   s6    t)))!DTF!LMM!!$''r*   c                    | j                   dk7  sJ d       || j                  v r| j                  |   S |j                  | j                  v rGt        | j                  |j                        }t        j                  t        j                  d |j                        }t        j                  t        j                  d |j                        } |j                  |i |}dt        dt        t           ffdg }|D ]'  }t               }	 ||	       |j!                  |	       ) |j"                  d	   }
| j%                  |
      }t'        |      D ]@  \  }}	||	v s| j)                  ||       |j*                  d
   j,                  |   | _         |c S  |j*                  d
   }t/        |t0        t2        f      r|D cg c]  }|j,                   }}n|j,                  g}d}|D ]L  }t5        |      d	k(  rd}t'        |      D ],  \  }}|| j                   k(  s| j)                  ||       |c c S  N |j"                  }|s| j)                  |d       yd}|D ]"  }
|
| j                  v s| j%                  |
      }$ |d	k  r| j)                  ||       |S |r| j)                  ||       |S | j)                  |d       yc c}w )z+Compute the batch dimension for the `node`.r   z"batch dim size is not initialized!c                      | j                   d   S Nvalmetans    r(   <lambda>z4BatchDimAnalyzer.compute_batch_dim.<locals>.<lambda>T   s    qvve}r*   c                      | j                   d   S r7   r9   r;   s    r(   r=   z4BatchDimAnalyzer.compute_batch_dim.<locals>.<lambda>V   s    166%=r*   cmd
input_dimsc                     t        | t              r|j                  | j                         | j	                         D ]  } ||        y r1   )
isinstancer   add	input_diminputs)r?   r@   inpcollect_input_dims      r(   rG   z=BatchDimAnalyzer.compute_batch_dim.<locals>.collect_input_dimZ   s7    c8,NN3==1::<C%c:6 (r*   )r@   r   r8   FT)r   r   targetr&   view_op_rulespytreetree_map_onlyfxNodeargskwargsdim_mapr   r   r%   setappendall_input_nodesr4   	enumerater2   r:   shaperB   listtuplelen)r'   r/   full_reductionview_op_ruleargs_val
kwargs_valoutput_dim_rulesoutput_dim_to_input_dimsrF   r@   operandoperand_batch_dim
output_dimnode_valr8   shapesrV   idim_sizeoperandsrG   s                       @r(   compute_batch_dimz"BatchDimAnalyzer.compute_batch_dimJ   s   ""b(N*NN(4%%%%%d++;;$+++():):4;;)GHL++BGG5LdiiXH--0$++J  4|33XLL7w 7CH 7 8:$''*u
!#*=(//
; (
 **1-G $ 2 27 ;*34L*M&
J$
2&&tZ8 +/))E*:*@*@*LD'%% +N 99U#hu.+348Ccii8F4nn%F E5zQ!%(/8t222&&tQ/H  0	  '' tR( !##d000(,(:(:7(C% $ !1$""4):;((""4):;(( ""4,a 5s   >K meshc                     | j                  |      }|dk(  rt        |t               f      }|S |dk(  rt        |t               f      }|S t        |t	        |      f      }|S )zuCompute the batch dimension for the current node, then generate the sharding spec that shards on the batch dimension.r   )ri   
placementsrH   )rh   r   r	   r   r
   )r'   r/   ri   node_batch_dimact_specs        r(   compute_act_specz!BatchDimAnalyzer.compute_act_spec   sm    //5R")+HH  r!"(*GH
  #%:O9QRHr*   )r   )F)__name__
__module____qualname____doc__r%   r)   r.   rM   rN   r2   r4   rh   r   r   rn    r*   r(   r   r      s    
# 
d 
(-# -$ --"'' -c -d -("'' (c (
Zbgg Z ZxRWW J ; r*   r   )typingr   r   r   r   r   torch.fxrM   torch.utils._pytreeutils_pytreerK   r   torch.distributed._tensorr   r	   r
   &torch.distributed._tensor.ops.view_opsr   r   r   rJ   )torch.distributed._tensor.placement_typesr   r   r   r   rs   r*   r(   <module>r|      sI    , ,   $ $  B B 
 Lyy~~] ]r*   