
    Ph)                     F   d dl mZ d dlmZ d dlmZmZmZmZ d dl	Z	d dl
mZ d dlmZmZmZmZmZmZ d dlmZ d dlmZ d d	lmZmZmZ e G d
 d             ZdefdZdeeef   deeef   fdZdee	j>                  j@                     fdZ! G d de	jD                        Z#y)    )	dataclass)partial)AnyListOptionalTupleN)_disabled_torch_function_impl)_ProxyTensorfetch_tensor_proxyget_innermost_proxy_modeget_proxy_slotset_proxy_slottrack_tensor_tree)_pytree)no_dispatch)tree_flattentree_maptree_map_onlyc                   ^    e Zd ZU ej                  ed<   ej                  j                  ed<   y)_CommResult_tensor_workN)__name__
__module____qualname__torchTensor__annotations__distributed_Work     nC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/_spmd/comm_tensor.pyr   r      s      \\"""r"   r   comm_resultc                 N    | j                   j                          | j                  S N)r   waitr   )r$   s    r#   
_wait_commr(      s"     r"   resultreturnc                 H    d }| d   }t        t        ||      | d         |fS )Nc                 \    t        |t        j                        sJ d       t        ||       S )NzeExcepting collection of tensors as the first element in the return value of communication operations.)
isinstancer   r   r   workes     r#   wrapz_wrap_comm_result.<locals>.wrap#   s1    !U\\* 	
8	
*
 1d##r"      r   )r   r   )r)   r1   r/   s      r#   _wrap_comm_resultr3   "   s/    $ !9DWT4(&)4d;;r"   c                  4    t               } | y | j                  S r&   )r   tracer)modes    r#   _get_tracerr7   2   s    #%D|;;r"   c                       e Zd ZU dZg dZee   ed<   ej                  ed<   e
ej                  j                     ed<   edej                  fd       Zd ZeZed	        Zedd       Zy
)
CommTensora!  
    A Tensor subclass to wrap input tensors for collective communications.

    This Tensor subclass works for both eager and tracing mode.
    In eager mode, it will record whether the inplace collective communication
    has been launched using this Tensor and remember the corresponding work
    handle. If yes, it will explicitly call wait() in the ``__torch_dispatch__``
    function before subsequent operations consuming the value of the Tensor.

    In tracing mode, ``CommTensor`` inserts two node into the graph using the
    ``__torch_dispatch__`` function.
    1. The first node is inserted right after the
    communication, wrapping both the inplace output tensor and the returned
    work handle into a custom ``_CommResult`` type. We have to do this because
    ``ProxyTorchDispatchMode`` only handles ``torch.Tensor``, ``_ProxyTensor``,
    and ``torch.nn.Parameter`` objects and will treat the work handle
    as a constant and embed that into the graph. As a result, during execution,
    it will use the work handle created during tracing and will lead to wrong
    result. The solution in this test is to manually create a proxy on the
    return value of ``allreduce_`` which is ``([tensor], work)``, and wrap that
    to ``[(_CommResult(tensor, work)), work]``. In this way, subsequent nodes can
    directly consume ``_CommResult``.
    2. The second node is inserted right before any subsequent node reads from
    ``_CommResult``. It will call ``wait()`` on the stashed work handle to ensure
    that computation waits for communication.
    )_allgather_base__reduce_scatter_base_
allreduce_
allgather_	alltoall_
broadcast_reduce_scatter_scatter__supported_commsr   r   tensorc                     t        |t              r|j                  n|}t               |S t        j
                  j                  | ||j                        }||_        d |_        |S )N)require_grad)	r-   r9   r   r   r   r   _make_subclassrequires_gradr   )clsrC   trs       r#   __new__zCommTensor.__new__c   s\    (<FNN&#%-M LL''QQ__'M 	 r"   c                 <    d| j                    d| j                   dS )NzCommTensor(z, work=))r   r   )selfs    r#   __repr__zCommTensor.__repr__u   s    T\\N'$**Q??r"   c                 @    t        fd| j                  D              S )Nc              3   &   K   | ]  }|v  
 y wr&   r!   ).0commop_names     r#   	<genexpr>z+CommTensor._is_supported.<locals>.<genexpr>~   s     D/Ct47?/Cs   )anyrB   )rH   rT   s    `r#   _is_supportedzCommTensor._is_supported|   s    Ds/C/CDDDr"   Nc           
         d d dt         ffd}dt         fd}dt        j                  j                  dt         fd}t	        ||      }t	        ||      }	| j                  |j                        rT!t        t        d t        t        j                  t              ||	f            \  }
} ||
i |}t        |t        j                  j                        sJ j                  dt        |fi d	      }t!               5   ||i |	}d d d        t#        |d 
       t%        j&                  t)        ||d         |d          t+        |d         \  }}t+        |d         \  }}t-        ||      D ]  \  }}t/        |t1        |              |S  ||i |	}t%        j&                  t)        ||d         |d          |S  ||i |	S t	        | ||i |	      S # 1 sw Y   xY w)Nr0   c                 h   t        | t              r| j                  t        | j                  t              s
t	               bPj                  dt        t        | j                        j                  fi d      }t        | j                  |       j                          | j                  S | S )Ncall_function	wait_commname)r-   r9   r   r   r7   create_proxyr(   r   proxyr   r'   )r0   	proxy_resr5   r/   s     r#   unwrapz-CommTensor.__torch_dispatch__.<locals>.unwrap   s    !Z( ww "!))Z8(]F#)$*$7$7+&+AIIv>DDF!, %8 %	 'qyy&)D IIK yy r"   c                 P    t        | t        j                        rt        |       S | S r&   )r-   r   r   r9   r0   s    r#   r1   z+CommTensor.__torch_dispatch__.<locals>.wrap   s    $.q%,,$?:a=FQFr"   r/   c                     t        |t              r	| |_        |S t        |t        j                        rt        d      |S )NzwType of output tensors from collective communication during tracing should always be CommTensor instead of torch.Tensor)r-   r9   r   r   r   RuntimeErrorr.   s     r#   set_workz/CommTensor.__torch_dispatch__.<locals>.set_work   sD    !Z( H Au||,"R  Hr"   c                     | j                   S r&   )r_   rc   s    r#   <lambda>z/CommTensor.__torch_dispatch__.<locals>.<lambda>   s    aggr"   rZ   r$   r\   )constantr5   r2   r   )r   r   r   r    r   rW   r   r   r
   r   r   r-   fxProxyr^   r3   r   r   pytree	tree_map_r   r   zipr   r   )rH   functypesargskwargsra   r1   rf   unwrapped_argsunwrapped_kwargs
proxy_argsproxy_kwargsr`   comm_result_proxyout	flat_args	args_specflat_outout_specaor5   r/   s                        @@r#   __torch_dispatch__zCommTensor.__torch_dispatch__   s    -126"	c "	H	GC 	G	5,,22 	s 	 "&$/#FF3T]]+!+8 %!*62')9:,(
L !*==	!)UXX^^<<< %+$7$7#%L& %8 %! !]C2BCC # "#'84PVW   3q6!:DGD (4N14E'F$	9%1#a&%9"(	84DAq"1fnQ.GH 5 
 N?.>?  3q6!:DGD
^@/?@@  dN&O>N&OPPC #]s   	G&&G/)r!   N)r   r   r   __doc__rB   r   strr   r   r   r   r   r    staticmethodrK   rO   r	   __torch_function__classmethodrW   r   r!   r"   r#   r9   r9   9   s    6	#d3i 	 \\E%%++,,U\\  "@
 7E E vQ vQr"   r9   )$dataclassesr   	functoolsr   typingr   r   r   r   r   torch._Cr	   "torch.fx.experimental.proxy_tensorr
   r   r   r   r   r   torch.utilsr   rl   torch.utils._mode_utilsr   torch.utils._pytreer   r   r   r   r(   r3   rj   Tracerr7   r   r9   r!   r"   r#   <module>r      s    !  - -  2  * / E E # # #K <eCHo <%S/ < Xehhoo. ~Q ~Qr"   