
    Ph&              
          U d Z ddlmZ ddlmZ ddlmZmZmZm	Z	m
Z
mZmZmZmZ ddlZddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) dgZ*ee+   e,d<   ee   Z-eee-f   Z.e
ee/   e/ef   Z0eree	d      Z1ee
e2ee
def   e0df   f      Z3neZ1eZ3dededdfdZ4dede$de$ddfdZ5dede$de$ddfdZ6de7de7deee
e7e7f         fdZ8 G d  d      Z9y)!z!The pipeline parallelism of Pipe.    )Queue)TracebackType)	TYPE_CHECKINGIterableListOptionalTupleTypeUnioncastSequenceN)Tensornn)record_function   )Checkpointing)CopyWait)forkjoin)Batch)
SkipLayout)SkipTrackerThroughPotalsuse_skip_tracker)AbstractStreamcurrent_stream
use_device)Taskcreate_workersPipeline__all__r   	fork_fromjoin_toreturnc                     | j                         }|j                         }t        | |         \  | |<   }t        ||   |      ||<   y N)find_tensor_idxr   r   )r"   r#   fork_from_idxjoin_to_idxphonys        sC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/pipeline/sync/pipeline.py_dependr,   +   sM    --/M))+K&*9]+C&D#Ime 4e<GK    batchprev_streamnext_streamc                     t        j                  ||g|  | d d  t        | D cg c]9  }t        j                  |      r |j                         s|j                         n|; c}      | d d  y c c}w r&   )r   applytupletorch	is_tensoris_floating_pointdetachr.   r/   r0   xs       r+   _copyr:   3   a    zz+{;U;E!Hejkej`aEOOA$6q?R?R?TahhjZ[[ejklE!Hk   >A-c                     t        j                  ||g|  | d d  t        | D cg c]9  }t        j                  |      r |j                         s|j                         n|; c}      | d d  y c c}w r&   )r   r2   r3   r4   r5   r6   r7   r8   s       r+   _waitr>   9   r;   r<   mnc              #      K   t        | |z   dz
        D ]@  }t        t        d|z   | z
  d      t        d|z   |            D cg c]	  }||z
  |f c} B yc c}w w)z(Generate schedules for each clock cycle.r   r   N)rangemaxmin)r?   r@   kjs       r+   _clock_cyclesrG   ?   sa      1q519#(QUQY):CAqM#JK#JaAqz#JKK Ks   A AAAc                       e Zd ZdZdeej                     deej                     deee	      de
deddfd	Zd
ee   ddfdZd
ee   deeeef      dee   ddfdZd
ee   deeeef      dee   ddfdZy)r    z"The pipeline parallelism for Pipe.
partitionsdevicescopy_streamsskip_layoutcheckpoint_stopr$   Nc                 z    || _         || _        || _        || _        || _        t        |      \  | _        | _        y r&   )rI   rJ   rK   rL   rM   r   	in_queues
out_queues)selfrI   rJ   rK   rL   rM   s         r+   __init__zPipeline.__init__U   s=     %(&.,:7,C)r-   batchesc                     | j                   }| j                  }| j                  }t        |      }t        |      }|D cg c]  }t	        |       }}t        ||      D ](  }	| j                  ||	|       | j                  ||	|       * yc c}w )zURuns pipeline parallelism.

        It modifies the given batches in place.

        N)rI   rJ   rL   lenr   rG   fencecompute)
rQ   rS   rI   rJ   rL   r?   r@   _skip_trackersschedules
             r+   runzPipeline.rund   s     __
,,&&L
OHOP11+>P%a+HJJw-8LL(M: , Qs   BrZ   rY   c           	      R   | j                   }| j                  }|D ]  \  }}|dk7  r|dk7  rt        ||dz
     ||          ||   |   }|j                  |      D ])  \  }	}
}||	   |   }||   j	                  ||   |||
|       + |dk7  sp||dz
     |   }t        ||   ||        y)zDCopy micro-batches after computation for the previous micro-batches.r   r   N)rK   rL   r,   copy_policycopyr:   )rQ   rS   rZ   rY   rK   rL   irF   r0   prev_jnsnamer/   s                r+   rV   zPipeline.fencew   s     ((&&DAq Av!q&A
3&q/!,K$/$;$;A$> D*6215a %%gaj+{BPTU %? Av*1q51!4gaj+{; r-   c                    | j                   }| j                  }| j                  }| j                  }| j                   d   j                  sd}t        |      }|D 	cg c]  }	t        |	       }
}	d}|D ]  \  }}||   }||   }|dk7  rt        |||   |   |
|          ||k  }|ri|||   ||ddt        j                  dt        dt        dt        dt        f
d	}t        ||      }t        |
|   |j                  |j                   
      }~~nM||||   ||fdt"        dt        j                  dt        dt        dt        dt"        fd}t        |
|   |d
      }~| j$                  |   j'                  |       	 |D ]  \  }}| j(                  |   j+                         \  }}|)|st-        t.        |      }<t-        t0        t        t"        f   |      \  }}||dz
  k7  rt        ||
|   ||   |          t3        ||         5  |j5                  |       ddd       |||<    ||d   j7                  |d   |d         yc c}	w # 1 sw Y   3xY w)z/Run tasks with synchronization to copy streams.r   N)	partitionskip_trackerchunk_idpart_idrd   re   rf   rg   r$   c                     t        |      5  t        d||fz        5   | | cd d d        cd d d        S # 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wNzchunk%d-part%d)r   r   )rd   re   rf   rg   inputss        r+   functionz"Pipeline.compute.<locals>.function   sH     *,7IY]egn\oIo9p(&1 :q9p779p9p777s   A
5	A
>	A

A)rW   finalizer.   c                     t        |      5  t        d||fz        5  | j                  |      cd d d        cd d d        S # 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wri   )r   r   call)r.   rd   re   rf   rg   s        r+   rW   z!Pipeline.compute.<locals>.compute   sL     *,7IY]egn\oIo9p$zz)4 :q9p779p9p777s!   AA	AA
	AAr      )rI   rJ   rK   rM   trainingrU   r   r>   r   Moduler   intTensorOrTensorsr   r   
checkpoint	recomputer   rO   putrP   getr   ExcInfor	   r   rl   with_traceback)rQ   rS   rZ   rY   rI   rJ   rK   rM   r@   dstreamsexc_infor_   rF   r.   rd   rt   rk   chktaskrW   okpayloads                          r+   rW   zPipeline.compute   s    __
,,((.. q!**O
O.56g>!$g6&*6 DAqAJE"1I Ave\!_Q/< _,J ,5=J1=M$%#$2!yy2 #;2 "	2
 !2 %2 $He4GAJWc
 $)+4=J1=M$%#$5 5!yy5 #;5 "	5
 !5 5 GAJ$G NN1!!$'U X DAq//!,002KB #1uT5[17;KD% AEzeWQZa);<
 GAJ'e$ ( GAJ/ 4 1+,,Xa[(1+FF  E 7x ('s   III	)__name__
__module____qualname____doc__r   r   
Sequentialr4   devicer   r   rr   rR   r   r[   r	   r   rV   rW    r-   r+   r    r    R   s    ,D'D ell#D 4/0	D
  D D 
D;4; ;4 ;&<E{<.25c?.C<TXYqTr<	<.qGE{qG.25c?.CqGTXYqTrqG	qGr-   ):r   queuer   typesr   typingr   r   r   r   r	   r
   r   r   r   r4   r   r   torch.autograd.profilerr   rt   r   r^   r   r   
dependencyr   r   
microbatchr   skip.layoutr   skip.trackerr   r   streamr   r   r   workerr   r   r!   str__annotations__Tensorsrs   BaseExceptionrx   InQueueboolOutQueuer,   r:   r>   rr   rG   r    r   r-   r+   <module>r      sj   (   ^ ^ ^   3 %  "  # D > > ( \c ! 6
(
]#]MA
B HV$%GU4uVU]';Wd'J!KKLMHGH=u =u = =m m^ m. mUY mm m^ m. mUY mLS LS LXd5c?.C%D L&mG mGr-   