
    Ph)                     N   d dl Z d dlZd dlZd dlmZ d dlZd dlmZ d dlm	Z	 d dl
mZmZmZmZ d dlmZ d dlmZmZmZmZ d Z	 ddej.                  d	ed
ededej2                  deej4                     defdZdej.                  d	ededefdZdedee   dej.                  fdZy)    N)Optional)distributed_c10d)ShardShardedTensorShardedTensorMetadataTensorProperties)ShardMetadata)
DeviceMeshDTensor	Replicater   c                 T    |j                         dk(  rd|  d| S d|  d| d| |z   S )Ncpuzrank:/:)lower)rankdevice_typenum_devices_per_nodes      nC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/fsdp/_shard_utils.py_get_remote_device_strr      sF    e#tfAk]++tfAk]!D3G,G+HII    tensorr   
world_sizer   pgdevicereturnc                    | j                  |d      }t        |      |kD  rx||   j                         }| j                         D cg c]  }d }	}t	        j
                  | j                         d   |z        |z  |	d<   t        j                  ||	|      g}
ng }
|D cg c]  }t        |j                                }}dgt        t        j                  |D cg c]  }|d   	 c}            dd z   }dgt        |d         dz
  z  }	|D cg c]  }|g|	z   
 }}|t        j                  |      j                  n|j                  }t        t        |            D cg c]  }t        |||       }}t        |      t        |      cxk(  rt        |      k(  sJ  J t!        |||      D cg c]  \  }}}t#        |||       }}}}t%        || j                         t'        | j(                  | j*                  dt,        j.                  | j1                                     }t3        j4                  |
||	      S c c}w c c}w c c}w c c}w c c}w c c}}}w )
z
    Shard a tensor to chunks along the first dimension. The local rank will gets its
    corresponding chunk as the local shard to create a ShardedTensor.
    r   )dimN   F)dtypelayoutrequires_gradmemory_format
pin_memory)shards_metadatasizetensor_properties)sharded_tensor_metadataprocess_group)chunklencloner'   mathceilr   from_tensor_and_offsetslist	itertools
accumulater   _get_pg_default_devicetyperanger   zipr	   r   r   r!   r"   torchcontiguous_format	is_pinnedr   +_init_from_local_shards_and_global_metadata)r   r   r   r   r   r   chunkslocal_shard_offsetslocal_shardsr+   chunk_sizes
chunk_sizedim0_offsetsd0chunk_offsetsr   r
placementsoffsetr'   	placementshard_metadatar)   s                            r   _create_chunk_sharded_tensorrK      sk    \\*!\,F
6{TTl((*$kkm,m1m,YYv{{}Q/*<=D
55k7DQR 4::6%4

%6K:3kJk
jmkJK	r L cSQ(1,-G.:;lbTG^lM; > 	//388[[  s;'((A 	q+/CD(   {s=1DS_DDDDD (+=+z'R'R#FD) 	fdI.'R   4&[[]*,,==11'')

 DD.EUW M - ;J <
s$   	I  III#I5Idevice_meshc                 h   | j                         j                         } t        |j                        D cg c]  }t	                }}t        |j                        D cg c]  }t	                }}t        d      |d<   t        j                  | ||d      j                  |      S c c}w c c}w )z
    Shard a tensor to chunks along the first dimension. The local rank will gets its
    corresponding chunk as the local tensor to create a DTensor.
    r   r   F)	run_check)rG   )	r-   detachr6   ndimr   DShardr   
from_localredistribute)r   r   rL   r>   replicate_placementsshard_placementss         r   _create_chunk_dtensorrV   T   s     \\^""$F 27{7G7G1HI1HAIK1HI-2;3C3C-DE-D	-DE!!9R1Ul#  	 JEs   B* B/parent_meshc                     |J t        t        j                  | j                              }t	               |d<   | j                  | j                  |      } | j                         S )zT
    All gather a DTensor in its sharded dimension and return the local tensor.
    r   )rL   rG   )r1   copydeepcopyrG   r   rS   rL   to_local)r   rW   rG   s      r   _all_gather_dtensorr\   m   sf     dmmF$5$567J [JrN  && ! F
 ??r   )N)rY   r2   r.   typingr   r8   torch.distributeddistributeddistr   'torch.distributed._shard.sharded_tensorr   r   r   r   &torch.distributed._shard.sharding_specr	   torch.distributed._tensorr
   r   r   rQ   r   TensorintProcessGroupr   rK   rV   r\    r   r   <module>rh      s           .  A U UJ &*7LL7
7 7 	7
 	7 U\\"7 7tLL
  	2*% \\r   