Ph ddlmZmZmZmZddlZddlmZddlm Z m Z m Z m Z ddl mZdededee d eed ffd Zdededee d eeed feed fffd Zd ej$dedee d eeeeeffdZy))castListSequenceTupleN) ShapeType)_Partial Placement ReplicateShard) DeviceMesh global_shapemesh placementsreturn.cv|j}|yt|}t|}t|D]x\}}|j |}t |t s(|j} | |ks Jd| d||j|| |||\} } t | tsJ| || <zt|S)zl Compute the shape of a local shard of the given DTensor on its current coordinate of the mesh. )r Sharding dim  greater than tensor ndim ) get_coordinatelistlen enumeratesize isinstancer dim_local_shard_size_on_diminttuple) r rr my_coordinate local_shapendimidx placement mesh_dim_size shard_dimlocal_shard_size_s kC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/_tensor/_utils.pycompute_local_shaper(s'')M<( < ' 3NC IIcNM)U+%MM $O"9+-GvNO$&/&H&H *M=;M'# !""2C888)9 I&4[!!c|j}|yt|}dgt|z}t|D]\}}|j |}t |t s(|j} dgt|z} | t|ksJd| dt||j|| |||d\} } | || <| | | <|| | | kr | | || <|| xx| | z cc<t|t|fS)a0 Compute the local tensor shape and the global offsets into the original tensor of a DTensor on its current global rank. This is useful for checkpointing purpose. Example (2 host with 4GPUs each): # Below is a DeviceMesh with mesh_shape of (2, 4) mesh = DeviceMesh(device_type="cuda", mesh=[ [0, 1, 2, 3], [4, 5, 6, 7] ], ) Let's say we distribute a global_tensor of shape (8,4) over the above DeviceMesh with a placements of [Shard(0), Shard(0)]. The local shape and global offset will be as follows: rank0 -- local_shape:[1, 4], global_offset:[0, 0] rank1 -- local_shape:[1, 4], global_offset:[1, 0] rank2 -- local_shape:[1, 4], global_offset:[2, 0] rank5 -- local_shape:[1, 4], global_offset:[5, 0] rank3 -- local_shape:[1, 4], global_offset:[3, 0] rank4 -- local_shape:[1, 4], global_offset:[4, 0] rank6 -- local_shape:[1, 4], global_offset:[6, 0] rank7 -- local_shape:[1, 4], global_offset:[7, 0] )r+rrrT) return_offset) rrrrrrr rrr) r rrrr global_offsetr!r"r#r$ local_offset shard_size shard_offsets r'%compute_local_shape_and_global_offsetr1.sL8'')M<( c,// ' 3NC IIcNM)U+%MM !sS%66  3$["9+-GKHXGYZ[,5+M+M *!!#&"& ,N,( L*4 I&*6 Y' !+|I/FF/;I/FM),!), Y0GG,346[!5#777r)tensorc t|j}t|j}t|D]\}}|j|}|j rt t |}|jdkrtd||j} | |jksJd| d|jd|d|| } | |z|| <tt|D]} | | k7s || || k\s|| |z|| <!t|ttfrtdt!|d||fS) aV Compute the global size and stride of a DTensor from the given local tensor. The local size is multiplited by `world_size` per Sharding dim. The local stride is multiplited by `world_size` per Sharding dim, as long as the dimension is outside sharding dim. For example, if we have a local tensor with size (4, 8, 2) and stride (16, 1, 8). If the DTensor placements are [Shard(2)] and world_size is 2; then the global size is (4, 8, 4) and stride is (16 * 2, 1, 8). Args: tensor (:class:`torch.Tensor`): Local tensor which DTensor will be constructed from. mesh (:class:`DeviceMesh`): Object which describes the mesh topology of devices for the DTensor. placements (Sequence[:class:`Placement`]]): The attribute of the DTensor that describes its layout on the mesh topology. Return: tensor_shape: A List of int which specifies the size of DTensor which build on top of the local tensor. tensor_stride: A List of int which specifies the stride of DTensor. rzOShard placements should have negative dims normalized in the user-facing APIs: rrz for placement number .zplacement type z not supported!)rrstrideris_shardrr rAssertionErrorr rangerrr r RuntimeErrortype) r2rr tensor_shape tensor_strider!r"r#shard_placementr$local_dim_sizeis r'compute_global_tensor_infor@qsf8 &L)M#J/Y #    "5)4O""Q&$--<,=?(++IFKK' nyk)CFKK=Pfgjfkklm n'*)4N&4}&DL #3}-. >mA&6- :R&R'4Q'7-'GM!$/I 8'<=i0AQR R304  &&r))typingrrrrtorchtorch._prims_commonr)torch.distributed._tensor.placement_typesrr r r torch.distributed.device_meshr rr(r1Tensorr@r+r)r'rGs.. ) 5""#-";CI;N" 38_">@8@8#-@8;CI;N@8 5c?E#s(O +,@8F8' LL8' *8'8@8K8' 49d3i  8'r)