Ph2lddlZddlmZmZmZmZmZddlZddlm Z ddl m cm ZddlmZddlmZddlmZmZ ddedee j*deej,d ej.fd Zddd ed d edededee j*deej,dedeedfd eeeffdZddd ed deeefdee j*deej,dedeedfd eeeff dZddeddeeefdee j*deej,deedfd eeeff dZy)N)AnyCallableDictOptionalTuple)distributed_c10d) ShardedTensor)DTensor Replicatesharded_tensorpgdevicereturncr|tj}tj|}|j }|j d}|j j }tj||z |z|z}|tj|n|}|r|djj} | jj|jk7r| j|} || j z } | dkDr;tj | d| g} n"t#j$||j&|} t#j(||z| j&|} tj*| | || j-dd|j/|j } | S)Nr)dtyper)group)r_get_default_groupdistget_world_size local_shardssizenumelmathceil_get_pg_default_devicetensorflattenrtypetoFpadtorchzerosremptyall_gather_into_tensornarrowreshape) r r r world_sizeshards dim_0_size tensor_numel chunk_size pg_device local_tensor num_paddingrs yC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/checkpoint/_state_dict_utils.py_all_gather_sharded_tensorr1 s  z  0 0 2$$R(J  ( ( *F$$&q)J!&&(..0L: 23lBjPJ7=~//36ay''//1    # #y~~ 5'??95L <#5#5#77 ?55;/?@L{{ n229 [[Z  F   B? ]]1a . 6 6~7J7J7L MF MFr r cpu_offload ranks_only iter_objectsharded_tensor_func dtensor_funcr4r5.c tjd}t|tr ||||}nt|tr ||||}nt|tj t ttfs||}nt|tr4|jD cic]\} } | t| ||||||}} } nht|ttfr;|D cgc]} t| ||||||}} t|tr#t|}ntdt||rt!j"||vr/t|tj r|r|j%|}|St|trind}|Scc} } wcc} w)Ncpur3zUnexpected value type )r"r isinstancer r Tensorintfloatstrdictitems_iterate_state_dictlisttuple ValueErrorrrget_rankr) r6r7r8r rr4r5 cpu_deviceretkeyvaluevs r0rBrB2se$J+}-!+r6: K );F3;sE3 ?@KDW K &*//1  2 U $#'% 2   K$ /!  ! #'% !   k5 )*C1${2C1DEFF r*j8 c5<< ([&&$C JsD)bt JI    s FF  state_dictc 2d}d}t|||||||S)aG Given a state_dict, this API gathers all the ShardedTensors or DTensors in the state_dict. Args: state_dict (Dict[str, Any]): the target sharded state_dict. pg (Optional[dist.ProcessGroup]): the process group that is used to gather ShardedTensor. Note that gathering a DTensor will use the DeviceMesh. So this argument will be ignored when gathering a DTensor. device: (Optional[torch.device]): the device that is used to perform allgather for ShardedTensor. Note that gathering a DTensor will use the DeviceMesh. So this argument will be ignored when gathering a DTensor. cpu_offload (bool): whether to offload the tensors to CPU memory. The default value is False. ranks_only: (Tuple[int, ...]): if this tuple is empty, all ranks will have the same state_dicts. Otherwise only ranks that in ``ranks_only`` have the same state_dicts. Other ranks will get empty state_dicts. Returns: The gathered state dictionary. ctjd}t|||}|jr'|jdjjn|}|j|k7r|j |}|S|}|S)Nr:r)r"rr1rrr)rJr rrG output_tensorlocal_shard_devices r0r7z/_gather_state_dict..sharded_tensor_funcs\\%( 25"fE !!#     # * * 1 1    #5 5!$$%78E "E r2c<|j|jjk7r%|j|jj}|jDcgc] }t }}|j |j|}|j}|Scc}w)N) device_mesh placements)rrR device_typerrSr redistributeto_local)rJr r_rSs r0r8z(_gather_state_dict..dtensor_funcs <<5,,88 8HHU..::;E ,1+;+;<+;aik+; <""))!#   =sBr3rB)rLr rr4r5r7r8s r0_gather_state_dictrYns1B"   r2)r rr5c *t|dd||d|S)Nc|SNrJr rs r0z,_offload_state_dict_to_cpu..%r2c|Sr\r]r^s r0r_z,_offload_state_dict_to_cpu..r`r2Tr3rX)rLr rr5s r0_offload_state_dict_to_cpurbs& ''  r2)NN) rtypingrrrrrr"torch.distributed distributedrtorch.nn.functionalnn functionalr r'torch.distributed._shard.sharded_tensorr torch.distributed._tensorr r ProcessGrouprr<r1rDboolr=r?rBrYrbr]r2r0rms 77 .A8 '+%)#!#""## U\\ "# \\ #V'+%)"''99!99 ""# 9 U\\ " 99c3h9 #s(^9~'+%)"'' JS#XJ ""#J U\\ " J  J c3h J #s(^J`'+%)"'' S#X ""# U\\ "  c3h   #s(^ r2