
    Ph                     l   d dl mZ d dlZd dlZd dlmZ d dlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZ dd	lmZ d
dgZ	 	 	 	 ddededeej*                     dededee
   defdZdd ddddededeej*                     dededee
   defdZ	 	 	 	 ddededeej*                     dededee
   defdZy)    )OptionalN)Stateful   )SavePlanner)DefaultSavePlanner)StorageWriter)MetadataSTATE_DICT_TYPE)_DistWrappersave_state_dictsaveF
state_dictstorage_writerprocess_groupcoordinator_rankno_distplannerreturnc                 L    t        j                  d       t        | |||||      S )z3This method is deprecated. Please switch to 'save'.zb'save_state_dict' is deprecated and will be removed in future versions. Please use 'save' instead.)warningswarn_save_state_dict)r   r   r   r   r   r   s         xC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/checkpoint/state_dict_saver.pyr   r      s-     MMl
 JGWY`bijj    )r   r   r   r   c                    t         j                  j                  d       i }| j                         D ]*  \  }}t	        |t
              r|j                         n|||<   , t        ||||||      S )a	  
    Save a distributed model in SPMD style.

    This function is different from ``torch.save()`` as it handles
    ``ShardedTensor`` , and ``DTensor`` by having each rank only save their local shards.

    For each ``Stateful`` object (having both a ``state_dict`` and a ``load_state_dict``),
    save will call ``state_dict`` before serialization.

    .. warning::
        There is no guarantees of Backwards Compatibility across PyTorch versions
        for saved state_dicts.

    .. warning::
        If using the `process_group` argument, make sure that only its ranks
        call `save_state_dict` and that all data in state_dict belong to it.

    .. note::
        When saving checkpoint for FSDP's `ShardingStrategy.HYBRID_SHARD`, only one of
        the shard_group should be calling `save_state_dict` and the corresponding process
        group needs to be passed in.

    .. note::
        This function can be used to save a state_dict without having a process group
        initialized by passing ``no_dist=True``.


    Args:
        state_dict (Dict[str, Any]): The state_dict to save.
        storage_writer (StorageWriter):
            Instance of StorageWrite use to perform writes.
        process_group (ProcessGroup):
            ProcessGroup to be used for cross-rank synchronization.
        coordinator_rank (int): Rank to use to coordinate the checkpoint.
            rank0 is used by default.
        no_dist (bool): If ``True``, distributed checkpoint will not save
            in SPMD style. (Default: ``False``)

    Returns:
        Metadata: Metadata object for the saved checkpoint.

    Example:
        >>> # xdoctest: +SKIP
        >>> my_model = MyModule()

        >>> model_state_dict = my_model.state_dict()

        >>> fs_storage_writer = torch.distributed.checkpoint.FileSystemWriter("/checkpoint/1")
        >>> torch.distributed.checkpoint.save_state_dict(
        >>>     state_dict=model_state_dict,
        >>>     storage_writer=fs_storage_writer,
        >>> )

    .. note::
        save_state_dict uses collectives to coordinate writes across ranks.
        For NCCL-based process groups, internal tensor representations of
        objects must be moved to the GPU device before communication takes place.
        In this case, the device used is given by ``torch.cuda.current_device()``
        and it is the user's responsibility to ensure that this is set so that
        each rank has an individual GPU, via ``torch.cuda.set_device()``.
    z!torch.distributed.checkpoint.save)torch_C_log_api_usage_onceitems
isinstancer   r   r   )	r   r   r   r   r   r   dumpable_state_dictkeyelems	            r   r   r   %   st    L 
HH  !DE%%'	T8B48R4??#4X\C  (  r   c                    
 t         j                  j                  d       t        || |      
t	               J d  fd}fd}j                  d||      

fd}fd}	j                  d||	      S )Nz,torch.distributed.checkpoint.save_state_dictc                      J j                  j                         j                  j                         j                         } j	                  |       } | S N)set_up_planneris_coordinatorset_up_storage_writercreate_local_planprepare_local_plan)
local_plandistWr   r   r   s    r   
local_stepz$_save_state_dict.<locals>.local_step   s]    """z5+?+?@,,U-A-AB..0
#66zB
r   c                 Z    J j                  |       \  } j                  |       } | S r&   )create_global_planprepare_global_plan)all_local_plansglobal_metatadatar   r   s    r   global_stepz%_save_state_dict.<locals>.global_step   sB     """-4-G-G.
** )<<_Mr   planc                      J j                        } j                  |       }|j                          |j                         S r&   )finish_plan
write_datawaitvalue)final_local_plan
all_writescentral_planr   r   s     r   r8   z$_save_state_dict.<locals>.write_data   sL    """"..|<#../?I
!!r   c                 6    J j                  |        S )N)metadataresults)finish)all_resultsr3   r   s    r   finish_checkpointz+_save_state_dict.<locals>.finish_checkpoint   s)     ,,,'8+N  r   write)r   r   r   r   r   reduce_scatter
all_reduce)r   r   r   r   r   r   r.   r4   r8   rC   r=   r-   r3   s   ``   `    @@@r   r   r   z   s     
HH  !OPG5EFE$& ''
KHL"!
 GZ1BCCr   )Nr   FN)typingr   r   r   torch.distributeddistributeddist%torch.distributed.checkpoint.statefulr   r   r   default_plannerr   storager   r?   r	   r
   utilsr   __all__ProcessGroupintboolr   r   r    r   r   <module>rT      sy        :   / 0 f
% 26%)kk!k D--.k 	k
 k k"k k( 26%)SS!S D--.	S
 S S k"S Sp 26%)3D3D!3D D--.3D 	3D
 3D k"3D 3Dr   