Ph: |UddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZddlZddlmZddlmZddlmZmZmZmZmZmZmZddlmZmZmZm Z m!Z!m"Z"m#Z#dd l$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,dd l-m.Z.dd l/m0Z0ddl1m2Z2ejfe4Z5ejle7d<gdZ8GddeZ9GddeZ:de e;e fde!defdZdefdZ? d(de ede>de e ee!ffdZ@de"de!fdZAd ed!ede>fd"ZBd#ejd$ede>fd%ZDd&e ede!de>fd'ZEy))N)ChainMap)reduce)ListTupleDictAnyUnioncast)narrow_tensor_by_index)DTensor) SavePlanner LoadPlannerSavePlanLoadPlanReadItem WriteItem WriteItemType)BytesStorageMetadataChunkStorageMetadataTensorStorageMetadata MetadataIndexMetadataSTATE_DICT_TYPE STORAGE_TYPES)_create_read_items_create_write_items"_create_default_metadata_only_plan)FLATTEN_MAPPINGflatten_state_dict)_flatten_sharded_tensors) dedup_tensors)find_state_dict_object) set_elementlogger)DefaultSavePlannerDefaultLoadPlannercreate_default_local_load_plancreate_default_global_load_plancreate_default_local_save_plancreate_default_global_save_planc eZdZUeed< ddedededdfdZded eddfd Zde fd Z d e e de e e e ffd Zde de fdZdedeej&ej*ffdZdedefdZdedefdZy)r%mappingsrflatten_sharded_tensorsdedup_replicated_tensorsreturnNc<||_||_||_i|_yN)rr-r.r,)selfrr-r.s wC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/checkpoint/default_planner.py__init__zDefaultSavePlanner.__init__Gs# #5'>$(@% state_dictis_coordinatorc|jrt|\}|_|jr t|}||_||_yr1)rr,r-r r6r7)r2r6r7s r3set_up_plannerz!DefaultSavePlanner.set_up_plannerRs@  " "(::(F %J  ' '1*=J$,r5ct|j|j}|jr!t j ||j }||_|jS)N planner_data)r)r6r7r dataclassesreplacer,plan)r2r?s r3create_local_planz$DefaultSavePlanner.create_local_plan\sM- OOT00   " "&&t$--HD yyr5 all_planscj|jr t|}t|\}}|jrA|Dcgc]}|j}}t t |}tj||}t||s td||_ ||_ |j|jfScc}w)Nr;zFailed to validate global plan) r.r!r*rr<dictrr=r>_validate_global_plan ValueError global_planmetadata)r2rArFrGpplanner_data_dictmerged_mappingss r3create_global_planz%DefaultSavePlanner.create_global_planfs  ( (%i0I ? J X  " " :E EA  E"8->#?@O"**H%[(;=> >&  ..!FsB0new_planc||_|Sr1)r?r2rLs r3 finish_planzDefaultSavePlanner.finish_plans r5 write_itemc\|j|j}|j||Sr1) lookup_objectindextransform_object)r2rPobjects r3 resolve_datazDefaultSavePlanner.resolve_datas-##J$4$45$$Z88r5rSc.t|j|SzSExtension from the planner interface to make it easy to extend the default planner.r"r6r2rSs r3rRz DefaultSavePlanner.lookup_object%doou==r5rUc|jtjk(r,tj}t j |||}|SrX)typerBYTE_IOioBytesIOtorchsave)r2rPrUbytess r3rTz#DefaultSavePlanner.transform_objects7 ??m33 3JJLE JJvu %F r5)TTT)__name__ __module__ __qualname__r__annotations__boolr4rr9rr@rrrrKrOrr raTensorr_r`rVrrrRrTr5r3r%r%Ds$((,)-   "& #'   -)-;?- -8/h/ tH~x' (/6H9#9 u||RZZ' (9 >=>S>9cr5r%c4eZdZUdZeed<eed< ddededdfdZd ed e d eddfd Z de fd Z de e de e fdZde de fdZdedej$ddfdZdefdZdedej,ddfdZdedej,fdZdedej,fdZy)r&z DefaultLoadPlanner that adds multiple features on top of LoadPlanner. In particular it adds the following: flatten_state_dict: Handle state_dict with nested dicts flatten_sharded_tensors: For FSDP in 2D parallel mode original_state_dictr,rr-r/Nc<||_||_i|_i|_yr1)rr-rlr,)r2rr-s r3r4zDefaultLoadPlanner.__init__s# #5'>$#%  r5r6rGr7c||_|jr t|}|jrt|\}|_||_||_||_yr1)rlr-r rr,r6rGr7)r2r6rGr7s r3r9z!DefaultLoadPlanner.set_up_plannersO $.  ' '1*=J  " "(::(F %J $  ,r5cBt|j|jSr1)r'r6rG)r2s r3r@z$DefaultLoadPlanner.create_local_plans-doot}}MMr5rFct|Sr1)r()r2rFs r3rKz%DefaultLoadPlanner.create_global_plans .{;;r5rLc|Sr1rjrNs r3rOzDefaultLoadPlanner.finish_plansr5 read_itemvaluec |jrLt|j|j|jj t j|yt j||j|jj <yr1) rr#rlr, dest_indexfqnraloadr6)r2rrrss r3 load_byteszDefaultLoadPlanner.load_bytesse  " " (( i22667 5!  9> 58IDOOI0044 5r5c\|j|j}|j||Sr1) lookup_tensorrutransform_tensorr2rrtensors r3resolve_tensorz!DefaultLoadPlanner.resolve_tensors+##I$8$89$$Y77r5r}cyr1rjr|s r3 commit_tensorz DefaultLoadPlanner.commit_tensors r5rSc.t|j|SrXrYrZs r3rzz DefaultLoadPlanner.lookup_tensorr[r5cDt||j|jSrX)r dest_offsetslengthsr|s r3r{z#DefaultLoadPlanner.transform_tensors"% I**I,=,=  r5)TT)rdrerf__doc__rrgrrhr4rr9rr@rrKrOrr_r`rxr~rarirrrzr{rjr5r3r&r&s)($((, "&  -#-- -  -$N8N=>U\\> ( ELL r5r&r6rGr/cg} |jD]`\}}|j|}t|tr,|jj @|t |||z }Q|t |||z }bt|Sr1)itemsstate_dict_metadata isinstancer device_meshget_coordinaterr)r6rGrequestsrvobjmds r3r'r'sH$$&S  ) )# . c7 #--/;.sB<< *3C8 8H' H r5rAc|S)z Create global load plan used by DefaultLoadPlanner. The default load behavior involved no global coordination and this function currently doesn't change the local plans. rj)rAs r3r(r(s  r5r7cg}|jD]l\}}t|tr+|jj 1|t ||z }At|t js|s^|t ||z }nt|S)a Create the ``SavePlan`` used by DefaultSavePlanner. On non-coordinator ranks, this function ignores tensors and non-tensor objects, only producing writes for ShardedTensor objects. On the coordinator rank, produce writes for all values. ) rrr rrrrarir)r6r7rrvrs r3r)r)sH$$&S c7 #--/;/S99 ell , +C5 5H' H r5rewrite_index_hintsc .i}g}|D]}g}|jD]}|jtjk(s|jj |vsJ|jtj k(r3t||jj <|j||jJtt|j|jj t|jj|jjg}|}|rKtj |jt#|j$} tj || }|j||jj&Jd|jj d|j$j|jj&|jtj |||t)|fS)a6 Create the global plan and metadata used by DefaultSavePlanner. Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans. The only global planning change is to update index hints in all ``MetadataIndex`` objects if ``rewrite_index_hints`` is True. ) propertiessizechunks)rSzZ Cannot create MD for tensor without bounds. FQN: z )r)rr]rSHARDrSrvr^rappend tensor_datar r setdefaultrrr=r>lenrchunkr) rArr new_plansr? new_itemsitem tensor_mdnew_item new_indexs r3r*r*&s$&BI JJD99 3 33zz~~R///yyM111%9%;4::>>"  &''333 )MM -'+'7'7'B'B!%!1!1!6!6#%   & + 3 3 #i.>.>*?!I +224yIH  *$$**6**..)*6   ''(8(8(>(>?EF ,,TCDKL x| $$r5c:t|}t|g\}}|S)zTReturn the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``.)rr*)r6r?_rs r3_create_default_local_metadatar]s! -j 9D +TF 3EAr Ir5box0box1ct|j}t|D]d}|j||j||j|zk\ry|j||j||j|zk\sdyy)z9Check if two boxes overlap. Tuples are (offset, lengths).FT)roffsetsrangesizes)rrndimsis r3_check_box_overlaprdsu  E 5\ <<?dll1o 1 = = <<?dll1o 1 = =  r5outer_box_size inner_boxctt|D]Q}|j|dkry|j|dkry|j||j|z||kDsQyy)NrFT)rrrr)rrrs r3_check_box_boundsrvsp3~& '   Q ! # ??1  !   Q )//!"4 4~a7H H ( r5rFc d}|jjD]5\}}t|trt |j dk(r1d}t |jD]\}}t|j |s$tjd||j |d}|ttj|jdz }|j|dzdD])}t||stjd|||d}+ttj|j d} || k7stjd|| |d}8|S)NTrz~ key:%s has out of bounds chunk: tensor-size:%s chunk: %s Fz$key:%s has overlapping chunks: %s %szq key:%s invalid fill tensor-volume: %s chunks-volume: %s )rrrrrr enumeraterrr$warningroperatormulrr) rFrGall_goodkeyrs chunks_volume chunk_idxchunk0chunk1 tensor_volumes r3rDrDs=H2288: U e1 2  uzz?a   !*5<VV %H 8"9*x||UZZ; M ) NN-   HG;J Or5)T)Fr=r_loggingr collectionsr functoolsrtypingrrrrr r ratorch.distributed._shard._utilsr torch.distributed._tensorr $torch.distributed.checkpoint.plannerr rrrrrr%torch.distributed.checkpoint.metadatarrrrrrr,torch.distributed.checkpoint.planner_helpersrrr)torch.distributed.checkpoint._nested_dictrr2torch.distributed.checkpoint._sharded_tensor_utilsr +torch.distributed.checkpoint._dedup_tensorsr!"torch.distributed.checkpoint.utilsr"&torch.distributed.checkpoint._traverser# getLoggerrdr$Loggerrg__all__r%r&strr'r(rhr)r*rrSizerrDrjr5r3rs  66 B- FE>***844 QQhK K \S#X8 H~  (^ S#X04 4!%4%H~4%4% 4>8 #$4%n8 &: $ JJ +?   )h)+3) )r5