PhO@ ddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddlm Z ddl m Z m Z mZmZmZmZddlZddlZddlmZddlmZddlmZddlmZdd lmZdd lmZmZdd lm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+dd l,m-Z-ddl.m/Z/ddgZ0e GddZ1e GddZ2dZ3de&de+fdZ4GddeZ5Gdde5Z6Gdde5Z7de&de8fdZ9d e8d!ee&deee&fd"Z:d#eeejvejxfd$eejzej.fd%e&d&e>fd'Z?d(ejd)ejd*e%d+e8d,ef d-ZAGd.de*ZBGd/de)ZCy)0N)ABCabstractmethod) dataclass)CallablecastDictListOptionalUnion)AbstractFileSystem) url_to_fs)Tensor)_get_device_module)narrow_tensor_by_index)Metadata MetadataIndex) LoadItemTypeLoadPlan LoadPlannerReadItemSavePlan SavePlanner WriteItem WriteItemType) StorageReader StorageWriter WriteResult)_create_file_view)Future FsspecWriter FsspecReaderc0eZdZUdZeed<eed<eed<y) _StorageInfoz#This is the per entry storage info. relative_pathoffsetlengthN)__name__ __module__ __qualname____doc__str__annotations__intzC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/checkpoint/_fsspec_filesystem.pyr#r#2s- K Kr/r#ceZdZUeed<y)_StoragePrefixprefixN)r'r(r)r+r,r.r/r0r2r2;s Kr/r2z.distcpitemreturnc2t|j||S)N)index size_in_bytes storage_data)rr7)r4r8r9s r0_result_from_write_itemr:Cs jj L r/cFeZdZededefdZedZedZy) _TensorLoadersizeobjcyNr.selfr=r>s r0addz_TensorLoader.addL r/cyr@r.rBs r0 start_loadingz_TensorLoader.start_loadingPrDr/cyr@r.rFs r0valuesz_TensorLoader.valuesTrDr/N) r'r(r)rr-objectrCrGrIr.r/r0r<r<KsJ  &      r/r<c4eZdZdefdZdedefdZdZdZ y) _SerialCpuLoader resolve_func ||_g|_yr@)rMitems)rBrMs r0__init__z_SerialCpuLoader.__init__Zs& r/r=r>c>|jj||fyr@)rOappendrAs r0rCz_SerialCpuLoader.add^s 4+&r/cyr@r.rFs r0rGz_SerialCpuLoader.start_loadinga r/c#K|jD]y\}}|j|j}|j}|j j |j k7r|j}||f{ywr@)rOrMdetachcpustorager=numelclonerB_r>tensors r0rIz_SerialCpuLoader.valuesdsujjFAs%%c*113FZZ\F~~$$&&,,.8  !sB B N) r'r(r)rrPr-rJrCrGrIr.r/r0rLrLYs+H''&'  r/rLceZdZ ddededej ejfde fdZ e dZ dZ dZd Zd e d efd Zd ZdZy)_OverlappingCpuLoaderNrMstreaminflight_threshholdc||_g|_||_d|_t j |_d|_d|_|r |jntjdj|_ t|j|_|xs|jj|_|j |jjk7r4|j j#|jjyy)NrFcuda)rMrOrain_flight_data collectionsdeque current_itemsidxstarted device_typetorchdevicetyper device_modulecurrent_streamr` wait_stream)rBrMr`ras r0rPz_OverlappingCpuLoader.__init__qs ' #6 0;0A0A0C 176--U\\&=Q=V=V/0@0@AC 2 2 A A C ;;$,,;;= = KK # #D$6$6$E$E$G H >r/cF|jt|jk\Sr@)rhlenrOrFs r0_donez_OverlappingCpuLoader._donesxx3tzz?**r/cg}|j|jk\r|jj|j|jk\r|jj }|xj|dj |djzzc_|j||j|jk\r|SNr) rdrar` synchronizergpopleftrY element_sizerR)rBdrainedvals r0_drainz_OverlappingCpuLoader._drains   $":": : KK # # %!!T%=%==$$,,.C   3q6<<>CF4G4G4I#I I  NN3 !!T%=%==r/c|jj|j5|jsz|j|jkr`|j |j \}}|xj dz c_|j|j}|jj|jk(r|jdd}na|jtjdk(r?|jj|j!k7r|j#}|j$j'||f|xj|j!|j)zz c_|js|j|jkr`dddy#1swYyxYw)NrWT)rl non_blocking)rnr`rsrdrarOrhrMrVrlrmrjtorkrXr=rYrZrgrRrxr[s r0_refillz_OverlappingCpuLoader._refillsH    & &t{{ 3JJ''$*B*BBDHH-3A ))#.557==%%)9)99#YYe$YGF]]ell5&99~~',,.&,,.@!'"")) ##v||~8K8K8M'MM#'JJ''$*B*BB4 3 3s FF77Gc|jsJt|jdkDr|jj |jSru)rsrrrgr`rvrFs r0_finishz_OverlappingCpuLoader._finishs=zzz t!! "Q & KK # # %!!!r/r=r>cl|jr td|jj||fy)Nz&cannot add items after loading started)ri RuntimeErrorrOrRrAs r0rCz_OverlappingCpuLoader.adds+ <<GH H 4+&r/c|jryd|_|jjd|jy)NTc |dSrur.xs r0z5_OverlappingCpuLoader.start_loading..sadr/key)rirOsortrrFs r0rGz#_OverlappingCpuLoader.start_loadings. <<   N+ r/c#K|j|js7|j}|j|Ed{|js7|j Ed{y7*7wr@)rGrsr{rr)rBrys r0rIz_OverlappingCpuLoader.valuessY **kkmG LLN  ** <<>!! !s*AA3A/A3A3)A1*A31A3)Ni@B)r'r(r)rr io RawIOBaserkStreamr-rPpropertyrsr{rrrJrCrGrIr.r/r0r_r_ps;?#, IIdBLL%,,67I! I&++N0" ''&' "r/r_cd}|jJ|jjD]}||z} |jjj}|tj j |zSNr}) tensor_datar= propertiesdtyperk_utils _element_size)r4r=srs r0 _item_sizersk D    '' '    " "  #    ' ' - -E %,,,,U3 33r/binsrOcj|dk(r|gS|Dcgc]"}|jtjk(s!|$}}|Dcgc]"}|jtjk7s!|$}}t|Dcgc]}g}}t|Dcgc]}d}}|j t dt |D]\}}|||zj||D]F}tt |dd} || j||| xxt |z cc<H|Scc}wcc}wcc}wcc}w)Nr}rT)rreversec |dSrr.rs r0rz)_split_by_size_and_type..s1r/r) rmrBYTE_IOrangerr enumeraterRmin) rrOwibytes_wtensor_wr\buckets bucket_sizesirhs r0_split_by_size_and_typers$ qyw!FEbRWW 0E0E%ErEGF"GUrbgg1F1F&FUHG27+%>+Qb+G%>$T{+{!A{L+ MMj$M/7#2D  $$)L)~>qA BSZ^+  N#GG%>+s!"D!D!"D&D&0 D+ D0r`data write_item storage_keyc |j}|jtjk(r'  ")(rBGG}7L7L,L[ M[[D1V&- &33J?%,,'j+N'. /5mmo* %}},}%,,' KP/>2    ] +O 21  ;;   sgA)G+"F5F58G "F:-F:1GG A9F?G %G?G G  GGG-,G-c eZdZdZ ddeeejfdede de ddf fd Z d eddfd Z d e de fd Z d ee dee fdZd e dedeeefdZdedeeeddfdZxZS)r a` Basic implementation of StorageWriter using FFspec. This implementation makes the following assumptions and simplifications: * The checkpoint path is an empty or non-existing directory. * File creation is atomic The checkpoint consist of one file per write request plus a `.metadata` file with the serialized metadata. pathsingle_file_per_rank thread_countper_thread_copy_aheadr5Nct|||_t|\|_}||_||_||_y)a Initialize the writer pointing to `path`. Args: path: diretory where the checkpoint will be writen to. single_file_per_rank: Produce one file per rank instead of one file per tensor/blob. Default to True. thread_count: Number of IO threads to use to write. Default to 1. per_thread_copy_ahead: How many bytes to copy from the GPU ahead of saving then. Default 10Mb. N)superrPrr rrrr)rBrrrrr\ __class__s r0rPzFsspecWriter.__init__Cs>"  t_ $8!(%:"r/is_coordinatorcyr@r.)rBrs r0set_up_storage_writerz"FsspecWriter.set_up_storage_writer[rTr/plancT|jj|jd|S)NT)exist_ok)rmakedirsrrBrs r0prepare_local_planzFsspecWriter.prepare_local_plan^s" T2 r/ global_planc t|Dcgc])\}}tj|td|d+}}}|Scc}}w)N__r\r9)r dataclassesreplacer2)rBrrr new_planss r0prepare_global_planz FsspecWriter.prepare_global_planbsT %[1 14   >Bqc)3L M1    s.Arc |jdfd}tj}|jrkt |j |j D]G}|}tjj|j|}|j|||fInW|j D]H}|}tjj|j|}|j|||gfJtj} g} td|j D]W} tjt|| ||j|j f} | j#| j%| Yt|| ||j|j | D]} | jg} | | j'z } #tj($r! t+}|j-| |cYSwxYw)Nrc>jt}dz |Sr)r3DEFAULT_SUFFIX)r file_count storage_plans r0gen_filez)FsspecWriter.write_data..gen_filess,'../ |N;KLI !OJ r/r})targetargs)rrrrar)r9rQueuerrrrOosrjoinrr threadingThreadrrrstartrRrrr set_result)rBrrrrbucketr file_pathr4rthreadsr\tresfutrrs @@r0 write_datazFsspecWriter.write_dataks (,'8'8    #(++-  $ $1!!4::%J GGLLI>  9f=>  $J GGLLI>  9tf=># %*KKM q$++,A  . ..GG  A GGI NN1 - !% $ : :ww  A FFH |..00{{  -3XC NN3 J  s=G1HHmetadataresultsct}|D]6}|j|Dcic]}|j|jc}8||_tj j |j d}|jj5tj|d5}tj||ddddddycc}w#1swYxYw#1swYyxYw)N .metadatar) dictupdater7r9rrrrrrrpickledump)rBrr storage_mdwr_listwr metadata_path metadata_files r0finishzFsspecWriter.finishsV G   7K7Rrxx87K L * TYY < WW ]D1] Hm42! L 21! s)C C$C;CC CC&)Tr}i)r'r(r)r*r r+rPathLikeboolr-rPrrrr rrrrrrr __classcell__rs@r0r r 5s  &*%/ ;C$%;#; ; # ;  ;0 D T xH> hAAA [! " AF 5  5+/[0A+B 5  5r/ceZdZdeeej fddffd ZdefdZ de de de dfd Z defd Zd ed eddfd Zde de fdZdee dee fdZxZS)r!rr5Nctt|||_t|\|_}t |_yr@)rrPrr rrr9)rBrr\rs r0rPzFsspecReader.__init__s.  t_ ?Cvr/sinfocDt||j|jSr@)rr%r&)rBfiler s r0 _slice_filezFsspecReader._slice_files u||U\\BBr/rrc t}|jD]H}|j|j}|j}|j |gj |J|jD]\}}tjj|j|} tj| d5} |D]|} |j| j}|j| |} | jtjk(rRt!j"| j%|j&} | j)d|j+| | t-t.t1j2| d}t5|| j6| j8}|j;| j=}|j?|j?k(s6Jd| jd|j?d|j?|jA||jC| | dddtE}|jGd|S#1swYxYw)NrbrrW) map_locationzreq z mismatch sizes z vs )$rrOr9 storage_indexr$ setdefaultrRrrrrrrrmrrrrreadr&seek load_bytesrrrkloadrstorage_offsetslengthsresolve_tensorrVr=copy_ commit_tensorrr)rBrrper_file read_itemitem_mdrr$reqsabs_pathrreq file_slicebytesr] target_tensorrs r0 read_datazFsspecReader.read_datas.2fI'' (?(?@G((D   b ) 0 0 ;$ $,>>#3 M4ww||DII}=HXt,C"//0A0ABG!%!1!1$!@Jxx<#7#77 " :??7>>+J K 1 **36!%"EJJz$N""8"C$7$7")0(>(>s(C(J(J(L *..0FKKMAo!#"3"3!44D]EWEWEYDZZ^_e_j_j_l^mnoA%++F3--c=A) -,$44h t 5-,s 9FI%%I/ ctjj|jd}tj|d5}t j |cdddS#1swYyxYw)Nrr)rrrrrrr)rBrrs r0 read_metadatazFsspecReader.read_metadatasB TYY < [[ -;;}-. - -s A  A)rrcB|j|_|jJyr@r)rBrrs r0set_up_storage_readerz"FsspecReader.set_up_storage_readers$%11  ,,,r/c|Sr@r.rs r0rzFsspecReader.prepare_local_plans r/rc|Sr@r.)rBrs r0rz FsspecReader.prepare_global_plans r/)r'r(r)r r+rrrPr#rrrrr(rr*rr,rr rr r s@r0r!r!sFU3 #34FF C|C$h$$$N.x. - -26- - xH> hr/)Drerrrrrrabcrrrtypingrrrr r r rrkr fsspec.corer r torch._utilsrtorch.distributed._shard._utilsr%torch.distributed.checkpoint.metadatarr$torch.distributed.checkpoint.plannerrrrrrrrr$torch.distributed.checkpoint.storagerrr"torch.distributed.checkpoint.utilsr torch.futuresr__all__r#r2rr:r<rLr_r-rrrrrr+rrrr r!r.r/r0r:s #!>> %!+BI    A          C  }.S"MS"l4Y434 9o $y/4 U2<<56 7  ELL( ) ,1  1 ++1 1  1  1 hD5=D5NB=Br/