Ph UdZddlZddlZddlmZmZmZmZmZddl Z ddl m Z ddl m Z ddl mZgZeeed<ee j$eefZee Zee efZde j.d e j$d ee j0ddffd Zd ed dfd Zde j.deeee fded e j$d eef dZde j.deeee fdeded e j$d eef dZy)zPer-layer profilers.N)Any GeneratorListUnionSequence)Tensor)Batch__all__moduledevicereturnc#K|D]<}tj|}|j||j|>yw)zOCopies layers for ease to profile. It doesn't modify the given module. N)copydeepcopytotrain)r r layer layer_copys {C:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/distributed/pipeline/sync/_balance/profile.pylayerwise_sandboxrs>]]5)  f sAAbatchct|D]1\}}|jj|j||<3y)zDetaches from autograd graph.N) enumeratedetachrequires_grad_ requires_grad)rixs rrr&s3% 188:,,Q__=a!sampletimeoutc0td|jDr tdt|}t |D]@\}}|j j |j|j||<B|Dcgc]}g}}tj} tj| z |kr&|} t t||D]\}} t | |jdk(rtjj|tj} | j| } t!d| D} | r tj"j%| | |jdk(rtjj|tj}||j'|| z tj| z |kr&d|Dcgc]}t)fd|Dc}Scc}wcc}w)z!Profiles elapsed times per layer.c38K|]}|jduywN)grad.0ps r z profile_times...s ;':!166 ':sz#some parameter already has gradientcudac3:K|]}|js|ywr%)r)r(ys rr*z profile_times..Fs$I1Qsi@Bc3:K|]}t|zywr%)int)r(tuss rr*z profile_times..Qs)SAF Ss)any parameters ValueErrorr rrrrrtimertypetorchr+ synchronizecalltupleautogradbackwardappendsum)r r!r"r _batchrr_ time_bufsbegun_atrrtickbackward_tensorstockbufr1s @r profile_timesrG,s ;v'8'8': ;;>?? 6]F&!1HHJMM&)88Iq "17#71BI#7yy{H ))+ 7 *!"3FF"CDHAu 5M{{f$ &&v.99;DJJu%E %$I$II ''(8:JK{{f$ &&v.99;D aL  t ,'E ))+ 7 *0 B5> ?YcC)S) )Y ??9$88 @s H0Hinputchunks param_scalec|jdk7r tdt|}g}|djd|z }t |D]C\}} | ddj j |j| j||<Et||D]} t |tjjtjj|} |j| }tjjtjj|} | | z } t!d| j#D}| |z||zz}|j%t'||S)z%Profiles CUDA memory usage per layer.r+z'size profiler supports only CUDA devicerNc3XK|]"}|jj$ywr%)_typed_storage_nbytesr's rr*z profile_sizes..ns%R?Q!))+335?Qs(*)r6r4r sizerrrrrrr7_C_cuda_clearCublasWorkspacesr+memory_allocatedr9r>r3r=r/)r rHrIrJr rsizes latent_scalerrr memory_before memory_after latent_size param_sizerPs r profile_sizesrZTsG{{fBCC %LEE8==#f,L% 1Ra5<<>$$V,;;AOOLa!#662u  ,,. 33F;  5! ,,.zz226: "]2 Ru?O?O?QRR \)J,DD SY!3$ Lr )__doc__rr5typingrrrrrr7rtorch.nnnn microbatchr r str__annotations__r r/DeviceTensorsTensorOrTensors SequentialModulerrfloatrGrZr rrisJ 88 c u||S#% & 6 (bmmU\\yQSQZQZ\`bfQfGg>%>D> %@"--%@tCy&7H1I%@TY%@chcoco%@uyz}u~%@P  MM "'S 6(9": DG V[ ejeqeq  #Y r