Phb5!ddlZddlmZddlmZmZmZmZmZmZm Z m Z m Z ddl m Z ddlmZmZddgZd!d ZGd deZd ed e d ed e d e_ d"deedeedeedeedeedeedeedededededededededef dZdeedeedeedeedeedeedededededededededefdZdeedeedeedeedeedeedededededededededefd Zy)#N)Tensor) Optimizer_use_grad_for_differentiable _get_value_default_to_fused_or_foreach_differentiable_doc _foreach_doc _maximize_doc_capturable_doc _view_as_real) is_compiling)ListOptionalASGDasgdcht|tjstj||S|S)Ndevice) isinstancetorchrtensor)xrs [C:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/optim/asgd.py _to_tensorr s& a &||Af-- Hc leZdZ d deedededeffd ZfdZdZed dZ xZ S) rforeachmaximizedifferentiable capturablec d|kstd|d|kstd||dur | r tdt|||||||| |  } t | || y)NgzInvalid learning rate: zInvalid weight_decay value: F0Capturable not supported with single tensor ASGD) lrlambdalphat0 weight_decayrrr r!) ValueErrordictsuper__init__) selfparamsr$r%r&r'r(rrr r!defaults __class__s rr,z ASGD.__init__sby6rd;< <l";L>JK K e  OP P%)!   *rc|t|||jD]J}|jdd|jdd|jdd|jddLt |j j }t|dk7xrtj|dd}|s;|D]6}tjt|dtj|d<8t|dk7xrtj|dd }|s2|D]-}tj|d tj|d </t|dk7xrtj|dd }|s<|D]6}tjt|d tj|d <8yy) NrrFr r!rstep)dtypeetamu) r+ __setstate__ param_groups setdefaultliststatevalueslenr is_tensorrfloatfloat32) r-r:group state_valuesstep_is_tensors eta_is_tensor mu_is_tensorr0s rr6zASGD.__setstate__4s U#&&E   Y -   Z /   -u 5   \5 1 ' DJJ--/0 l+q0 eoo OF #7 !!LLqy)9O& "\*a/ U__ OE "6 ! <<% F%"L)Q. EOO OD !5 !,,uQtW~U]]K$"rchd}|dD]} | j|tj| z}|j| | jjr t d|j| j|j | } t| dk(rtjd| jtj| d<tj|d| jtj| d <tjd| jtj| d <tj| tj | d <|j| d |j| d |j| d |j| d|S) NFr.z&ASGD does not support sparse gradientsr)rr3r2r$r4r5) memory_formatax)gradr is_complexappend is_sparse RuntimeErrorr:r<zerosrr?rones zeros_likepreserve_format) r-r@params_with_gradgradsmusaxsetas state_steps has_complexpr:s r _init_groupzASGD._init_groupOsO xAvv!u//22  ''*66##&'OPP QVV$ 1 u:?$)KK1885==$YE&M#(<<d AHHTYTaTa#bE%L"'**R "VE$K"'"2"2)>)>#E$K 5;' 5;' E%L)""5=1+!,rcBd}|$tj5|}ddd|jD][}g}g}g}g}g}g} |j||||||| } t |||||| |d|d|d|d|d|d|d|d |d | ]|S#1swYuxYw) zPerform a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nr%r$r'r&r(rrr r!) r%r$r'r&r(rrr r!rY)r enable_gradr7r[r) r-closurelossr@rSrTrUrVrWrXrYs rr2z ASGD.stepis  ""$y%&&E! ECCDK**52BE3PSUY[fgK  Gn;;Gn">2i(z*$%56 .'! ': A%$s BB) g{Gz?g-C6?g?g.ArNFFFN) __name__ __module__ __qualname__rboolr,r6r[rr2 __classcell__)r0s@rrrsq  "&$  +$ + + + +DL64")")rahImplements Averaged Stochastic Gradient Descent. It has been proposed in `Acceleration of stochastic approximation by averaging`_. Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-2) lambd (float, optional): decay term (default: 1e-4) alpha (float, optional): power for eta update (default: 0.75) t0 (float, optional): point at which to start averaging (default: 1e6) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) z z For ASGD, capturable is only supported when foreach is True. .. _Acceleration of stochastic approximation by averaging: https://dl.acm.org/citation.cfm?id=131098 r.rTrVrUrWrXrrr r!rYr%r$r'r&r(c 6|t||d\}}|r)tjjr t d|r%tjjst }n| rt s t dt}|||||||| | | ||||| | y)znFunctional API that performs asgd algorithm computation. See :class:`~torch.optim.ASGD` for details. NF) use_fusedz6torch.jit.script not supported with foreach optimizersr#) r%r$r'r&r(rr r!rY)rrjit is_scriptingrN_multi_tensor_asgdr_single_tensor_asgd)r.rTrVrUrWrXrrr r!rYr%r$r'r&r(_funcs rrrs21&.TYZ 7599))+STTuyy--/! lnQR R"     !%rc <t|D]\}}||}| s|n| }||}||}||}||}tj|r?tj|}tj|}tj|}|dz }t |}| dk7r|j || }t |}|j d||zz |j|| ts|jdk7r0|j|j|j|n|j|t|d||z|zz| zz }|j|tdtd||z z }|j|y)Nrrr&) enumeraterrK view_as_realraddmul_add_ritemsubmulcopy_rmax)r.rTrVrUrWrXr%r$r'r&r(rr r!rYiparamrJr5rIr4step_tr2 eta_valuenew_etanew_mus rrkrks$f%5Qx#t$ V V1gQ   E "%%d+D&&u-E##B'B ! &! 1 88E86DsO  1uy(()  4 z * >RWWY!^ GGEIIbM%%b) * HHUORA T(9$9e#CDE 'AAtby 112 I&rc ~t|dk(ry| rJdtj||||||g}|jD]\\}}\\}}}}}}}| rt j |}t |}|r t||||djr.t j|t jdddnt j|d| dk7rN| rt j||| |}nt j||| }t j|||nt j|||}t j|||d ~t j||}t j|||~| rt j||}t j|dt j|t j ||~t j"|| }t j$||t j$||t j|dt j|t j$||t j ||@|dj'}g}g}t)t|D]]}t+|d||z|| zzzz |}|j-|t+dt/d||z z |}|j-|_t j ||t j ||y) Nrz#_foreach ops don't support autogradg?cpurror)value)r<r"_group_tensors_by_device_and_dtypeitemsr _foreach_negr9r is_cpu _foreach_add_r _foreach_add_foreach_addcmul_ _foreach_sub_foreach_maximum__foreach_reciprocal__foreach_copy_ _foreach_pow _foreach_mul_rurangerrLry)r.rTrVrUrWrXr%r$r'r&r(rr r!rYgrouped_tensorsrrlgrouped_params grouped_grads grouped_axs grouped_mus grouped_etasgrouped_state_steps intermediatenew_musnew_etasr2rzr~rs rrjrj s& 6{aDDD BBFESVX[]acnCopO4C4I4I4K 1&!0, {K *Q !..}=M]+  .- E q ! ( (    3U\\#e5T\_ `    3Q 7 1 ##M>V, $11-Wcd    nE J --m^SXYL  lRTU )).+F   \;G  (()rrkrjrGrrrs jjj%! 6  B9BJ      ! B# 6 L6 <6 f6 f 6 v, 6 f 6d^66666 6 !6" #6$ %6&'6r6 L6 <6 f6 f 6 v, 6 f 6 6 6 6 66666 !6rh7 Lh7 <h7 fh7 f h7 v, h7 f h7 h7 h7 h7 h7h7h7h7h7 !h7r