
    FPh                     t   d Z ddlZddlZddlZddlZddlZddlmZ ddlmZm	Z	 ddl
mZ ddlZddlZddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?  G d d      Z@y)z|
Train a model on a dataset.

Usage:
    $ yolo mode=train model=yolov8n.pt data=coco128.yaml imgsz=640 epochs=100 batch=16
    N)deepcopy)datetime	timedelta)Path)distributed)nnoptim)amp)DistributedDataParallel)get_cfgget_save_dir)check_cls_datasetcheck_det_dataset)attempt_load_one_weightattempt_load_weights)
DEFAULT_CFGLOGGERRANKTQDM__version__	callbacks	clean_urlcolorstremojis	yaml_save)check_train_batch_size)	check_amp
check_filecheck_imgsz
print_args)ddp_cleanupgenerate_ddp_command)get_latest_run)EarlyStoppingModelEMAde_parallel
init_seeds	one_cycleselect_devicestrip_optimizerc                       e Zd ZdZeddfdZdefdZdefdZdefdZ	d Z
d	 Zd
 Zd#dZd Zed        Zd Zd Zd Zd Zd$dZd Zd%dZd&dZd'dZd Zd Zd Zd Zd Zd Zd Z d(dZ!d Z"d  Z#d! Z$d)d"Z%y)*BaseTrainera  
    BaseTrainer.

    A base class for creating trainers.

    Attributes:
        args (SimpleNamespace): Configuration for the trainer.
        check_resume (method): Method to check if training should be resumed from a saved checkpoint.
        validator (BaseValidator): Validator instance.
        model (nn.Module): Model instance.
        callbacks (defaultdict): Dictionary of callbacks.
        save_dir (Path): Directory to save results.
        wdir (Path): Directory to save weights.
        last (Path): Path to the last checkpoint.
        best (Path): Path to the best checkpoint.
        save_period (int): Save checkpoint every x epochs (disabled if < 1).
        batch_size (int): Batch size for training.
        epochs (int): Number of epochs to train for.
        start_epoch (int): Starting epoch for training.
        device (torch.device): Device to use for training.
        amp (bool): Flag to enable AMP (Automatic Mixed Precision).
        scaler (amp.GradScaler): Gradient scaler for AMP.
        data (str): Path to data.
        trainset (torch.utils.data.Dataset): Training dataset.
        testset (torch.utils.data.Dataset): Testing dataset.
        ema (nn.Module): EMA (Exponential Moving Average) of the model.
        lf (nn.Module): Loss function.
        scheduler (torch.optim.lr_scheduler._LRScheduler): Learning rate scheduler.
        best_fitness (float): The best fitness value achieved.
        fitness (float): Current fitness value.
        loss (float): Current loss value.
        tloss (float): Total loss value.
        loss_names (list): List of loss names.
        csv (Path): Path to results CSV file.
    Nc           	      ~   t        ||      | _        | j                  |       t        | j                  j                  | j                  j
                        | _        d| _        d| _        d| _        i | _	        t        | j                  j                  dz   t        z   | j                  j                         t        | j                        | _        | j                  j                   | j                  _        | j                  dz  | _        t        dv rm| j"                  j%                  dd       t'        | j                        | j                  _        t)        | j                  dz  t+        | j                               | j"                  d	z  | j"                  d
z  c| _        | _        | j                  j0                  | _        | j                  j
                  | _        | j                  j4                  | _        d| _        t        dk(  rt9        t+        | j                               | j                  j:                  dv rd| j                  _        | j                  j                  | _        	 | j                  j>                  dk(  r%tA        | j                  jB                        | _!        n| j                  jB                  jE                  d      d   dv s| j                  j>                  dv rPtG        | j                  jB                        | _!        d| jB                  v r| jB                  d   | j                  _!        | jQ                  | jB                        \  | _)        | _*        d| _+        d| _,        d| _-        d| _.        d| _/        d| _0        d| _1        dg| _2        | j                  dz  | _3        g d| _4        |xs tk        jl                         | _5        t        dv rtk        jn                  |        yy# tH        $ r=}tK        tM        dtO        | j                  jB                         d|             |d}~ww xY w)z
        Initializes the BaseTrainer class.

        Args:
            cfg (str, optional): Path to a configuration file. Defaults to DEFAULT_CFG.
            overrides (dict, optional): Configuration overrides. Defaults to None.
        N   )deterministicweightsr   T)parentsexist_okz	args.yamlzlast.ptzbest.ptr   r2   )cpumpsclassify.)yamlyml)detectsegmentpose	yaml_filez	Dataset 'u   ' error ❌ Losszresults.csv)r   r.      )8r   argscheck_resumer)   devicebatch	validatormodelmetricsplotsr'   seedr   r/   r   save_dirnamewdirmkdirstrr   varslastbestsave_period
batch_sizeepochsstart_epochr    typeworkerstaskr   datasplitr   	ExceptionRuntimeErrorr   r   get_datasettrainsettestsetemalf	schedulerbest_fitnessfitnesslosstloss
loss_namescsvplot_idxr   get_default_callbacksadd_integration_callbacks)selfcfg	overrides
_callbackses        eC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\ultralytics/engine/trainer.py__init__zBaseTrainer.__init__K   s     C+	)$#DII$4$4diiooF

499>>A%,DII<S<ST %TYY/++		MMI-	7?IIOOD4O8!$T]]!3DIIdmmk14		?C#yy94dii)6K	499900))//ii&&2:tDII' ;;~- !DII YY__
	fyy~~+-diinn=	%%c*2./ATYY^^WtEt-diinn=	$))+%)YY{%;DIIN '+&6&6tyy&A#t|  !	
!(===0! $Hy'F'F'H7?//5 +  	fv	)DIINN2K1LLYZX[&\]^dee	fs   ,CO6 6	P<?8P77P<eventc                 @    | j                   |   j                  |       y)zAppends the given callback.N)r   appendrl   rs   callbacks      rq   add_callbackzBaseTrainer.add_callback   s    u$$X.    c                 $    |g| j                   |<   y)z9Overrides the existing callbacks with the given callback.N)r   rv   s      rq   set_callbackzBaseTrainer.set_callback   s    !)
ury   c                 V    | j                   j                  |g       D ]
  } ||         y)z>Run all existing callbacks associated with a particular event.N)r   getrv   s      rq   run_callbackszBaseTrainer.run_callbacks   s$    **5"5HTN 6ry   c                 B   t        | j                  j                  t              rNt	        | j                  j                        r/t	        | j                  j                  j                  d            }nmt        | j                  j                  t        t        f      r t	        | j                  j                        }n#t        j                  j                         rd}nd}|dkD  rdt        j                  vr| j                  j                  r&t        j                  d       d| j                  _        | j                  j                   dk(  r&t        j                  d       d	| j                  _        t#        ||       \  }}	 t        j$                  t'        d
       ddj)                  |              t+        j,                  |d       	 t1        | t        |             y| j3                  |       y# t.        $ r}|d}~ww xY w# t1        | t        |             w xY w)zIAllow device='', device=None on Multi-GPU systems to default to device=0.,r.   r   
LOCAL_RANKuX   WARNING ⚠️ 'rect=True' is incompatible with Multi-GPU training, setting 'rect=False'Fr2   uk   WARNING ⚠️ 'batch=-1' for AutoBatch is incompatible with Multi-GPU training, setting default 'batch=16'   zDDP:z debug command  T)checkN)
isinstancerA   rC   rN   lenrZ   tuplelisttorchcudais_availableosenvironrectr   warningrD   r"   infor   join
subprocessrunr[   r!   	_do_train)rl   
world_sizecmdfilerp   s        rq   trainzBaseTrainer.train   s   dii&&,TYY5E5E1FTYY--33C89J		((5$-8TYY--.JZZ$$&JJ >l"**<yy~~yz!&		yy"$  4 5"$		 -Z>IC-x/0OPs$/ D#d), NN:&   D#d),s%   A	G4 4	H=G??HH Hc                 (   t         j                  j                  t               t        j                  dt              | _        dt
        j                  d<   t        j                  t        j                         rdndt        d      t        |       y	)
zIInitializes and sets the DistributedDataParallel parameters for training.r   1NCCL_BLOCKING_WAITncclglooi0*  )seconds)timeoutrankr   N)r   r   
set_devicer   rC   r   r   distinit_process_groupis_nccl_availabler   )rl   r   s     rq   
_setup_ddpzBaseTrainer._setup_ddp   sa    

d#ll640+.

'(,,.FFe,!		#ry   c           	      6     j                  d        j                         } j                  j                   j                         _         j                          t         j                  j                  t              r j                  j                  nDt         j                  j                  t              rt         j                  j                        ng }dg}|D cg c]  }d| d
 c}|z   } j                  j                         D ]g  \  }t        fd|D              r!t        j                  d d       d|_        ;|j                   rHt        j                  d	 d
       d|_        i t#        j$                   j                  j&                        j                   j                         _         j&                  rjt(        dv rbt*        j,                  j/                         }t#        j$                  t1         j                         j                         _        |t*        _        t(        dkD  r&|dkD  r!t3        j4                   j&                  d       t7         j&                         _        t'        j8                   j&                         _        |dkD  r!t=         j                  t(        g       _        t?        t        tA         j                  d      r$ j                  jB                  j?                         nd      d      }	tE         j                  jF                  |	|	d       j                  _#         jH                  dk(  rTt(        dk(  rKtK         j                   j                  jF                   j&                        x j                  _&         _$         jH                  t?        |d      z  }
 jO                   jP                  |
t(        d       _)        t(        dv r։ jO                   jT                  |
dz  dd       _+         jY                          _-         jZ                  j\                  j^                   ja                  d      z   }tc        te        |dgtg        |      z               _.        ti         j                         _5         j                  jl                  r jo                          t?        tq         j                  jr                   jH                  z        d       _:         j                  jv                   jH                  z   jt                  z   j                  jr                  z  }ty        jz                  tg         jR                  j|                        t?         jH                   j                  jr                        z         j~                  z  } j                   j                   j                  j                   j                  j                   j                  j                  ||       _A         j                  j                  r1t        d j                  j                   j~                         _G        n
 fd _G        t        j                  j                   j                   j                         _K        t         j                  j                        dc _N         _O         j                  |        j                  dz
   j                  _R         j                  d        y!c c}w )"z9Builds dataloaders and optimizer on correct rank process.on_pretrain_routine_startz.dflzmodel.r8   c              3   &   K   | ]  }|v  
 y wN ).0xks     rq   	<genexpr>z+BaseTrainer._setup_train.<locals>.<genexpr>   s     6#5a16#5s   zFreezing layer ''Fu>   WARNING ⚠️ setting 'requires_grad=True' for frozen layer 'zE'. See ultralytics.engine.trainer for customization of frozen layers.Tr1   )rC   r2   r.   r   )src)enabled)
device_idsstride    )r   floormax_dimr   )rS   r   moder@   val)prefix)rF   rK   lrmomentumdecay
iterationsc                     d| j                   z  z
  dj                  j                  z
  z  j                  j                  z   S )Nr.   g      ?)rT   rA   lrf)r   rl   s    rq   <lambda>z*BaseTrainer._setup_train.<locals>.<lambda>  s6    Q_!4tyy}}9L MPTPYPYP]P] ]ry   )	lr_lambda)patienceon_pretrain_routine_endN)Sr~   setup_modelrF   torC   set_model_attributesr   rA   freezer   intrangenamed_parametersanyr   r   requires_gradr   tensorr
   r   r   default_callbackscopyr   r   	broadcastbool
GradScalerscalerDDPmaxhasattrr   r   imgszrS   r   rD   get_dataloaderr^   train_loaderr_   test_loaderget_validatorrE   rG   keyslabel_loss_itemsdictzipr   r%   r`   rH   plot_training_labelsroundnbs
accumulateweight_decaymathceildatasetrT   build_optimizer	optimizerlr0r   cos_lrr(   r   ra   r	   lr_schedulerLambdaLRrb   r$   r   stopperstopresume_trainingrU   
last_epoch)rl   r   ckptfreeze_listalways_freeze_namesr   freeze_layer_namesvcallbacks_backupgsrS   metric_keysr   r   r   s   `             @rq   _setup_trainzBaseTrainer._setup_train   s)    	67!ZZ]]4;;/
!!# +5IId+$dii&&DNtyyO_O_adDetyy/?/?)@km 	%h5@A[qcm[ADWWJJ//1DAq6#566.qc34"'__\]^\_ `a a b"& 2 <<		.11$++>88(::??A||Idjj$9$++NDH*:I'"9aNN488+>nnTXX6>TZZTF;DJ 

H0MTZZ&&**,SUVXZ[%diioobTUV		 ??b TRZ0FtzzSWS\S\SbSbdhdldl0mmDIIOdo __J(::
 //*[_fm/n7?#224<<JYZNacjo2pD!//1DN..00558M8MUZ8M8[[KK!s;7G1G HIDL

+DHyy))+ eDIIMMDOO$CDaHyy--?$//QTXT]T]TaTaaYYs4#4#4#<#<=DOOUYU^U^UbUb@ccdgkgrgrr
--DJJ37993F3F157;yy7I7I4@9C . E 99499==$++>DG]DG++44T^^tww4W"/9K9K"LediT"$($4$4q$8!45 Bs   $\c                    |dkD  r| j                  |       | j                  |       d| _        t        j                         | _        t        j                         | _        t        | j                        }| j                  j                  dkD  r,t        t        | j                  j                  |z        d      nd}d}| j                  d       t        j                  d| j                  j                   d| j                  j                   d	| j                  j                   |xs dz   d
t#        d| j$                         d| j&                   d       | j                  j(                  rJ| j&                  | j                  j(                  z
  |z  }| j*                  j-                  ||dz   |dz   g       | j&                  }t/        | j0                  | j&                        D ]%  }|| _        | j                  d       | j4                  j7                          t8        dk7  r%| j                  j:                  j=                  |       t?        | j                        }|| j&                  | j                  j(                  z
  k(  rt        j                  d       tA        | j                  jB                  d      rd| j                  jB                  _"        tA        | j                  jB                  d      r0| j                  jB                  j)                  | j                         | j                  jG                          t8        dv rCt        j                  | jI                                tK        t?        | j                        |      }d| _&        | jN                  jQ                          |D ]  \  }}	| j                  d       |||z  z   }
|
|k  rd|g}t        dtS        jT                  |
|d| j                  jV                  | jX                  z  g      j                               | _-        t?        | jN                  j\                        D ]  \  }}tS        jT                  |
||dk(  r| j                  j^                  nd|d   | ja                  |      z  g      |d<   d|v sWtS        jT                  |
|| j                  jb                  | j                  jd                  g      |d<    tf        jh                  jj                  jm                  | jj                        5  | jo                  |	      }	| j5                  |	      \  | _8        | _9        t8        dk7  r| xjp                  |z  c_8        | jL                  "| jL                  |z  | jr                  z   |dz   z  n| jr                  | _&        ddd       | jt                  jw                  | jp                        jy                          |
|z
  | jZ                  k\  r| j{                          |
}tf        jh                  j}                         r!tf        jh                  j                         dz  nddd}t        | jL                  j                               r| jL                  j                  d   nd}|dkD  r| jL                  ntg        j                  | jL                  d      }t8        dv r|j                  dd d|z   z  z   |dz    d!| j&                   |g||	d"   j                  d   |	d#   j                  d   z         | j                  d$       | j                  j                  r |
| j*                  v r| j                  |	|
       | j                  d%        t?        | jN                  j\                        D ci c]  \  }}d&| |d    c}}| _F        t        j                         5  t        j                  d'       | j                  j                          ddd       | j                  d(       t8        dv r:| j                  j                  | j4                  g d)*       |dz   | j&                  k(  xs | j                  j                  }| j                  j                  s|r| j                         \  | _R        | _S        | j                  i | j                  | jL                        | j                  | j                  +       | j                  |dz   | j                        | _V        | j                  j                  s|dz   | j&                  k(  r!| j                          | j                  d,       t        j                         }|| j                  z
  | _        || _        | j                  d-       tf        jh                  j                          t8        dk7  rAt8        dk(  r| j                  ndg}t        j                  |d       t8        dk7  r
|d   | _V        | j                  s& n t8        dv rt        j                  d.|| j0                  z
  dz    d/t        j                         | j
                  z
  d0z  d1d2       | j                          | j                  j                  r| j                          | j                  d3       tf        jh                  j                          | j                  d4       y# 1 sw Y   5xY wc c}}w # 1 sw Y   xY w)5z=Train completed, evaluate and plot if specified by arguments.r.   Nr   d   r2   on_train_startzImage sizes z train, z val
Using z' dataloader workers
Logging results to boldz
Starting training for z
 epochs...r@   on_train_epoch_startClosing dataloader mosaicmosaicFclose_mosaichypr1   )totalon_train_batch_start        
initial_lrr   r   g    eAz.3gGz%11s%11sz%11.4g/clsimgon_batch_endon_train_batch_endzlr/pgignoreon_train_epoch_end)r9   ncrA   namesr   class_weights)include)rG   on_model_saveon_fit_epoch_end
z epochs completed in i  z.3fz hours.on_train_endteardown)^r   r   
epoch_timetimeepoch_time_starttrain_time_startr   r   rA   warmup_epochsr   r   r~   r   r   r   num_workersr   rJ   rT   r  ri   extendr   rU   epochrF   r   r   sampler	set_epoch	enumerater   r   r  resetprogress_stringr   rf   r   	zero_gradnpinterpr   rS   r   param_groupswarmup_bias_lrra   warmup_momentumr   r   r   r
   autocastpreprocess_batchre   
loss_itemsr   scalebackwardoptimizer_stepr   memory_reservedsizeshape	unsqueezeset_descriptionrH   plot_training_samplesr   warningscatch_warningssimplefilterrb   stepr`   update_attrr   possible_stopr   validaterG   rd   save_metricsr   r   save
save_modelempty_cacher   broadcast_object_list
final_evalplot_metrics)rl   r   nbnwlast_opt_stepbase_idxr!  pbarirD   nixijr   memloss_lenlossesirfinal_epochtnowbroadcast_lists                        rq   r   zBaseTrainer._do_train  s   >OOJ'*% $		 $		""#>Bii>U>UXY>YStyy..34c:_a+,l499??"38DIIOO;L M!..::joANO P**264==*I)J K--1[[ME 	F 99!!dii&<&<<BHMM  (HqL(Q,!GH4++T[[9EDJ56JJrz!!))33E:T../Dtyy'='==>784,,44h?7<D%%--44,,44nE%%--::tyy:I!!'')wD0023Id&7&78CDJNN$$& 5""#9:e^8RB&)!RYYr2499==SWSbSbCb?c-d-j-j-l&mDO )$..*E*E F1"$))aTYY%=%=SRST`RadhdkdklqdrRr$s#u$%?,.IIb"tyy?X?XZ^ZcZcZlZl>m,nAjM !G ZZ^^,,TXX6 11%8E15E1B.DItrz		Z/	QUQ[Q[Qg$**q.4??"Bq1u!M!__ J 7 !!$)),557 %8'')$&M @Ezz?V?V?X335;^_`cddef25djjoo6G2H4::++A.a'/!|UV9W7?((#h!h,&??!AI;a}5ssVsU5\EWEWXYEZs\abg\h\n\noq\rstu &&~6yy2+>225"=""#78W !Z ;DDNND_D_:`a:`Qrd|QtW,:`aDG((*%%h/##% + 34w $$TZZ9s$t$qyDKK7VDLL<V<V99==K15.DL$,!!*jT-B-B4::-N*jRVR^R^*jbfbibi*j!k LLDLLA	 99>>eai4;;&>OO%&&799;D"T%:%::DO$(D!12JJ""$ rz/3qy$))d!C**>1=19 .q 1DIyyM :P 7?KK"UT%5%559::O IIK$*?*??4GLGU VOOyy!!#~.

 :&[ 76< b**s   Bkk&0k,k#,k6	c                 F   ddl }i | j                  d| j                  i}|j                  | j                        j                  d      j                         D ci c]  \  }}|j                         | }}}| j                  | j                  t        t        | j                              j                         t        | j                  j                        j                         | j                  j                  | j                   j#                         t%        | j&                        ||t)        j*                         j-                         t.        d}t1        j2                  || j4                         | j                  | j                  k(  r t1        j2                  || j6                         | j8                  dkD  r_| j                  dkD  rO| j                  | j8                  z  dk(  r2t1        j2                  || j:                  d| j                   dz         yyyyc c}}w )	z9Save model training checkpoints with additional metadata.r   Nrd   r   )orient)r!  rc   rF   r`   updatesr   
train_argstrain_metricstrain_resultsdateversionr!  .pt)pandasrG   rd   read_csvrh   to_dictitemsstripr!  rc   r   r&   rF   halfr`   rY  r   
state_dictrO   rA   r   now	isoformatr   r   rA  rP   rQ   rR   rL   )rl   pdrG   r   r   resultsr   s          rq   rB  zBaseTrainer.save_model  s   ?T\\?i%>?,.KK,A,I,IQW,I,X,^,^,`a,`DAq1779a<,`aZZ --k$**56;;=DHHLL)..0xx''224tyy/$$LLN,,."$ 	

4#,JJtTYY'q tzzA~DJJIYIY<Y]^<^JJtTYY5C)@@A =_~ % bs   Hc                 T    | d   | j                  d      xs | j                  d      fS )zz
        Get train, val path from data dict if it exists.

        Returns None if data format is not recognized.
        r   r   test)r}   )rY   s    rq   r]   zBaseTrainer.get_dataset  s)     G}dhhuoA&1AAAry   c                 8   t        | j                  t        j                  j                        ry| j                  d}}d}t        |      j                  d      rt        |      \  }}|d   j                  }n|}| j                  ||t        dk(        | _        |S )z(Load/create/download model for any task.Nr_  rF   r2   )rm   r0   verbose)r   rF   r   r   ModulerN   endswithr   r9   	get_modelr   )rl   rF   r0   r   rm   s        rq   r   zBaseTrainer.setup_model  s    djj%((//2Twu:u%3E:MGTw-$$CC^^Wdbj^Q
ry   c                    | j                   j                  | j                         t        j                  j
                  j                  | j                  j                         d       | j                   j                  | j                         | j                   j                          | j                  j                          | j                  r&| j                  j                  | j                         yy)zVPerform a single step of the training optimizer with gradient clipping and EMA update.g      $@)max_normN)r   unscale_r   r   r   utilsclip_grad_norm_rF   
parametersr<  updater'  r`   rl   s    rq   r2  zBaseTrainer.optimizer_step  s    T^^,&&tzz'<'<'>&N(  "88HHOODJJ' ry   c                     |S )zRAllows custom preprocessing model inputs and ground truths depending on task type.r   )rl   rD   s     rq   r.  zBaseTrainer.preprocess_batch  s    ry   c                     | j                  |       }|j                  d| j                  j                         j	                         j                                }| j                  r| j                  |k  r|| _        ||fS )z
        Runs validation on test set using self.validator.

        The returned dict is expected to contain "fitness" key.
        rd   )rE   popre   detachr5   numpyrc   )rl   rG   rd   s      rq   r?  zBaseTrainer.validate  sm     ..&++i$))*:*:*<*@*@*B*H*H*J)JK  D$5$5$? 'Dry   c                     t        d      )z>Get model and raise NotImplementedError for loading cfg files.z3This task trainer doesn't support loading cfg filesNotImplementedError)rl   rm   r0   rn  s       rq   rq  zBaseTrainer.get_model  s    !"WXXry   c                     t        d      )zHReturns a NotImplementedError when the get_validator function is called.z1get_validator function not implemented in trainerr  ry  s    rq   r   zBaseTrainer.get_validator      !"UVVry   c                     t        d      )z6Returns dataloader derived from torch.data.Dataloader.z2get_dataloader function not implemented in trainerr  )rl   dataset_pathrS   r   r   s        rq   r   zBaseTrainer.get_dataloader  s    !"VWWry   c                     t        d      )zBuild dataset.z1build_dataset function not implemented in trainerr  )rl   img_pathr   rD   s       rq   build_datasetzBaseTrainer.build_dataset  r  ry   c                     |d|iS dgS )z=Returns a loss dict with labelled training loss items tensor.re   r   )rl   r/  r   s      rq   r   zBaseTrainer.label_loss_items  s     (2'=
#KF8Kry   c                 @    | j                   d   | j                  _        y)z2To set or update model parameters before training.r  N)rY   rF   r  ry  s    rq   r   z BaseTrainer.set_model_attributes  s    99W-

ry   c                      y)z.Builds target tensors for training YOLO model.Nr   )rl   predstargetss      rq   build_targetszBaseTrainer.build_targets      ry   c                      y)z.Returns a string describing training progress. r   ry  s    rq   r&  zBaseTrainer.progress_string  s    ry   c                      y)z,Plots training samples during YOLO training.Nr   )rl   rD   rM  s      rq   r8  z!BaseTrainer.plot_training_samples  r  ry   c                      y)z%Plots training labels for YOLO model.Nr   ry  s    rq   r   z BaseTrainer.plot_training_labels  r  ry   c           	         t        |j                               t        |j                               }}t        |      dz   }| j                  j                         rdn&d|z  t        dg|z         z  j                  d      dz   }t        | j                  d      5 }|j                  |d|z  t        | j                  dz   g|z         z  j                  d      z   dz          d	d	d	       y	# 1 sw Y   y	xY w)
z%Saves training metrics to a CSV file.r.   r  z%23s,r!  r   r  az%23.5g,N)r   r   valuesr   rh   existsr   rstripopenwriter!  )rl   rG   r   valsnsfs          rq   r@  zBaseTrainer.save_metrics  s    ',,.)40@+AdL1((//#B7Q;y4?O9P+P*X*XY\*]`d*d$((C AGGAQ

Q/?$/F)GGOOPSTTW[[\ !  s   AC**C3c                      y)z"Plot and display metrics visually.Nr   ry  s    rq   rF  zBaseTrainer.plot_metrics!  r  ry   c                 b    t        |      }|t        j                         d| j                  |<   y)z2Registers plots (e.g. to be consumed in callbacks))rY   	timestampN)r   r  rH   )rl   rK   rY   paths       rq   on_plotzBaseTrainer.on_plot%  s$    Dz$(tyy{C

4ry   c                    | j                   | j                  fD ]  }|j                         st        |       || j                  u s.t	        j
                  d| d       | j                  j                  | j                  j                  _        | j                  |      | _	        | j                  j                  dd       | j                  d        y)zIPerforms final evaluation and validation for object detection YOLO model.z
Validating z...)rF   rd   Nr  )rP   rQ   r  r*   r   r   rA   rH   rE   rG   r|  r~   )rl   r  s     rq   rE  zBaseTrainer.final_eval*  s    DII%Axxz"		>KK-s# 6704		DNN''-#'>>>#:DLLL$$Y5&&'9: &ry   c                 j   | j                   j                  }|r	 t        |t        t        f      xr t	        |      j                         }t	        |rt        |      n	t                     }t        |      j                   }t	        |d         j                         s| j                   j                  |d<   d}t        |      | _         t        |      | j                   _        dD ]!  }||v st        | j                   |||          # 	 || _        y|| _        y# t        $ r}t        d      |d}~ww xY w)zCCheck if resume checkpoint exists and update arguments accordingly.rY   T)r   rD   zzResume checkpoint not found. Please pass a valid checkpoint to resume from, i.e. 'yolo train resume model=path/to/last.pt'N)rA   resumer   rN   r   r  r   r#   r   rY   r   rF   setattrr[   FileNotFoundError)rl   rn   r  r  rP   	ckpt_argsr   rp   s           rq   rB   zBaseTrainer.check_resume6  s   !!a#FS$K8RT&\=P=P=R&Jv.n>NO 16;;	If-.557(,		If%#I.	"%d)		)AI~		1il; * f  a' )Y Z_`aas   CD +D 	D2!D--D2c           	         |yd}|d   dz   }|d   #| j                   j                  |d          |d   }| j                  ri|j                  d      rX| j                  j                  j                  |d   j	                         j                                |d   | j                  _        | j                  r|d	kD  sBJ | j                  j                   d
| j                   d| j                  j                   d       t        j                  d| j                  j                   d|dz    d| j                   d       | j                  |k  rMt        j                  | j                   d|d    d| j                   d       | xj                  |d   z  c_
        || _        || _        || j                  | j                  j                  z
  kD  rt        j                  d       t!        | j"                  j$                  d      rd| j"                  j$                  _        t!        | j"                  j$                  d      r1| j"                  j$                  j                  | j                         yyy)z7Resume YOLO training from given epoch and best fitness.Nr  r!  r.   r   rc   r`   rY  r   z training to zf epochs is finished, nothing to resume.
Start a new training without resuming, i.e. 'yolo train model=r   zResuming training from z from epoch z to z total epochsz has been trained for z epochs. Fine-tuning for z more epochs.r   r  Fr  r  )r   load_state_dictr`   r}   floatrf  rY  r  rA   rF   rT   r   r   rc   rU   r  r   r   r   r  )rl   r   rc   rU   s       rq   r   zBaseTrainer.resume_trainingO  s$   <7ma'(NN**4+<=/L88HHLL((e):):)<)G)G)IJ#IDHH;;? d99??#= >QQUQZQZQ`Q`Paabdd? KK)$))//):,{UVFWW[\`\g\g[hhuvx;;$KK::,4T']OC\]a]h]h\iivwyKK4=(K(&$++		(>(>>?KK34t((00(;38!!))0t((00.A!!))664996E B	 @ry   c                    g g g f}t        d t        j                  j                         D              }|dk(  rt	        j
                  t        d       d| j                  j                   d| j                  j                   d       t        |dd      }	t        d	d
|	z   z  d      }
|dkD  rdnd|
df\  }}}d| j                  _        |j                         D ]x  \  }}|j                  d      D ]^  \  }}|r| d| n|}d|v r|d   j                  |       *t!        ||      r|d   j                  |       K|d   j                  |       ` z |dv r/ t        t"        |t"        j$                        |d   ||dfd      }nR|dk(  rt#        j&                  |d   ||      }n1|dk(  rt#        j(                  |d   ||d      }nt+        d | d!      |j-                  |d   |d"       |j-                  |d   dd"       t	        j
                  t        d       d#t/        |      j0                   d$| d%| d&t3        |d          d't3        |d          d(| d)t3        |d          d*       |S )+a  
        Constructs an optimizer for the given model, based on the specified optimizer name, learning rate, momentum,
        weight decay, and number of iterations.

        Args:
            model (torch.nn.Module): The model for which to build an optimizer.
            name (str, optional): The name of the optimizer to use. If 'auto', the optimizer is selected
                based on the number of iterations. Default: 'auto'.
            lr (float, optional): The learning rate for the optimizer. Default: 0.001.
            momentum (float, optional): The momentum factor for the optimizer. Default: 0.9.
            decay (float, optional): The weight decay for the optimizer. Default: 1e-5.
            iterations (float, optional): The number of iterations, which determines the optimizer if
                name is 'auto'. Default: 1e5.

        Returns:
            (torch.optim.Optimizer): The constructed optimizer.
        c              3   0   K   | ]  \  }}d |v s|  yw)NormNr   )r   r   r   s      rq   r   z.BaseTrainer.build_optimizer.<locals>.<genexpr>  s     D!4A!1!4s   autoz
optimizer:z' 'optimizer=auto' found, ignoring 'lr0=z' and 'momentum=zJ' and determining best 'optimizer', 'lr0' and 'momentum' automatically... r  
   {Gz?      i'  )SGDr  ?AdamWr  r  F)recurser8   biasr@   r.   r   )AdamAdamaxr  NAdamRAdamg+?)r   betasr   RMSProp)r   r   r  T)r   r   nesterovzOptimizer 'z' not found in list of available optimizers [Adam, AdamW, NAdam, RAdam, RMSProp, SGD, auto].To request support for addition optimizers please visit https://github.com/ultralytics/ultralytics.)paramsr   r   z(lr=z, momentum=z) with parameter groups z weight(decay=0.0), z weight(decay=z), z bias(decay=0.0))r   r   __dict__rc  r   r   r   rA   r   r   getattrr   r+  named_modulesr   ru   r   r	   r  RMSpropr  r  add_param_grouprV   __name__r   )rl   rF   rK   r   r   r   r   gbnr  lr_fitmodule_namemodule
param_nameparamfullnamer   s                    rq   r   zBaseTrainer.build_optimizern  s   & BJD!2!2!4DD6>KK8L12 3))-7G		HZHZG[ \_` a b)B9B/3F7AE7I!3PWY_adOeD"h'*DII$#(#6#6#8K%+%<%<U%<%K!
E<Gk]!J<8ZX%aDKK&+aDKK&aDKK& &L $9 @@8tUZZ81"XW\L]lopIYadrHEIU]		!A$24PI%dV $v vw w
 	!!QqT5"IJ!!QqT3"GH%&aY(@(@'AbTU]T^^v1Q4yk-c!A$i[ugSQTUVWXUYQZP[[km	n ry   )r.   )NNT)r   r   r   )r   N)Nr   r   )r  gMbP?r  gh㈵>g     j@)&r  
__module____qualname____doc__r   rr   rN   rx   r{   r~   r   r   r   r   rB  staticmethodr]   r   r2  r.  r?  rq  r   r   r  r   r   r  r&  r8  r   r@  rF  r  rE  rB   r   r   r   ry   rq   r,   r,   &   s    "H '$4 E6N/# /+# +3 
!'F
#L6\F'PB2 B B(
 YWXWL
.
]D

;2F>9ry   r,   )Ar  r   r   r   r  r9  r   r   r   r   pathlibr   r~  r(  r   r   r   r   r	   
torch.cudar
   torch.nn.parallelr   r   ultralytics.cfgr   r   ultralytics.data.utilsr   r   ultralytics.nn.tasksr   r   ultralytics.utilsr   r   r   r   r   r   r   r   r   r   ultralytics.utils.autobatchr   ultralytics.utils.checksr   r   r   r    ultralytics.utils.distr!   r"   ultralytics.utils.filesr#   ultralytics.utils.torch_utilsr$   r%   r&   r'   r(   r)   r*   r,   r   ry   rq   <module>r     sx     	     (    %   < 1 G N* * * > S S D 2< < <A
 A
ry   