
    Ph                    i   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
mZmZmZmZmZmZ d dlZd dlZd dlZd dlmc mZ d dlmZmZ d dlmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d dl&m'Z'm(Z( d dl)m*Z*m+Z+m,Z, d	d
l-m.Z. ddl/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6 ddl2m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZA ddlmBZBmCZCmDZDmEZEmFZFmGZG ddlHmIZImJZJ  ej                  eL      ZMi ZNi ZO eP       ZQej                  j                  ZRej                  j                  ZSej                  j                  ZT eP       ZU eP       ZV eP       ZW eX       ZYd ZZd Z[d Z\ e[eRj                  eRj                  eRj                  eRj                  eRj                  eRj                  eRj                  eRj                  eRj                  eRj                  eRj                  eRj                  eRj                  g       ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  dZvdewfdZxd Zyd Zzde!fdZ{d Z|d Z}d Z~d Zde!j                   dfd Zd! Zd6d"Z	 	 	 	 	 d7d#Zd8d$Zd8d%e?dej                  fd&Z eeTj                  d'      d%e?dej                  fd(       Zdd)d%e?dej                  fd*Z eeRj                  j                  d'      d%e?dej                  fd+       Zdd)d%e?d,ej                  fd-Z eeTj                  d'      d%e?d,ej                  fd.       Zdd/e!j                   dddddfd0Z	 d8d1Z eeRj&                  dd2      d3        Z eeRj(                  dd2      d4        Z eeRj*                  eRj,                  eRj.                  eRj0                  eTj2                  g      d5        Z eeRd6      r  eeRj8                        e        eeRj:                  d'      d6d7       Z eeRj<                  d'      d6d8       Z eeRj>                  g      d6d9       Z eeRj@                        d:        Z eeRjB                        d;        Z eeRjD                        d<        Z eeRjF                        d=        Z eeRjH                        d>        Z eeRjJ                        d?        Z eeRjL                  d'      d@        Z eeTjN                  d'      dA        Z eeRjP                  d'      dB        Z eeRjR                        dC        Z eeRjT                  d'       eeRj                  d'       eeRjV                  d'      dD                      Z eeRjX                  d'      dE        Z eeRjZ                  d'      d9dF       Z eeRj^                  d'       e       fdG       Z eeRj                  d'      d6dH       Z] eeRjb                  d'      d6dI       Z eeRjd                  d'      d6dJ       Zd:dKZ eeRjh                        d:dL       Z eeRjj                  d'      d;dMewdNewdOewfdP       Z eeRjl                  d'      d;dMewdNewdOewfdQ       Z eeRjn                  d'      d;dMewdNewdOewfdR       Z eeRjp                  d'      dS        Z eeRjr                  d'      d:dT       Z eeRjt                  d'      d:dU       Z eeRjv                  d'      d:dV       Z eeRjx                  d'      dW        Z eeRjz                  d'      dX        Z eeRj|                  d'      dY        Zd:dZZ eeRj                        d<d[       Zd\ Z e        d=d]Z e j                  d      d^        ZĐd6d_ej                  j                  fd`Zǐd6d_ej                  j                  fdaZȐd=dbej                  j                  fdcZːd>ddZde Z eej                  j                  j                  d'      df        Z eeRj                  d'      dg        Z eeRj                  d'      dh        Z eeRj                  j                  d'      di        Z eeRj                        dj        Z e j                  d      dk        Zdl Z eeRj                  j                        Z eeRj                  j                        Z eeRj                  j                        Z eeRj                  j                        Z eeRj                          eeRj                        dm        Z eeRj                        dn        Z ee1j                  d'      do        Z ee1j                  d'      d,ej                  fdp       Z ee1j                  d'      dq        Z ee1j                  d'      dr        Z ee1j                  d'      d dsdteew   due?dvedMewfdw       Z ee1j                  d'      d dsdxewdyewdteew   due?dMewf
dz       Z eeRj                  d'      ddd{d|e?d}e?d~etdetfd       Zd Zd Zd Zd ZdhZ eeRj                  e        eeRj                  e        eeRj                  e        eeRj                  e        eeRj                  ed        eeRj                  ed        eeRj                  e        eeRj                  e        eeRj                          eeRj                          eeRj                  e        eeRj                         d Z eeRj                  j                  ed        eeRj                   j                  ed        eeRj                  j                  ed        eeRj                  j                  ed        eeRj                  j                  e        eeRj                  j                  e        eeRj
                  j                  e        eeRj                  j                  e        eeRj                          eeRj                  j                          eeRj                          eeRj                  e        eeRj                          eeRj                  e        eeRj                  j                  e        eeRj                  e        eeRj                          eeRj                           eeRj"                          eeRj$                          eeRj&                          eeRj(                          eeRj*                          eeRj,                          eeRj.                  d        eeRj0                  d        eeRj2                          eeRj4                          eeRj6                          eeRj8                          eeRj:                          eeRj                  d        eeRj<                  d        eeRj>                          eeRj@                          eeRj>                          eeRj@                          eeRjB                          eeRjD                          eeRjF                          eeRjH                          eeRjJ                          eeRjL                          eeRjN                  d        eeRjP                  d        eeRjR                          eeRjT                          eeRjV                          eeRjX                          eeRjZ                          eeRj\                          eeRj^                          eeRj`                          eeRjb                          eeRjd                          eeRjf                          eeRjh                          eeRjj                          eeRjl                          eeRjn                          eeRjp                          eeRjr                          eeRjt                          eeRjv                          eeRjx                          eeRjz                          eeRj|                          eeRj~                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                  d        eeRj                  d        eeRj                  d        eeRj                          eeRj                          eeRj                          eeRj                  d        eeRj                          eeRj                          eeRj                  d        eeRj                  d        eeRj                  d        eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                  d        eeRj                          eeRj                          eeRj                  d        eeRj                  d        eeRj                          eeRj                          eeRj                  d        eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                  d        eeRj                  j                          eeRj                  j                          eeRj                  j                          eeRj                          eeRj                          eeRj                          eeRj                           eeRj                  j                          eeRj                  j                          eeRj                  j                          eeRj
                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                  j                  d        eeRj                          eeRj                  j                          eeRj                  j                          eej                   j"                  j$                          eej                   j"                  j&                          eeRj(                  j                  d        eeRj*                  d'      d8d       Z eeRj,                        ddd       Z eeRd      r  eeRj.                        e        eeTj0                        d        Z eeRj2                  d'      dewdewfd       Z eeRj4                  d'      d?d       Zd Z eej8                  eRj:                  g      dddddd       Z eej<                        d@d       Z eej>                        d        Z eeRjB                        d        Zd Z eeRjF                  d'      d        Zd Z eejJ                  eRjJ                  g      dddddddd       Zd Zd Z  eeRjP                         ee            Z e ed            Z e ed             Zd Z eeRjX                        dddddd       Z eeRjZ                        dddddd       Z eeRj\                        dddddd       Z eeTj^                  j                        d        Z eej`                  eRj`                  g      d        Z eeRjb                  d'      d8d       Z eeRjd                  d'      dAd       Zd Zd Zd Z eeRjl                  d'      d        Z eeRjn                  d'      d        Z eeRjp                        d8d       Z eeRjr                        d8d       Zd Zd Z eeRjx                  d'      d8d       Z ee1jz                  d'      d8d       Zd Zd Z ee1j                  dd      d        Z eeRj                  d'      d6d       Z eeRj                  d'      dewfd       Zdd/ddewdee   detfdZ eeRj                  d'      dddewdee   fd       Z eeRj                  d'      dewfd       Z eeRj                  d'      dewfd       Z eeRj                  d'      dewfd       Z eeRj                  d'      d/ddewdetfd       Z	 	 dBdeee   df   dewdetfdǄZ eeRj                  j                        d6dee   fdɄ       Z eeRj                  j                        d6dee   fdʄ       Z eeRj                  j                        	 d@dee   dee   fd̈́       Zf eeRj                  j                        	 d@dee   dee   fd΄       Zg eeRj                  j                        	 	 	 dCdee   dee   dee   fdЄ       Z eeRj                  j                        	 	 	 dCdee   dee   dee   fdф       Zd҄ Z eeRj                  j                        	 	 d@detdee   dee   fdԄ       Z eeRj                        dՄ        Z eeRj                        dք        Z eeTj                  j                        dׄ        Z eeRj                  d'      d:d؄       Zdej                  dxeej                  ewf   fdڄZdej                  dyej                  fdۄZdej                  dyej                  dxej                  fd܄ZؐdDd݄Zdބ Z eeRj                  j                  d߫      Z eeRj                  d'      	 dEd       Zc eeRj                  j                  d߫      Z eeRj                  d'      d        Zdd Zd Z eeRj                  j                  d߫      Z eeRj                        d        Z eeRj                  j                        	 dFd       Z eeRj                  j                  d߫      Z eeRj                  d'      	 	 	 	 	 dGd       Z^ eeRj                  j                  d߫      Z eeRj                  d'      	 d6d       Z_d Zd Zd6defdZ eeRj                        dHddd       Zd Zd Zd Z eeRj                  eTj                  g      d6dddd       Z eeRj                        d6dddd       Zd Zed        Z eeRj                  j                  d߫      Z eeRj                  j                  d߫      Z eeRj                  j                  d߫      Z eeRj                  d/      d        Zd8dZ eeRj                        d        Z eeRj                  d'      d8d       Zed        Zed        Z eeRj                  d/      d6d       Z eeRj                  gd/      d        Z eeTj                  gd/      d        Z  eeRj                   eRj                  j                  gd/e!j                  2      e      Z eeRj                  eTj                  gd/      d         Z eeRj                        d        Z eeRj
                  eTj
                  g      dHddd       Z eeRj                        dHddd       Z eeRj                        dHd       Z	 eeRj                  d'      dHd       Z eeRj                  d'      dHd       Z  eeTj                         ed               eeRj                         ed            Z  eeRj"                         ed	            Z  eeRj&                         ed
ej                              Z  eeRj*                         edej                              Z eeRj.                  d/d      Zd Zd Z eeRj4                        Z eeRj6                        Z eeRj8                        Z eeRj:                        Z eeRj<                        Z eeRj>                        Z eeRj@                        Z  eeRjB                  d/      Z! eeRjD                          eeRjF                          eeRjH                        Z$ eeRjJ                        Z% eeRjL                        Z& eeRjN                  d      Z' eeRjP                        Z( eeRjR                        Z) eeRjT                        Z* eeRjV                          eeRjX                        Z,  eeRjZ                  e!j                  '      e,        eeRj\                          eeRj^                          eeRj`                          eeRj                          eeRjb                  dd/ej                        Z1 eeRjd                  dd/ej                        Z2 eeRjf                  dd/ej                        Z3 eeRjh                  dd/ej                        Z4 eeRjj                        Z5 eeRjl                        Z6  eeRjn                        e5         eeRjp                        e6        eeRjr                        Z9 eeRjt                        Z: eeRjv                          eeRjx                  d      Z< eeRjD                          eeRjz                  ej                           eeRj|                        e9        eeRj~                  ej                          eeRj                  ej                          eeRj                  ej                          eeRj                  ej                        ZB eeRj                  ej                          eeRj                  ej                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                          eeRj                  j                  ed/      ZU eeRj                  j                  ed/      ZV eeRj                  j                  ed/        eeRj                  j                  e      ZX eeRj                  j                  e      ZY eeRj                  j                  e!        eeRj                  j                  e!        eeRj                  j                  e9        eeRj                  j                  e$        eeRj                  j                  e        eeRj                  j                  e        eeRj                  j                  e      Z` eeRj                  j                  e      Za eeRj                  e        eeRj                  j                  e5        eeRj                  j                  e5        eeRj                  j                  e6        eeRj                  j                  e6        eeRj                  j                  e5        eeRj                  j                  e5        eeRj                  j                  e6        eeRj                  j                  e6        eeRj                  e:        eeRj                  e<        eeRj                  e       d Zj ejeRj                  j                  eRj                  j                  eU        ejeRj                  j                  eRj                  j                  eV        ejeRj                  j                  eRj                  j                  eX        ejeRj                  j                  eRj                  j                  eY        ejeRj                  j                  eRj                  j                  e`        ejeRj                  j                  eRj                  j                  ea       d Zn eneRj                  e        eneRj                  e%        eneRj                  e&        eneRj                  e'        eneRj                  e(        eneRj                  e)        eneRj                  e*        eneRj                  e        eneRj                  j                  e        eneRj                  j                  e        eneRj                  e1        eneRj                  e2        eneRj                  e3        eneRj                  e4        eneRj                  e!        eneRj                  e        eneRj                  e         eeRj                         e%         eeRj                        e&         eeRj                        e(         eeRj                        e)         eeRj                        e*        eneRj
                  eRj                           eneRj                  eRj                          eneRj                  eRj                          eneRj                  eRj                          eneRj                  eRj                          eeRj                        d        Z eeRj                  j                        d        Z eeRj                  j                        d        Z eeRj                        d        Z e'j                         D ]  \  ZZ  e e(e            e         eeRj                        d        Z eej                  j$                  j&                        d        Z eej                  j*                  j,                        d        Z ee      d        Z ee      d         Z	 d dlZej                  j4                  Z eej6                        d!        Z eej:                        d"        Z eej<                        d#        Z eej@                        d$        Z eejB                        d%        Z eejD                        d&        Z eejF                        d'        Z eejH                        d(        Z eejJ                        d)        Zej                  jL                  Z eej<                        d*        Z eejP                        d+        Z eejD                        d,        Z eejV                        d-        Z eej@                        d.        Z eejF                        d/        Z eejB                        d0        Z eejH                        d1        Z eej6                        d2        Zdd4l/mZ  e.e       dd5l/mZ  ejl                          y# e$ r eMjg                  d3       Y Jw xY w(I      N)defaultdict)Iterable)AnyCallableDictListOptionalTupleUnion) triton_kernel_wrapper_functionaltriton_kernel_wrapper_mutation)
canonicalize_dimcanonicalize_dimscheckdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator)CeilDivFloorDivModularIndexing   )import_submodule   )configinductor_primsirtest_operators)decompositionsget_decompositions)
ExpandViewIndexingConstant	is_tritonops_wrapperPermuteView	Pointwise	ReductionSqueezeView	TensorBoxvalidate_irView)ceildivdecode_device
is_dynamicis_pointwise_usepad_listlikesympy_product)opsVc                 $    | st        d|       y )Nzinductor does not support NotImplementedError)condmsgs     cC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/_inductor/lowering.py
assert_nyir?   I   s    !$>se"DEE     c                 R   t        | t        t        t        f      r| D cg c]  }t	        |       c}S t
        j                  |        t        | t        j                  j                        r5| j                         D ]!  }t
        j                  t        | |             # y y c c}w N)
isinstancelisttuplesetadd_needs_realized_inputsneeds_realized_inputsaddtorch_opsOpOverloadPacket	overloadsgetattr)fnxoverloads      r>   rG   rG   N   s    "tUC()689b)!,b99b!"ejj112H!%%gb(&;< ' 3 :s   B$c                     t        | t        j                  j                        r)| j	                         D ]  }|t
        t        | |      <    y |t
        | <   y rB   )rC   rJ   rK   rL   rM   layout_constraintsrN   )rO   
constraintrQ   s      r>   add_layout_constraintrU   W   sE    "ejj112H8Bwr845 ' ",2r@   )r   r   r                     	   
         dtypec                 b    t        | t              s| S | t        v sJ d|  d       t        |    } | S )Nzid z missing from DTYPE_ID_LOOKUP)rC   intDTYPE_ID_LOOKUPr`   s    r>   decode_dtypere      s=    eS!O#Os5'1N%OO#E"ELr@   c                     t        | t              r4t        | j                               xs t	        | j                               S t        | t
        j                        r| j                  du S t        | t              S NT)	rC   r.   r   	get_dtyper   sympySymbol
is_integerrb   rP   s    r>   is_integer_typerm      sX    !Y.Q2B1;;=2QQ	Au||	$||t##!S!!r@   c                 t    t        | t              rt        | j                               S t        | t              S rB   )rC   r.   r   rh   boolrl   s    r>   is_boolean_typerp      s*    !Y..!T""r@   type_promotion_kindc                 \    d }|D cg c]
  } ||       }}t        |d| i\  }}|S c c}w )Nc                     t        | t        t        j                  f      r| S t	        | d      sJ t        | j                               }t        j                  dg|z  | j                               S )Nrh   r   rd   )
rC   r   ri   rj   hasattrlenget_sizerJ   zerosrh   )inpdims     r>   construct_inputz+get_promoted_dtype.<locals>.construct_input   sW    cFELL12J3,,,clln%C;;sSy@@r@   rq   )r   )rq   argsrz   arginps_r`   s          r>   get_promoted_dtyper      sA    A -11DSOC DD1!4Q=PQHAuL 2s   )c                 0   t        | t        t        f      s| g} nt        |       } t        |       D ]b  }t        |t        j                  j
                        s(|j                         D ](  }t        ||      }|t        vs| j                  |       * d | S rB   )
rC   rD   rE   rJ   rK   rL   rM   rN   	loweringsappend)aten_fnrO   rQ   other_fns       r>   get_overloadsr      sy    ge}-)w-7mb%**556LLN"2x09,NN8, +  Nr@   c           
         	
 t               D cg c]  \  }}t        |t              s| c}}
|s|ri
rg|rt        j                  	n7 D cg c]!  }t        |t
              st        |d      s |# }}t        |d|i	 	
fd} D cg c]
  } ||       c} |r
rt        
t        
D cg c]  } |   	 c}       D ]
  \  }}| |<    t        t                     D ]Y  }t         |   t        j                        s!t        j                   |   t!         
d      j#                                      |<   [  S c c}}w c c}w c c}w c c}w )Nrh   rq   c                     t        | t              rt        |       S t        | t        j                        r5t        j                  | j
                  d      j                               S | S Nr   )rC   r.   to_dtyper"   Constantvalue
get_device)r|   r{   r`   indicess    r>   promoteztransform_args.<locals>.promote   sV    #y)U++C-{{399eT'!*5E5P5P5RSS
r@   r   )	enumeraterC   r.   rJ   ro   r   rt   r   zipbroadcast_tensorsrangeru   r"   r   r&   createrD   rv   )r{   	broadcastrq   convert_input_to_boolirP   apromoting_argsr   r`   r   s   `        @@r>   transform_argsr      sO   &t_I_TQ
1i0Hq_IG4' JJE  a:a#8GA{<S4   '5HE
	 %))Dq
D)W!2g4NgT!Wg4N!OPDAqDG Qs4y!A$q'2;;/$++DGT$wqz:J:S:S:U5VWQ " K= J  *4Ns"   EE!E2EE3Ec                     t        j                        fd       }t        |       }t        j	                  |       t
        j	                  |D ci c]  }|| c}       |S c c}w )a  
    Add a foreach lowering to lowerings dict.

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                  N    t        |       dk  sJ  | i |}t        |       |S )Nr   )ru   r/   )r{   kwargsout	decomp_fns      r>   wrappedz+_register_foreach_lowering.<locals>.wrapped   s/    4yA~~((C
r@   )	functoolswrapsr   foreach_opsupdater   )r   r   r   aten_fnsrO   s    `   r>   _register_foreach_loweringr      se     __Y   W%Hx H5Hbb'kH56N 6s   
A%c                      t        j                         fd       }t                t        j	                   D ci c]  }|| c}       |S c c}w )a  
    Add a lowering to lowerings dict

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                     t        |       } d}t        |       dk(  r t        | d   t         t        f      rd}| d   } t	        d |j                         D              rJ d       t	        d |j                         D              rt        d D              sJ t        |       } |r| g}  | i |}t        |       |S )	NFr   r   Tc              3   &   K   | ]	  }|d k(    yw)r   N .0rP   s     r>   	<genexpr>z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s      
 -1AJ   zout= ops aren't yet supportedc              3   <   K   | ]  }t        |t                y wrB   )rC   r.   r   s     r>   r   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s     IAz!Y/s   c              3   ,   K   | ]  }|t         v   y wrB   )	fallbacks)r   rO   s     r>   r   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s      Q
&-B)Ogs   )
rD   ru   rC   rE   anykeysvaluesallr   r/   )	r{   r   unpackedr   r   r   r   r   rq   s	       r>   r   z#_register_lowering.<locals>.wrapped
  s    BFt*t9>ja4-@H7D  
 &
 
 	+*	+ 
 IIIS Q
&-Q
 N
 	
 
 )02G
 6D((C
r@   )r   r   r   r   r   )r   r   r   rq   r   r   rO   s   `````  r>   _register_loweringr      sW     __Y  : G$GG4Gbb'kG45N 5s   
AFc                 >    t        j                  t        | |||      S )z+
    Shim to support decorator syntax.
    r   rq   r   )r   partialr   )r   r   rq   r   s       r>   register_loweringr   .  s&     /3 r@   c                 P   g }t        j                  t        |       t        |      t        j                  d            D ]  \  }}|dk(  r|j                  |       |dk(  r|j                  |       4t        j                  j                  j                  ||       t        t        j                  |      j                        t        t        j                  |      j                        k  r|j                  |       |j                  |        t        t        |            S )z
    Broadcasting logic based on symbolic shapes.

    We give the shapes 0 and 1 concrete values, while all other shapes
    are symbolic sympy formulas.
    r   )	fillvalue)	itertoolszip_longestreversedri   Integerr   r8   graphsizevarsguard_equalsru   expandfree_symbolsrE   )r   boutputrP   ys        r>   broadcast_symbolic_shapesr   @  s     F%%Xa[EMM!,<1 6MM!!VMM!GG))!Q/5<<?//03u||A7S7S3TTa a  &!""r@   c                 0   t        d | D              s| S t        d | D              r8|xs t        | dt        j                  ifd}| D cg c]
  } ||       c}S t        d | D              }g }| D ]  }t        |t        t        f      rn|j                  t        j                  t        j                  ||j                         |j                               t!        |j#                                            t        |t$        j&                        re|j                  t        j                  t)        ||j                         |j                               t!        |j#                                            |j                  |        |S c c}w )Nc              3   f   K   | ])  }t        |t        j                  t        t        f       + y wrB   )rC   ri   Exprrb   floatr   s     r>   r   z$promote_constants.<locals>.<genexpr>Y  s#     G1z!ejj#u56   /1c              3   f   K   | ])  }t        |t        t        t        j                  f       + y wrB   )rC   rb   r   ri   rj   r   s     r>   r   z$promote_constants.<locals>.<genexpr>[  s#     
Ef:a#uell34fr   rq   c                     t        | t        j                        r t        j                  | t        d             S t        j                  | t        d             S rB   )rC   ri   rj   r"   r'   r2   r   rP   r`   s    r>   
const_funcz%promote_constants.<locals>.const_func`  sC    !U\\***1e]45HII{{1e]4-@AAr@   c              3   N   K   | ]  }t        |t        t        f      s|  y wrB   )rC   r.   r&   r   s     r>   r   z$promote_constants.<locals>.<genexpr>g  s     JA:a)Z1H#Ias   %%)r   r   r   r   DEFAULTnextrC   rb   r   r   r&   r   r"   r   rh   r   rD   rv   ri   r   r'   )inputsoverride_return_dtyper   rP   exr   r`   s         @r>   promote_constantsr   X  sA   GGG

Ef
EE% 
);*
)H)P)P*
	B (..v!
1v..	JJ	JB
Ca#u&JJ!!KK2<<>2==?CT"++-EX
 5::&JJ!!$QH' JJqM " J) /s   Fc                 H     d ddt         t           f fd}|S )Nalphar   c           	      (  	
 t        |      }r%| '| dk7  r"t        |      }t        |d   |       |d<   n| J |D cg c]  }|j                          c}	|d   j	                         
xs |d   j                         t        |d   j                               j                  dk(  |dd  D ]`  }t        |t        j                        rt        
      t        |j	                               k(  rDJ d d
 d|j	                                  	
fd}sMd }|D ]1  }|j                         j                  dk(  s!|j                         } n |s|d   j                         }xs }t        j                  ||
      S c c}w )	Nr   r   cudazndim mismatch  c                 X   t        |       t              k(  sJ d|  d        t        j                  k(  r D cg c]
  } ||        c} S r.r,t        j                  k(  r D cg c]
  } ||        c} S  D cg c]
  } ||        c} S c c}w c c}w c c}w )Nzwrong ndim r   )ru   rJ   ro   float64)	indexloadr`   rO   is_cudaloadersoverride_fn_when_cuda_float64override_fn_when_input_boolrangess	     r>   inner_fnz/make_pointwise.<locals>.inner.<locals>.inner_fn  s    u:V,LE7!F8.LL,

"'B'N2W4UWTT%[W4UVV.7u?U4w6WwttE{w6WXXG<GDDKG<==	 5V6W<s   B1B"
B'devicer`   r   r   )r   rD   mulmake_loaderrv   rh   r2   r   typerC   r"   BaseConstantru   r+   r   )r   r   rP   otherr   r   r   r`   r   r   r   allow_alpharO   override_devicer   r   r   s          @@@@r>   innerzmake_pointwise.<locals>.inner  s   "6+@A UaZf U3r
= =,23Fq1==?F3##%%>)<)<)>q	 4 4 67<<FABZEeR__5V I : At1VHAenn.>-?@A   
	> 	> F<<>&&&0\\^F  --/ *F	
 	
= 4s   Fr   r.   )rO   r   r   r   r   r   r   s   `````` r>   make_pointwiser   }  s%     /3 +
tI +
 +
Z Lr@   c                 F     dddt         t         t              f fd}|S )Nr   r   r   c                     d }t        t        j                  j                  j                        dk(  xs* t        j                  j                  j
                  t        v }t        j                  j                  j                  D ]7  }|j                  D ]&  }|j                  dk(  r|j
                  t        v r%d}( 9 d }|D ]  }t        |t        t        f      s|} n |J d       g }|D ]H  }t        |t        t        f      s|j                  |gt        |      z         8|j                  |       J  |t        |       }	d gt        |      z  }
|	j                         D ]  \  \  }}}g }|D ]O  \  }}r	 |d| i}n | }||
|<   |j                  dk(  s+|s.|s1|j                  |j!                                Q |sbt        j                  j#                  |        t%        d |
D              sJ |
S )	Nc                    t        t              }t        |       D ]d  \  }}t        |  }d }|D ].  }t	        |t
              s|j                  j                         } n |J d       |||f   j                  ||f       f |S )Nz.foreach op should have at least one tensor arg)	r   rD   r   r3   rC   r.   datar   r   )	arg_pairsr   r   r{   use_foreachr   ts          r>   
group_argsz9make_foreach_pointwise.<locals>.inner.<locals>.group_args  s    d#C$Y/4",d"33A!!Y/!"!2!2!4 
 &DCD&V[)*111d)< 0 Jr@   r   call_functionTz1at least one input must be a list to a foreach opr   r   c              3   $   K   | ]  }|d u 
 y wrB   r   r   s     r>   r   z8make_foreach_pointwise.<locals>.inner.<locals>.<genexpr>  s     2'Q1D='s   )ru   r8   r   current_nodeuserstargetinplace_foreach_opsopr   rC   rD   rE   r   r   itemsr   realizeregister_listr   )r   r   r   realize_outputsnodeusera_list_inputinputbroadcast_inputsgroupsoutputsr   r   groupbuffer_list
output_indr{   r   r   pw_fns                     r>   r   z%make_foreach_pointwise.<locals>.inner  s   	  $$**+q0 Bww##**.AA 	 GG((..D

?2{8R&*O # /
 E%$/$ 
 $	?>	?$ EedE]3 ''#l2C(CD ''.	  C!123&3|,,,2LLN(!V[5K "D66F"D\F&,
#;;&([_&&v~~'78  %%k2# -;& 2'2222r@   r   )r  r   r   s   `` r>   make_foreach_pointwiser    s%    45 CtDO, CJ Lr@   rP   c                     | j                         k(  r|rt        |       S | S fd} t        |      |       S )Nc                 4    t        j                  |       S )N)	src_dtype)r7   r   )rP   r`   r  s    r>   	_to_dtypezto_dtype.<locals>._to_dtype  s    ||Au	::r@   r   )rh   cloner   )rP   r`   copyr  r  s    `  @r>   r   r     sC    IEuQx&Q&; B>)5A!DDr@   rq   c                     t        | |d      S NTr  )r   r   s     r>   _convert_element_typer!  	  s    Au4((r@   r   c                   | j                         k(  r|rt        |       S | S d } || j                               } |      }||k7  r t        d| j                          d d      fd} t        |      |       S )Nc                     | j                   rt        j                  |       j                  S t        j                  |       j                  S rB   )is_floating_pointrJ   finfobitsiinford   s    r>   _get_primitive_bitwidthz1to_dtype_bitcast.<locals>._get_primitive_bitwidth  s5    "";;u%***;;u%***r@   zbitcast z to different bitwidth type z is not supported yet.c                 0    t        j                  |       S rB   )r7   to_dtype_bitcastr   s    r>   _to_dtype_bitcastz+to_dtype_bitcast.<locals>._to_dtype_bitcast  s    ##Au--r@   r  )rh   r  r;   r   )rP   r`   r  r(  src_bitsdst_bitsr+  s    `     r>   r*  r*    s    {{}uQx&Q&+ 'q{{}5H&u-H8!q{{}o%A%H^_
 	
. J>+5I!LLr@   c                 *   |j                   s| j                         j                   r`t        j                  t        j
                  j                  t        j                  j                  j                  j                  | |            S t        | |d      S r  )
is_complexrh   r.   r   r"   ComplexViewrJ   r7   atenviewr`   r*  r   s     r>   _view_dtyper3  %  sg    1;;=33NN!!%)).."5"5";";QF
 	
 Au400r@   r   c                    t        |      }| j                         |k(  r|rt        |       S | S t        j                  t
        j                  j	                  | |            S rB   )r2   r   r  r.   r   r"   
DeviceCopy)rP   r   r  s      r>   	to_devicer6  .  sM    6"F||~uQx&Q&BMM00F;<<r@   c                     t        | |d      S r  )r6  )rP   r   s     r>   _device_putr8  5  s    QT**r@   Tc	                 ,   |xs | j                   }t        |      }	|rt        d|z         }
|t        |      }t        |	|||r
nd|      }	 t        | |||      |	      }	t	        t
        |      r" t        t        t
        |      d|      |	       |	S )z3A pointwise function that maps ops.{name} to inputs
libdevice_N)r   r   r   r   r   )rq   r   )__name__r)   r   r   rt   primsrN   )r   namer   rq   r   r   r   r   use_libdevice_for_f64rO   fn_libdevices              r>   register_pointwiser@  :  s     #7##D	T	B"<$#67".&12M&N#	
3$?6KlQU
B
	/3	

 	

B ud	
E4  $"7	
 		
 Ir@   c                 8    t        ||      }t        | |      }|S )Nr   )r  r   )r   pointwise_lowering_fnr   rO   s       r>   register_foreach_pointwiserD  d  s"    
 
  5;	OB	#GR	0BIr@   )r   rq   c           
         d }t        |t        t        f      r t        |      |      }t        |t        t        f      r t        |      |      }| ||g}t	        |d   |d   t
        j                        }t        |      D cg c]  \  }}t        |t              s| }}}t        |t        |D cg c]  }||   	 c}       D ]
  \  }}|||<    t        t        |            D ]Y  }t        ||   t        j                        s!t        j                   ||   t#        ||d      j%                                     ||<   [  t'        ||      |d   t)        |d   |      t)        |d   |            S c c}}w c c}w )Nc                  &    t        j                  |  S rB   )r7   wherer{   s    r>   rO   zwhere.<locals>.fnp  s    yy$r@   r   r   r  r   r  )rC   r   rb   constant_liker   r   r   r   r.   r   r   r   ru   r"   r   r&   r   rD   rv   r   r   )	r<   r   r   rO   r{   r`   r   rP   r   s	            r>   rG  rG  n  si     !eS\"M!Q!eS\"M!Q!Q<DQa.M.U.UE 't_I_TQ
1i0Hq_GIG.'0J'Qa'0JKL1Q M3t9d1gr{{+ ''Qd71:6F6O6O6Q1RSDG  ;>"E:Q$q'5)8DGU+C  J0Js   E9 E96E?c                     t        |       dk(  r$t        | d   t        t        f      rt	        | d    S t        j                  t        | D cg c]  }|j                          c}g       }g }| D ]b  }|j                         }t        |      t        |      k7  st        d t        ||      D              rt        ||      }|j                  |       d |S c c}w )Nr   r   c              3   V   K   | ]!  \  }}|d k(  xr |d k7  xs |d k7  xr |d k(   # ywr   Nr   r   r   r   s      r>   r   z$broadcast_tensors.<locals>.<genexpr>  s;      ,
EWTQa1fa7Q!V%6Q7EWs   '))ru   rC   rD   rE   r   r   reducer   rv   r   r   r   r   )r   rP   r  r  sizess        r>   r   r     s    
6{aJvay4-@ &),,(//!&#A&QAJJL&#A2 F G

u:V$ ,
EHPVEW,
 )
 q&!Aq  N $Bs   C
c                     | S rB   r   rl   s    r>   noprQ    s    Hr@   
lift_freshc                 *   t        | t              sJ |(t        t        j                  | j                              S t        t        | j                               |      }t        t        |t              s|fn|      }g }t        | j                               D ]X  \  }}||v r>t        j                  j                  j                  t        j                   |d            rH|j#                  |       Z || j                         k7  rt%        | |      S | S Nr   )rC   r.   r-   r   r   r   ru   rv   rF   rE   r   r8   r   r   evaluate_exprri   Eqr   r2  )rP   ry   dims	new_shapedss         r>   squeezer[    s    a###
{++AFF344
C

-s
3CZU3v=DI!**,'1T	agg..<<UXXa^LQ (
 "+ajjl!:49AAr@   c                 ,    t        t        | |            S rB   )r  r[  )rP   ry   s     r>   squeeze_copyr]    s    C!!r@   c                     t        | |      }t        | t              sJ t        |t              sJ |j                  | _        | S rB   )r[  rC   r.   r   rP   ry   vals      r>   squeeze_ra    s=    
!S/Ca###c9%%%XXAFHr@   c                     t        |       rt        | dt        j                        S t	        d      } t        |t        j                        |       S )NFrd   isinfr  rm   	full_likerJ   ro   r)   r   rP   rO   s     r>   rc  rc    ?    qE44	W	B?>"EJJ?BBr@   c                     t        |       rt        | dt        j                        S t	        d      } t        |t        j                        |       S )NFrd   isnanr  rd  rf  s     r>   ri  ri    rg  r@   c                 f    t        |       rt        |       S t        d      } t        |      |       S )Nceilrm   r  r)   r   rf  s     r>   rk  rk    s/    qQx	V	B>"a  r@   c                 f    t        |       rt        |       S t        d      } t        |      |       S )Nfloorrl  rf  s     r>   rn  rn    /    qQx	W	B>"a  r@   c                 f    t        |       rt        |       S t        d      } t        |      |       S )Nroundrl  rf  s     r>   rq  rq    ro  r@   c                 f    t        |       rt        |       S t        d      } t        |      |       S )Ntruncrl  rf  s     r>   rs  rs    ro  r@   c                    t        | g      \  } t        | t        j                        rt	        j
                  | t        |            S t        | t              sJ t        |t        t        f      sJ t        | j                               t        |      k(  r| S t        d | j                         D              st        j                  j                  j                  t        | j                                     }|dkD  rVt        d |D              sD| j!                  t        j                  j                  j                  t        |            |z         t        t	        j
                  | j"                  t        |                  S )Nc              3      K   | ]7  }t         j                  j                  j                  j	                  |       9 y wrB   r8   r   r   	shape_envis_unbacked_symintr   rZ  s     r>   r   zexpand.<locals>.<genexpr>  s,     VAqww))<<Q?   =?r   c              3      K   | ]7  }t         j                  j                  j                  j	                  |       9 y wrB   rv  ry  s     r>   r   zexpand.<locals>.<genexpr>  s/      *
FKAGG&&99!<erz  )r   rC   r"   r   r&   r   rE   r.   rD   rv   r   r8   r   r   	size_hintr6   
mark_reuser   )rP   rO  x_size_products      r>   r   r     s   aS!DQ!R__%  E%L11a###edE]+++QZZ\eEl*VVV))33M!**,4OP
 Ac *
FK*
 '
 LL  **=+?@NR Z&&qvvuU|<==r@   c                     t        |      }|D ]  }d||<   	 | }t        |      D ]  \  }}|dk7  st        ||      } t        ||      S )Nr   )rD   r   	unsqueezer   )r   shapebroadcast_dimensionsrZ  broadcast_dimensionvidxrP   s           r>   broadcast_in_dimr    s\    UA3!#
  4 	
AA,Q7!S!A  !Ur@   c                 6    t        | |j                               S rB   )r   rv   )rP   r   s     r>   	expand_asr    s    !QZZ\""r@   c                 "   t        | j                               t              t              kD  rFt        j                  d      gt              t              z
  z  z   t        | t                    } t              t        | j                               k(  sJ t        | j                               }d}t        t                    D ]  }|   dk(  rd}||   |   z  ||<    |r*t        || j                         | j                               S t        d t              D              rt        | |      S fd}t        j                  j                  j!                  t#                    }|dkD  rD| j%                  t        j                  j                  j!                  t#        |            |z         | j'                         t)        j*                  | j                         | j                         |t        |            S )	Nr   Fr   Tr`   r   c              3   :   K   | ]  \  }}|d k(  xs |d k(    ywrL  r   rM  s      r>   r   zrepeat.<locals>.<genexpr>1  s'     
A*@$!QAFa1f*@s   c                    t        |       t              k(  sJ t        |       } t        t                    D ]B  }|   dk7  s|   dk(  rt        j                  d      | |<   -t        | |   d|         | |<   D  |       S Nr   r   )ru   rD   r   ri   r   r   )r   r   old_sizerepeatsx_loaders     r>   r   zrepeat.<locals>.inner_fn6  s    5zS\)))Us7|$AqzQA;!#$}}Q/E!H.uQxHQKHE!H % r@   r   )rD   rv   ru   ri   r   r2  r   emptyrh   r   r   r   r   r8   r   r   r|  r6   r}  r   r+   r   )	rP   r  new_sizezero_tensorr   r   old_size_productr  r  s	    `     @@r>   repeatr    s   AJJL!H
7|c(m#MM!$%WH)EFQDN#w<3qzz|,,,,AJJL!HK3w< 1:?KqkGAJ. !
 XQ[[]1<<>JJ

A#gx*@
AAa""	 ww''11-2IJ!	GG&&}X'>?CSS	
 }}H||~kkmH~	 r@   c                     t        | t              sJ t        |t        t        f      sJ t        t	        j
                  | j                  |            S rB   )rC   r.   rD   rE   r0   r   r   )rP   rO  s     r>   r2  r2  Q  sB     a###edE]+++T[[/00r@   c                     t        | t              sJ t        |t        t        f      sJ t        t	        j
                  | j                  t        |                  S rB   )rC   r.   rD   rE   r*   r   r   )rP   rW  s     r>   permuter  Z  sF    a###dT5M***[''d<==r@   c           	         t        | t              sJ t        | |d      }| j                         |   }t        j
                  j                  j                  t        j                  ||z   d            rd}t        j
                  j                  j                  t        j                  ||z   d            rd}t        t        j                  j                  | j                  ||||            S r   )rC   r.   _validate_dimrv   r8   r   r   rU  ri   Ltr"   	SliceViewr   r   )rP   ry   startendstepdim_sizes         r>   slice_r  a  s    a###
3
"Czz|C Hww%%ehhux/?&CDww%%ehhsX~q&ABR\\((eS$GHHr@   c                    t        |t              s|f}t        |t              s|f}|D cg c]  }t        | |       }}t        | j	                               dk(  rt        |       S t        |      }t        |      }|dk7  s|dk7  r|dk(  rt        d      |dk(  rY|dk(  rTt        | t        | j	                               g      }t        ||d      }t        |t        | j	                                     S ||k7  rt        d| d|       |dd }|dd }	t        | |d   |d         }
t        |
||	      S |\  t        j                  j                  j                  | j	                                  |d   z
  z  | j                         fd}t!        j"                  | j%                         | j'                         || j	                               S c c}w )	z
    This is based on torch._refs.roll(), but uses ModularIndexing().

    We can't use the ref here because it is based on multiple calls to
    torch.cat() that this will result in terrible code.
    r   r   z`shifts` requiredz*shifts and dimensions must align. shifts: z, dims: Nc                     t        |       } t        |    z   t        j                  d      t        j                              | <    |       S rT  )rD   r   ri   r   r   )r   a_loaderry   sizer  s    r>   rO   zroll.<locals>.fn  sH    U$#Ja 0%,,t2D
c
 r@   r   )rC   r   r  r6   rv   r  ru   RuntimeErrorr2  rollrD   r8   r   r   evaluate_static_shaper   r+   r   r   rh   )r   shiftsrW  rY  
len_shiftslen_dimsflatrolledtail_shifts	tail_dimsfirst_dim_rolledrO   r  ry   r  r  s               @@@@r>   r  r  m  s    fh'dH%w)-.AM!QD.QZZ\"a'QxVJ4yHQ(a-?233 q=Z1_M!**,789D$*FQZZ\ 233!<ZLQYPZ[  QRjH	6!9d1g6$k9==FS7711!**,s2CDDF1I%E}}H ||~kkmzz|	 M /s   Gc                    t        | t              r>t        | j                  t        j                        r| j                  j                         } | j                          t        j                  |       st        d|  d      t        j                  |       \  }}t        j                  |j                  |j                  |D cg c]  }t        j                  |       c}|D cg c]  }t        j                  |       c}t        j                  |xs d            }t        t        j                  ||            S c c}w c c}w )Nzunrealized as_strided(z, ...)r   )rC   r.   r   r"   BaseViewunwrap_viewr  is_storage_and_layoutr;   as_storage_and_layoutFixedLayoutr   r`   ri   r   ReinterpretView)rP   r  stridestorage_offsetstorage
old_layoutrZ  
new_layouts           r>   
as_stridedr    s    !YJqvvr{{$CFF IIK##A&!$:1#V"DEE2215GZ"&'$Qa$'"()&Qa&)^(q)J R''<==	 	()s   E &Ec                 d    t        | t              sJ t        | |||      j                  | _        | S rB   )rC   r.   r  r   )rP   r  r  r  s       r>   as_strided_r    s/    a###48==AFHr@   c                 4    t        | |||      }t        |      S rB   )r  r  )rP   r  r  r  results        r>   as_strided_copyr    s    48F=r@   c                     g d} D ]1  }j                  |||j                            z   f       d   d   }3  D cg c]  }|j                          c} fd}t         d   j                               }d   d   |<   t	        j
                   d   j                          d   j                         ||      S c c}w )Nr   r   c           	      $  	
 t        j                  |    t        j                        }g g 
t	        t                    D ]8  dk(  r$t        j                  dt        j                        n)t        j                     d   t        j                        }t        j                     d   t        j                        }t        j                  ||      }t        j                  ||      }dk(  r|}n*t              dz
  k(  r|}nt        j                  ||      }j                  |       t        |       		xx      d   z  cc<   
j                  t        j                  |	fdd             ; 
fd d      S )Nr   r   c                                 S rB   r   )r   idx_loadinputs_loaderss   r>   <lambda>z1pointwise_cat.<locals>.inner_fn.<locals>.<lambda>  s    -N1-h7r@           c                 |    | t              dz
  k7  r&t        j                  |    |     | dz               S d   S )Nr   r   )ru   r7   rG  )r   get_masked_valr   masked_loadsmaskss    r>   r  z7pointwise_cat.<locals>.inner_fn.<locals>.get_masked_val  sK    CK!O#yy!H O"1q5)  $B''r@   )r7   
index_exprrJ   int64r   ru   constantgeltand_r   rD   masked)r  idx_dimr  r  
start_condend_condmaskr  r   r  r  r  ry   r   r  inputs_rangess          @@@@@r>   r   zpointwise_cat.<locals>.inner_fn  sL   ..S5;;7s6{#A 6 Q,^^M!$4Q$7E 
 ..q!1!!4ekkBC/Jvvgs+HAvc&kAo%!xx
H5LLCyH SM]1-a00M

71 $@	( a  r@   r   )r   rv   r   rD   r+   r   r   rh   )r   ry   prev_endrx   r   r  r  r  s   ``    @@r>   pointwise_catr    s    9;MHh3<<>#3F(FGH $R(  4::6Ccoo'6:N/!b F1I&&()H!"%b)HSMay##%Qi!!#	 m ;s   Cc                 8  	 t        d | D              rp| D ]  }|j                           t        d | D              rt        t        j                  g|  \  } } t        t        j                  j                        | |      S t        |       dk(  rt        | d         S t        | d   |d      }t        | dt        j                  i}| D cg c]  }t        ||       } }dt        f	fd	t        |       t        j                   k  rdt        d t"        j$                  j&                  D              }t        	fd	| D              }t)        	fd
| D              }|s|r|rt+        | |      S t-        t.        j0                  j3                  | |            S c c}w )Nc              3   \   K   | ]$  }|j                         t        j                  u  & y wrB   )rh   rJ   uint8r   r  s     r>   r   zcat.<locals>.<genexpr>  s!     
@5??+s   *,c              3   T   K   | ]   }t        |j                               d k(   " yw)rW   N)ru   rv   r  s     r>   r   zcat.<locals>.<genexpr>  s"     >ves5>>#$)vs   &(r   r   rq   returnc                    t        j                  |       r:t        j                  | d      \  }}t         j                  j	                  |       S t        | t              rVt        | j                  t         j                        r  | j                  j                               S  | j                        S t        | t         j                        r | j                        S t        | t         j                        ryy)NF)freezeT)r"   r  r  ConcatKernelcan_realize_into_without_copyrC   r.   r   r  r  
StorageBoxr+   )rP   r  r~   should_lower_cat_inputs      r>   r  z#cat.<locals>.should_lower_cat_input"  s     ##A&11!EBJGQDDWMMMa#!&&"++.-aff.@.@.BCC-aff55a')!&&11a&r@   c              3   2   K   | ]  }t        |        y wrB   )r4   )r   uses     r>   r   zcat.<locals>.<genexpr>8  s     S>Rs-c2>Rs   c              3   .   K   | ]  } |        y wrB   r   r   rx   r  s     r>   r   zcat.<locals>.<genexpr>9       "Q&3#9##>&   c              3   .   K   | ]  } |        y wrB   r   r  s     r>   r   zcat.<locals>.<genexpr>:  r  r  )r   r  require_channels_lastr1  catfallback_handlerdefaultru   r  r  r   r   r   r   ro   r    max_pointwise_cat_inputsr8   r  r  r   r  r.   r"   r  r   )
r   ry   r  r~   r`   rx   pointwise_usesall_pointwise_inputsany_pointwise_inputsr  s
            @r>   r  r    sa   

@
@@ EMMO >v>>-dhh@@IFA1 0 01&#>>
6{aVAY
q	3
*C	%D%L%LE /55fshsE"fF5T * 6{f555Sann>R>RSS""Q&"QQ""Q&"QQ$8^ --R__++FC899? 6s   Foffsetdim1dim2c                   
 | j                         t              t              t              t        k7  fd       t        j
                  j                  j                  t        j                  |d            }|r t        t           |z            d      }nt        t              |z
        d      }d
|r| df
nd|f
t              D cg c]  \  }}|fvs| }}}|j                  |       
fd}	t        t        j                   j#                  | ||	            S c c}}w )N)r  rankc                      d  d S )Nz(diagonal dimensions cannot be identical z, r   )r  r  s   r>   r  zdiagonal.<locals>.<lambda>J  s     HbQUPVWr@   r   )r   r   c                     | d   }dgt        	      z  }d}t              D ]1  }|k(  r|d   z   ||<   |k(  r|d   z   ||<   %| |   ||<   |dz  }3 |t        	      dz
  k(  sJ |S )Nr   r   r   r   )ru   r   )
r  diag_idxoriginal_idxcur_dimrY  base_idxr  r  num_dimsoriginal_shapes
        r>   	reindexerzdiagonal.<locals>.reindexer\  s    r7sS00xADy"*Xa["8Qd"*Xa["8Q"%g,Q1 ! #n-1111r@   )rv   ru   r   r   r8   r   r   rU  ri   r  maxminr   r   r.   r"   GenericViewr   )r  r  r  r  offset_negative	diag_sizer   rZ  rO  r  r  r  r  s     ``      @@@r>   diagonalr
  B  s5   ^^%N>"H84D84D	W gg&&44UXXfa5HION4069>$;OPRST	N40.2F2OPRST	HGQ<v;$^4N441a$8MQ4EN	LL   R^^**5%CDD' Os   (E7Ec                 0    t        t        | |||            S rB   )r  r
  )r  r  r  r  s       r>   diagonal_copyr  o  s    %t455r@   c                 P    t        |       }t        ||||      }t        ||       |S rB   )r  r
  	mutate_to)r  srcr  r  r  r   r  s          r>   diagonal_scatterr  t  s*    5\FffdD1FfcMr@   c           	          t        j                  || j                         |         }t        t	        | |||dz         |      S rT  )r0   handle_negative_indexrv   r[  r  )rP   ry   r  s      r>   selectr  |  s;    

$
$S!**,s*;
<C6!S#sQw/55r@   c           	         t        | |d      }t        j                  j                  j	                  | j                         |         }t        |t        j                        r)t        j                  j                  j	                  |      }t        |t        t        j                  f      r|g||z   dz
  |z  z  }g }d}|D ]&  }||z   }|j                  t        | |||             |}( |S Nr   r   )r  r8   r   r   r  rv   rC   ri   r   rb   r   r   r  )rP   rO  ry   x_sizer  r  r  r  s           r>   splitr    s    
3
"CWW33AJJL4EFF%$   66u=%#u}}-.FUNQ.589FEdlfQUC01  Mr@   c                     t        | ||      S rB   )r  )rP   rO  ry   s      r>   split_with_sizesr    s    E3r@   c                     t        | |d      }t        j                  j                  j	                  | j                         |         }g }t        |      D ]  }|j                  t        | ||               |S r   )	r  r8   r   r   r  rv   r   r   r  )rP   ry   r  r  r   s        r>   unbindr    sb    
3
"CWW33AJJL4EFFF6]fQQ'( Mr@   c                 >   | j                         }t        |      }t        ||      |dk(  rt        t	        | d      |      S |   }t
        j                  j                  }|j                  ||       |j                  d       t        ||z
        dz   }|j                  |      dkD  r-| j                  |j                  t        ||z  |                   g |d  ||dz   d  |}	fd}
t        t        j                   j#                  | |	|
            S )Nr   )r  r   c                 D    | d   |    z  z   }g | d  || dz   d S )Nr   r   r   )r  dim_idxry   r  s     r>   r  zunfold.<locals>.reindexer  s?    b'CHtO+8Tc8G8c#'B&788r@   )rv   ru   r   r  r  r8   r   r   	guard_leqguard_ltr   r|  r}  r   r.   r"   r  r   )rP   	dimensionr  r  rO  ndimr  r   new_dim_sizeout_sizer  ry   s      `       @r>   unfoldr%    s   JJLEu:D
4
+Cqyi1o400SzHwwHtX&aHtOT2Q6L(#a'	X''t0CX(NOPDtDlDU379-=DtDH9 R^^**1h	BCCr@   c                     t        | |d      }t        | j                               }|j                  |t	        j
                  d             t        | |      S rT  )r  rD   rv   insertri   r   r2  )rP   ry   rX  s      r>   r  r    sF    
3
"CQZZ\"IS%--*+9r@   c                     t        | |      }t        | t              sJ t        |t              sJ |j                  | _        | S rB   )r  rC   r.   r   r_  s      r>   
unsqueeze_r)    s>    
As
Ca###c9%%%XXAFHr@   c                     t        |t              sJ t        | j                               }|dk  r|||z   z  }d|cxk  r	||z   k  sJ  J |S r   )rC   rb   ru   rv   )rP   ry   r  r"  s       r>   r  r    sY    c3qzz|D
Qwtf}#dVm#####Jr@   c                     t        | |d      }t        j                  j                  j	                  | j                         |         dz  }t        | |d|      }t        | |||dz        }t        |t        |            S )Nr   r   )	r  r8   r   r   r  rv   r  r   sigmoid)rP   ry   new_lenr   r   s        r>   glur.    so    
3
"Cgg44QZZ\#5FG1LGq#q'"Aq#w!,Aq'!*r@   c            !      	   t         j                  j                  rt         j                  j                  j
                  t         j                  j                  j                  t         j                  j                  j                  t         j                  j                  j                  t        j                  j                  t         j                  j                  j                  g} t        t         j                  j                  j
                        dt        dt        dt        fd       }t        t         j                  j                  j
                  j                         dt        dt        dt        dt        fd       }t        t         j                  j                  j                  j                         dt        dt        dt        dt        fd       }t        t         j                  j                  j                        dt        dt        d	t        fd
       }t        t         j                  j                  j                  j                         dt        dt        dt        d	t        fd       }t        t         j                  j                  j                        dt        dt        dt        fd       }t        t        j                  j                        dt        dt        dt        dt        dt        dt        dt        dt"        dt$        t&           dt&        dt&        dt&        dt"        dt"        dt"        dt"        f d       }t        t         j                  j                  j                  d       dt        dt        d t        d!t        dt        f
d"       }t        t         j                  j                  j                  j                   d       dt        d#t        dt        d t        d!t        dt        fd$       }	t        t         j                  j                  j(                  d       dt        dt        d t        d!t        dt        f
d%       }
t         j                  j*                  r| j-                  t         j                  j.                  j0                         t        t         j                  j.                  j0                        dt        d&t        d't        d	t        fd(       }t3        |        y y ))NrP   weightbiasc
                 x    t        j                  t        j                  j                  | |||||||||	
            S rB   )r.   r   r"   ConvolutionUnary)
rP   r0  r1  paddingr  dilationr  attrscalars	algorithms
             r>   convolution_unaryz5register_onednn_fusion_ops.<locals>.convolution_unary  sI     ####** r@   r   c                 ~    t        j                  t        j                  j                  | |||||||||	|
||            S rB   )r.   r   r"   ConvolutionBinaryrP   r   r0  r1  r4  r  r5  r  binary_attrbinary_alpha
unary_attrunary_scalarsunary_algorithms                r>   convolution_binaryz6register_onednn_fusion_ops.<locals>.convolution_binary
  sR      ##$$++ !# r@   c                 ~    t        j                  t        j                  j                  | |||||||||	|
||            S rB   )r.   r   r"   ConvolutionBinaryInplacer<  s                r>   convolution_binary_inplacez>register_onednn_fusion_ops.<locals>.convolution_binary_inplace,  sR      ##++22 !# r@   wr   c           
      p    t        j                  t        j                  j                  | |||||            S rB   )r.   r   r"   LinearUnary)rP   rF  r   r6  r7  r8  s         r>   linear_unaryz0register_onednn_fusion_ops.<locals>.linear_unaryN  s3     ##%%aAtWiH r@   r   c           	      n    t        j                  t        j                  j                  | ||||            S rB   )r.   r   r"   LinearBinary)rP   r   rF  r   r6  s        r>   linear_binaryz1register_onednn_fusion_ops.<locals>.linear_binaryV  s*    ##BOO$:$:1aAt$LMMr@   c                 z    t        j                  t        j                  j                  | |||||||||	|
            S rB   )r.   r   r"   ConvolutionTransposeUnary)rP   r0  r1  r4  output_paddingr  r5  r  r6  r7  r8  s              r>   convolution_transpose_unaryz?register_onednn_fusion_ops.<locals>.convolution_transpose_unaryZ  sL     ##,,33" r@   w0w1w2w3hxcxreversebatch_sizesmodehidden_size
num_layers
has_biasesbidirectionalbatch_firsttrainc                     t        j                  t        j                  t        j
                  j                  | |||||||||	|
|||||            S rB   )pytreetree_mapr.   r   r"   MkldnnRnnLayer)rP   rQ  rR  rS  rT  rU  rV  rW  rX  rY  rZ  r[  r\  r]  r^  r_  s                   r>   mkldnn_rnn_layerz4register_onednn_fusion_ops.<locals>.mkldnn_rnn_layerx  sb    & ??  !!((!! r@   r  packed_weightw_scalew_zpc                     t        j                  t        j                  j                  | |||||||||	|
||||||            S rB   )r.   r   r"   QConvPointWisePT2E)rP   x_scalex_zpre  rf  rg  r1  r  r4  r5  r  o_inv_scaleo_zero_pointoutput_dtyper6  r7  r8  s                    r>   qconvolution_unaryz6register_onednn_fusion_ops.<locals>.qconvolution_unary  s^    ( ##%%,,!  # r@   accumc                     t        j                  t        j                  j                  | |||||||||	|
|||||||||||            S rB   )r.   r   r"   QConvPointWiseBinaryPT2E)rP   rj  rk  rp  accum_scaleaccum_zpre  rf  rg  r1  r  r4  r5  r  rl  rm  rn  r=  r   r?  r@  unary_algorithmms                         r>   qconvolution_binaryz7register_onednn_fusion_ops.<locals>.qconvolution_binary  sm    6 ##++22!  !$- r@   c                 ~    t        j                  t        j                  j                  | |||||||||	|
||            S rB   )r.   r   r"   QLinearPointwisePT2E)rP   rj  rk  re  rf  rg  r1  rl  rm  rn  r6  r7  r8  s                r>   qlinear_unaryz1register_onednn_fusion_ops.<locals>.qlinear_unary  sR      ##''..!   r@   packed_worig_wc                     t        j                  t        j                  j                  | |||            }|t	        ||      }|S rB   )r.   r   r"   MKLPackedLinearrI   )rP   rz  r{  r   
batch_sizer  s         r>   mkl_packed_linearz5register_onednn_fusion_ops.<locals>.mkl_packed_linear&  sE     #))&&--a6:N = ^Fr@   )rJ   _C_has_mkldnnr7   mkldnn_convolution_pointwise_convolution_pointwise_ _convolution_transpose_pointwise_linear_pointwiser1  rd  r  onednnqconv2d_pointwiser   r.   binaryro   r   rb   qlinear_pointwisehas_mklr   mkl_mkl_linearrG   )cpu_needs_realized_inputsr9  rB  rE  rI  rL  rP  rd  ro  rv  ry  r  s               r>   register_onednn_fusion_opsr    s   xxII33II44II==II..!!))II..%
! 
599++BB	C			 	 
D	6 
599++BBII	J			 	 		 
K	B 
599++CCJJ	K			 	 		 
L	B 
599++==	>		&	+4	 
?	 
599++==DD	E	NY 	N9 	N 	Ny 	N 
F	N 
599++LL	M			 	 
N	: 
40088	9&	&	&	 &	 	&	
 &	 &	 &	 &	 c&	 &	 &	 &	 &	  &	 &	  !&	 
:&	P 
599++==SW	X'	'	 %	'	
 '	 '	 '	 
Y'	R 
II..554

1	1	 	1	 %1	 1	 1	 1	

1	f 
599++==SW	X		 %		
 	 	 	 
Y	B 88%,,UYY]]-F-FGuyy}}889# " 	 : 	"";<r@   c                 @     |rt         j                           fd}|S )Nc                      t        j                  t        j                  t	        j
                  j                  g| i |      S rB   )ra  rb  r.   r   r"   FallbackKernel)r{   r   kernels     r>   handlerz!fallback_handler.<locals>.handlerA  s:    b//66vOOO
 	
r@   )r   rI   )r  add_to_fallback_setr  s   `  r>   r  r  =  s    f

 Nr@   c                  .    t        j                  d       y )NzjTorchinductor does not support code generation for complex operators. Performance may be worse than eager.)warningswarnr   r@   r>   _warn_complex_not_supportedr  I  s    MMtr@   r   c                     | j                         rI|r<|j                  t        j                  j                  j
                  j                  k(  ryt                yy)z0Do not support reading or writing to this tensorFT)r/  r  rJ   r7   r1  r2  r`   r  r   parents     r>   unsupported_input_tensorr  R  s<    ||~fmmuyy~~':':'@'@@#%r@   c                 X    t        | |      ry| j                  xr t        j                  S )z2Do not support writing tensor but can read from itT)r  is_cpur    disable_cpp_codegenr  s     r>   unsupported_output_tensorr  ]  s$    6*882222r@   r
  c                 0   | j                   t        j                  j                  u ry| j                   t        j                  j                  u ryd }t        j                  | j                  i | j                  D ]  } ||| d      s y  || | d      S )NFc                 B   t        | t        j                  j                        syd| j                  vryt        j                  | j                  d         D ]F  }t        |t        j                  j                        s(|rt        ||      s7 yt        ||      sF y y)NFr`  T)rC   rJ   fxNodemetara  tree_leaves_subclasses
FakeTensorr  r  )r
  r  	is_outputr  s       r>   check_skip_conditionzCfallback_node_due_to_unsupported_type.<locals>.check_skip_conditionm  s    $.		!&&tyy'78DdE$5$5$@$@A,T6:+D&9 9 r@   )r  T)	r  r1  view_as_complexr  lift_fresh_copyra  arg_tree_leavesr{   r   )r
  allow_cpu_inputsr  r|   s       r>   %fallback_node_due_to_unsupported_typer  d  s    {{d**222 {{d**222* %%tyy@DKK@TU; A  dd;;r@   c                 "   | t         vs
J d|         |rt        t        j                  d            rt	        | g      rt
        j                  r&| t        j                  j                  j                  v sgt        j                  j
                  j                  r4dt        j                  j
                  _        t        j                  d       t        d|  d      fd}t!        | t        j"                  j$                        r*| j'                         D ]  }t)        | |      } ||        y t!        | t        j"                  j*                  t        j"                  j,                  f      r	 ||        y t/        d|  d	t1        |              )
Nz*both a fallback and a decomp for same op: CIFznA make_fallback error occurred in suppress_errors config, and suppress_errors is being disabled to surface it.zmake_fallback(a.  ): a decomposition exists, we should switch to it. To fix this error, either add a decomposition to core_aten_decompositions (preferred) or inductor_decompositions, and delete the corresponding `make_fallback` line. Get help from the inductor team if unsure, don't pick arbitrarily to unblock yourself.c                 n    t        |        t        |         t        | d       t        |             S Nr  )rG   rU   r   r  )op_overloadlayout_constraints    r>   register_fallbackz(make_fallback.<locals>.register_fallback  s;    !+.(!+/@AG $G[)
 	
r@   zUnsupported fallback z with type )r$   ro   osgetenvr%   r    fallback_randomrJ   _decompdecompositions_for_rngextra_random_decomps_dynamosuppress_errorslogwarningAssertionErrorrC   rK   rL   rM   rN   
OpOverloadHigherOrderOperatorr  r   )r  r  r  r  olr  s    `    r>   make_fallbackr    sC   ^#V'QRTQU%VV#4!t$ ""emm::OOO ==//38EMM  0KKH RD !f f
 	

 "ejj112,,.B!"b/Kk* ! 
B..

0N0NO	P"22$k$r(LMMr@   c                 T    d}| D ]  }||z  }	 t        |t        j                        S )z
    TorchInductor offset calculation differs from PyTorch eager offset
    calculation for random ops (tl.rand vs torch.rand). In future, we should
    strive for same impl for tl.rand and torch.rand.
    r   rd   tensorrJ   r  )r  numelrZ  s      r>   philox_rand_offsetr    s-     E	 %u{{++r@   c                 B  	
 t        j                  || t         j                  j                  |             j	                         
|j                         |j                         		
fd}t        j                  ||t        |             }t        |       }||fS )Nc                    t        j                   g       t        j                        }t        j                   g       t        j                        }t        j                  t        j
                   |       t        j                        |      }t        j                  ||      }t        j                  |      S rB   )r7   r   rJ   int32rI   r  rand)	r   seed_index_exproffset_index_exprrand_index_exprr  r`   offset_loader
random_posseed_loaders	        r>   r   zphilox_rand.<locals>.inner_fn  s     ,,{2DLLr):EKKH''NN:e,ekk:<M
 
 ||FE**r@   r   )
r"   r  FlexibleLayoutcontiguous_stridesmake_indexerr   r+   r   rD   r  )r  seedr  r  r   r`   r   random_values_nodeoffset_noder  r  r  s        `   @@@r>   philox_randr    s     
,,T2	
 ln  ""$K&&(M+ #))Dz	 %T*K{**r@   c           	          t         j                  r\t        j                  t        j
                  t        j                  j                  t        j                  j                  | ||            S t        d      )Nz&should be handled in replace_random.py)r    r  ra  rb  r.   r   r"   r  r1  native_dropoutr  r  )rP   pr_  s      r>   r  r    sY    $$T%8%8%@%@!QN
 	

 EFFr@   c                     t         j                  s-| j                         t        j                  d      k(  sJ d       | j                          t        j                  | g|  | S NcpuzTthis should be handled in decomps unless config.fallback_random or the device is CPU)r    r  r   rJ   r   r  r"   InplaceBernoulliFallbackrP   r{   s     r>   
bernoulli_r    s]    !!Q\\^u||8 & ^]^  IIK)D)Hr@   c                     t         j                  s-| j                         t        j                  d      k(  sJ d       t        t        |       g| S r  )r    r  r   rJ   r   r  r  r  s     r>   bernoulli_pr     sP    !!Q\\^u||8 & ^]^  eAh&&&r@   c                     t               rB   r  )r~   s    r>   _foobarr  	  s    

r@   c                 .    t         j                  d       y )Nz1using triton random, expect difference from eager)r  info)salts    r>   _warn_triton_randomr    s    HH@Ar@   c                  J    t        t        j                  j                         y rB   )r  r8   r   creation_timer   r@   r>   warn_triton_randomr    s    --.r@   c                      |j                  dd       t        | i |S t        j                  r|j	                  dd        t        | i |S t        d      N	generatorz-should have been handled in replace_random.py)getfallback_rand_generatorr    r  popfallback_rand_defaultr  r{   r   s     r>   r  r    sV    zz+t$0&777			

;%$d5f55
H
IIr@   c                      |j                  dd       t        | i |S t        j                  r|j	                  dd        t        | i |S t        d      r  )r  fallback_randn_generatorr    r  r  fallback_randn_defaultr  r  s     r>   randnr  )  sV    zz+t$0'888			

;%%t6v66
H
IIr@   c                 l    t        j                  |      }t         j                  j                  | |      S rB   )r"   get_stride_orderExternKernelrequire_stride_order)input_tensorr  stride_orders      r>   inductor_force_stride_orderr  3  s)    &&v.L??//lKKr@   c                     t        d      )Nz.should be handled in fuse_seed_creation_pass()r  )r   s    r>   inductor_seedr  9  s    
I
JJr@   c                 z    t                t        j                  t        j                  | t        |                  S rB   )r  r.   r   r"   RandomSeedsr2   )countr   s     r>   inductor_seedsr
  >  s)    BNN5-2GHIIr@   c                 |      fd}t        j                   j                          j                         |g       S )Nc                 L    t        j                  j                               S rB   )r7   	load_seedget_name)r~   r   seedss    r>   r   z&inductor_lookup_seed.<locals>.inner_fnF  s    }}U^^-u55r@   r   )r+   r   r   rh   )r  r   r   s   `` r>   inductor_lookup_seedr  D  s9    6 !oo	 r@   r  r  r  rY  c                  	 t         j                  rJ dv sJ g | } t        j                  }|j	                         }t        j                  ||| t
        j                  j                  |       |      j                         |j                         		fd}t        j                  |||g |       }|j                          |S )N)r  r  r  c                      t        t               g       t        j                   |       t        j                              S rB   )rN   r7   r  rJ   r  )r   rY  r  r  s    r>   r   z!inductor_random.<locals>.inner_fn]  s7    !wsD!ONN:e,ekk:
 	
r@   r   )r    r  rJ   float32r   r"   r  r  r  r  r   r+   r   r  )
r  r  rY  r  r`   r   r   r  r  r  s
     `     @@r>   inductor_randomr  Q  s    %%%%$$$$T7DMME__FtR..AA$GPVln  ""$K
 w	F NNMr@   lowhighc                f   	 t         j                  rJ g |}t        j                  }|j	                         }t        j                  |||t
        j                  j                  |      |      j                         |j                         	 	fd}t        j                  |||g |      S )Nr  c                     t        j                   g       t        j                   |       t        j                              S rB   )r7   	randint64r  rJ   r  )r   r  r  r  r  s    r>   r   z"inductor_randint.<locals>.inner_fnz  s8    }}ONN:e,ekk:	
 	
r@   r   )r    r  rJ   r  r   r"   r  r  r  r  r   r+   r   )
r  r  r  r  r  r`   r   r   r  r  s
   ``      @@r>   inductor_randintr  m  s     %%%%T7DKKE__FtR..AA$GPVln  ""$K
 w	 r@   	out_int32rightr  
boundariesr  r  c                  	 t        j                               dk(  sJ t        |       rt              s/ t        t        j
                  j                  d      | |      S j                          j                         d   j                         }| j                         }| j                         	|rt        j                  nt        j                  	fd}t        j                  ||| j                               S )Nr   Fr  r  r   c                 f     |       }t        j                  |j                               }|S rB   )r7   	bucketizer  )r   r`  r   r  boundaries_sizeindex_dtypeinput_loaderr  s      r>   r   zbucketize.<locals>.inner_fn  s:    5!--!
 r@   r   )ru   rv   r(   r  r1  r#  Tensorr  r   r   rJ   r  r  r+   r   )
r  r  r  r  boundaries_loaderr   r   r$  r%  r&  s
    ` `   @@@r>   r#  r#    s     z""$%***e:!6Q 5 55Q:%
 	
  ))+A.O"..0F$$&L!*%++K
 
 ~~	 r@   c                     t        j                  t        j                  t        j                  j
                  ||f      \  }}||fS rB   )ra  tree_map_onlyr"   IRNoder   require_stride1r~   r{   r   s      r>   require_denser.    s;    ''
		2??22T6NLD& <r@   c                     t        j                  t        j                  t        j                  j
                  ||f      \  }}||fS rB   )ra  r*  r"   r+  r   require_contiguousr-  s      r>   r0  r0    s;    ''
		2??55f~LD& <r@   c                     t        j                  t        j                  t        j                  j
                  ||f      \  }}||fS rB   )ra  r*  r"   r+  r   r  r-  s      r>   r  r    s;    ''
		2??884.LD& <r@   c           
          d t        fdt        || j                        D              }|j                         D ci c]  \  }}| || j                  |          }}}||fS c c}}w )Nc                     t        | t        j                        rPt        j                  |j                  d   j                               }t        j                  j                  | |      S | S Nr`  )rC   r"   r+  r  r  r  r   r  )r|   fx_argr  s      r>   apply_constraintz1constrain_to_fx_strides.<locals>.apply_constraint  sN    c299%..v{{5/A/H/H/JKL??77\JJ
r@   c              3   6   K   | ]  \  }} ||        y wrB   r   r   r|   r5  r6  s      r>   r   z*constrain_to_fx_strides.<locals>.<genexpr>  "      9P+#vf%9P   rE   r   r{   r  r   fx_noder{   r   kr  r6  s        @r>   constrain_to_fx_stridesr?    st      9<T7<<9P D EKLLNSNDAqa!!W^^A%677NFS< Ts    !A(ztorchvision::roi_align)r  c           
            fdt        fdt        | j                        D              }|j                         D ci c]  \  }}| | j                  |          }}}||fS c c}}w )Nc           
         t        | t        j                        s| S |j                  d   }|j                  s| S t        j
                  |j                               }|r<|d   dk7  r4t        t        t        t        | j                                                 }dj                  t        j                  j                  t        j                   j                  fv }fd}t        | t"              sJ t        | j$                  t        j&                  t        j(                  f      r  ||       s || j+                               r| S fd}t        | j$                  t        j,                        rL| j$                  j/                         r2|r0t        | j$                  j                               dk(  r
 ||       r| S t        j0                  j3                  | |      S )Nr`  r   r   r[   c                     t         j                  j                  j                  | j	                         d         z  dk(  S )Nr   r   )r8   r   r   r|  rv   )rP   	ALIGNMENTs    r>   
is_alignedz=sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned  s4    GG$$..qzz|B/?@9LQRRRr@   c           	           t         fdt        t         j                               dz
        D              }t        j
                  j                  j                   j                         d         dk(  xr |S )Nc              3      K   | ]D  }t         j                  j                  j                  j	                         |         z  d k(   F ywr   N)r8   r   r   r|  
get_stride)r   r   rC  rP   s     r>   r   zYsdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned_backward.<locals>.<genexpr>  sC      "7A !!++ALLN1,=>JqP7s   A
Ar   r   )r   r   ru   rH  r8   r   r   r|  )rP   aligned_stridesrC  s   ` r>   is_aligned_backwardzFsdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned_backward  si    ! "s1<<>2Q67" O
   **1<<>"+=> '&'r@   rW   )rC   r"   r+  r  r   r  r  rD   r   r   ru   rv   r  r1  0_scaled_dot_product_efficient_attention_backwardr  ,_scaled_dot_product_flash_attention_backwardr.   r   r  r&   r  r  is_input_bufferr   r  )	r|   r5  meta_valr  is_backwardrD  rJ  rC  r=  s	          @r>   r6  z)sdpa_constraint.<locals>.apply_constraint  sk   #ryy)J;;u%J**8??+<=L,1s3<<>/B)C DEL 	nnAAII==EE)
 

	S #y))) chhr}} =>c? coo/0J	' sxx/((*388$$&'1,1DS1I
33CFFr@   c              3   6   K   | ]  \  }} ||        y wrB   r   r8  s      r>   r   z"sdpa_constraint.<locals>.<genexpr>'  r9  r:  r;  r<  s   `    @r>   sdpa_constraintrQ    sv    4Gl  9<T7<<9P D EKLLNSNDAqa!!W^^A%677NFS< Ts   !A+c                    |}| j                         |j                         k7  rt        || j                               }| j                         |j                         k7  rt        || j                               }| j	                         |j	                         k7  r%t        || j	                               }t        |      S t        |      S rB   )r   r6  rh   r   rv   r   r  )selfr  non_blockingrP   r   s        r>   r  r    s    ACNN,,a*+~~3==?*Q()}}#,,.(Q(Sz8Or@   )memory_formatc          	          t        j                  | j                         | j                         | j	                         t        | j                                     S )Nr   )r+   r   r   rh   r   rD   rv   )rP   rU  s     r>   r  r    s@     ||~kkmAJJL!	 r@   r  c                \    fd}t        j                  t        |      || g      S )Nc                 D    t        j                  | d   z  z         S )Nr   rd   r7   r  )r   r`   r  r  s    r>   rO   ziota.<locals>.fn	  s!    ~~dU1Xo5UCCr@   r   )r+   r   r2   )lengthr  r  r`   r   requires_gradrO   s    ```   r>   iotar\    s1    D V$x	 r@   ry   r   c           	      
   | j                         |j                         k(  sJ | j                         t        | d      t        j                  j
                  j                  t        j                  d            r| j                            z   t        j                  j
                  j                  d       t        j                  j
                  j                  | j                                   t        t        |      | j                               }|j                         fd}t        j                  | j!                         | j                         |t#        | j                                     S )Nr   c           	          t        j                  t        j                  t        j                  |    t        j
                        t        j                  t        j
                               |        |             S rB   )r7   rG  eqr  rJ   r  )r  ry   r   
src_loaderr  s    r>   r   z select_scatter.<locals>.inner_fn	  sX    yyFFs3x5uekk2 sOSM
 	
r@   r   )rh   r   r  r8   r   r   rU  ri   r  rv   r  r   r   r  r+   r   r   rD   )rP   r  ry   r   r   r`  r  s     `` @@r>   select_scatterra  	  s   ;;=CMMO+++}}H
3
"Cww%%ehhua&89

S))GGq%(GGeQZZ\#%67
3$ajjl
3C"J
 ||~kkmAJJL!	 r@   c           	         	
  j                         |j                         k(  sJ  j                         
t         d       j                            Bt        j
                  j                  j                  t        j                  d            rz   Bt        j
                  j                  j                  t        j                  d            rz   d;t        j
                  j                  j                   j                                  rt         j                               }t        z
  dz
  z         |<   t        ||      }|j                         		 
fd}t        j                   j!                          j                         |t         j                                     S )Nr   r   c                 .   	dk(  rk(  r
dk(  r |       S t        j                  |    t        j                        }t	        |       t        |    	z
  
      <   g }	dk7  rZ|j                  t        j                  |t        j                  t        j                  	      t        j                                     k7  rZ|j                  t        j                  |t        j                  t        j                        t        j                                     
dk7  r|j                  t        j                  t        j                  t        |    	z
  d
      t        j                        t        j                  dt        j                  j                                     |sJ t        j                  t         j                   |      }t        j"                  |fdt%              rdnd      }t        j&                  || |             S )Nr   r   c                              S rB   r   )src_idxr`  s   r>   r  z1slice_scatter.<locals>.inner_fn.<locals>.<lambda>i	  s
    Jw'r@   r  )r7   r  rJ   r  rD   r   r   r  ri   r   r  r_  r   r  r   rN  r  r  rm   rG  )r  r  r  src_valre  ry   r  r  r`  r  r  rP   r  s       @r>   r   zslice_scatter.<locals>.inner_fnD	  s   A:#/daic?"..S5;;7s)C5 0$7A:KKNN5<<#6D (?KKNN5<<#4ekkB 19KKNN'C5(8!TBEKK LLEKK$5$56	 t$/**' #A

 yySM
 	
r@   r   )rh   r   r  rv   r8   r   r   rU  ri   r  statically_known_leqrD   r   r   r+   r   r   )rP   r  ry   r  r  r  src_sizer   r  r`  r  s   ` ````  @@@r>   slice_scatterri  0	  sh   ;;=CMMO+++}}H
3
"Czz|C HQWW--;;EHHUA<NO 
177++99%((3:JKHn}
{agg&&;;AJJL<MsSAJJL!HS5[D1H5t<HSM
h
C"J,
 ,
\ ||~kkmAJJL!	 r@   c                 j    t        | t        t        f      rt        |       dkD  rt	        | d         S | S r   )rC   rD   rE   ru   _unwraprl   s    r>   rk  rk  z	  s-    !dE]#A
qt}Hr@   r`   r   layout
pin_memoryc                    t        |d t        j                  fv d|        t        | d       t        t	               t
              rxs t        j                  nxs t        j                         g }t         t        j                        r fd}nt         t        t
        f      r fd}nt               dk(  s't         d   t        t
        f      rBt               dk  r4|j                  t        j                  t                             fd}n5t        j                  j!                  t        j"                   |            S t%        j&                  t)        |      ||	      S )
Nlayout=rn  c                 0    t        j                        S rB   rY  r   r   r`   s    r>   r   ztensor.<locals>.inner_fn	  s    >>$..r@   c                 0    t        j                        S rB   r7   r  rr  s    r>   r   ztensor.<locals>.inner_fn	  s    <<e,,r@   r   r[   c                       fdt              dk(  rt        j                  d      S  dt                    S )Nc           	      j   | |k  sJ || z
  dk(  rt        j                  |          S || z
  dz  | z   }t        j                  t        j                  t        j                  d   t
        j                        t        j                  |t
        j                               | |       ||            S )Nr   r   r   )r7   r  rG  r  r  rJ   r  )r  r  midbinary_searchr   r`   r   s      r>   rx  z/tensor.<locals>.inner_fn.<locals>.binary_search	  s    s{"{;!#<<UU;;U{q(50yyFFuQx=S%++6 "%-!#s+ r@   r   )ru   r7   r  )r   rx  r   r`   s   `@r>   r   ztensor.<locals>.inner_fn	  s7     4yA~||Au-- CI..r@   r  r   )r?   rJ   stridedrC   rk  rb   r  get_default_dtyperi   r   r   ru   r   r   r8   r   add_tensor_constantr  r+   r   r2   )r   r`   r   rm  rn  r   r   s   ``     r>   r  r  	  s   v$..'&0BC:~|,'$-%$2002!F$

#	/ 
D5#,	'	- 
Ta:d1gs|<TaemmCI./	/( ww**LLU6:
 	
 V$	 r@   c                 z    t        | t              r|t        | |      } |t        | |      } | S t	        | ||      S )Nr  )rC   r.   r   r6  r  )r   r`   r   s      r>   	as_tensorr}  	  sD    $	"D%(DT6*D$eF33r@   c                 8    t        | t        j                        S )Nrd   r  )r   s    r>   long_tensorr  	  s    $ekk**r@   c                     t         j                  j                  j                  d   j                  j
                  }t        j                  ||       }t         j                  j                  |      |_	        |S r4  )
r8   r   r  r  r
  exprr"   DynamicScalarregister_bufferr=  )r   symbuffers      r>   _local_scalar_denser  	  sW     ''


#
#E
*
/
/
4
4Cc4(F''))&1FKJr@   c                    | t        | t        t        f      st        d      rj                  t        t        t        f      rfd}nTt        t
        j                        rfd}n3t        j                               dk(  sJ j                         fd}t        j                  ||t        |            S )Nr   c                 0    t        j                        S rB   rt  r   r`   r   s    r>   r   z_full.<locals>.inner_fn	  s    <<u--r@   c                 0    t        j                        S rB   rY  r  s    r>   r   z_full.<locals>.inner_fn	  s    >>%//r@   r   c                      g       S rB   r   )r   value_loaders    r>   r   z_full.<locals>.inner_fn	  s    ##r@   r   )rC   rb   r   rt   r   ri   r   ru   rv   r   r+   r   rD   )
fill_valuer   r`   r  r   r   r  s     `  @@r>   _fullr  	  s    Ej3,/GE74K%#u&	. 
E5::	&	0 5>>#$)))((*	$ Dz	 r@   c                 8     t        t        |            | fi |S rB   create_tensor_liketensor_constructor)rP   r  r   s      r>   re  re  	  s     =0<=aJ6JJr@   c                 "     d d d d dd d fd
}|S )NF)namesr`   r   rm  rn  rU  c                    t        | d u d       t        |d t        j                  fv d|        t        | d       t        |      }|xs t        j                         }t        |      dk(  r6t        |d   t        t        t        j                  f      rt        |d         }|D cg c]  }t        j                  |       }}t        |||      S c c}w )Nnamed tensorsrp  rn  r   r   )r?   rJ   ry  r2   rz  ru   rC   rD   rE   Sizeri   r   r  )	r  r`   r   rm  rn  rU  r  rZ  r  s	           r>   r   z!tensor_constructor.<locals>.inner 
  s     	5D=/26dEMM22gfX4FGz><0v&2002t9>ja4

2KLa>D)-.AQ.Z55 /s   )Cr   )r  r   s   ` r>   r  r  	  s#     6& Lr@   )r  r`   rm  r   rn  rU  c                     t        | d u d       t        |      }t        |      dk(  r6t        |d   t        t
        t        j                  f      rt        |d         }t        |d ||||      S )Nr  r   r   r`   rm  r   rn  )	r?   r2   ru   rC   rD   rE   rJ   r  empty_strided)r  r`   rm  r   rn  rU  r  s          r>   r  r  
  sf     u}o.6"F
4yA~*T!WtUEJJ.GHT!W~d%v* r@   c                       dddddd fd
}|S )zZ
    Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
    NF)r`   r   rm  rn  rU  c                   t        | d       t        |d t        j                  fv d|        || j                         }nt	        |      }|xs | j                         }t        | j                               } |||||      S )Nrn  rp  rl  )r?   rJ   ry  rh   re   r   rD   rv   )rP   r`   r   rm  rn  rU  r  creation_fns          r>   _constant_likez*create_tensor_like.<locals>._constant_like.
  s     	z><06dEMM22gfX4FG=KKME 'E)1<<>AJJL!fV

 	
r@   r   )r  r  s   ` r>   r  r  )
  s     duTX
 r@   c                 *    t        t        |             S rB   r  )r  s    r>   rI  rI  @
  s    0<==r@   c                      d d d d d fd
}|S )Nr  c                \   t        |t        t        f      sJ t        | d       t        |d t        j
                  fv d|        t        |      xs | j                         }|xs | j                         }|D cg c]  }t        j                  |       }}t        |||      S c c}w )Nrn  rp  )rC   rD   rE   r?   rJ   ry  re   rh   r   ri   r   r  )rP   r  r`   rm  r   rn  rZ  r  s          r>   _new_constantz#new_constant.<locals>._new_constantJ
  s     $u...z><06dEMM22gfX4FGU#4q{{})1<<>*./$Qa $/Z55 0s   =B)r   )r  r  s   ` r>   new_constantr  I
  s    tDT	6 r@   r  c                l    || j                         }|| j                         }t        |d ||||      S Nr  rh   r   r  )rP   r  r`   rm  r   rn  s         r>   	new_emptyr  X
  s<    }~d%v* r@   c                X   t        | t        t        f      sJ t        |t        t        t        d       f      sJ t	        | d       t	        |d t
        j                  fv d|        t        |      xs t        j                         }|xs t        j                  d      j                  }t        d|||       }|j                          |j                  j                  }dgt        |       z  |j                  _        t        |t         j"                        sJ | D cg c]  }t%        j&                  |       } }|r#|D cg c]  }t%        j&                  |       c}nt         j(                  j+                  |       }t!        j,                  ||| |      |_        |S c c}w c c}w )Nrn  rp  r  r   )r  r   r`   r  )r   r`   r  r  )rC   rD   rE   r   r?   rJ   ry  re   rz  r  r   r  r  r   ru   r   r"   ComputedBufferri   r   r  r  r  rm  )	r  r  r`   rm  r   rn  	pointwiser  rZ  s	            r>   r  r  c
  sj    dT5M***ftUDJ7888:~|,v$..'&0BC<5#:#:#<E/u||C(//F6TJI^^  Fs4yFKKfb//000%)*TELLOTD*  #))&Qa&)11$7 
 NN	FM  +)s    F"F'c                l    || j                         }|| j                         }t        ||||||      S r  r  )rP   r  r  r`   rm  r   rn  s          r>   new_empty_stridedr  
  s>     }~fE&J r@   c                 
   |D cg c]+  }t         j                  j                  j                  |      - }}t	        t        t        |            |j                        }t        j                  j                  | |      S c c}w )N)key)r8   r   r   r|  sortedr   ru   __getitem__r"   r   r  )rP   r  rZ  r  s       r>   copy_stridedr  
  sb    5;<Vagg((+VF<%F,&2D2DEL??//<@@ =s   0B c                 V    |j                  d      J d        t        |      | fi |S )Nr`   z(dtype should be handled by decomposition)r  r  )r  r  r   s      r>   fullr  
  s5    ::g*V,VV*)j)$9&99r@   c                    t        | t              sJ |j                         t        j                  k(  sJ | j                         t              dk(  }t        | |      | j                         |j                         fd}t        j                  | j                         | j                         ||j                               S )Nr   c                     t        |       } t        |       dk7  r"t        j                   |                | <    |       S r   )rD   ru   r7   indirect_indexing)r  ry   index_loaderr  r  s    r>   rO   zgather.<locals>.fn
  s@    3is8q=,,\#->S	JCH}r@   r   )rC   r.   rh   rJ   r  rv   ru   r  r   r+   r   r   )	rP   ry   r   sparse_gradr  rO   r  r  r  s	    `    @@@r>   gatherr  
  s     a###??+++::<DY!^F
3
'C}}H$$&L ||~kkm~~	 r@   c                   	
 |rJ t        | t              sJ t        |t              sJ dt        |j                               v sJ | j	                         	|j	                         t        |j                               | j                         
g |j                         
dd  	
fd}t        j                  | j                         | j                         |      S )Nrb   r   c                     t        |       t              k(  sJ |  d         | d        }t        j                  |d         gg | d  z   } |      S )Nz != r   )ru   r7   r  )r  	var_index
weight_idxindices_loaderindices_ndimr  weight_loaderweight_sizes      r>   rO   zembedding.<locals>.fn
  s{    3x3x=(@SEhZ*@@("3}#56	++I{1~FG K
K
 

 Z((r@   r   )
rC   r.   strrh   r   ru   rv   r+   r   r   )r0  r   padding_idxscale_grad_by_freqsparserO   r  r  r  r  r  s         @@@@@r>   	embeddingr  
  s    :fi(((gy)))C))+,,,,&&(M((*Nw'')*L//#K6!!#6k!"o6H) )   " 	 r@   c           
         t        d | D              s)J d| D cg c]  }||j                          c}        t        d | D              rt        d      t	        |       D cg c]  \  }}t        |t              s| }}}t        |      dkD  sJ d       d gt        |       z  }t        |t        |D cg c]  }| |   	 c}       D ](  \  }}|j                         |k7  rt        d      |||<   * ||fS c c}w c c}}w c c}w )Nc              3      K   | ]T  }|P|j                         t        j                  t        j                  t        j                  t        j
                  fv  V y wrB   )rh   rJ   r  r  ro   r  r   r   s     r>   r   z.check_and_broadcast_indices.<locals>.<genexpr>
  sA      A= 	
%++u{{EJJLLs   AAz)indices must be int64, byte or bool. Got c              3      K   | ]7  }||j                         t        j                  t        j                  fv  9 y wrB   )rh   rJ   ro   r  r  s     r>   r   z.check_and_broadcast_indices.<locals>.<genexpr>
  s0      <Cqq}%**ekk22Gs   ?5?zFallback for bool indicesr   z"requires at least 1 non-None indexz.Fallback when indices is on a different device)r   rh   r   r;   r   rC   r.   ru   r   r   r   )r   r   r   rP   
valid_idxsnew_indicess         r>   check_and_broadcast_indicesr  
  s8      g 
373d7aVWVcAKKM73d2ef	g 
  <C  ""=>> )' 2O 21jI6N! 2JOz?QD DD&3w<'KJ 1
3S
1GAJ
3S TU1 <<>V#%&VWWA V 
""! 4e P 4Ts   C=C='D D:Dc           	      D   dt        dd        D ]  \  }}	|	|z
  dk7  sd t              D 
cg c]  \  }
}|	| |
    c}}
g | t              t              z   d  d   }rz   nd | z   |d  z   f	d}|fS c c}}
w )NFr   Tr   c                   	 t        |       t              k(  sJ t              t        
      k(  sJ t              }g }d   }rdn|}d}t        d   dz         D ]|  }||k(  r||z  }|   *|t        |       k  sJ |j                  | |          |dz  }<|   }|J 
|   }|j                  t        j                   || |||z          |	             ~ g || |d  }|S  |      S )Nr   r   r   r   )ru   r   r   r7   r  )r  r  	new_indexfirst_tensor_indexstart_offsetnext_idxr   loaderr  r   indexed_sizer   indices_loadersnon_consecutive_tensorsoutput_sizetensor_indicestensor_sizer  s            r>   rO   z*index_output_size_and_inner_fn.<locals>.fn  sF   3x3{++++?#s<'8888;	+A.3q9K~b)A-.AL D qz!#c(***  X/A(+)))#A  ))s<,2EFG# /$

^
	 %,yE(92EEr@   )r   r   ru   )r  r   r  r  r  r  r  r   previouscurrentr   r`  r  rO   r  r  s    ```````      @@r>   index_output_size_and_inner_fnr  
  s    " $ 1CD'X"&*# E ,5W+=M+=C6!9+=MKSKS&[)9C<O)O)Q"RSK'*!K/ ++,,-./ 	F FB ?] Ns   
BBc           
         t        |t        t        f      sJ | j                         }t	        || j                               \  }}t        |      dkD  sJ d       |D cg c]  }||j                         nd  }}t        ||d      j                               }| j                         }t        t        |            D cg c]  }||   	||    }	}d|	v rd|vrt        d      t        t        |            D cg c]  }||   	 }	}t        ||||||	||      \  }
}t        j                  | j                         | j                         ||
      S c c}w c c}w c c}w )Nr   z Must have at least one valid idxz0index is out of bounds for dimension with size 0r  r   )rC   rD   rE   r   r  r   ru   rv   r   
IndexErrorr  r+   r   rh   )rP   r   r   r  r  r   r  r  r  r  r  r   s               r>   
index_implr  3  sm   ge}---}}H9'1<<>RG^~"F$FF"KRS7a!-q}}TA7OS w~a01::<=KZZ\F',S\':U':!gaj>TF1I':LULQk1KLL',S\':;':!F1I':L;:	K ||~kkm	 1 T V <s   E
EE>E!c                     	 t        | |d      S # t        $ r? | j                           t        t        j
                  j                  d      | |      cY S w xY w)NTr  Fr!  )r  r;   r  r  r1  r   r'  rP   r   s     r>   r   r   Y  sT    
!WD11 
			M

 1 1uMw
 	

s    AAAc                     t        | |d      S NFr  )r  r  s     r>   _unsafe_indexr  e  s    a..r@   c                 0    t        t        |       |||      S rB   )
index_put_r  rP   r   r   
accumulates       r>   	index_putr  r  s    eAh<<r@   c                 4    t        t        |       |||d      S r  )index_put_impl_r  r  s       r>   _unsafe_index_putr  w  s    58WfjNNr@   c                     |j                         | j                         k7  rt        || j                               }|rt        | |      }t        | t	        |d   ||             S r   )r   r6  rI   r  rG  )rS  r   r   r  s       r>   index_put_as_masked_fillr  |  sV    T__..%!23D% T5UD9::r@   c                 V   t        j                         }t        |      rq|s|rmdt        j                  _        |sdnd}t        j                  j                  j                  j                  dd       x}r| d| }|t        j                  _	        t        j                  | |||       | S )NTzindex put with accumulate.zdeterministic index put.stack_tracez Found from : 
 )rJ   $are_deterministic_algorithms_enabledr(   r8   r   disable_cudagraphsr  r  r  disable_cudagraphs_reasonr"   IndexPutFallback)rS  r   r   r  deterministicr=   r  s          r>   index_put_fallbackr    s    >>@MjM%)" ! )+ 	
 ''..3377tLL;LE*;-8C,/)gvz:Kr@   c                 "    t        | |||d      S )NTr  r  rS  r   r   r  s       r>   r  r    s    4&*DIIr@   c                 "    t        | |||d      S r  r  r  s       r>   _unsafe_index_put_r    s    4&*EJJr@   c                 d    | t         j                  t         j                  t         j                  hv S rB   )rJ   r  ro   bfloat16rd   s    r>   ,needs_fallback_due_to_atomic_add_limitationsr     s     U[[%**enn===r@   c           
         |j                         dk(  rt        |      dk(  r|d   j                         t        j                  t        j
                  hv r_|d   }t        t        |j                               t        | j                                     D ]  }t        |d      } t        | |g||      S t        j                         rt        | |||      S |D ]E  }||j                         t        j                  t        j
                  hv s7t        | |||      c S  | j                         }t        |      }	|rLt        | j                               r3|	dk(  rt        | dg      } t        | |||      } |	dk(  rt        | g       } | S t        || j                               }	 t        || j!                               \  }}
|D cg c]  }||j%                         nd  }}t'        | t(              sJ | j+                          |	dk(  rt        | dg      } t-        ||
d      j                               }t        t        |            D cg c]  }||   	 }}t/        |||
|||d |      \  }}t1        ||      }t3        j4                  | j!                         | j                         |j%                         |||rdnd       }t3        j6                  d t3        j8                  |       |      }t:        j<                  j?                  |      |_         |	dk(  rt        | g       } | S # t"        $ r t        | |||      cY S w xY wc c}w c c}w )Nr   r   r   r  
atomic_addr   r`   r   r   output_indexerscatter_mode)!	get_numelru   rh   rJ   ro   r  r   rv   r  r  r  r  r   r2  r   r  r   r;   r   rC   r.   r  rD   r  r   r"   Scatterr  MutationLayoutr8   r   r  r=  )rS  r   r   r  r   r  r~   r   r  x_ndimr  r   r  r  r  expected_vals_sizer   scatterr  s                      r>   r  r    s    	aLAAJ  "uzz5;;&??qzs4==?+S-ABAT2&D C'tffjII 113!$DD !2uzz5;;6O!O%dGVZHH  ]]_F[FB4>>CSTQ;qc?D!$DQ;b>Dfdnn./FE"=T__&#
 LSS7a!-q}}TA7OSdI&&&LLN {D1# w~a01::<=K',S\':;':!F1I':L;#A	$  F./F jj nn##%!%/\TG 
$F
 ''))&1FK{D"~K_  E!$DDE T <s   $L% MM%L?>L?)rq   r   c                 B  	
 d | ||fD        \  }}}| j                         |j                         k(  sJ |j                         t        j                  t        j                  hv sJ |j                         
|j                         	|j                         |j                         |j                         	
fd}t        j                  | j                         | j                         ||j                               }t        || j                               S )Nc              3   4   K   | ]  }t        |d         yw)r   N)r2  r   s     r>   r   z,masked_scatter_with_index.<locals>.<genexpr>  s     (VAUAaAUs   c                              }t        j                          t        j                        } 	fd}t        j                  ||d      }t        j
                  |||      S )Nc                  R           } t        j                  |       } |g      S rB   )r7   r  )source_idx_valr   r  source_idx_loadersource_loadersource_numels     r>   load_source_valzDmasked_scatter_with_index.<locals>.inner_fn.<locals>.load_source_val  s,    .s3N%%nlCA !%%r@   r   )r7   r   rJ   ro   r  rG  )
r  self_valmask_valr  
source_valmask_loaderself_loaderr  r  r  s
   `    r>   r   z+masked_scatter_with_index.<locals>.inner_fn  sR    s#<<C 0%**=	&
 ZZ/1=
yy:x88r@   r   )rv   rh   rJ   ro   r  r   r  r+   r   r   r2  )rS  r  
source_idxsource	self_flat	mask_flatsource_flatr   result_flatr  r  r  r  r  s            @@@@@r>   masked_scatter_with_indexr!    s     )W$fAU(V%Iy+==?dmmo--->>

EKK8888'')K'')K"..0++-M##%L
9 
9 "" nn!!#	K T]]_--r@   c                 P    t        |       }t        ||||      }t        ||       |S rB   )r  r  copy_)rS  r  r  r  r  r   output_views          r>   as_strided_scatterr%  "  s*    4[FVT6>BK	+sMr@   c                 2    t        t        |       |||fi |S rB   )scatter_r  )rP   ry   r   r  r   s        r>   r  r  *  s    E!Hc5#888r@   rN  include_selfrN  r)  c          	      l   | dk(  rdnd}|d |hvst        |t              rS|j                         j                  t	        j
                  d      j                  k(  rt        |j                               s| dk(  rU|dk(  rPt        |t              r@|j                         t	        j
                  d      k(  rt        j                  j                  sI||k(  r0|j                         t        j                  t        j                  hv st	        j                         rt        j                  | ||||||       |S y )Naten.scatter_rI   sumr   aten.scatter_reduce_r  r(  )rC   r.   r   r   rJ   r   r   rh   r    cppfallback_scatter_reduce_sumro   r  r  r"   ScatterFallback)rO   rS  ry   r   r  rN  r)  	reduce_tys           r>   scatter_fallbackr2  /  s     .EItY''sI& %%f)=)B)BB<S]]_M ((%3	* ELL$77

66iDNN$4U[[8Q$Q557
c5#f<	
 r@   rN  c                t    |dv sJ t        d| ||||      }|r|S |dk(  rd}n|dk(  rd}t        | ||||      S )N>   NrI   multiplyr+  r3  rI   r,  r5  prod)r2  scatter_reduce_)rS  ry   r   r  rN  fallback_results         r>   r'  r'  S  s_    ....&sE3vO 	:	4eS&99r@   c                 0    t        t        |       |||      S rB   )scatter_add_r  rP   ry   r   r  s       r>   scatter_addr<  f  s    a#uc22r@   c                      t        | |||d      S )Nr,  )r7  r;  s       r>   r:  r:  k  s    1c5#u55r@   c                 4    t        t        |       ||||fi |S rB   )r7  r  )rP   ry   r   r  reduction_typer   s         r>   scatter_reducer@  p  s    58S%nOOOr@   )r)  c          
      "    |dv sJ t        d |||      }|r|S t         t              sJ dt        |j	                               v sJ t         j                               }|dk(  rt         dg       t        t              r)t        j                               dk(  rt        dg      t        |t              r)t        |j                               dk(  rt        |dg      }t                j                          |j                         t        t              rj                         nd  fd} fd}	d	 }
|st        j                   j                          j	                          fd
|j                         |d       }t        j                  d t        j                         |      }t         j"                  j%                  |      |_        t        j                   j                          j	                         |	|j                         | |
|            }t        j                  d t        j                         |      }t         j"                  j%                  |      |_        |dk(  rt         g         S )N>   Nr,  amaxaminmeanr6  r-  r(  rb   r   r   c                     j                         }t        |      }t        |       }t        j                   |       |dk(  rdn|         |<   |S r  )rv   ru   rD   r7   r  )r  r  r"  indirect_idxry   r  rS  s       r>   r  z'scatter_reduce_.<locals>.output_indexer  sS    5zCy11DAIq5:
S r@   c                 `    r |       S t        j                  j                               S rB   r7   r  rh   )r  rS  r  r`  s    r>   rO   zscatter_reduce_.<locals>.fn  s*    c?" <<T^^%566r@   c                     | dk(  ry| J y )Nr,  r  r   r3  s    r>   backend_reduce_strz+scatter_reduce_.<locals>.backend_reduce_str  s    U? >!>r@   c                 L    t        j                  dj                               S r   rH  )r   rS  s    r>   r  z!scatter_reduce_.<locals>.<lambda>  s    3<<4>>3C#Dr@   r  )r2  rC   r.   r  rh   ru   rv   r2  r  r  r   r"   r  r   r  r  r8   r   r  r=  )rS  ry   r   r  rN  r)  r8  r"  r  rO   rJ  zero_outr  r  r  r`  s   `` `          @@r>   r7  r7  u  s0   BBBB&!O dI&&&C)****t}}DqyD1##y!c#,,.&9Q&>3n%#ENN,<(=(BUQC 
c
"CLLN$$&L&0i&@"dJ7 ::??$.."D>>#)
 ""d#

 gg--f5
 jj nn~~%'/G 
$F
 ''))&1FKqyD"~Kr@   scales_x.nexactc                 N   | j                          | j                         | j                          d  | j                         d   }D cg c]+  }t        j                  j
                  j                  |      - c}t        |      k(  sJ |}t        |      D cg c]
  \  }}||z   c}}t        |      D ]  \  }}	|	s	|	|<    fdfd}
t        j                  | j                         | j                         |
g ||      S c c}w c c}}w )Nc                    t        j                  | t        j                        } r8t        j                  | t        j
                  dt        j                              } t        j                  | t        j
                  |t        j                              } t        j                  | t        j                        } t        j                  | |d      S )N      ?Fr  )
r7   r  rJ   r  rI   r  r   r   r  r  )rP   scaler  rO  s      r>   scale_fnz$upsample_nearestnd.<locals>.scale_fn  s     NN1emm,3<<U]];<AGGAs||E5==9:LLEKK($$QE::r@   c                     |  d  }| d   } 
g |t        |	      D cg c]  \  }}} |||       c}}}      S c c}}}w rB   )r   )r  rP   r   r   rZ  r  i_sizesrN  rT  scalesr  s         r>   rO   zupsample_nearestnd.<locals>.fn  sc    H!HSaS#a:QR:QJAq$8Aq$':QRS
 	
Rs   Ar   )realize_hintr   rv   r8   r   r   r  ru   r   r   r+   r   r   rh   )rP   r  rM  rN  rO  batchr   o_sizesorS  rO   rV  rT  rW  r  s      ``      @@@@r>   upsample_nearestndr\    s    NN}}HjjlA23GJJL1"EBIJ'Qqww55a8'JGx=AG #GW 56 51a!e 56Fh'5F1I (	;
 
 ||~kkm!!!	 9 K
 7s   0D'D!rW  c                 "    t        | ||fd      S )Nr   rN  r\  rP   r  rW  s      r>   upsample_nearest1dra  	  s    avi1==r@   c                 $    t        | ||fdd      S )Nr   TrN  rO  r_  r`  s      r>   _upsample_nearest_exact1drd    s    avi1DIIr@   scales_hscales_wc                 $    t        | |||fd      S )Nr   r^  r_  rP   r  re  rf  s       r>   upsample_nearest2dri    s     ax.BaHHr@   c                 &    t        | |||fdd      S )Nr   Trc  r_  rh  s       r>   _upsample_nearest_exact2drk    s     ax.BatTTr@   scales_dc                 &    t        | ||||fd      S )NrV   r^  r_  rP   r  rl  re  rf  s        r>   upsample_nearest3dro  !  s     ax8.LPQRRr@   c                 (    t        | ||||fdd      S )NrV   Trc  r_  rn  s        r>   _upsample_nearest_exact3drq  ,  s#     	;8X6!4 r@   c                 ,     t         fd|D              S )Nc              3   J   K   | ]  }t        j                  |        y wrB   rt  )r   r   r`   s     r>   r   z$_create_constants.<locals>.<genexpr>:  s     6Aa'    #rE   )r`   r{   s   ` r>   _create_constantsrv  9  s    6666r@   align_cornersc                 v   | j                          | j                         | j                         \  }}|\  }}t        j                  j
                  j                        t        j                  j
                  j                        d dd}	d d d fdfd |	||       |	||      d f
d	}
t        j                  | j                         | j                         |
||t        j                  |      t        j                  |      g
      S )Nc                     | t        j                  t         j                        j                  kD  rt         j                  S t         j                  S rB   )rJ   r'  r  r  r  )maxvals    r>   get_int_dtypez1upsample_bicubic2d_default.<locals>.get_int_dtypeN  s1    EKK,000;;{{r@   c                 L    |r|dkD  r| dz
  |dz
  z  S dS |
|dkD  rd|z  S | |z  S r  r   )in_sizer$  rw  rS  s       r>   compute_scalez1upsample_bicubic2d_default.<locals>.compute_scaleS  sB    5=\GaKHqL1HqH % 1eai1u9WWxEWWr@   c                     t        j                  |t        j                        }t        j                  | t        j                        } |rt        j
                  | |      S t        j                  dt        j                        }| ||z   z  |z
  S )NrR  )r7   r  rJ   r  r  r   )rS  	dst_indexrw  dst_index_iehalfs        r>   compute_source_indexz8upsample_bicubic2d_default.<locals>.compute_source_indexY  sg    ~~i?UEMM2775,//<<U]]3DL4/0477r@   c                 r    t        |dz   |dz   dt        j                        \  }}}|| z  |z
  | z  | z  |z   S )Nr   rV   r   rd   rv  rJ   r  )rP   A_Ap2_Ap3_1s        r>   cubic_convolution1z6upsample_bicubic2d_default.<locals>.cubic_convolution1b  sA    *1q5!a%%--PdBq41$q(2--r@   c                     t        |d|z  d|z  d|z  t        j                        \  }}}}|| z  |z
  | z  |z   | z  |z
  S )NrW   rX   r[   rd   r  )rP   r  _A_4A_5A_8As         r>   cubic_convolution2z6upsample_bicubic2d_default.<locals>.cubic_convolution2f  sT    -q1ua!eQU%--
Cc a#"S(A-33r@   c                     d}t        j                  dt        j                        } 	t        j                  | |      |      } | |      }t        j
                  ||       } ||      } 	t        j                  ||      |      }||||fS )Ng            ?)r7   r  rJ   r  rI   sub)
r   r  r  c0c1x2c2c3r  r  s
           r>   get_cubic_upsample_coefficientszCupsample_bicubic2d_default.<locals>.get_cubic_upsample_coefficientsl  s~    \\#u}}-221%WWR^A&B3BBr@   c                 r     |      }| d   |d   z  | d   |d   z  z   | d   |d   z  z   | d   |d   z  z   S )Nr   r   r   rV   r   )xsr   csr  s      r>   cubic_interp1dz2upsample_bicubic2d_default.<locals>.cubic_interp1dw  sR    ,Q/!ur!u}r!ur!u},r!ur!u}<r!ur!u}LLr@   c                 V    t        j                  |t        j                  ||             S rB   )r7   maximumminimum)r  r  r  s      r>   clampz)upsample_bicubic2d_default.<locals>.clamp  s    {{3C 344r@   c                    

 | \  
}} |      }t        j                  |      }t        j                  ||       |      }t        j                  |      }t        j                  ||      }
fdt        j                  | dz               t        j                  | dz               t	        fddD              }t	        fddD              fdt	        fd|D              }	 |	|      S )Nc                    t        j                  dt        j                        }t        j                  	dz
  t        j                        }t        j                  
dz
  t        j                        }t        j                   | ||      	d      }t        j                   |||      
d      } ||g      S )Nr   r   Fr  )r7   r  rJ   r  r  )fyr  _0iHm1iWm1iyixcr  iHiWrN  r  s          r>   load_boundedz<upsample_bicubic2d_default.<locals>.fn.<locals>.load_bounded  s    a-B<<Q4D<<Q4D&&uRT':BeLB&&uRT':BeLBQ2rN++r@   r   c              3   J   K   | ]  }t        j                  |        y wrB   r7   rI   )r   ofsr  s     r>   r   z9upsample_bicubic2d_default.<locals>.fn.<locals>.<genexpr>       BMSC(Mrt  )r   r   r   r   c              3   J   K   | ]  }t        j                  |        y wrB   r  )r   r  r  s     r>   r   z9upsample_bicubic2d_default.<locals>.fn.<locals>.<genexpr>  r  rt  c                 B     t         fdD              } |      S )Nc              3   0   K   | ]  } |        y wrB   r   )r   rP   r  r   s     r>   r   zOupsample_bicubic2d_default.<locals>.fn.<locals>.get_x_interp.<locals>.<genexpr>  s     AA\!Q/s   ru  )r   coeffs_xr  ixs_ofsr  t_xs   ` r>   get_x_interpz<upsample_bicubic2d_default.<locals>.fn.<locals>.get_x_interp  s     AAAH!(C00r@   c              3   .   K   | ]  } |        y wrB   r   )r   r   r  s     r>   r   z9upsample_bicubic2d_default.<locals>.fn.<locals>.<genexpr>  s     :'Qa'r  )r7   rn  r  r   rE   )r  oyoxreal_xin_xreal_yin_yt_yiys_ofscoeffs_yr  r  r  r  r  r  rN  r  rw  r  r  r  r{  height_scaler  r  width_scaler  s             @@@@@@@@r>   rO   z&upsample_bicubic2d_default.<locals>.fn  s    1b"%k2}Eyy ggfd#%lBFyy ggfd#	, 	, \\$b1f 56\\$b1f 56BMBBBMBB	1 :'::h,,r@   r   rB   )rX  r   rv   r8   r   r   r  r+   r   r   rh   ri   r   )rP   r  rw  re  rf  NCoHoWr~  rO   r  r  r  r  r  r  r{  r  r  r  r  r  s     `        @@@@@@@@@@@@r>   upsample_bicubic2d_defaultr  =  s
    NN}}H::<LAq"bFB	
			/	/	3B	
			/	/	3B
X8.4	 M
 !RALBx@K5- -@ ||~kkm1emmB'r):;	 r@   c                 ,  	
 t        |      dk(  sJ |\  }	}| j                         | j                         ^ }
d 	
fd}t        j                  | j                         | j                         |g |	z   |z   
z   |z         S )NrW   c                    |}t        j                  |dz
  t        j                        }t        j                  | t        j                        } t        j                  | t        j                  |t        j                              } t        j                  |t        j
                  t        j                  |t        j
                  |                         } t        j                  | |d      S )Nr   Fr  )r7   r  rJ   r  r  absr  )rP   r  r  size_nums       r>   reflectz!reflection_pad2d.<locals>.reflect  s    ~~dQh4NN1ekk*GGAs~~fekk:;GGD#''#''$
";<=$$Q>>r@   c                 T    | ^ }}} |      } |      } 	g |||      S rB   r   )
r  r   rP   r   hleftr  toprF  r  s
       r>   rO   zreflection_pad2d.<locals>.fn  sA    AqAq#Aq$
!
Q

##r@   r   )ru   r   rv   r+   r   r   rh   )rP   r4  r  botrY  rO   r  r  r  r  rF  r  s         @@@@@@r>   reflection_pad2dr    s    w<1#D%c}}H::<LUAq?$ $ ||~kkm88C#8q4x%'78	 r@   c           	        	
 t        |      dk(  sJ |\  
|j                         ^ }	t        j                  j                  j                  	      dz
  	t        j                  j                  j                        dz
  | j                         | j                         ^ }}}	
fd}t        j                  | j                         | j                         |t        |j                                     S )NrW   r   c           	         | ^ }}fdd |z   |z   c|z
  |z
  }}dz  z   |z
  dz  z   |z
  }}dz   z   f}dz   z   f}t        j                   |       |            }	t        j                  |	fdd      d	fd	}
 |
|||df        |
|||z
  dz
  f        |
||df|        |
||z
  dz
  f|        |
|||df|df        |
|||df|z
  dz
  f        |
|||z
  dz
  f|df        |
|||z
  dz
  f|z
  dz
  f       S )
Nc                       g | |      S rB   r   )rP   r   r   grad_loaders     r>   load_from_outputz?reflection_pad2d_backward.<locals>.fn.<locals>.load_from_output  s    zzAzqz**r@   c                 b   | \  }}}t        j                  |t        j                        }t        j                  |t        j                        }t        j                  |t        j                        }t        j
                  t        j                  ||      t        j                  ||            S rB   )r7   r  rJ   r  r  r  r  le)index_ranger   lbubs       r>   index_range_conditionzDreflection_pad2d_backward.<locals>.fn.<locals>.index_range_condition  sn    #IAr2q%++.AEKK0BEKK0B88CFF1bM366!R=99r@   r   r   c                              S rB   r   )center_xcenter_yr  s   r>   r  z7reflection_pad2d_backward.<locals>.fn.<locals>.<lambda>  s    (88(Lr@   r  c                 2    |d   |d   k  }t        |t              r|ry  	|      }|:|d   |d   k  }t        |t              r|ry t        j                  | 	|            }t        j                  |
 fdd      }t        j
                  |      y )Nr   r   c                              S rB   r   )r  out_xout_ys   r>   r  zKreflection_pad2d_backward.<locals>.fn.<locals>.accumulate.<locals>.<lambda>  s    )9%)Gr@   r  )rC   ro   r7   r  r  rI   )r  r  index_range1index_range2upper_less_than_lower1r<   upper_less_than_lower2ggradr  r  s   ``      r>   r  z9reflection_pad2d_backward.<locals>.fn.<locals>.accumulate  s    
 &2!_|A%F"0$7<R(6D')5a<?)J&4d;@Vxx&;L&IJ

4!GMA774#Dr@   r   rB   )r7   r  r  )r  rP   r   top_reflect_xleft_reflect_ybot_reflect_xright_reflect_yrange_cxrange_cyr<   r  r   r  r  r  r  r  r  r  r  r  r  r  rF  s              @@@@@@r>   rO   z%reflection_pad2d_backward.<locals>.fn  s   Aq	+	:" Wa$h((+a~)*Qq!a%$,:J aS3/aTE!12xx!(+-B8-L
 zz$ LcR	$" 	8^X1d|D8_hAIq1u8MN=(Q3KB=(QCQ,?J=.1a+1d|L=/Aq#;AIqSTu@UV=.1a#gq1u2E1d|T?QCQ,?!QYPQTUPUAV	
 r@   r   )ru   rv   r8   r   r   r  r   r+   r   r   rh   rD   )grad_outputrP   r4  r~   h_gradw_gradrO   r  r  r  r  r  r  rF  s          @@@@@@@r>   reflection_pad2d_backwardr    s    w<1#D%czz|HQ1	..q1A5A	..q1A5A))+K$--/Q@ @D %%'##%AJJL!	 r@   c                     | j                         | j                         fd}t        j                  | j	                         | j                         |      S )Nc                     t        |       } t        |       t              k(  sJ D ]  }|   dz
  | |   z
  | |<     |       S rT  )rD   ru   )r  ry   rW  rO  r  s     r>   r  zrev.<locals>.loader#  sR    3i3x3u:%%%Cc
Q#c(2CH  }r@   r   )r   rv   r+   r   r   rh   )rP   rW  r  rO  r  s    ` @@r>   revr    sM     }}HJJLE ||~kkm	 r@   c                    t        |      dz  dk(  sJ t        d |D              rt        |       S | j                         }t	        t        t	        t        |d d d   |dd d                           t        |      t              z
  g D ]i  \  }}t        |t        j                        r5|j                  s)t        j                  j                  j                  |      n|}j                  ||f       k t	        |d        }g t        |d        D ]C  \  \  }}	}
j                  |
       |j                  t        j                   |
|z   |	z                E t        |      t        |      k(  sJ  t#        | j%                                     fdfd}| j'                         t)        j*                  | j-                         | j%                         ||      S )Nr   r   c              3   &   K   | ]	  }|d k(    ywrG  r   )r   r  s     r>   r   z"constant_pad_nd.<locals>.<genexpr>6  s     
#7a167r   r   c                 B    g }t         	d        D ]J  \  }\  }}}|dk7  r|j                  t        |d             |dk7  s0|j                  t        ||             L t	        j
                  t        j                  |      }t        j                  | 
fd      S )Nr   c                              S rB   r   )r   r  s   r>   r  z/constant_pad_nd.<locals>.mask.<locals>.<lambda>X  s	    r@   )	r   r   range_mask_lowrange_mask_highr   rN  r7   r  r  )r   r  r  r  r  rZ  boundsr  
mask_sizesrN  r  s   `     r>   r  zconstant_pad_nd.<locals>.maskP  s    (+E!"Ivz(J$C#tfaxN323qyOC89	 )K
 $/zz$ 7DDr@   c                     t        | d        }t        | d        D ]  \  }\  }}|j                  ||z
          t        |      t        |       k(  sJ  |      S rB   )rD   r   r   ru   )r   r  r  r  r  bounds_precompr  rN  s        r>   	offset_fnz"constant_pad_nd.<locals>.offset_fnZ  sd    rO	 #E!"I~ >C#tS3Y' !?9~U+++Ir@   r   )ru   r   r  rv   rD   r   r   rC   ri   r   	is_numberr8   r   r   lookup_precomputed_sizer   r   r   rh   r   r+   r   r   )rP   r4  r  rO  lr  	l_precompr  r  r  r  r  r   r  r  r  rN  r  s     `         @@@@@@r>   constant_pad_ndr	  3  s   L1"""

#7
##QxJJLE(4GCaCL'!$Q$- @ABCFE
S[ A 68N1 !UZZ( GG44Q7 	
 	y!n-  uRay/KJ qr3dT$5<<s
T(9:; 4 {s5z)))-q{{}-j9JE E }}H||~kkm	 r@   r   c                     t        j                  t        j                  | t        j                        t        j                  t        j                  |      t        j                              S rB   )r7   r  r  rJ   r  ri   r   )r   r  s     r>   r  r  j  s@    66q%++&u}}S)5;;7 r@   c                     t        j                  t        j                  | t        j                        t        j                  |t        j                              S rB   )r7   r  r  rJ   r  )r   r  s     r>   r  r  q  s7    66q%++&tU[[) r@   c                 V    t        j                  t        | |      t        | |            S rB   )r7   r  r  r  )r   r  r  s      r>   
range_maskr  x  s'    88q#4  r@   c           	          	
  j                         ^ }	 j                         
rd   ndrd   nd	 
f	d}|S )Nr   r   c                    	 | ^ t        j                  t        	z   	       t        
z   
             }rt        j                  |fd      S t        j                  |fd      S )Nc                  4     t              g        S rB   )constant_boundary_condition_2d)ihiwpad_fill_valueprefixrP   s   r>   r  z>constant_boundary_condition_2d.<locals>.load.<locals>.<lambda>  s'    I6q.I%f%b%"%r@   c                        g        S rB   r   )r  r  r  r  s   r>   r  z>constant_boundary_condition_2d.<locals>.load.<locals>.<lambda>  s    (3DV3DR3D3D*Er@   )r7   r  r  r  )r   r  r  r  r  r  r  r  r4  	padding_h	padding_wrF  rP   r  s     @@@r>   r   z,constant_boundary_condition_2d.<locals>.load  s}    Rxxr1y=9*5r1y=9*5
  JJ 
	
 D"EzR
	
r@   rv   r   )rP   r  r4  r  r~   r   r  r  r  rF  r  s   ````  @@@@@r>   r  r    sK    zz|HQ1}}H%
1I%
1I
 
& Kr@   c                 x   t        | d||   z  z   ||   dz
  z
  ||   dz
  z   ||         }|r
t        | d||   z  z   ||   dz
  z
  d||   dz
  z  z   ||         }t        j                  j                  j	                  |dz
  ||   z  | z
  ||   z
        dk\  r>|dz  }t        j                  j                  j                  d|||   z  | z
  ||   z
         t        j                  j                  j	                  ||z
        dk(  r0t        j                  j                  j                  ||       d}||fS |}||fS )Nr   r   r   F)r   r8   r   r   r|  r  r   )rP   r   kernel_sizer  r4  	ceil_modex_outx_alts           r>   pooling_sizer    sW   	A
Nk!nq01VAY]CVAYE GAJ+a.1"45VAY]8KKVTUY
 77%%uqyF1I&=&AGAJ&NOSTTQJEGG&&q%&)*;a*?'!**LM77%%eem49GG))%7I ) E)r@   r!  c                 8   dk(  rddg|dk(  rddg}st        d      t        d      t        d      t        |d      }t        | t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |      dk(  sJ t        | j	                               dv sJ | j                          | j	                         ^ }}t        |d|      \  }}	t        d|      \  }
}d   s	d   s|	s|rt        | t        d            n| j                         t        |      ||
gz   }d   d   z  }|dkD  st        d |D              rt        | ||      S fd}t        j                  | j                         | j!                         t#        j$                  |d	
      |      }t        j                  | j                         t&        j(                  t#        j$                  |d
      |      }||fS )Nr   r   r   rV   rW   z-inf   c              3   &   K   | ]	  }|d k7    ywrL  r   r   rY  s     r>   r   z*max_pool2d_with_indices.<locals>.<genexpr>  s     8x!qAvxr   c                    | ^ }}}d }d }t        j                  t        d         t        d               D ]  \  }}|d   z  |z   d   z
  }|d   z  |z   d   z
  } g |||      }	|rZt        j                  |z  |z   t
        j                        }
||
}n+t        j                  t        j                  |	|      |
|      }||	}t        j                  |	|      } |r|S |S r  )
r   productr   r7   r  rJ   r  rG  gtr  )r  return_indexr  bhbwrz  maxindexr  r  r`  r   r  r4  r  rF  r  s              r>   rO   z#max_pool2d_with_indices.<locals>.fn  s   R''k!n(=u[QR^?TUFBfQi"$wqz1BfQi"$wqz1B,V,R,,-CrAv{EKK@#$H"yyV)<eXNH~S&1 V OMr@   F)r(  r   T)r5   rC   r.   ru   rv   rX  r  r  r   r   rD   r    fallback_max_pool2d_with_indicesr+   r   r   rh   r   r   rJ   r  )rP   r  r  r4  r5  r  rY  r  h_out
ceil_mode1w_out
ceil_mode2r  window_sizerO   r1r2rF  r  s    ```             @@r>   max_pool2d_with_indicesr4    s    !|a&1}q6{A.K&!$F7A&GHa(Ha###{q   v;!w<1x=Aqzz|&&&NN::<LUAq$Q;SE:$Q;SE:qzWQZ:1!U6]C==?E{eU^+Ha.;q>1KR38x88/{FGXy
 	
 . 
		||~kkm""2E:	
B 
		||~kk""2D9	
B r6Mr@   c                 b   dk(  rddg|dk(  rddg}st        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |      dk(  sJ t        |j                               dv sJ | j	                          	 | j                         }t        |t              rt        |j                  j                  t              r|j                  j                  }	t        j                  d t        j                  |	j                         |	j                         |	j                               |	      }
|
j                          |
j                         }n	 |j                         }|d uxr |d   dk(  xs |d uxr |d   dk(  }t        j                   xs" t        j"                  xs t        j$                  }t'        d |D              s|r|st)        | ||||      S |j	                          |j                         ^ }}| j                         ^ }|j+                         | j+                         t-        |j                               }t/        t1        d   dz        D cg c]-  }t/        |d   z  t/        d|d   z
  d   z        z
  d      / c}      t/        t1        d   dz        D cg c]-  }t/        |d   z  t/        d|d   z
  d   z        z
  d      / c}      z  }|dkD  rt)        | ||||      S |j                         fd	}t        j2                  | j                         | j                         ||
      S # t        $ r d }Y w xY w# t        $ r d }Y w xY wc c}w c c}w )Nr   r   r   r!  )r   r`   r  )r=  rm  r   c              3   &   K   | ]	  }|d k7    ywrL  r   r$  s     r>   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>C  s     
$8a168r   r"  c                    | ^ }}}t        j                  |z  |z   t        j                        }|d   z   }|d   z   }t        j                  t	        |d   z
  d   z   d         t        j                        }t        j                  t	        |d   z
  d   z   d         t        j                        }t        j                  t	        |d         dz   t        j                        }t        j                  t	        |d         dz   t        j                        }t        j
                  |t        j                  dt        j                              }t        j
                  |t        j                  dt        j                              }t        j                  |t        j                  t        j                              }t        j                  |t        j                  t        j                              }d }	t              D ]3  }
t              D ]!  }t        j                  |t        j                  |
t        j                              }t        j                  |t        j                  |t        j                              }g |t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d   d      t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d   d      } |      } |      }t        j                  ||      }|	;t        j                  ||t        j                  dt        j                              }	t        j                  t        j                  t        j                   ||      t        j                   ||            |      }t        j                  |t        j                  |	|      |	      }	$ 6 |	J |	S )Nr   r   Fr  r   r  )r7   r  rJ   r  r   r  r  r  r   rI   r  r  r_  rG  r  r  r  )r  r  r  rF  
index_testphstartpwstartphendpwendgradientph_pw_phpw
grad_indexindex_actual	grad_partr   r  r  h_window_sizer  indices_sizer  r4  pooled_heightpooled_widthr  w_window_sizewidths                      r>   rO   z,max_pool2d_with_indices_backward.<locals>.fni  s3   A^^AIM5;;?

N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM'C]+WWWcll3&DEWWWcll3&DE))Bs||Au{{7S(TU$R(# ))Bs||Au{{7S(TU$R(#
  .j9'
3	|Z8#"yyy#,,sEMM*J H 88FF2u-FF2u- D  #yyswwx/KXVHE , (H ###r@   r   )rC   r.   ru   rv   rX  rH  AttributeErrorr   r+   r"   r  r  r   rh   decide_layoutr    coordinate_descent_tuningmax_autotunemax_autotune_pointwiser   )fallback_max_pool2d_with_indices_backwardr   rD   r  r   r   )r  rP   r  r  r4  r5  r  r   	gO_strider   x_bufferx_strideis_channels_lastautotunerY  heightr~   r  r  rF  r1  rO   r  rF  r  rG  rH  rI  rJ  rK  s     ```                 @@@@@@@@r>    max_pool2d_with_indices_backwardrX    s    !|a&1}q6a###{q   v;!w<1x=Aqzz|&&& **,	
 !YJqvv{{I$Fvv{{$$$$(nn&]]_
 
 	 &&(	||~H !,A!1A 3)A,!"3  	(( 	)	)(( 
 
$8
$$)9(8K(Iw
 	
 JJLUFE&1&:&:&<#Q|((*N))+KAJJL!H ;q>A-.	
. VAYQ[^);q	(I!JJAN.	
M  ;q>A-.	
. VAYQ[^);q	(I!JJAN.	
M  -/KR8K(Iw
 	
 ##%L9 9v %%'##%	 A   	(  	H	6	
	
s0   "N 6N 	2N'2N,NNN$#N$c                 Z    | j                         ^ }}}| j                         fd}|S )Nc           
         	
 |\  	|\  
|\  }}t        j                  t        j                  t        j                  z   t        j
                        t        j                  |t        j
                              t        j                  t        j                  
	z   t        j
                        t        j                  |t        j
                                    }t        j                  |	 
fdd      S )Nc                  ,     g  z   z         S rB   r   )h_start_indexr  r  r  w_start_indexr  s   r>   r  z3pad_adaptive_loader.<locals>.load.<locals>.<lambda>  s$    HNvN}r'9N=2;MNOr@   r  )r7   r  r  r  rJ   r  r  )r  
incrementsstart_indicesend_indicesh_end_indexw_end_indexr  r\  r  r  r]  r  s   `      @@@@r>   r   z!pad_adaptive_loader.<locals>.load  s    B'4$}#. [xxFF}r15;;?{EKK8 FF}r15;;?{EKK8	
 zzOO
 	
r@   r  )rP   r~   r  rF  r   r  s        @r>   pad_adaptive_loaderrc    s+    zz|HQ1}}H
, Kr@   c                 6     |\  |\   fd}|S )Nc                    | ^ }}} |      } |      } |      } |      }d }	t        j                  t        d         t        d               D ]1  \  }
} |||
|g||g||g      }|	|}	t        j                  ||	      }	3 |	S r  r   r&  r   r7   rI   )r  r  r  r)  r*  r\  ra  r]  rb  totalr  r  r`  h_end_index_fnh_start_index_fnkernel_maxesw_end_index_fnw_start_index_fns                r>   fn_sumz)_adaptive_pooling_idx_sum.<locals>.fn_sum  s    R(,$R((,$R(''l1o(>lSTo@VWFBR.k*	C }U+ X r@   r   )rj  start_index_fnsend_index_fnsrm  rh  ri  rk  rl  s   `   @@@@r>   _adaptive_pooling_idx_sumrp    s*    )8&&%2"NN . Mr@   c                 ^    t         t              sJ t        |      dk(  sJ  j                           j	                         ^ }}}t
        j                  j                  j                  |      }t
        j                  j                  j                  |      }|\  }}||k(  r||k(  rt               S |dk(  s|dk(  r2g |||}t        | j                          j                               S ||z  dk(  r||z  dk(  r||z  ||z  g}t         |      S t        ||z   dz
  |      }	t        ||z   dz
  |      }
t        |      ||gz   } j                         }d }d }t!        j"                  |||      }t!        j"                  |||      }t!        j"                  |||      }t!        j"                  |||      }|	|
z  }|dkD  rt%         |      S t'        |	|
g||g||g      t)        t+                      fd	}t-        j.                   j                         |||
      }|S )Nr   r   r  r   c                      t        | |z  |      S rB   r   r   out_diminp_dims      r>   start_indexz)_adaptive_avg_pool2d.<locals>.start_index
  s    733r@   c                 2    t        | dz   |z  |z   dz
  |      S rT  rs  rt  s      r>   	end_indexz'_adaptive_avg_pool2d.<locals>.end_index  s"    g-7!;WEEr@   ru  rv  r"  c                 ^    t        j                   | t                     |             S rB   )r7   truedivrc  )r  rm  ones_loaderrP   s    r>   rO   z _adaptive_avg_pool2d.<locals>.fn#  s-    {{3+A./[1I
 	
r@   r   )rC   r.   ru   rX  rv   r8   r   r   r  r  r  rh   r   
avg_pool2dr1   rD   r   r   fallback_adaptive_avg_pool2drp  rc  	ones_liker+   r   )rP   r  rY  h_inw_inr-  r/  o_sizer  h_kernel_maxw_kernel_maxr  r`   rw  ry  r\  ra  r]  rb  r1  rO   rvrm  r}  s   `                     @@r>   _adaptive_avg_pool2dr    s.   a###{q   NNUD$7711$7D7711$7DLE5 u}QxzUaZ'5'%''V1;;=HHe|qTE\Q.u}dem4![))D5L1,u5LD5L1,u5LE{eU^+HKKME4F %%k5$OM##IudKK%%k5$OM##IudKK-KR+A{;;&	|$	&	k"F &il3K

 
		||~	
B Ir@   c                      j                           j                         ^ }}}t        j                  j                  j                  |      }t        j                  j                  j                  |      }|^ }}}	||z  dk(  r||	z  dk(  rt         ||z  ||	z  gd      S t        ||      }
t        ||	      }d fd}t        j                  ||      }t        j                  |||      }t        j                  |	|      }t        j                  ||	|      }t        |
|g||g||g       fd}t        j                   j                          j                         |t        |            }|S )	Nr   r   )divisor_overridec                      t        | |z  |      S rB   )r   rt  s      r>   rw  z0upsample_nearest2d_backward.<locals>.start_indexD  s    uw00r@   c                      | dz   ||      S rT  r   )r   ru  rv  rw  s      r>   ry  z.upsample_nearest2d_backward.<locals>.end_indexG  s    EAI99r@   rz  c                 (     | t                    S rB   )rc  )r  rm  rP   s    r>   rO   z'upsample_nearest2d_backward.<locals>.fnV  s    c.q122r@   r   )rX  rv   r8   r   r   r  r~  r1   r   r   rp  r+   r   r   rh   rD   )rP   r  
input_sizere  rf  rY  inp_hinp_wout_hout_wr  r  ry  r\  ra  r]  rb  rO   r  rm  rw  s   `                  @@r>   upsample_nearest2d_backwardr  2  sf    NN::<UE5GG2259EGG2259E%UE5u}eemq0!euneun=PQRR5%(L5%(L1: %%k5%PM##IueLK%%k5%PM##IueLK&	|$	&	k"F3 
		||~kkmJ	
B Ir@   c           	         ssddgt        d      t        d      t        d      t        | t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        | j	                               dv sJ | j                          | j	                         ^ }}}	t        |d|      \  }
}t        |	d|      \  }}d   s	d   s|s|rt        | d      d}n| j                         d}t        |      |
|gz   }| j                         d   d   z  }|dkD  rt        | |||      S fd	|r|r|rd|z  nd
d   d   z  z  fd}n!t        t        |       d|rnd       fd}t        j                  | j                         ||      }|S )Nr   r   r!  r   r  TFr"  c                    | ^ }}}d }t        j                  t        	d         t        	d               D ]P  \  }}|d   z  |z   
d   z
  }|d   z  |z   
d   z
  } |g |||      }||};t        j                  ||      }R |S r  rf  )r  r  r  r)  r*  rg  r  r  r`  r  r4  r  s            r>   rm  zavg_pool2d.<locals>.fn_sum  s    R''k!n(=u[QR^?TUFBfQi"$wqz1BfQi"$wqz1B*6*2*r*+C}U+ V r@   r  c                 f    t        j                   |       t        j                              S rB   )r7   r   r  )r  r`   rm  rS  r  s    r>   rO   zavg_pool2d.<locals>.fn  s&    776#x0#,,ue2LMMr@   c                 L    t        j                   |        |             S rB   r7   r|  )r  rm  r}  r  s    r>   rO   zavg_pool2d.<locals>.fn  s"    ;;vc84fS+6NOOr@   r   )r5   rC   r.   ru   rv   rX  r  r  r   rD   rh   fallback_avg_pool2dr  r+   r   r   )rP   r  r  r4  r  count_include_padr  rY  r  rF  r-  r.  r/  r0  had_paddingr  r1  rO   r  r`   rm  r}  rS  r  s    ```               @@@@@r>   r~  r~  h  s     a&{A.K&!$F7A&Ga###{q   v;!w<1qzz|&&&NN::<LUAq$Q;SE:$Q;SE:qzWQZ:1!S9==?E{eU^+HKKMEa.;q>1KR"
 	
 *((E;q>KN:;E	N 	N 5aL#*;w
	P 
		||~	
B Ir@   c                    dk7  sJ d       ssddgt        | t              sJ t        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |j                               dv sJ | j	                          |j                         ^ }t        d|      \  }	}
t        d|      \  }}| j                         d   xs d   xs |
xs || j                         ^ }t        |j                               }|j                         }t        t        d   dz        D cg c]-  }t        |d   z  t        d|d   z
  d   z        z
  d      / c}      t        t        d   dz        D cg c]-  }t        |d   z  t        d|d   z
  d   z        z
  d      / c}      z  }|dkD  rt        | ||      S fdfd}t        j                  | j                         |||	      }|S c c}w c c}w )
Nr   zdivisor must be not zeror   r!  r   r"  c           	      @   t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  t        j
                  | |      |      }t        j                  t        j
                  ||      |      }	t        j                  t        j                  ||      t        j                  t        j                  t        j                        |            }
t        j                  t        j                  |	|      t        j                  t        j                  t        j                        |            }t        j                  |t        j                  dt        j                              }t        j                  |	t        j                  dt        j                              }	t        j                  |
t        j                  t        j                              }
t        j                  |t        j                  t        j                              }t        j
                  t        j                  |
|      t        j                  ||	            }|S )z{
        This computes the scaling factor that we will divide an element
        by when `count_include_pad=False`
        r   r   )
r7   r  rJ   r  r  r   r  rI   r  r  )rA  rB  stride_hstride_wpad_hpad_wkernel_hkernel_whstartwstarthendwenddivide_factorrW  r  r4  r  rK  s                r>   !compute_pool_size_without_paddingz>avg_pool2d_backward.<locals>.compute_pool_size_without_padding  s   
 <<q	5;;7<<q	5;;7WQZ5WQZ5<<A<<<A<X.6X.6{{GGFH%GGCNN65;;7?
 {{GGFH%GGCNN5%++6>
 VS\\!U[[%ABVS\\!U[[%AB{{4!DE{{4u{{!CDf 5swwtV7LMr@   c                 8   | ^ }}}|d   z   }|d   z   }t        j                  t        |d   z
  d   z   d         t        j                        }t        j                  t        |d   z
  d   z   d         t        j                        }t        j                  t        |d         dz   t        j                        }t        j                  t        |d         dz   t        j                        }t        j
                  |t        j                  dt        j                              }t        j
                  |t        j                  dt        j                              }t        j                  |t        j                  t        j                              }t        j                  |t        j                  t        j                              }d }t              D ]+  }	t              D ]  }
t        j                  |t        j                  |	t        j                              }t        j                  |t        j                  |
t        j                              }}nssd   d   z  }n	 ||      }t        j                   g |t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d      t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d            |      }t        j                  t        j                  ||      t        j                  ||            }|;t        j                  ||t        j                  dt        j                               }t        j                  |t        j                  ||      |      } . |J |S )Nr   r   Fr  r  )r7   r  r   rJ   r  r  r  r  r   rI   r|  r  r  r  r  rG  r  )r  r  r  rF  r:  r;  r<  r=  r>  r?  r@  rA  rB  rS  partr  r  r  r  r  rF  r  r  r4  rH  rI  r  rJ  s                   r>   rO   zavg_pool2d_backward.<locals>.fn)  s$   A
N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM'C]+WWWcll3&DEWWWcll3&DE#/,E&k'N[^;E=b"EE{{#11 #$&s||Au{{7S(T!" !.&+  11 #$&s||Au{{7S(T!" !-&+& ). xxFF2u%FF2u% #"yytS\\#u}}5UVH"yyswwx/FQHS , (V ###r@   r   )rC   r.   ru   rv   rX  r  r   rD   rh   r  r   fallback_avg_pool2d_backwardr+   r   r   )r  rP   r  r  r4  r  r  r  rY  r-  r.  r/  r0  r~   r  r`   r  rF  r1  rO   r  r  r  rF  r  rW  rH  rI  rJ  rK  s     ``` ``             @@@@@@@@@r>   avg_pool2d_backwardr    s    #'71'<X>XX<a&k9---a###{q   v;!w<1qzz|&&&JJLUFE$VQVWiXE:$UA{FGYWE:))+K!*F
FjFJK&1&:&:&<#Q|AJJL!HKKME ;q>A-.	
. VAYQ[^);q	(I!JJAN.	
M  ;q>A-.	
. VAYQ[^);q	(I!JJAN.	
M  -/KR+	
 		
 8? ?B 
		%%'	
B I{	
	
s   2I,2Ic                    | j                         }t        |t              r|g}n|st        t	        |            }t	        |      dk(  rt        |      dv s
J d|        g S t        |      }t        t	        |            D ]_  }||   dk  r#||xx   t	        |      rt	        |      ndz  cc<   d||   cxk  rt	        |      k  rFn t	        |      dk(  r	||   dk(  r_J  t	        t        |            t	        |      k(  sJ d       |S )Nr   )r   r   r  zinvalid axis: r   zreduction axis not unique)rv   rC   rb   r   ru   rE   rD   rF   )rP   axisr  r   s       r>   _validate_reduction_axisr  s  s    ::<D$vSY
4yA~T{//H>$1HH/	:D3t97Q;GCIs4y14GDG'c$i'CINtAw!|LL  s4y>SY&C(CC&Kr@   c                `  
 |t        | |      } | j                         t        t        | |            }g }g g }g t	        t                    D ]Q  }||v r&j                  |       |j                  |          -j                  |       |j                  |          S 
fd}r+t              }	D ]  }t        j                  d      |	|<    n|}	| j                         
t        | j                         |xs | j                         | j                         ||	|      S )Nc                    t        |      t        	      k(  sJ r+t        |       t        
      k(  sJ D cg c]  }| |   	 } }t        |       t              k(  sJ d gt        |       t        |      z   z  }t        j                  t        |       t        	|            D ]
  \  }}|||<     |      S c c}w rB   )ru   r   chainr   )r   reduction_indexr   r  r  varinner_loaderkeepdimskept_idxreduced_idxr  s         r>   r  z%_make_reduction_inner.<locals>.loader  s    ?#s;'7777u:T***'/0x!U1XxE05zS]***Fc%j3+??@	!% #k?"C
HC !IcN
 I&& 1s   B;r   )r   	dst_dtyper  r   r   reduction_ranges)r   rv   rF   r  r   ru   r   rD   ri   r   r   dictr   rh   )rP   r  r  r`   r   
kept_sizesreduced_sizesr   r  r  r  r  r  r  s     `       @@@@r>   _make_reduction_innerr    s   Q::<D'401DJHMK3t99q!  a)OOAd1g& ' ' :A--*HQK  ==?L||~'81;;=++-& r@   r?  c                      dd d fd}|S )Nrd   c                    t        | |||      }t        j                  d| d|}t        |j                  j                  t              r|j                          |S )Nr  r  r`   r   )r?  
input_noder   )r  r,   r   rC   r   r  )rP   r  r  r`   r   r  r   r?  s         r>   r   zmake_reduction.<locals>.inner  sa    &"7
 !!XAXQWXKKi
 NNr@   NFr   )r?  r   r   s   `` r>   make_reductionr    s    T  Lr@   rd   c                   |t        | |      } | j                         t        | |      }| j                         }|t        j
                  t        j                  fv rt        | t        j                        } t        | ||      }t        fd|D              }t        j                  || j                         | j                               }t        j                  |t        |j                                     }t        t!        ||      |      S )Nc              3   (   K   | ]	  }|     y wrB   r   r   r   r  s     r>   r   zmean.<locals>.<genexpr>       04a$q'4   )r   rv   r  rh   rJ   float16r  r   sum_r6   r"   r'   r   r&   r   rD   div)rP   r  keepdimr`   rn  
sum_resultdenomr  s          @r>   rD  rD    s    Q::<D#At,D;;=Lu~~66Q$aw'J0400Eq{{}allnEEeT**=*=*?%@AEC
E*L99r@   c                 .  
 |d}| j                         
t        | |      }t        | |d      }|r|j                          t	        t        | |            }t        |||      }t        
fd|D              }|rt        j                  ||z
  d      }t        j                  || j                         | j                               }t        j                  |t!        |j                                     }t#        ||      }	|s|	S |r|nt%        ||      }|	|fS )Nr   T)r  c              3   (   K   | ]	  }|     y wrB   r   r  s     r>   r   z var_mean_sum_.<locals>.<genexpr>  r  r  r   )rv   r  rD  r  squarer  r  r6   ri   Maxr"   r'   rh   r   r&   r   rD   r  r[  )rP   r  
correctionr  return_meanx_meandiffsr  r  x_varr  s             @r>   var_mean_sum_r    s    
::<D#At,D!T4(F3q&>"EeT7+J0400E		%*,a0q{{}allnEEeT**=*=*?%@AE
E"EVGFD$9F&=r@   c                     t        | |      }t        | ||d d       }|d   }t        |d         }t        |t        j
                        xr, t        |      t        j                  k  xr t        |      dk7  S )Nr  r   r  r   )	r  r  r6   rC   ri   r   rb   r    unroll_reductions_threshold)rP   r  r  r   r   reduction_numels         r>   use_two_step_variancer    s|    #At,D"	wd$F HF#F+=$>?O?EMM2 	' 6#E#EE	'&!Q&r@   c                   dt        | ||d d       }|j                  d      }|j                  d       |j                  d       t        j                  j                  d|fd| j                         d|\  }}}	|j                          | j                         | j                         t        | |      }t        fd|D              d	 fd
}
 t        |
      |      }|r|j                          ||fS |S )Nr   r  r   r  r  welford_reduce)	inner_fnsr?  r`   c              3   (   K   | ]	  }|     y wrB   r   r  s     r>   r   z$var_mean_welford_.<locals>.<genexpr>  s     1Dq47Dr  c                     t        | t        j                        rD| j                  s8t	        j
                  t	        j                  | t        j                        |      S t	        j                  | |      S rB   )
rC   ri   r   r  r7   r   r  rJ   r  r  r   s     r>   get_constant_or_index_exprz5var_mean_welford_.<locals>.get_constant_or_index_expr  sG    a$Q[[<<q%++ >FF||Au%%r@   c                            }       }t        j                  d      }| t        j                  |||z
        z  S r   )r7   r  r  )r   r  r  zeror  r`   r  rnumels       r>   rT  z#var_mean_welford_.<locals>.scale_fn!  sE    &z59&vu5||Au%ckk$A...r@   r   )r  r  r"   WelfordReductionr   rh   r  rv   r  r6   r   )rP   r  r  r  r  r   r  rD  m2r~   rT  r  r`   r  r  r  s     `         @@@@r>   var_mean_welford_r    s    
"	wd$F ZZ
#F
JJ{
JJ{%%,, )'kkm 	KD"a JJLKKME::<D#At,D1D11F&
/ #.
"2
&CDyJr@   )r  r  c                ^    t        | ||      rt        | |||d      S t        | |||d      S )Nr  r  Fr  r  r  r  r  r  r  rP   r  r  r  s       r>   var_r  /  s@    QT7;DZe
 	
 	W% r@   c                ^    t        | ||      rt        | |||d      S t        | |||d      S )Nr  Tr  r  r  s       r>   var_meanr  ;  s@    QT7;DZd
 	
 	W$ r@   c                    |dk  r!t        t        j                  |       | |      S |dk(  rt        j                  d|      S |dk(  r| S t        | |dz  |      }t        j                  ||      }|dz  dk(  rt        j                  ||       }|S )Nr   r   r   )pow_recursiver7   
reciprocalr  r   )rP   r   r`   r  s       r>   r  r  G  s    1uS^^A.E::Av||Au%%Av1a1fe,FWWVV$F	A!|#Mr@   c                 .    t        j                  | |      S rB   )r7   powr   r   s     r>   
pow_nativer  V  s    771a=r@   )r   c                 v    t        t              r#t              k(  rt         t                    S t        t              rdk(  rt	               S t        t              rdk(  rt               S t        d  fD              }t        |      }t        t              xr dcxk  xr dk  nc xs	 |xr dk\  }|rZ j                          fd}t        j                   j                          j                         | j                               S t         t              r: dk(  rt        d      S  d	k(  r$t!        j                               rt#              S |rDt         t              rt%               S t        t              rt'               S t)               S t+               S )
NrR  r   c              3   r   K   | ]/  }t        |t        j                        s|j                          1 y wrB   )rC   r"   r.   rh   r   s     r>   r   zpow.<locals>.<genexpr>n  s$     N1*Q2Ms   77i    r   c                 F    t         |       j                               S rB   )r  rh   )r  r   r   r  s    r>   rO   zpow.<locals>.fnx  s     a??r@   r   r   )rC   r   rb   r  sqrtr  r   r   r   r+   r   r   rh   rv   r   re  r   exp2fallback_pow_scalarfallback_pow_tensor_scalarfallback_pow_tensor_tensorr  )r   r   r`   is_integer_powembed_exponentrO   r  s   ``    @r>   r  r  d  sx   !USV1c!f~	Au	!s(Aw	As	QQx NANNE%e,N  3' a"32AF  	@ <<>++-::<	
 	
 !V6Q?"6nQ[[]37Na &q!,,6"-a33-a33ar@   c                    t        | t              r| j                  }n| }t        |t              r|j                  }t        |t        j                        swt        j                  | j                         | j                         |j                         | j                               j                  }t        |t        j                        sJ t        |t        j                        rW|j                         sGt        |j                  t        j                        s#|j                          |j                  |_        | S t        j                  j                  |||       | S )Nr   unsafe_alias)rC   r.   r   r"   r  r+   r   r   rh   r   rv   rM  	NopKernelr  r  realize_into)changedr`  r  changed_datas       r>   r  r    s   '9%||#y!hhc2==)%%'##%__&##%	

 $ 	 #r}}---,.$$&*\5F5F*U 	HH""3<"PNr@   c                 .    t        | t        | |            S rB   )r  re  )rP   r  s     r>   fill_r    s    Q	!Z011r@   c                     t        || j                               }t        || j                               }t	        || j                               }t        | |      S rB   )r6  r   r   rh   r   rv   r  )dstr  rT  s      r>   r#  r#    sF    
C)
*C
3
(C
clln
%CS#r@   c                 .    t        j                  | |      S rB   )r7   floordivr  s     r>   r  r        <<1r@   c                 .    t        j                  | |      S rB   )r7   truncdivr  s     r>   r  r    r  r@   c                 >   t        |       xr t        |      }t        |       xr t        |      }|dk(  r,|rJ d       |rt        | |      S t        t	        | |            S |dk(  r,|rJ d       |rt        | |      S t        t	        | |            S t	        | |      S )Nrn  z5floordiv operands can not be boolean at the same timers  z5truncdiv operands can not be boolean at the same time)rm   rp   r  rn  r  r  rs  )r   r   rounding_modeboth_integerboth_booleans        r>   div_moder    s    "1%</!*<L"1%</!*<L X!XX!-x1~C5Q3CCX!XX!-x1~C5Q3CCq!9r@   c                     t        |       xr t        |      }|rt        | |      S t        t        j                  j
                        } t        |      | |      S rB   )rp   logical_andr)   r1  r   r;  r   )r   r   	both_boolrO   s       r>   r   r     sN    "9q'9I1a  **+!~b!!Q''r@   c                 p    t        d | |fD              }|rt        | |      S d } t        |      | |      S )Nc              3   L   K   | ]  }t        |      xs t        |        y wrB   )rp   rm   r   s     r>   r   zdiv_prim.<locals>.<genexpr>  s#     O1oa(>OA,>>s   "$c                  &    t        j                  |  S rB   r  rH  s    r>   rO   zdiv_prim.<locals>.fn  s    {{D!!r@   )r   r  r   r   r   is_integralrO   s       r>   div_primr"    s@    OAOOK1~" >"a##r@   c                 h    t        |       xs t        |       }|rd }nd } t        |      | |      S )Nc                 .    t        j                  | |      S rB   )r7   modr  s     r>   rO   zfmod.<locals>.fn  s    771a= r@   c                 .    t        j                  | |      S rB   )r7   fmodr  s     r>   rO   zfmod.<locals>.fn  s    88Aq>!r@   )rp   rm   r   r   s       r>   r'  r'    s9    !!$:(:K	!
	" >"a##r@   c                     | j                         }t        |      st        |      rt        | t	        j
                               } d } t        |      |       S )Nc                 ,    t        j                  |       S rB   )r7   rsqrtrl   s    r>   _rsqrtzrsqrt.<locals>._rsqrt  s    yy|r@   )rh   r   r   r   rJ   rz  r   )rP   r`   r+  s      r>   r*  r*  
  sK    KKME"25"9Q//12 ">&!!$$r@   c                    t        | j                               st        | j                               r|t        j                  }t        d|      } || |||      S )Nr,  r  rd   r   rh   r   rJ   r  r  rP   r  r  r`   rO   s        r>   r  r    sK     	'+;AKKM+J
-	U	;Baxu--r@   c                    t        | j                               st        | j                               r|t        j                  }t        d|      } || |||      S )Nr6  r  rd   r-  r.  s        r>   r6  r6  !  sK     	'+;AKKM+J
-	e	<Baxu--r@   c                 ^    t        | t        j                        }  t        d      | ||      S )Nr   r  r  )r   rJ   ro   r  rP   ry   r  s      r>   
reduce_anyr3  ,  s(    EJJA >% w??r@   c                 Z    |t        | ||      t        | ||      fS t        | d |      S Nr1  )reduce_amaxreduce_argmaxr2  s      r>   
reduce_maxr8  2  :    
g6!#8
 	

 qtg66r@   c                 Z    |t        | ||      t        | ||      fS t        | d |      S r5  )reduce_aminreduce_argminr2  s      r>   
reduce_minr=  =  r9  r@   xor_sumr  r  argmaxr  argmin
logical_or)r   r   c                 8    t        | t        j                        S r  r@  r   INT_TO_FLOATr  s    r>   register_pointwise_numericrF  W  s    
 ? L L r@   c                 :    t        | t        j                  d      S )NT)rq   r>  rC  rE  s    r>    register_pointwise_numeric_ldf64rH  ]  s    
;HH" r@   rB  logical_not)r   )rq   r   r   identityc                 d    | t         |<   t        j                  |        fd}t        | |       y )Nc                       | i |}g }t        | d   |      D ]"  \  }}|j                  t        ||d             $ |S )Nr   Tr  )r   r   r  )r{   r   resultsmut_resultsr|   r  outplace_ops         r>   rO   z$register_foreach_inplace.<locals>.fn  sO    t.v.tAw0KCyf4HI 1 r@   )inplaceable_foreach_opsr  rI   r   )aten_opoutplace_aten_oprO  rO   s     ` r>   register_foreach_inplacerS    s.    07,-G$ w+r@   c                 2    t        | d       fd       }|S )Nr  c                  l     | i |}t        || d   j                               }t        | d   |      S r   )r   rh   r  )r{   r   r  rO  s      r>   rO   zregister_inplace.<locals>.fn  s<    d-f-&$q'"3"3"56a&))r@   )r   )rQ  rO  rO   s    ` r>   register_inplacerV    s#    wD9* :*
 Ir@   c                     t         j                  j                  j                         }| |j                  j
                  j                  v sJ | S rB   )rJ   _guardsTracingContextr  	fake_moderw  var_to_range)r   r  r  tracing_contexts       r>   sym_constrain_ranger]  -  s?    mm22668O))33@@@@@Hr@   c                     t         j                  j                  j                  d   }t	        |t
        j                        sJ |j                  j                  S r4  	r8   r   r  r  rC   rJ   SymIntr
  r  r   ry   r`  s      r>   sym_sizerb  4  s@    
''


#
#E
*C c5<<(((88==r@   c                     t         j                  j                  j                  d   }t	        |t
        j                        sJ |j                  j                  S r4  r_  ra  s      r>   
sym_striderd  E  s>    
''


#
#E
*Cc5<<(((88==r@   c                 "    | j                         S rB   )r  )r   s    r>   	sym_numelrf  M  s    ;;=r@   c                     t        d      )NzHelpful for debuggingr:   )rS  r{   r   s      r>   foobarrh  V  s    
5
66r@   c                 8    | j                          t        |       S rB   )r  r  rl   s    r>   _realizerj  [  s    IIK8Or@   c                 r    | j                          |j                          t        j                  | |       | S rB   )r  r"   AccumulateGrad)variablenew_grads     r>   accumulate_grad_ro  a  s1     h)Or@   c                     t        j                  | ||       |j                         D ci c]  \  }}t        |t              s|| c}}S c c}}w )N)
kernel_idxgridkernel_args)r"   UserDefinedTritonKernelr  rC   r.   )rq  rr  r   r  r`  s        r>   triton_kernel_wrap_ru  j  sC    *4VT%+\\^R^cz#y7QCH^RRRs   AAc                     |j                         D ci c]  \  }}|||v rt        |      n| }}}t        | ||      S c c}}w )N)rq  rr  r   )r  r  ru  )rq  rr  r   tensors_to_cloner  rP   s         r>   triton_kernel_wraprx  p  sU     IOHVfc1#!11eAhq8   *4OOs   Ac                 f    t        j                  t        j                  j                  |             S rB   )r.   r   r"   Wait)r  s    r>   waitr{  }  s     u 566r@   c                 H    t         j                  j                  | ||||      S rB   )r"   	Broadcastr   )r  r  tagranks
group_sizes        r>   r   r     s    ||""5#sE:FFr@   c                 H    t         j                  j                  | ||||      S rB   )r"   	AllReducer   r  	reduce_opr~  r  r  s        r>   	allreducer    s    ||""5)S%LLr@   c                     t        j                  t        j                  j                  t        j                  j                  |       |||            S rB   )r.   r   r"   AllGatherIntoTensorr   r0  )shardr~  r  r  s       r>   all_gather_into_tensorr    sA    ""))22593z
 	
r@   c           	      n    t        j                  t        j                  j                  | ||||            S rB   )r.   r   r"   ReduceScatterTensorr  s        r>   reduce_scatter_tensorr    s1    ""))%C
S
 	
r@   c                 H    t         j                  j                  | ||||      S rB   )r"   AllReduceCoalescedr   r  s        r>   all_reduce_coalescedr    s!    $$++E9c5*UUr@   c                     t         j                  j                  | |||      }t        t	        t
        j                  |            S rB   )r"   AllGatherIntoTensorCoalescedr   rD   mapr.   )rS  r~  r  r  r  s        r>    all_gather_into_tensor_coalescedr    s6    0077c5*UC	((&122r@   c                     t         j                  j                  | ||||      }t        t	        t
        j                  |            S rB   )r"   ReduceScatterTensorCoalescedr   rD   r  r.   )rS  reduceOpr~  r  r  r  s         r>   reduce_scatter_tensor_coalescedr    s=    0077(C

 C	((&122r@   c           
      p    t        j                  t        j                  j                  | |||||            S rB   )r.   r   r"   AllToAllSingle)rS  output_split_sizesinput_split_sizesr~  r  r  s         r>   all_to_all_singler    s:     $$(*;S%
 	
r@   c                     t        |       } t        j                  j                  t        j
                  j                  | ||       | S rB   )r  r"   _CollectiveKernelcreate_inplace_c10d_functionalall_reduce_r  rx   r  
group_names      r>   _all_reducer    s;    Cj
++((00#y*	
 
r@   c                 z    t         j                  j                  t        j                  j
                  | ||       | S rB   )r"   r  r  r  r  r  r  s      r>   _all_reduce_r    s2    
++((00#y*	
 
r@   c                     | D cg c]  }t        |       } }t        j                  j                  t        j
                  j                  | ||       | S c c}w rB   )r  r"   r  r  r  all_reduce_coalesced_r  )r   r  r  rx   s       r>   _all_reduce_coalescedr    sR    (./%*/
++22::		
  0s   Ac                 z    t         j                  j                  t        j                  j
                  | ||       | S rB   )r"   r  r  r  r  r  )r   r  r  s      r>   _all_reduce_coalesced_r    s5    
++22::		
 r@   c                     t         j                  j                  t         j                  j	                  t
        j                  j                  | ||            S rB   )r"   r.   r   r  create_out_of_placer  r  r  )rx   r  r  s      r>   _all_gather_into_tensorr    sE    ||""  44 77??	
 	
r@   c           	          t        j                  t        j                  j                  t        j
                  j                  t        j                  j                  | ||            S rB   )
ra  rb  r"   r.   r   r  r  r  r  r  )r   r  r  s      r>   !_all_gather_into_tensor_coalescedr    sL    LL  44 AAII	
 	
r@   c           	          t         j                  j                  t         j                  j	                  t
        j                  j                  | |||            S rB   )r"   r.   r   r  r  r  r  r  )rx   r  r  r  s       r>   _reduce_scatter_tensorr    sH    ||""  44 66>>
 	
r@   c           
          t        j                  t        j                  j                  t        j
                  j                  t        j                  j                  | |||            S rB   )
ra  rb  r"   r.   r   r  r  r  r  r  )r   r  r  r  s       r>    _reduce_scatter_tensor_coalescedr    sO    LL  44 @@HH	
 		
r@   c                 v    t         j                  j                  t        j                  j
                  |        | S rB   )r"   _WaitKernelcreate_waitr  wait_tensorr  )rx   s    r>   _wait_tensorr    s(    
""#3#?#?#G#GM
r@   zRInductor support for distributed collectives depends on building torch.distributed)r  )quantized_loweringsrB   )NNNNF)F)r   r   l            r   r  )r   r   r   r  )Trg   )r   NNr   )NN)r   FF)r   F)NNN)Nr  )Nr   r   F)NNNN)r   r   FTNr  (  r   r   loggingr  r  collectionsr   collections.abcr   typingr   r   r   r   r	   r
   r   ri   rJ   torch.fxtorch.utils._pytreeutils_pytreera  *torch._higher_order_ops.triton_kernel_wrapr   r   torch._prims_commonr   r   r   r   r   r   r   r   r   r   torch.fx.experimental.sym_noder   r   torch.utils._sympy.functionsr   r   r   _dynamo.utilsr    r    r!   r"   r#   decompositionr$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   virtualizedr7   r8   	getLoggerr;  r  r   rS   rF   r   r1  tr_c10dr<  rH   r   r  r  rP  r?   rG   rU   r  r~  r  bmmconvolutionconvolution_backwardr4  rX  mmri  rk  upsample_bicubic2d_int_mmr  int8int16r  r  r  r  r   	complex32	complex64ro   r  rc   rb   re   rm   rp   r   r   r   r   r   r   r   r   r   r   r  r`   r   convert_element_typer!  r*  r2  r3  r   r6  
device_putr8  r@  rD  rG  r   aliasdetachdetach_liftview_ofrQ  rt   rR  r[  r]  ra  rc  ri  rk  rn  rq  rs  r   r  r  r  _unsafe_viewreshaper  slicer  r  rE   r  r  r  r  r
  r  r  r  r  r  r  r%  r  r)  r  r.  r  r  	lru_cacher  r  r  r  r  r  r  r  r  r  rngprimsr  r  r  	bernoullir  r  r  r  r  r  r  r  r  r  r  r  r  randintforce_stride_orderr  r  r  r  r
  lookup_seedr  randomr  r  r  r#  r.  r0  r  r?  FALLBACK_ALLOW_LIST_adaptive_avg_pool2d_backward
_cudnn_rnn_cudnn_rnn_backwardcumsumcumprod_embedding_bag_embedding_bag_forward_only_fused_moving_avg_obs_fq_helper*_fused_moving_avg_obs_fq_helper_functionalgrid_sampler_2d_backwardrandpermrQ  '_scaled_dot_product_efficient_attentionrK  #_scaled_dot_product_flash_attentionrL  _flash_attention_forward_flash_attention_backward_efficient_attention_forward_efficient_attention_backwardsortstable(_sparse_coo_tensor_with_dims_and_tensors_thnn_fused_lstm_celltopkupsample_bicubic2d_backward
_scaled_mmr  upsample_linear1dupsample_trilinear3dupsample_linear1d_backwardupsample_trilinear3d_backward_adaptive_avg_pool3dadaptive_max_pool2dadaptive_max_pool3daddbmmaddmv_addmm_activation
avg_pool3d
block_diag_cdist_forwardcummaxcummindigamma_efficientzerotensor*_embedding_bag_per_sample_weights_backwardfractional_max_pool2dfractional_max_pool3dfrexpgeqrfhistci0igammaigammacisinkthvaluelinalg_cholesky_exlinalg_cross_linalg_detlinalg_householder_productlinalg_inv_exlinalg_ldl_factor_exlinalg_ldl_solve	linalg_lulinalg_lu_factor_exlinalg_lu_solvelinalg_matrix_exp	linalg_qr_linalg_slogdet_linalg_solve_exlinalg_solve_triangular_linalg_svdlogcumsumexp	lu_unpackmax_pool3d_with_indicesmax_unpool2dmax_unpool3dmedianrY  	nanmedianormqr_pdist_forwardpixel_shufflepixel_unshuffle	polygammaputreflection_pad1dreplication_pad1dresizeresize_	resize_as
resize_as_searchsortedspecial_airy_aispecial_bessel_j0special_bessel_j1special_bessel_y0special_bessel_y1special_chebyshev_polynomial_tspecial_chebyshev_polynomial_uspecial_erfcxspecial_hermite_polynomial_hspecial_hermite_polynomial_hespecial_i0e
special_i1special_i1especial_laguerre_polynomial_lspecial_modified_bessel_i0special_modified_bessel_i1special_modified_bessel_k0special_modified_bessel_k1special_ndtri!special_scaled_modified_bessel_k0!special_scaled_modified_bessel_k1special_spherical_bessel_j0special_zetatake
_trilinearuniform_adaptive_avg_pool3d_backwardadaptive_max_pool2d_backwardadaptive_max_pool3d_backwardavg_pool3d_backward_cdist_backward_embedding_bag_dense_backwardfractional_max_pool2d_backwardfractional_max_pool3d_backward_linalg_check_errors max_pool3d_with_indices_backward_pdist_backwardreflection_pad1d_backwardreplication_pad1d_backwardreplication_pad2d_backwardsoft_margin_loss_backwardlinalg_pinvatol_rtol_tensorsegment_reduce_segment_reduce_backwardanglecholesky_inversecholesky_solve_fft_r2c	histogrambin_ct_histogramdd_bin_edges_histogramdd_from_bin_ctsindex_reducemasked_scattermasked_scatter_backward	to_sparse
_to_sparsetriangular_solvegcd_linalg_eighrw   r  nonzero_prims	rng_primsrun_and_save_rng_staterun_with_rng_stateexponentialr  r  r  r\  ra  ri  rk  r  scalar_tensorr}  
LongTensorr  r  r  re  r  r  r  rI  
empty_liker  
zeros_liker  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r   r  r!  r%  r  r2  r'  r<  r:  r@  r7  r   r\  ra  rd  ro  rq  rv  r  r  r  r  r	  r   r  r  r  r  r  r,  rQ  rc  rp  r  r  r  r  r  r  r  r  rD  r  r  r  r  r  r  r  r  r  Tensor_Tensorr  Scalarr   Tensor_Scalarr  r  r  r#  r  r  r  r  r   r"  true_divider'  rD  r'  r*  r,  r  r6  r   r3  r  r8  r  r=  r>  rB  r6  rC  r;  r?  r7  r@  r<  rI   rF  rH  expr  expm1relur,  r  r  r  cossinr  bitwise_andbitwise_left_shiftbitwise_not
bitwise_orbitwise_right_shiftbitwise_xorlgammaerfspecial_erflog1ptantanhr  rI  rA  logical_xorr  r  	clamp_min	clamp_maxnegr  	remaindersignsignbit	_neg_viewr  r  r  r'  r_  necoshsinhacosacoshasinasinhatan2atanatanhcopysignerfcerfinvhypotlog10	nextafter_foreach_addforeach_add_listforeach_add_scalar_foreach_mulforeach_mul_listforeach_mul_scalar_foreach_sub_foreach_neg_foreach_abs_foreach_powScalarAndTensor_foreach_divforeach_div_listforeach_div_scalar_foreach_sqrt_foreach_maximum_foreach_minimum_foreach_clamp_min_foreach_clamp_max_foreach_reciprocal_foreach_sign_foreach_copyrS  _foreach_add__foreach_mul__foreach_div_rV  add_bitwise_and_bitwise_left_shift_bitwise_not_bitwise_or_bitwise_right_shift_bitwise_xor_mul_div_Tensor_modelogical_and_logical_not_logical_or_logical_xor_sub_relu_sigmoid___and__
__lshift____or__
__rshift____xor____iand____ilshift____ior____irshift____ixor__r]  rb  rd  rf  r  methodfuncrh  _inductor_testr  rj  inductorro  ru  rx  )torch.distributed._functional_collectivesc10d_functionalr  r{  r   
all_reducer  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  ImportErrorr  r  r  register_quantized_opsr   r@   r>   <module>r     s0      	  # $ D D D    $ $   M K K , 8 8 =      g!	 E	yy~~
))

		 ee & F
=,   !!$$--&&* {{zz{{{{{{}}}}}}

. "#3R  D2/h 7??	$#0"N  $"&5pFRE	 E%++ E 5--4H)Y )u{{ ) I) @E M	 M%++ M. 499??=19 1U[[ 1 >1 ;@ = =ELL = 5##>+9 +ell + ?+ 
7?? $'Z  4::DI J0 4))UPTU V" DJJT\\499emmTU V 4&doo&s+ 4<<T:B ;B" 4$$$?" @" DMM?# $ 4::C C 4::C C 499! ! 4::! ! 4::! ! 4::! ! 4;;D9> :>2 5))tD
 E
 4>>t<# =# 4;;.  .b 4$$$?499$74<<T:1 ; 8 @1 4<<T:> ;> 4::48I 9I 499$7 7 87t 4??=> >>$ 4##> ? 4''TB C
CL 4880: 0:f 4==d;)EC )E3 )E# )E <)EX 4%%4@6 6 6s 6 A6 4((dC  s  D 4;;D96 :6
 4::48 9$ 4((dC  D  4;;D9 : 4;;D9D :D4 4>>t< = 4??= > 488 Tn
  	 T  1 1 < < 3!2!2!=!= 3"< "<J,N^	, 599%%11tL#+ M#+L 4&&DAG BG 4??= > 4>>##>' ?' 4<<  ! QB B/
 )):):; *499+>+>? )$***<*<= +DJJ,@,@A  dll  499J J 4::J J >44$OL PL
 >&&DAK%,, K BK >''TBJ CJ
 >--4H	 I	 >((dCRS $s) 9 C C  D6 >))tDLM	#Cy09FI E8 4>>t<
 +++ 	+
 + =+\"   d00- @ d'')@ A doo} - d&&(: ; dkk=u 5 dllM 6 d!!#5 6 d..0B C d22 3 d== > d++] ; dmm <~ 0088	
 99AA	
 ,,44	
 55==	
 d++33_ E d,,44o F d//77 I d0088/ J dii  dii  d;; < d((- 8 dii  d..0B C doo%%'> ? d""$6 7 d$$ % d'' ( d-- . d00 1 d'' ( d&& ' d&& ' dkk  djju % d$$5 1 doo  doo  d!! " dkk  dkk  dll ' dll ' d'' ( d== > d'' ( d== > d(( ) d(( ) djj  djj  djj  dgg  dkk & dll ' dii  dmm  d%% & d   d  d-- . d   ! d'' ( d## $ dnn  d&& ' d"" # d$$ % dnn  d"" # d## $ d** + d  d   dnn  d** + d   d   dkk  dii  dnn  djj  d!! " d   ! d"" # dnn  dhh  d## $ d$$ % dkk  dll  dnn  doo  d   d"" # d$$5 1 d$$5 1 d$$5 1 d$$ % d11 2 d11 2 d  u - d// 0 d00 1 dU + dooE * dU + d00 1 d-- . d-- . d-- . d-- . d  u - d44 5 d44 5 d..U ; de , dii  doo  dll ' d00 1 d// 0 d// 0 d&& ' d"" # d00 1 d11 2 d11 2 d'' ( d33 4 d"" # d,, - d-- . d-- . d,,5 9 d// 0 d!!)) * d++33 4 djj  d## $ d!! " dmm  dnn## $ d))11 2 d,,44 5 d   d!! " d** + dnn  doo  d## $ dhhU + d   djj  dll"" # ell$$;; < ell$$77 8 d&&U 3
 499$7
 8
 4::"   4"#+d**+E2 5:: ( 4&&DA C  B: 4%%4@F AFR ELL$"4"456tDU 5 75p 5??#4 $4 5##$+ %+ 4++, -"< 4>>t<K =K0 EKK,- 
 .$.> 0t/0B50IJ
1!45	 21 56
 4>>" $T$4  # 4%%&Td '< 4))*"4	 +	 5%%--.A /A EJJ		*+: ,:
 4;;D9 :4 4>>t< =8#0DN#L 4::48
 9
 4%%4@/ A/ 4>>"= #= 4))*O +O;" 4??=J >J >44$OK PK>
Wt ,,$RW..B 4**E F 4<<T:9C 9 ;9 !! 
! SM! !H 4==d;DH : :HSM : <:$ 4##>3 3 ?3 4$$$?6 6 @6 4&&DAP3 P BP 4''TBPT as a a CaP , HUOS(), 	,
 ,^ 4**223>x > 4> 41199:Jhuo J ;J 4**223RVI&uoI@HI 4I 41199:RVU&uoU@HU ;U 4**223 !% $ $S uoS uo	S
 uoS 4S 41199: !% $ $	 uo	 uo		
 uo	 ;	7 4**223
 !% $i i uo	i
 uoi 4iX 4(() *: 4112Q 3Qh 599$$% &* 4''TB3 C3lejj uUZZ_'= uzz  %** EJJ UZZ 8, $4  (($   4//TJBGM KM` -=))11- ) 488dSZ TZz:<  0%%5  
 4,,-B .BJ 433;;<BF- =-` 'OO 
 4??= Z >Zz  0$$%  
 4++F d GdN$.b3 & 499:t : : 4 (V DHHeii()T5  * 4==!u  "   .HH  'txxER -HH 
 488t,, -,^< 4::2 2 4::48 9     488t, - DHH:.( /( EII;$/	$ 0	$	txx'7DD 	 DIIuzz*d;$ <$  4::% % DHHeii(). . *. 499. . . 488@ @
 48867 77 48867 77 ! %--  	!: ;*		*>%+@A*		*>%+@A.!$++.85;;? /!$++.85;;? HH$L
 'txx0!$)),"4::.$))$
*4<<
8'		2	DKK	(t4   *   *" !1!12'(?(?@  -  0
()A)AB  !1!12 4;; ' * *I*V*V 4:: & 488 $ 499 %   * **	 !**	  OO**	
 !**	 T\\
*
T\\
* ! $.. !' * ! $.. !' *"'8
 4>> "$))L 499  4<<uzz B ! $.. !# & 477%** = 477%** = 477%** =uzzB 477%** = 477%** = 499 % 499 % 499 % 4:: & 499 % 4:: & 4:: & 499 % 4:: & 4== ) 499 % 4;; ' 4:: & 4:: & 4>> *-CT  0ct  4,,33Sd K-d.?.?.D.DcJ /0A0A0H0H#N  4,,113 7 4,,33S 9 4,,44c : 4,,44c : 4,,33S 9 4,,<<c B-d.?.?.D.DcJ /0A0A0H0H#N  4--t 4 40055w ? 40077 A 40055w ? 40077 A 42277 A 422997 C 42277 A 422997 C 433Z @ 4--t 4 4--t 4, T..335E t00779K T..335E t00779K T..335E t00779K
 C   ""K 0 ))+= > ""K 0 !!: . **,? @ ""K 0 C   !!3 ' && 1 ""K 0 ""K 0 !!: . ""K 0 C   T "  (   $,,  , " $// "#5 6  $++ z * " $// "#6 7  $,,  ,  - !!4?? 3 t{{ + !!4?? 3  - 4++, - 4==$$% &  4??&&' ( 4>>" # (M'')LFD1(01$7 * 4<< 7 !7 599++334 5
 599%%667 8 12S 3S
 34P 5PW4ii//O2237 47 001G 2G 112M 3M ==>
 ?
 <<=
 >

 ;;<V =V GGH3 I3 FFG3 H3 889
 :
 yy11'223 4 '334 5 '<<= > '==> ? '>>?
 @
 'HHI	
 J	
 '==>	
 ?	
 'GGH

 I

 '334 5     ! ! *  * * ,  HH\s   IH-CR5 R5CSSCS