PhUddlZddlZddlZddlmZmZmZmZmZm Z m Z m Z m Z m Z ddlmZddlZddlZddlZddlZddlZddlmZddlmZddlmZddlmZddlmZdd lm Z  ddl!Z!ddl"Z"dd l#m$Z%ejLd ddl'Zddl(Zddl)Zddl*Zddl+Zddl,Zdd l-m.Z.dd l/m0Z0m1Z1dZ2gdZ4de5fdZ6de e7fdZ8de7fdZ9dZ:de7fdZ;ejxe=Z>GddeZ?dejjddfdZBdejjddfdZCde e7dffd ZDdejjde edffd!ZEdejjdefd"ZFdejjde e7dffd#ZGd$e e7dfde e7dffd%ZHd&e ejdfde d'fd(ZJd)e ejdfd*e d'de ejdffd+ZKd,edejfd-ZLd.d/d0e e7dfd1e ejdfd2e d'd3e e7dfd4e ejdfd5e d'd6e5de ejdffd7ZMd.d/d0e e7dfd1e ejdfd2e d'd3e e7dfd4e ejdfd5e d'd6e5de ejdffd8ZNGd9d:ZOejGd;d<ZQe e7e e7ee7efffZReeSd=< ejd>ed?Gd@dAZTed?GdBdCZUed?ddDdejjdEe e eTee7efffdFZVy#e3$rdZ2YwxYw)GN) AnyDictFinalListMappingOptionalSequenceSetTupleUnion) TypeAlias) FakeTensor) compatibility)FakeTensorProp)OperatorSupport)CALLABLE_NODE_OPS)_pytree) _pybind_state onnxscript)fx_onnx_interpreter)_TORCH_DTYPE_TO_NUMPY_DTYPE(_TORCH_DTYPE_TO_ONNX_TENSOR_ELEMENT_TYPETF)is_onnxrt_backend_supportedtorch_compile_backendOrtExecutionProviderOrtBackendOptions OrtBackendreturnctS)a!Returns ``True`` if ONNX Runtime dependencies are installed and usable to support TorchDynamo backend integration; ``False`` otherwise. Example:: # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ONNX) >>> import torch >>> if torch.onnx.is_onnxrt_backend_supported(): ... @torch.compile(backend="onnxrt") ... def f(x): ... return x * x ... print(f(torch.randn(10))) ... else: ... print("pip install onnx onnxscript onnxruntime") ... )_SUPPORT_ONNXRTkC:\Users\daisl\Desktop\realtime-object-detection\venv\Lib\site-packages\torch/onnx/_internal/onnxruntime.pyrrCs " r"cdgS)NCPUExecutionProviderr!r!r"r#_infer_default_epsr&Ws  # ##r"namectjjr*tjjj |yy)zIf PyTorch is installed with CUDA support, this starts NVTX range. Check torch.cuda.nvtx.range_push's document for more details. N)torchcuda is_availablenvtx range_pushr's r#_nvtx_range_pushr/]s/  zz  ""4(!r"ctjjr)tjjj yy)zIf PyTorch is installed with CUDA support, this terminates NVTX range. Check torch.cuda.nvtx.range_pop's document for more details. N)r)r*r+r, range_popr!r"r#_nvtx_range_popr2fs-  zz  !!#!r" device_typec|dk(rtjjS|dk(rtjjS|dk(rtjj St d|z)Nr*cpuortzUnsupported device type: )ORTC OrtDevicer*r5npu ValueError)r3s r#_get_ort_device_typer;osdf~~""$$e~~!!##e~~!!## 0;> ??r"ceZdZdZdeedeeefffd Zde ee jjfde jjdeffd ZxZS) OrtOperatorSupporta0Operator support for ONNXRuntime backend. It has two-level of support decision. One is via support_dict and the other one is via extra_support_dict. The logic of using support_dict is implemented in OrtOperatorSupport and extra_support_dict is used by OperatorSupport.is_node_supported. support_dictextra_support_dictc2t||||_yN)super__init___onnx_support_dict)selfr>r? __class__s r#rCzOrtOperatorSupport.__init__s +,".r" submodulesnoderc>|jtvry|jdk(rM|j|jvr5tj d|jt |jytj d|jt |jt|!||r5tj d|jt |jytj d|jt |jy)NF call_functionz0support_dict supports node.target: %s (type: %s)Tz7support_dict doesn't support node.target: %s (type: %s)z6extra_support_dict supports node.target: %s (type: %s)z>extra_support_dict doesn't supports node.target: %s (type: %s)) oprtargetrDloggerwarningtyperBis_node_supported)rErGrHrFs r#rPz$OrtOperatorSupport.is_node_supporteds 77+ + 77o %$++9P9P*P NNB T[[!   E KK    7 $Z 6 NNH T[[!   L KK   r")__name__ __module__ __qualname____doc__r rrstrrCrr)nnModulefxNodeboolrP __classcell__)rFs@r#r=r=sc/SX/4S>/$!#uxx"67$?Dxx}}$ $$r"r= graph_modulec|j}g}d}|jD]7}|jdk(r|j||&|jdk7s6|}9|y|D]}|j |y)z In torch.fx.Graph, placeholder is a special assignment node. If it's not executed in the beginning, it could overwrite values computed by upstream nodes. N placeholder)graphnodesrKappendprepend)r\r_ placeholdersfirst_not_placeholderrHr^s r#_move_placeholder_to_frontres|   EL   77m #    % (TWW -E$( !  $# %%k2$r" fx_modulec|jjD]}t|jtj j s3|jjtjjjk(sod}d}d}d}d|jvr"|jdtjk7rd}d|jvr8|jd|jdjdjk7rd}d|jvrd}|jD] }|dvsd} t!|jd k(rS|rQ|rO|rM|rKd|jdi|_ tjjj"j$|_t'd |jDcgc]}|jc}d |j|j)ycc}w) NTlayoutFdevicervaldtype)rhrirkzaaten._to_copy must be replaced with other ONNX-supported aten ops. args=z , kwargs=)r_r` isinstancerLr)_ops OpOverloadoverloadpacketopsaten_to_copykwargsstridedargsmetarilentork RuntimeError recompile)rfrHis_default_layoutis_on_same_deviceis_castare_kwargs_supportedkwargargs r#_replace_to_copy_with_tors%% t{{EJJ$9$9 : **eiinn.E.EE $  $ G#' 4;;&4;;x+@EMM+Q$)!DKK'KK)TYYq\->->u-E-L-LL$)!dkk) ==+0(% DII!#%%( ' G(<= #iinn//55 "48II>ISI>?y WI&P ?s+G& .cg}|D]]}t|ds|j}|jdk(r|jd=|jdk(sM|jd_t |S)zBReturn the first valid device (i.e., GPU or CPU) in argument list.rir*CUDAExecutionProviderr5r%)hasattrrirOratuple)rvepsrris r#_infer_ep_from_devicersb C 3 !ZZF{{f$ 23% 12  :r"cg}|jjD]f}|jdk(st|dr7d|jvr)t |jdt jsJ|j|ht|S)Nr^rwrj) r_r`rKrrwrmr)Tensorrar)r\rcrHs r#_extract_graph_module_inputsrsvL""(( 77m #tV$$)));!$))E"2ELLAAA    % )  r"c|jjD]"}|jdk(s|jdcSt d)zHCollect "val" fields from outputs metadata in this torch.fx.GraphModule.outputrz2No output node found in this torch.fx.GraphModule.)r_r`rKrvr:)r\rHs r#_extract_graph_module_outputsrsB""(( 77h 99Q<  ) I JJr"ctjt|\}}|Dcgc]+}t|drd|jvr|jd-}}t |Scc}w)z[Return the all valid devices (i.e., GPU or CPU) among outputs of this torch.fx.GraphModule.rwrj)r tree_flattenrrrwr)r\flattened_output_args_ output_argselected_output_argss r#_infer_ep_from_graph_modulersu&33%l3 1 0/J J 'EZ__,D / !"6 77s0A rcfdtdtfd}t|}tt ||dS)z:Sort execution providers in eps based on pre-set priority.eprc|dk(ry|dk(ryy)Nr%rrlrr!)rs r#get_execution_provider_priorityz2_sort_eps..get_execution_provider_priority-s ' ' ( (r"T)keyreverse)rUintsetrsorted)rr unique_epss r# _sort_epsr*s6 C C SJ  (GQUV WWr"valueszORTC.OrtDevice.ctfdDsJddtdtfdtfdD}|S)Nc3VK|] }|jdjk("yw)rN)ri).0valuers r# z$_get_onnx_devices..=s(6._device_id_or_zeroAs ~Ar"c3K|]n}tjt|jjtjj |jj pywrA)r7r8r;rirOdefault_memoryindex)rrrs r#rz$_get_onnx_devices..DsZ2 E  !2!2 3 NN ) ) + u||11 2 sA4A7)allrr)rdevicesrs` @r#_get_onnx_devicesr<s_ 6< 0/0 cc-22  2-G Nr"tensorsrcftj}|jt|g}g}g}|D]b}|j t |j |j |j|j |jd|j||||||SrA) r7OrtValueVectorreserverxrarrksizedata_ptrpush_back_batch)rr ortvaluesdtypesshapes data_ptrstensors r#!_get_ortvalues_from_torch_tensorsrOs##%I c'l# F FI 1&,,?@ fkkm$*+gy&&'J r"rc|jr tdtj|j |j |j }|S)Nz#sparse tensor is not yet supported.)rkri) is_sparser:r)emptyrrkri)routs r#_to_real_tensorr`s< >?? ++fkkm6<< NC Jr"sessonnxruntime.InferenceSession input_namesinputs input_devices output_namesoutputsoutput_devicespreallocate_outputc:tdtd|D}ttdt||}|rtd|D} t| |} nt j } ttdt j} | jdd|j| |||| |t|r Stdt jjjj| } t| S) N contiguousc3<K|]}|jywrA)r)ras r#rz8_run_onnx_session_with_ortvaluevector..rs26a1<<>6src3VK|]!}t|tr t|n|#ywrA)rmrr)rts r#rz8_run_onnx_session_with_ortvaluevector..}s' LSq*Q ";OA  BGs')run_with_ortvaluevector'disable_synchronize_execution_providers1zafter run_with_ortvaluevector)r/rr2rr7r onnxruntime RunOptionsadd_run_config_entryrtraining ortmodule_utils_ortvalues_to_torch_tensor) rrrrrrrr ort_inputs pth_outputs ort_outputs run_optionss r#%_run_onnx_session_with_ortvaluevectorrgs\" 262 2F&'26=IJ  LS  8 ^T ))+ ./((*K$$%NPST  [*lK89!**44;;VV   r"c t||D cic]A\}} |tjj| j j C} }} |j || } tdt| |D} | Scc} }w)Nc3~K|]5\}}tj|j|j7ywrA)r) from_numpyryri)rrrs r#rz/_run_onnx_session_with_fetch..s66ME6 ""6==16s;=)ziprOrtValueortvalue_from_numpyr5numpyrunr) rrrrrrrrr'rfeedrrs r#_run_onnx_session_with_fetchrs V4 4LD& k""66vzz|7I7I7KLL4  ((<.K g6K  sAB ceZdZdZdddeedfdeddeedfd edd ed d ed d eeejdfejffdZ dZ y)OrtExecutionInfoPerSessionzWInformation required to execute torch.fx.GraphModule using onnxruntime.InferenceSessionsessionrr.input_value_infos)zonnx.ValueInfoProto.routput_value_infosrrrexample_outputsc t||_||_||_||_||_||_||_||_yrArrrrrrrr) rErrrrrrrrs r#rCz#OrtExecutionInfoPerSession.__init__sM6= -8BS-9CU  0 0 > E Ed Kr"N) rQrRrSrCr)rX GraphModulerrr r!r"r#rrs@!HH00 L!HH00L8RLr"rr)frozen)is_backward_compatibleceZdZUdZdZeeeed< dZ e ed< dZ eeeed< dZ e ed< dZ e ed < dZed ed < dZed ed <y)raJOptions for constructing an ``OrtBackend``, the ONNX Runtime backend (``"onnxrt"``) for ``torch.compile``. Example:: >>> @torch.compile( ... backend="onnxrt", ... options=torch.onnx._OrtBackendOptions(...), ... ) ... def ort_function(x): ... return x ** x Npreferred_execution_providersTinfer_execution_providersdefault_execution_providersFruse_aot_autogradztorch.onnx.ExportOptionsexport_optionszonnxruntime.SessionOptionsort_session_options)rQrRrSrTrrr r__annotations__rrZrrrrrr!r"r#rrs OS!8H5I,J#KR'+t*mLP(3G*H!IP %$V!d!<@NH78?UBF">?FVr"rc eZdZUdZddeefdZdejjde e e e e efffdZdejjfdZdejjdejjfd Zdejjdejjfd Zd Zeed <gZeeded <e ddeeee e effddfdZedZedZy)ra A backend compiles (sub-)graphs in torch.fx.GraphModule to onnxruntime.InferenceSession calls. The compiler entry point is OrtBackend.compile, which 1. partitions the original graph into supported sub-graphs (type: torch.fx.GraphModule) and unsupported sub-graphs. 2. For each supported sub-graph, it replaces its _wrapped_call function with _ort_accelerated_call. 3. Inside _ort_accelerated_call, it creates onnxruntime.InferenceSession and calls it to execute the sub-graph. Noptionsc| tn||_tjjj j |jjtjjn|jj|_ tjjjjj|jj}ddd}t|||_i|_t#|_d|_d|_t+t,dr t.|_yt0|_y)N)getattrz_operator.getitemFrr)r_optionsr)onnx _internalexporterResolvedExportOptionsr ExportOptions_resolved_onnx_exporter_optionsrXdecomposition_table'_create_onnx_supports_op_overload_table onnx_registryr=_supported_ops_partitioner_cacher_all_ort_execution_info_assert_allclose_to_baselineexecution_countrr7rrr)rErr>r?s r#rCzOrtBackend.__init__^s6=o027 JJ ) ) ? ?==//7 ((*]]11  ,&zz++..BBjj  0 0 > > !%.  1?QRUW(J'K$,1)  t./ 2 . r"r\rct}|jjrt|x}r|}nt |x}r|}g}g|jj xsgt ||jjxs tD]L}t|tr|if}nt|tr |d|dif}|7||vs<|j|N|S)Nrlr) rrrrrrrrr&rmrUra)rEr\rv inferred_eps eps_from_argseps_from_graph_module selected_epsrs r# _select_epszOrtBackend._select_epss). == 2 2 5t <<}< - *El*SS&S 5   mm99?R | $ mm77O;M;O B "c""XB&2a5=eR[~"L"8##B' r"c H |jj|g|}|rJ|j}|j}|j}|j }|j } |j} ntjjjjj|jj|j!}|jj"r,d|_t'|} d} t)j*| | } n t-|j.|i|} t7j8|jj} tjjjjj;|jj|j!}| j!||jj<|jj>}|jA|jjBjD}tGjH|jK|jLjN|jP|g|}tSd|jTjVD}tSd |jTjXD}t[|}t]| tRr t[| } n t[| f} t_||tSd |jTjVD|tSd |jTjXD|| | }|jja|||xjbd z c_1t]| tjd}|r| fn| }t]|tRsJtgd|DsJtid|j!||||||| |jLj$}tk|jlrhtjnjpjr|g|ddi}|r|fn|}tu||D]%\}}tjvjy||'|r|dS|S#t0$rt2j5d|d|_wxYw)aThis function replaces GraphModule._wrapped_call in compiled model. The _wrapped_call is the underlying implementation of forward method. Replacing it means we delegate the computation to _ort_acclerated_call and therefore onnxruntime.InferenceSession. FcXt|drd|jvr|jdS|S)Nrwrj)rrw)rs r#maybe_map_to_meta_valz>OrtBackend._ort_acclerated_call..maybe_map_to_meta_vals-uf-%5::2E %zz%00$ r"zFakeTensorProb failed for %s)diagnostic_context)fx_graph_moduleonnxfunction_dispatcherop_level_debug) opset_version) path_or_bytes sess_options providersc34K|]}|jywrAr.rinputs r#rz2OrtBackend._ort_acclerated_call..1sO8Nu 8Nc34K|]}|jywrAr.rrs r#rz2OrtBackend._ort_acclerated_call..2s S;R;Rr>c3 K|]}|ywrAr!r<s r#rz2OrtBackend._ort_acclerated_call..?s'R;Q%;Q c3 K|]}|ywrAr!r@s r#rz2OrtBackend._ort_acclerated_call..As(V>UF>UrBrrlc3PK|]}t|tj ywrA)rmr)r)relems r#rz2OrtBackend._ort_acclerated_call..UsV>Ud:dELL1>Us$&$run_onnx_session_with_ortvaluevectorexecutorrrr)=r'rrrrrrrr)rrrXpassesMovePlaceholderToFrontr!r3rdynamic_shapesrrrtree_mapr propagate ExceptionrMrNrFxOnnxInterpreterInsertTypePromotionr5r6to_model_protor$r7rInferenceSessionSerializeToStringrrr/rr_r=rrrmrr r)rrr/r2r(_primsrGexecutertesting assert_close)rEr\rvrt!cached_execution_info_per_session onnx_sessionrrrr prim_outputsextracted_outputsr2fx_interpreterexported onnx_modelexecution_info_per_sessionis_single_tensor_outputnormalized_prim_outputs onnx_outputsbaseline_outputsnormalized_baseline_ouptuts onnx_outputbaseline_outputs r#_ort_acclerated_callzOrtBackend._ort_acclerated_callsW PD ( ( O O #  * -<DDL;GGK<IIL=KKM>MMN<LLL !:://2299PP44GGce 33BB*/'$A,$O!% '//)+<   #I>,#?#I#I$!'$L1BB#'#G#G#Z#ZN !:://2299MM44GGce  &)) ,(,(L(L(d(d#CCRR*H "00"BBPP^^1J'77(::<!]]>>*$**<?$?L O 8H8H8N8NOOK S:;K;K;R;R SSL-d3M,.!2N>N>U>U(V#V+- , * &  ( ( E E8  ! #-\5<<"H6\OL 15999V>UVVVV?@xx      #  MM , ,     , ,$||44<< # .4  (?!#DT (1491, _ **;H1#:|AK|Ka!NN#A<P/4D+s Q99(R!cddlm}||jvr|j|}|S|}t||||jd}|j }||j|<|j jD]H}|jdk(sd|jvs"t||j}|j|_ J|S)Nr)CapabilityBasedPartitionerT)allows_single_node_partition call_modulefused_) !torch.fx.passes.infra.partitionerrhr&rr%partition_and_fuser_r`rKr'rrf _wrapped_call) rEr\rvrhpartitioned_prim_graph_moduleprim_graph_module partitionerrH fused_modules r#compilezOrtBackend.compilers Q, 422 2,0,C,CL,Q )6-,3!-  %%6 74!##-1K -8,J,J,L )4QD # #L 1 6;;AA77m+DII0E#*+H$))#TL261J1JL.B-,r"c|jjr.reusables//13R3RR..!2M2MM00A4Q4QQ''1+?+??%%););; $$0A4I4I4U1#3#33+0@0@0L$$33q7G7G7V7VVW((77''667W((;;''::;W ((66!:J:J:X:XX W ((559I9I9V9VVr"c3JK|]}|js|ywrA)r)rrrrs r#rz=OrtBackend.get_cached_instance_for_options..s! U31x G7TQ3s##Nz No more than z instances of z allowed. Please instantiate `z` explicitly to pass to `torch.compile`. See https://github.com/pytorch/pytorch/pull/107973#discussion_r1306144795 for discussion.r!)rrmnextrrrxr~ra)rbackendrs` @r#get_cached_instance_for_optionsz*OrtBackend.get_cached_instance_for_optionss" )" .?" H'#45':7=b:G U 33 U   ?J//0:3X3XX  E EFn,|T RRr")W dataclasses importlibloggingtypingrrrrrrr r r r typing_extensionsr r)torch._C torch._opstorch._prims.executortorch.fxtorch._subclasses.fake_tensorrtorch.fx._compatibilityr torch.fx.passes.fake_tensor_propr torch.fx.passes.operator_supportrtorch.fx.passes.tools_commonr torch.utilsrrronnxruntime.capirr7 import_module torch.onnxtorch.onnx._internal torch.onnx._internal.diagnosticstorch.onnx._internal.exporter+torch.onnx._internal.fx.decomposition_tabletorch.onnx._internal.fx.passestorch.onnx._internal.fxr"torch.onnx._internal.fx.type_utilsrrr ImportError__all__rZrrUr&r/r2r; getLoggerrQrMr=rXr rerrrrrrrrrrrrr dataclassrrrrrrr!r"r#rs   ( 41;<:6IL)+(6); O T($HSM$ )3)$@c@   8 $ 44n3UXX-A-A3d3**(<(<**Z E#s(O uxx/C/CcSVhK0D0DKK 8ehh.B.B 8uSRUX 8 X5c?XuS#XX$eELL#$565AV;W& 5<<$ %056K0L 5<< "J5<<. (.sCx. %,,# $../ . S/ . 5<<$ % ./0.. 5<< .b (sCx %,,# $./  S/  5<<$ % /0 5<< ,88v LLLB#(U3S8I3I-J(J"KiK d#e,6W6W-$6Wre,z2z2-z2z e, FJ S((&&Se-wsCx/@@A B S-SwOs%AMMM