
    Yj;S                     0   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZd dl	Z	d dl
mZ d dlmZmZ eeeee         z  f         Z ej        e          Z G d d          Z G d	 d
          Z G d d          Z G d de          Z G d d          ZdS )    N)OrderedDict)Mapping)Any)TensorProto)InferenceSession
RunOptionsc                      e Zd Zedededefd            Zededefd            Zedefd            Zedefd            Z	edede
eef         fd	            Zedefd
            Zedej        fd            Zedej        fd            Zedede
eej        f         fd            Zedede
eej        f         fd            ZdS )
TypeHelperort_sessionnamereturnc                     t          |                                           D ]\  }}|j        |k    r	|j        c S t	          d| d          )Nzinput name 
 not found)	enumerate
get_inputsr   type
ValueError)r   r   _iinputs       g/var/www/FlaskApp/flask-venv/lib/python3.11/site-packages/onnxruntime/transformers/io_binding_helper.pyget_input_typezTypeHelper.get_input_type   sc    ";#9#9#;#;<< 	" 	"IBzT!!z!!! "7t777888    c                     t          |                                           D ]\  }}|j        |k    r	|j        c S t	          d| d          )Nzoutput name r   )r   get_outputsr   r   r   )r   r   r   outputs       r   get_output_typezTypeHelper.get_output_type   se    #K$;$;$=$=>> 	# 	#JB{d""{""" # 8888999r   ort_typec                 n   t           j        t           j        t           j        t           j        t
          t           j        t           j        t           j        t           j	        t           j
        t           j        t           j        t           j        t           j        d}| |vrt          |  d          ||          S )N)tensor(int64)tensor(int32)tensor(float)tensor(float16)tensor(bool)tensor(uint8)tensor(int8)tensor(double)tensor(int16)tensor(uint16)tensor(uint32)tensor(uint64)tensor(complex64)tensor(complex128) not found in map)numpyint64int32float32float16booluint8int8float64int16uint16uint32uint64	complex64
complex128r   )r   ort_type_to_numpy_type_maps     r   ort_type_to_numpy_typez!TypeHelper.ort_type_to_numpy_type#   s     #["["]$} "[!J#m"[#l#l#l!&"'"2&
 &
"  555;;;<<<)(33r   c                 Z   i dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        d	t           j	        d
t           j
        dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        t           j        t           j        t           j        t           j        d}| |vrt-          |  d          ||          S )Nr   r    r!   r"   tensor(bfloat16)r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   tensor(float8e4m3fn)tensor(float8e4m3fnuz))tensor(float8e5m2)tensor(float8e5m2fnuz)tensor(int4)tensor(uint4)r-   )torchr/   r0   r1   r2   bfloat16r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   float8_e4m3fnfloat8_e4m3fnuzfloat8_e5m2float8_e5m2fnuzint4uint4r   )r   ort_type_to_torch_type_maps     r   ort_type_to_torch_typez!TypeHelper.ort_type_to_torch_type:   sN   &
U[&
U[&
 U]&
 u}	&

 &
 EJ&
 U[&
 EJ&
 em&
 U[&
 el&
 el&
 el&
  &
 !%"2&
  #E$7!&
" %e&;#&
$ #("3&+&;!J"[+&
 &
 &
". 555;;;<<<)(33r   c                    i }|                                  D ])}t                              |j                  ||j        <   *|                                 D ])}t                              |j                  ||j        <   *|S )z9Create a mapping from input/output name to onnx data type)r   r
   ort_type_to_onnx_typer   r   r   )r   name_to_onnx_typer   r   s       r   get_io_onnx_type_mapzTypeHelper.get_io_onnx_type_mapX   s      ++-- 	Y 	YE,6,L,LUZ,X,Xej))!--// 	[ 	[F-7-M-Mfk-Z-Zfk**  r   c                    i dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        d	t           j	        d
t           j
        dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        dt           j        t           j        t           j        t           j        t           j        t           j        t           j        d}| |vrt1          |  d          ||          S )Nr   r    r!   r"   r@   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   rA   rB   )rC   rD   ztensor(float4e2m1)rE   rF   ztensor(string)r-   )r   INT64INT32FLOATFLOAT16BFLOAT16BOOLUINT8INT8DOUBLEINT16UINT16UINT32UINT64	COMPLEX64
COMPLEX128FLOAT8E4M3FNFLOAT8E4M3FNUZ
FLOAT8E5M2FLOAT8E5M2FNUZ
FLOAT4E2M1INT4UINT4STRINGr   )r   ort_type_to_onnx_type_maps     r   rR   z TypeHelper.ort_type_to_onnx_typec   sj   %
[.%
[.%
 [.%
 {2	%

  4%
 K,%
 [.%
 K,%
 k0%
 [.%
 k0%
 k0%
 k0%
  !6%
 !+"8%
  #K$<!%
" %k&@#%
$ #."8&1&@"-"8',(.)0/%
 %
 %
!2 444;;;<<<(22r   
numpy_typec                    t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t          t          j        t           j        t          j        t           j        t          j        t           j	        t          j	        t           j
        t          j
        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        i}| |vrt!          |  d          ||          S Nr-   )r.   r/   rG   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r   )rn   numpy_type_to_torch_type_maps     r   numpy_type_to_torch_typez#TypeHelper.numpy_type_to_torch_type   s     KKM5=M5=%*KJ
M5=KL%,L%,L%,OU_e.(
$" 999
===>>>+J77r   
torch_typec                    t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          t           j        t          j        t           j        t          j        t           j	        t          j	        t           j
        t          j
        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        t           j        t          j        i}| |vrt!          |  d          ||          S rp   )rG   r/   r.   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r   )rs   torch_type_to_numpy_type_maps     r   torch_type_to_numpy_typez#TypeHelper.torch_type_to_numpy_type   s     KKM5=M5=JKJ
M5=KL%,L%,L%,OU_e.(
$" 999
===>>>+J77r   c                    i }|                                  D ])}t                              |j                  ||j        <   *|                                 D ])}t                              |j                  ||j        <   *|S )z:Create a mapping from input/output name to numpy data type)r   r
   r>   r   r   r   )r   name_to_numpy_typer   r   s       r   get_io_numpy_type_mapz TypeHelper.get_io_numpy_type_map          ++-- 	[ 	[E-7-N-Nuz-Z-Zuz**!--// 	] 	]F.8.O.OPVP[.\.\v{++!!r   c                    i }|                                  D ])}t                              |j                  ||j        <   *|                                 D ])}t                              |j                  ||j        <   *|S )z:Create a mapping from input/output name to torch data type)r   r
   rP   r   r   r   )r   name_to_torch_typer   r   s       r   get_io_torch_type_mapz TypeHelper.get_io_torch_type_map   rz   r   N)__name__
__module____qualname__staticmethodr   strr   r   r>   rP   dictintrT   rR   r.   dtyperr   rG   rv   ry   r}    r   r   r
   r
      s       9$4 9C 9C 9 9 9 \9 :3 :3 : : : \: 4 4 4 4 \4, 4 4 4 4 \4: !*: !tCH~ ! ! ! \! 3 3 3 3 \3> 8U[ 8 8 8 \8. 8U[ 8 8 8 \8. "+; "S%+EU@V " " " \" "+; "S%+EU@V " " " \" " "r   r
   c            
           e Zd Zedefd            Zedej        dej        dej        deej                 fd            Z	edd	            Z
d
S )IOBindingHelperr   c                     i }|                                 D ]f\  }}t                              | |          }t                              |          }t	          j        t          j        |          ||          ||<   g|S )zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.)r   device)itemsr
   r   rP   rG   emptyr.   prod)r   output_shapesr   output_buffersr   shaper   rs   s           r   get_output_buffersz"IOBindingHelper.get_output_buffers   s     (..00 	c 	cKD%!11+tDDH#::8DDJ#(;uz%/@/@
[a#b#b#bN4  r   	input_idsposition_idsattention_maskpastc                    t                               |           }|                                 }|                                sJ |                    d|j        j        d|d         t          |                                          |	                                           |t          |          D ]\  }	}
|
                                sJ |
	                                }|dk    r|	                                }|                    d|	 |
j        j        d|d|	          t          |
                                          |           |q|                                sJ |                    d|j        j        d|d         t          |                                          |	                                           |q|                                sJ |                    d|j        j        d|d         t          |                                          |	                                           |                                 D ]}|j        }||         }t                              | d|j        j         dt          |                                                      |                    ||j        j        d||         ||         |	                                           |S )	zdIO binding for a session: bind inputs (input_ids, position_ids, attention_mask, past_*) and outputs.r   r   Npast_r   r   z device type=z shape=)r
   rT   
io_bindingis_contiguous
bind_inputr   r   listsizedata_ptrr   r   r   loggerdebugbind_output)r   r   r   r   r   r   r   rS   r   ipast_ir   r   output_nameoutput_buffers                  r   prepare_io_bindingz"IOBindingHelper.prepare_io_binding   s    ';;KHH !++--
 &&(((((!k*!!""  	
 	
 	
 &t__  	6++-----!??,,q==  )1133H%%AKKM&%kakk2''    %!//11111!! %*!"23^((**++''))   #--/////!!#(!.1\&&(())%%''   "--// 	 	F +K*;7MLLKttm6J6OttX\]j]o]o]q]qXrXrttuuu""$)!+.k*&&((    r   Tc                    g }|                                  D ]}|j        }||         }||         }|dt          j        |                                       |                                                                          }	|r:|                    |	                                                                           |                    |	           |S )z3Copy results to cpu. Returns a list of numpy array.r   )	r   r   r.   r   reshapeclonedetachappendcpu)
r   r   r   return_numpyort_outputsr   r   bufferr   copy_tensors
             r   "get_outputs_from_io_binding_bufferz2IOBindingHelper.get_outputs_from_io_binding_buffer*  s     !--// 	0 	0F +K#K0F!+.E UZ%6%6!67??FFLLNNUUWWK 0"";??#4#4#:#:#<#<===="";////r   N)T)r~   r   r   r   r   r   rG   Tensorr   r   r   r   r   r   r   r      s        (8    \ R<R lR 	R
 5< R R R \Rh    \  r   r   c                       e Zd ZdZddedej        fdZdedefdZ	d	 Z
d
edej        fdZdefdZddeeej        f         dedefdZeddedededeeef         fd            ZdS )CudaSessionzLInference Session with IO Binding for ONNX Runtime CUDA or TensorRT providerFr   r   c                    || _         d | j                                         D             | _        d | j                                         D             | _        t
                              | j                   | _        t
                              | j                   | _	        | j         
                                | _
        || _        t                      | _        t                      | _        || _        i | _        d S )Nc                     g | ]	}|j         
S r   r   ).0r   s     r   
<listcomp>z(CudaSession.__init__.<locals>.<listcomp>?  s    RRR5EJRRRr   c                     g | ]	}|j         
S r   r   )r   r   s     r   r   z(CudaSession.__init__.<locals>.<listcomp>@  s    VVVVV[VVVr   )r   r   input_namesr   output_namesr
   rT   io_name_to_onnx_typer}   io_name_to_torch_typer   enable_cuda_graphr   input_tensorsoutput_tensorsr   buffer_sharing)selfr   r   r   s       r   __init__zCudaSession.__init__=  s    &RRD4D4O4O4Q4QRRRVVt7G7S7S7U7UVVV$.$C$CDDT$U$U!%/%E%EdFV%W%W"*5577!2(]])mm /1r   
input_namer   c                 Z    || j         v sJ || j        v sJ || j        |<   || j        |<   d S N)r   r   r   )r   r   r   s      r   set_buffer_sharingzCudaSession.set_buffer_sharingM  sJ    T-----d/////*5J'+5K(((r   c                     | ` | `| `d S r   )r   r   r   )r   s    r   __del__zCudaSession.__del__S  s    OOOr   r   tensorc           	         |j         j        |j         j        nd}t          |j                  dk    rdgnt	          |j                  }| j                            ||j         j        || j        |         ||	                                           || j
        v rh| j                            | j
        |         |j         j        || j        |         ||	                                           || j        | j
        |         <   d S d S )Nr      )r   indexlenr   r   r   r   r   r   r   r   r   r   )r   r   r   	device_idtensor_shapes        r   bind_input_and_buffer_sharingz)CudaSession.bind_input_and_buffer_sharingX  s	   +1=+>+JFM''PQ	!&,//144ss$v|:L:L""M%d+OO	
 	
 	
 4&&&O''#D)")$/!!   >DD 3D 9::: '&r   
shape_dictc                 2   | j         r|                                D ]\  }}|| j        v r|| j        v r@t	          | j        |         j                  t	          |          k    rHt          d          | j        |         }t          j	        t	          |          |          
                    | j                  }|| j        |<   |                     ||           |                                D ]$\  }}|| j        v r|| j        v r1t	          | j        |         j                  t	          |          k    rJ|| j        v rT| j        |         }t          j	        t	          |          |          
                    | j                  }|| j        |<   | j                            ||j        j        |j        j        |j        j        nd| j        |         t+          |                                          |                                           &dS )z Allocate tensors for I/O Bindingz(Expect static input shape for cuda graph)r   )r   Nr   )r   r   r   r   tupler   RuntimeErrorr   rG   r   tor   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   torch_dtyper   s         r   allocate_bufferszCudaSession.allocate_buffersp  s   ! 	E)//11 E Ee4+++t111 !3D!9!?@@E%LLPP$*+UVVV"&"<T"BK"[u[IIILLTXT_L``F/5D&t,66tVDDD%++-- 	 	KD%t(((4...59LT9R9X3Y3Y]bch]i]i3i3i4..."8>U5\\EEEHHPTP[H\\,2#D)++M&+1=+>+JFM''PQ-d3''OO%%  	 	r   NT	feed_dictrun_optionssynchronizec                    |                                 D ]\  }}t          |t          j                  r|                                sJ || j        v r| j        r| j        |                                         |                                k    sJ | j        |         j	        |j	        k    sJ |j
        j        dk    sJ | j        |                             |           |                     ||           |rS| j                                         | j                            | j        |           | j                                         n | j                            | j        |           | j        S )z$Bind input tensors and run inferencecuda)r   
isinstancerG   r   r   r   r   r   nelementr   r   r   copy_r   r   synchronize_inputsr   run_with_iobindingsynchronize_outputsr   )r   r   r   r   r   r   s         r   inferzCudaSession.infer  sp   %OO-- 		E 		ELD&fel33N8L8L8N8NNNNt''') E-d3<<>>&//BSBSSSSS-d39V\IIII!=-7777&t,226::::66tVDDD 	NO..000//MMMO//1111//MMM""r   r   r   r   streamr   c                 B    | d|d}|dk    rt          |          |d<   |S )NkSameAsRequested)r   arena_extend_strategyr   r   user_compute_stream)r   )r   r   r   optionss       r   get_cuda_provider_optionsz%CudaSession.get_cuda_provider_options  s:     #%7!2
 
 Q;;-0[[G)*r   F)NT)r   )r~   r   r   __doc__r   rG   r   r   r   r   r   r   r   	ShapeDictr   r   r   r3   r   r   r   r   r   r   r   r   r   r   :  sE       VV1 1$4 1el 1 1 1 1 6S 6s 6 6 6 6  
D# Du| D D D D0$9 $ $ $ $L# #tC$56 #Z #ei # # # #,  S T SV _cdgildl_m    \  r   r   c                        e Zd Z	 	 	 	 ddedej        deded	ed
ede	e
e
f         dz  f fdZddedefdZdde	e
ej        f         def fdZ xZS )
GpuBindingFr   Nr   r   r   enable_gpu_graphgpu_graph_idr   r   c                 @   t                                          |||           |r0|                                D ]\  }}	|                     ||	           |                     |           || _        |rt          j        |          nd | _        || _	        d | _
        d S r   )superr   r   r   r   r   copydeepcopyr   r   last_run_gpu_graph_id)r   r   r   r   r   r   r   r   r   r   	__class__s             r   r   zGpuBinding.__init__  s     	f.>??? 	A+9+?+?+A+A A A'
K''
K@@@@j)))(7GQ$-
333T%)"""r   disable_cuda_graph_in_runr   c                     t                      }|rdn| j        }|                    dt          |                     || _        |S )Nr   r   )r   r   add_run_config_entryr   r   )r   r   r   r   s       r   get_run_optionszGpuBinding.get_run_options  sG    ,,6MrrD<M$$^S5F5FGGG%1"r   r   c                     |                      |          }| j        r|                    dd           t                                          ||          S )N'disable_synchronize_execution_providers1)r   r   r   r   r   )r   r   r   r   r   s       r   r   zGpuBinding.infer  sR    **+DEE; 	],,-VX[\\\ww}}Y444r   )Fr   r   Nr   )r~   r   r   r   rG   r   r   r3   r   r   r   r   r   r   r   r   __classcell__)r   s   @r   r   r     s        "'04* *%* * 	*
 * * * S#X-* * * * * *.	 	 	* 	 	 	 	5 5tC$56 5SW 5 5 5 5 5 5 5 5 5 5r   r   c            	       j    e Zd ZdZddedej        dedefdZ	 	 dde	de
deeef         d
z  defdZd
S )GpuBindingManagerzA manager for I/O bindings that support multiple CUDA Graphs.
    One cuda graph is reused for same input shape. Automatically add a new cuda graph for new input shape.
    r   r   r   r   r   max_cuda_graphsc                 Z    || _         || _        g | _        d | _        || _        || _        d S r   )r   r   graph_bindingsno_graph_bindingr   r  )r   r   r   r   r  s        r   r   zGpuBindingManager.__init__  s:    & ! !%.r   FNr   use_cuda_graphr   r   c           	         | j         D ]}|j        |k    r|c S t          | j                   | j        k    s|sQ| j        )t          | j        | j        || j        |          | _        n| j        	                    |           | j        S t          | j        | j        |dt          | j                   | j        |          }| j         
                    |           |S )N)r   r   T)r   r   r   r   )r  r   r   r  r  r   r   r   r   r   r   )r   r   r  r   gpu_graph_bindings        r   get_bindingzGpuBindingManager.get_binding  s    "&!4 	) 	) +z99(((( : t"##t';;;N;$,(2$dk:dkbp) ) )%% %66zBBB(( 'K!T011;)
 
 
 	""#4555  r   )r   r   )FN)r~   r   r   r   r   rG   r   r   r   r   r3   r   r   r   r  r   r   r   r  r    s         / /$4 /el /TW /nq / / / /"  %04	 !  ! !  ! S#X-	 !
 
 !  !  !  !  !  !r   r  )r   loggingcollectionsr   collections.abcr   typingr   r.   rG   onnxr   onnxruntimer   r   r   r   r   r   r   	getLoggerr~   r   r
   r   r   r   r  r   r   r   <module>r     s     # # # # # # # # # # # #               4 4 4 4 4 4 4 4 Cc**+			8	$	$t" t" t" t" t" t" t" t"nm m m m m m m m`~ ~ ~ ~ ~ ~ ~ ~B)5 )5 )5 )5 )5 )5 )5 )5X3! 3! 3! 3! 3! 3! 3! 3! 3! 3!r   