
    Yj|[                     z    d dl Z d dlZd dlmZmZ d dlmZ d dlm	Z	  e j
        e          Z G d de          ZdS )    N)AttentionMaskFusionAttention)helper)	OnnxModelc                   :     e Zd ZdZdedededef fdZd Z xZ	S )FusionBartAttentionz?
    Fuse Bart Attention subgraph into one Attention node.
    modelhidden_size	num_headsattention_maskc                 P    t                                          ||||           d S )N)super__init__)selfr	   r
   r   r   	__class__s        k/var/www/FlaskApp/flask-venv/lib/python3.11/site-packages/onnxruntime/transformers/fusion_bart_attention.pyr   zFusionBartAttention.__init__   s)     	YGGGGG    c                    | j                             |g dg d          }d }|4| j                             |g dg d          }||d         }|dd          }|	|\  }}}	}
}nt                              d           d S |8|j        d         |j        d         k    r|j        d         }ni|j        d         }n[g }|j        D ]4}||vr||d         j        d         k    r|                    |           5t          |          dk    rd S |d         }	 ||         }|j        dv r | j         	                    |          d         }|j        D ]6}|s||         }d	 |D             }|
                    d
          dk    r|} n7d | j                                         j        D             }d | j                                         j        D             }| j                             |g dg d          }| j                             |g dg d          }| j                             |g dg d          }d\  }}g d d }}}||}|\  }}}}||j        d                  }|D ]W} | j        dk    rJ| j                             | ddgddg          }!|!|!d         j        d         }|| j        d                  } nX|D ]@} || j        d                  }"|"D ] }#|#j        d         |v r|#j        d         } n!|dk    r nAnx|%|}|\  }$}}}}|$j        d         }|$j        d         }nQ|j        d         |v r|j        d         }n4||}|d         j        d         }nt                              d           d S ||v r|nd}||v r|nd}| j                             |dd
gddg          }%| j                             |g dg d          }&| j                             |g dg d          }'| j                             |g dg d           }(g d }*})|%|%\  }+},|%})n>|&	|&\  }+}*},|&})n3|'	|'\  }+}+},|'})n(|(
|(\  }+}+}*},|(})nt                              d!           d S | j                             |,g d"g d#          }-| j                             |,g d$g d#          }.| j                             |,g d$g d%          }/g }0|-|-}0|0\  }1}2}3}4}5n6|.|.}0|0\  }3}1}2}4}5n)|/|/}0|0\  }3}1}2}4}5nt                              d&           d S | j                             |,g d'g d(          }6| j                             |,g d)g d*          }7| j                             |,g d+g d          }8| j                             |,g d,g d-          }9| j                             |,g d.g d/          }:d\  };}<g d d }?}>}=|6@|6}=|=\  }@}A}?||Aj        d                  }B|BD ] }C|Cj        d         |v r|Cj        d         }< n!nq|7&|7}=|=\  }+}D}@}A}?|Dj        d         };|Dj        d         }<nI||,j        d                  j        d         |v r)||,j        d                  g}=|=d         j        d         };n |:|:}=|=\  }+}+}+}+}@}A}>}?n|8|8}=|=\  }+}@}A}?||?j        d                  }|D ]W} | j        dk    rJ| j                             | ddgddg          }E|E|Ed         j        d         };|| j        d                  } nX|D ]@} || j        d                  }"|"D ] }#|#j        d         |v r|#j        d         }< n!|<dk    r nAn4|9|9}=|=d         j        d         };nt                              d0           d S |;|v r|;nd};|<|v r|<nd}<|?|>| j                             |j        d                   }F|Fj        d         }G|Fj        }Hd1}I| j                             |I          }J|JD|                     |I|H|Ggt!          j        d2g|Gz  t%          j        |H          3          4           | j                             d5          }Kt%          j        d5|I|?j        d         g|Aj        g|K          }>t/          |;          ot/          |          o|?d u o|d u }L|L o2|5j        d         |k    o!|?j        d         |k    o|j        d         |k    }M|L oH|5j        d         |k    o7|?j        d         |j        d         k    o|?j        d         |5j        d         k    }N|)|&|(fv }O|O }P|Mo|P}Q|Mo|O}R|No|P}S|Rot/          |;          ot/          |          }T|Lo|P}U|O}Vg }W|Vr| j                             |*d6gdg          }X| j                             |*g d7g d8          }Y| j                             |*g d9g d:          }Z| j                             |*d;d;gddg          }[|Y|Y}Wn+|Z|Z}Wn&|[|[}Wn!|X|X}Wnt                              d<           d S t          |W          dk    sJ |Qs	|Rs|Ss|Ts|Ur|	}\|                     |2          \  }]}^|]dk    s|^dk    rL| j        dk    rA| j        dk    r6t                              d=|]|^| j        | j                   | j        }]| j        }^|]dk    s|^dk    s	|^|]z  dk    rt                              d>           d S d }_|Ts|Ss|UrX| j        rN|                     |5|Ss|Tr|?n|;|Ss|Tr|n||4|Ss|Tr|>nd |Ss|Tr|nd |]|^|\j        d         |V|Tr|;nd|Tr|nd|<|?          nd }_nE| j        }`d@| _        |                     d |5|?||4|>||]|^||\j        d         |V|;||<|A          }_|`| _        |_t                              dB           d S | j                            |_           | j        | j         |_j        <   | j!        "                    |\|
|g           | j!        "                    |)           |Ts|Ss|UrWt          |0          dk    r%|0d         j        d
k    r|0#                                 t          |=          dk    r%|=d         j        d
k    r|=#                                 t          |          dk    r%|d         j        d
k    r|#                                 | j$        rt          |0          dk    r%|0d         j        d5k    r|0#                                 t          |=          dk    r%|=d         j        d5k    r|=#                                 t          |          dk    r%|d         j        d5k    r|#                                 | j!        "                    |0           | j!        "                    |=           | j!        "                    |           dC| _%        d S d S )DN)AddMatMulReshape	Transposer   )   r   r   r   r   )r   r   r   r   r   r   )r   Nr   r   r   r   r   r   z(fuse_attention: failed to match qkv path>   r   Clipc                     g | ]	}|j         
S  )op_type).0childs     r   
<listcomp>z,FusionBartAttention.fuse.<locals>.<listcomp>e   s    BBBemBBBr   r   c                     h | ]	}|j         
S r   namer   nodes     r   	<setcomp>z+FusionBartAttention.fuse.<locals>.<setcomp>j   s    LLL4TYLLLr   c                     h | ]	}|j         
S r   r"   r$   s     r   r&   z+FusionBartAttention.fuse.<locals>.<setcomp>k   s    NNNDdiNNNr   )r   r   r   r   )r   r   r   N)Concatr   r   r   r   )r   r   r   r   N)r   r   r   r   )r   r   r   r   ) r)   r(   r   r   r)   z&fuse_attention: failed to match v pathSoftmax)r+   r   r   )r   r   r   )Wherer+   r   )r      r   )r,   r+   r   r   )r   r-   r   r   z'fuse_attention: failed to match qk path)r   r   Mulr   r   )r   r   r   r   r   )r.   r   r   r   r   )r   r   r   r   Nz&fuse_attention: failed to match q path)r   r   r   )r   r   r   )r   r(   r   r   r   )r   r   r   r   r   )r.   r   r   r   )r.   r   r   r   r   )r   r   r   r   r   )r.   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   Nz&fuse_attention: failed to match k path
empty_biasg        )dtype)dimsvalsr   r,   )SliceExpandr,   )r   r   r   )r3   	UnsqueezeGatherShaper   )r   r-   r   r   r   r3   z*fuse_attention: failed to match mask nodeszfuse_attention: reshape dims invalid (num_heads=%d, hidden_size=%d), falling back to user-specified num_heads=%d, hidden_size=%dz9fuse_attention: failed to detect num_heads or hidden_size)q_matmulk_matmulv_matmulq_addk_addv_addr   r
   outputunidirectionalpast_kpast_v	present_k	present_vF)
mask_indexr8   r9   r:   r;   r<   r=   r   r
   first_inputr>   causalr@   rA   rB   rC   z+fuse_attention: failed to create fused nodeT)&r	   match_parent_pathloggerdebuginputr>   appendlenr   get_childrencountgraphget_initializerr1   	data_typeadd_initializernparrayr   tensor_dtype_to_np_dtypecreate_node_name	make_noder#   boolget_num_heads_and_hidden_sizer   r
   use_multi_head_attentioncreate_multihead_attention_nodecreate_attention_nodenodes_to_addthis_graph_namenode_name_to_graph_namenodes_to_removeextendpop!disable_multi_head_attention_biasprune_graph)ar   normalize_nodeinput_name_to_nodesoutput_name_to_node	qkv_nodesadd_before_layernormqkv_nodes_with_residualadd_out
matmul_outreshape_qkvtranspose_qkv
matmul_qkv
root_inputother_inputsinput_skip_layernormr>   childrenchildren_typesgraph_input_namesgraph_output_namesv_nodes_past_or_presentv_nodes_with_pastv_nodes_past_only_oairA   rC   v_nodesadd_vmatmul_vtranspose_v	reshape_vstart_child_nodesstart_child_nodeconcat_v_nodesstart_grandchild_nodesstart_grandchild_nodeconcat_vqk_nodes_no_maskqk_nodes_with_maskqk_nodes_sdpa_no_maskqk_nodes_sdpa_with_maskqk_nodesadd_qk_	matmul_qk
q_nodes_hfq_nodes_oaiq_nodes_sdpaq_nodestranspose_q	reshape_qmul_qadd_qmatmul_qk_nodes_no_past_hfk_nodes_with_past_hfk_nodes_past_or_present_oaik_nodes_past_only_oaik_nodes_sdpar@   rB   k_nodesadd_kmatmul_ktranspose_k	reshape_ktranspose_k_nodestranspose_k_nodeconcat_kconcat_k_nodesadd_v_tensorbias_dimr0   empty_bias_nameempty_tensoradd_namethree_root_inputsone_root_inputtwo_root_inputshas_maskno_maskencoder_attentiondecoder_self_attentiondecoder_cross_attention decoder_self_attention_with_past!decoder_cross_attention_with_pastcausal_mask
mask_nodesmask_nodes_bartmask_nodes_whisper_hfmask_nodes_whisper_oai mask_nodes_whisper_oai_unit_testattention_last_noder   r
   new_node%use_multi_head_attention_ground_truthsa                                                                                                    r   fusezFusionBartAttention.fuse   s+   J00???OO
 
	  $&*j&B&BJJJ%%%' '#
 '2'>q'A$3ABB7	  
 LLCDDDF+#)!,q0AAA17:

17:

L(. , ,!444Yq\0333##F++++<  A%%%aJ	 -Z8 !_44!Z44^DDQGN$+ 	 	F *62HBBBBBN##H--22#
 3 ML4:3C3C3E3E3KLLLNNDJ4D4D4F4F4MNNN"&*">">555OO#
 #

 !J88???
 

 !%
 < <<<<LL!
 !

 #	#%tT".-G8?5[)UH !4ELO D$5 
 
 #+x77%)Z%A%A("K0A& &N
 &1!/!3!9!!<(;<L<STU<V(W%E 8 %6   )<=M=TUV=W)X&-C  ),3A6:LLL$9$@$C	 M ??E #*'GBI?X{Iuh^A&F *IIa $555%a(FF".+GR[&q)FFLLABBBF!%666B!*.@!@!@IIb	:77
YPXDY\]_`[abb!Z99*FbFbFbdmdmdmnn !%
 < <ZIgIgIgiririr s s"&*">">===|||#
 #
 t&'+LAy'HH+#5 Avy)HH".3OAq),HH$0&=#Aq&).HHLLBCCCFZ11<<<OO
 


 j22<<<OO
 
 z33<<<
 

 ! G?F<[)UE88$!G?F<UKE88%"G?F<UKE88LLABBBF!Z99...II
 

  $z;;EEEOO 
  

 '+j&B&B555LL'
 '
#
 !%
 < <CCCOO!
 !
 z33___'''
 

 #	#%tT)(G18.[)X !4I4DQ4G H$5   #*1-1CCC 0 7 :IE D "-*G>E;Q+y(^A&F *II !34:1=ARRR*9?1+=>?GQZ%a(FF%"GDKAQ1ai(41G4;1QY !4HOA4F G$5 
 
 #+x77%)Z%A%A("K0A& &N
 &1!/!3!9!!<(;<L<STU<V(W%E 8 %6   )<=M=TUV=W)X&-C  ),3A6:LLL$9$@$C	 M ??E #".+GR[&q)FFLLABBBF!%666B!*.@!@!@IIb	EM:55ek!nEEL#(+H *E*O:55oFFL#$$#"3%("2&:YZ_:`:`aaa	 %    z22599H$U_hoa>P,QT]TbScemnnE LLcT&\\ch$>NcS[_cSc!! 0q!Z/0q!Z/0 q!Z/	 	 "! 7q!Z/7q!X^A%667 q!X^A%66	 	   24KLL,*6w!/!<H"1"=g+A+cd6ll+cW[\bWcWc(,=,I') 
  	'"j::	 O
 %)J$@$@,,,		% %!
 &*Z%A%A@@@& &"
 04z/K/K'"A0 0,
 %02

'33

1==

 ,,

IJJJz??Q&&&& k	$%k	$ 'k	$ 0	k	$
 1k	$ #.%)%G%G	%R%R"I{ Q+"2"28J8JtO_bcOcOcRN$   !N	".A~~!1!1kI6MRS5S5SXYYYH/ .V3J .VNo .V* 4!D88!)-D!tHh!tnt-D!tHh!tnt#'>lBbleehl'>lBbleehl"+$/29!<'2)IQvvr)IQvvr"+"+ 9   " % , 9=8U505-55#%%%' + *.5a8&!!''! 6  $ 1V-JKKK$$X...:>:ND(7 '')<mZ(XYYY ''111 0 &3J &No &w<<!##(;x(G(GKKMMMw<<!##(;x(G(GKKMMMw<<!##(;x(G(GKKMMM9 &7||a''GBK,?5,H,H7||a''GBK,?5,H,H7||a''GBK,?5,H,H ''000 ''000 ''000  $DWk	$ k	$r   )
__name__
__module____qualname____doc__r   intr   r   r   __classcell__)r   s   @r   r   r      s         HH H 	H
 &H H H H H H]$ ]$ ]$ ]$ ]$ ]$ ]$r   r   )loggingnumpyrS   fusion_attentionr   r   onnxr   
onnx_modelr   	getLoggerr   rH   r   r   r   r   <module>r      s   
      ; ; ; ; ; ; ; ;                  		8	$	$k$ k$ k$ k$ k$/ k$ k$ k$ k$ k$r   