File tree 2 files changed +6
-1
lines changed
2 files changed +6
-1
lines changed Original file line number Diff line number Diff line change @@ -177,7 +177,7 @@ def varlen_attention(
177
177
out : torch .Tensor ,
178
178
seqlen_q : torch .Tensor ,
179
179
seqlen_k : torch .Tensor ,
180
- alibi_slopes : torch .Tensor ,
180
+ alibi_slopes : Optional [ torch .Tensor ] ,
181
181
max_seqlen_q : int ,
182
182
max_seqlen_k : int ,
183
183
pdropout : float ,
@@ -193,6 +193,8 @@ def varlen_attention(
193
193
if ipex .__version__ .endswith ("cpu" ):
194
194
if logits_soft_cap != 0.0 :
195
195
raise ValueError ("IPEX CPU does not support logits_soft_cap" )
196
+ assert alibi_slopes is None
197
+ assert window_size_left < 0 and window_size_right < 0
196
198
ipex .llm .functional .varlen_attention (query .contiguous (),
197
199
key .contiguous (),
198
200
value .contiguous (), out ,
Original file line number Diff line number Diff line change @@ -273,6 +273,9 @@ def _forward_prefill(
273
273
return_softmax = False ,
274
274
gen_ = None ,
275
275
logits_soft_cap = 0.0 ,
276
+ window_size_left = - 1 ,
277
+ window_size_right = - 1 ,
278
+ alibi_slopes = None ,
276
279
)
277
280
278
281
# remove padding
You can’t perform that action at this time.
0 commit comments