@@ -1106,6 +1106,51 @@ def _sage_attention_backward_op(
11061106 raise NotImplementedError ("Backward pass is not implemented for Sage attention." )
11071107
11081108
1109+ def _npu_attention_forward_op (
1110+ ctx : torch .autograd .function .FunctionCtx ,
1111+ query : torch .Tensor ,
1112+ key : torch .Tensor ,
1113+ value : torch .Tensor ,
1114+ attn_mask : Optional [torch .Tensor ] = None ,
1115+ dropout_p : float = 0.0 ,
1116+ is_causal : bool = False ,
1117+ scale : Optional [float ] = None ,
1118+ enable_gqa : bool = False ,
1119+ return_lse : bool = False ,
1120+ _save_ctx : bool = True ,
1121+ _parallel_config : Optional ["ParallelConfig" ] = None ,
1122+ ):
1123+ if return_lse :
1124+ raise ValueError ("NPU attention backend does not support setting `return_lse=True`." )
1125+
1126+ out = npu_fusion_attention (
1127+ query ,
1128+ key ,
1129+ value ,
1130+ query .size (2 ), # num_heads
1131+ input_layout = "BSND" ,
1132+ pse = None ,
1133+ scale = 1.0 / math .sqrt (query .shape [- 1 ]) if scale is None else scale ,
1134+ pre_tockens = 65536 ,
1135+ next_tockens = 65536 ,
1136+ keep_prob = 1.0 - dropout_p ,
1137+ sync = False ,
1138+ inner_precise = 0 ,
1139+ )[0 ]
1140+
1141+ return out
1142+
1143+
1144+ # Not implemented yet.
1145+ def _npu_attention_backward_op (
1146+ ctx : torch .autograd .function .FunctionCtx ,
1147+ grad_out : torch .Tensor ,
1148+ * args ,
1149+ ** kwargs ,
1150+ ):
1151+ raise NotImplementedError ("Backward pass is not implemented for Npu Fusion Attention." )
1152+
1153+
11091154# ===== Context parallel =====
11101155
11111156
@@ -2131,6 +2176,7 @@ def _native_math_attention(
21312176@_AttentionBackendRegistry .register (
21322177 AttentionBackendName ._NATIVE_NPU ,
21332178 constraints = [_check_device , _check_qkv_dtype_bf16_or_fp16 , _check_shape ],
2179+ supports_context_parallel = True ,
21342180)
21352181def _native_npu_attention (
21362182 query : torch .Tensor ,
@@ -2146,22 +2192,36 @@ def _native_npu_attention(
21462192 raise ValueError ("`attn_mask` is not supported for NPU attention" )
21472193 if return_lse :
21482194 raise ValueError ("NPU attention backend does not support setting `return_lse=True`." )
2149- query , key , value = (x .transpose (1 , 2 ).contiguous () for x in (query , key , value ))
2150- out = npu_fusion_attention (
2151- query ,
2152- key ,
2153- value ,
2154- query .size (1 ), # num_heads
2155- input_layout = "BNSD" ,
2156- pse = None ,
2157- scale = 1.0 / math .sqrt (query .shape [- 1 ]) if scale is None else scale ,
2158- pre_tockens = 65536 ,
2159- next_tockens = 65536 ,
2160- keep_prob = 1.0 - dropout_p ,
2161- sync = False ,
2162- inner_precise = 0 ,
2163- )[0 ]
2164- out = out .transpose (1 , 2 ).contiguous ()
2195+ if _parallel_config is None :
2196+ out = npu_fusion_attention (
2197+ query ,
2198+ key ,
2199+ value ,
2200+ query .size (2 ), # num_heads
2201+ input_layout = "BSND" ,
2202+ pse = None ,
2203+ scale = 1.0 / math .sqrt (query .shape [- 1 ]) if scale is None else scale ,
2204+ pre_tockens = 65536 ,
2205+ next_tockens = 65536 ,
2206+ keep_prob = 1.0 - dropout_p ,
2207+ sync = False ,
2208+ inner_precise = 0 ,
2209+ )[0 ]
2210+ else :
2211+ out = _templated_context_parallel_attention (
2212+ query ,
2213+ key ,
2214+ value ,
2215+ None ,
2216+ dropout_p ,
2217+ None ,
2218+ scale ,
2219+ None ,
2220+ return_lse ,
2221+ forward_op = _npu_attention_forward_op ,
2222+ backward_op = _npu_attention_backward_op ,
2223+ _parallel_config = _parallel_config ,
2224+ )
21652225 return out
21662226
21672227
0 commit comments