Merged
test(quant): 补齐 dequant_swiglu_quant/swiglu_mx_quant/dual_axis 的 golden 通路并修正契约偏差 #13332
test(quant): 补齐 dequant_swiglu_quant/swiglu_mx_quant/dual_axis 的 golden 通路并修正契约偏差 #13332
Merged
柴海伦Add on 19 days ago
Total: 6 files changed+3083-959
@@ -0,0 +1,13 @@
1+# -----------------------------------------------------------------------------------------------------------
2+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+# CANN Open Software License Agreement Version 2.0 (the "License").
5+# Please refer to the License for details. You may not use this file except in compliance with the License.
6+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+# See LICENSE in the root of the software repository for the full text of the License.
9+# -----------------------------------------------------------------------------------------------------------
10+ 
11+__all__ = ["swiglu_mx_quant_with_dual_axis"]
12+ 
13+from .swiglu_mx_quant_with_dual_axis import swiglu_mx_quant_with_dual_axis
@@ -0,0 +1,137 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file swiglu_mx_quant_with_dual_axis.cpp
13+ * \brief Torch binding of the forward-only SwigluMxQuantWithDualAxis operator.
14+ */
15+ 
16+#include <torch/extension.h>
17+#include "aclnnop/aclnn_swiglu_mx_quant_with_dual_axis.h"
18+#include "aclnn_common.h"
19+ 
20+namespace cann_ops_nn {
21+namespace quant {
22+namespace {
23+constexpr int64_t BLOCK_SIZE = 32;
24+constexpr int64_t ROW_BLOCK_SIZE = 64;
25+constexpr int64_t MXSCALE_TAIL_DIM = 2;
26+constexpr int64_t FP4_PER_BYTE = 2;
27+constexpr int64_t DST_TYPE_FP8_E5M2 = 35;
28+constexpr int64_t DST_TYPE_FP8_E4M3FN = 36;
29+constexpr int64_t DST_TYPE_FP4_E2M1 = 40;
30+constexpr int64_t DST_TYPE_FP4_E1M2 = 41;
31+ 
32+bool IsFp4DstType(int64_t dstType) { return dstType == DST_TYPE_FP4_E2M1 || dstType == DST_TYPE_FP4_E1M2; }
33+ 
34+aclDataType GetAclDataTypeFromDstType(int64_t dstType)
35+{
36+ switch (dstType) {
37+ case DST_TYPE_FP8_E5M2:
38+ return ACL_FLOAT8_E5M2;
39+ case DST_TYPE_FP8_E4M3FN:
40+ return ACL_FLOAT8_E4M3FN;
41+ case DST_TYPE_FP4_E2M1:
42+ return ACL_FLOAT4_E2M1;
43+ case DST_TYPE_FP4_E1M2:
44+ return ACL_FLOAT4_E1M2;
45+ default:
46+ TORCH_CHECK(false, "dst_type must be 35(FP8_E5M2), 36(FP8_E4M3FN), 40(FP4_E2M1) or 41(FP4_E1M2), but got ",
47+ dstType);
48+ }
49+}
50+ 
51+at::ScalarType GetScalarTypeFromDstType(int64_t dstType)
52+{
53+ switch (dstType) {
54+ case DST_TYPE_FP8_E5M2:
55+ return at::ScalarType::Float8_e5m2;
56+ case DST_TYPE_FP8_E4M3FN:
57+ return at::ScalarType::Float8_e4m3fn;
58+ default:
59+ return at::kByte; // FP4 packed as uint8
60+ }
61+}
62+} // namespace
63+ 
64+std::tuple<at::Tensor, at::Tensor, at::Tensor, at::Tensor> swiglu_mx_quant_with_dual_axis(
65+ const at::Tensor& x, const c10::optional<at::Tensor>& groupIndex, bool activateLeft, c10::string_view roundMode,
66+ int64_t scaleAlg, int64_t dstType, double maxDtypeValue)
67+{
68+ TORCH_CHECK(x.device().type() == at::kPrivateUse1, "x must be on NPU device, but got ", x.device());
69+ TORCH_CHECK(!x.requires_grad(), "swiglu_mx_quant_with_dual_axis is forward-only and does not support autograd");
70+ TORCH_CHECK(x.dim() == 2, "x must be 2-dimensional, but got ", x.dim());
71+ TORCH_CHECK(x.size(1) % FP4_PER_BYTE == 0, "x last dim must be even, but got ", x.size(1));
72+ const auto inputDtype = x.scalar_type();
73+ TORCH_CHECK(inputDtype == at::kHalf || inputDtype == at::kBFloat16, "x dtype must be float16 or bfloat16, but got ",
74+ inputDtype);
75+ 
76+ // NOTE: the checks here mirror register_meta() in
77+ // swiglu_mx_quant_with_dual_axis.py and must stay in lockstep with it
78+ // (same order, same accept/reject semantics); they sit behind the NPU
79+ // device check, so only the Meta copy is reachable without hardware.
80+ const aclDataType yAclType = GetAclDataTypeFromDstType(dstType);
81+ const at::ScalarType yScalarType = GetScalarTypeFromDstType(dstType);
82+ TORCH_CHECK(scaleAlg == 0 || scaleAlg == 1, "scale_alg must be 0 (OCP) or 1 (cuBLAS, FP8 only), but got ",
83+ scaleAlg);
84+ if (IsFp4DstType(dstType)) {
85+ TORCH_CHECK(scaleAlg == 0, "scale_alg must be 0 (OCP) for FP4 dst_type, but got ", scaleAlg);
86+ TORCH_CHECK(roundMode == "rint" || roundMode == "floor" || roundMode == "round",
87+ "round_mode must be rint, floor or round for FP4 dst_type, but got ", roundMode);
88+ } else {
89+ TORCH_CHECK(roundMode == "rint", "round_mode must be rint for FP8 dst_type, but got ", roundMode);
90+ }
91+ 
92+ const int64_t rows = x.size(0);
93+ const int64_t cols = x.size(1) / FP4_PER_BYTE;
94+ int64_t groups = 0;
95+ if (groupIndex.has_value() && groupIndex->defined()) {
96+ TORCH_CHECK(groupIndex->device().type() == at::kPrivateUse1, "group_index must be on NPU device");
97+ TORCH_CHECK(groupIndex->dim() == 1, "group_index must be 1-dimensional, but got ", groupIndex->dim());
98+ TORCH_CHECK(groupIndex->scalar_type() == at::kLong || groupIndex->scalar_type() == at::kInt,
99+ "group_index dtype must be int32 or int64, but got ", groupIndex->scalar_type());
100+ groups = groupIndex->numel();
101+ }
102+ 
103+ // y1/y2: [M, N]; FP4 packs two elements per byte along the last dim.
104+ std::vector<int64_t> yShape{rows, IsFp4DstType(dstType) ? cols / FP4_PER_BYTE : cols};
105+ // mxScale1 pairs the -1 axis blocks: [M, ceil(ceil(N/32)/2), 2]
106+ const int64_t colBlocks = (cols + BLOCK_SIZE - 1) / BLOCK_SIZE;
107+ std::vector<int64_t> scale1Shape{rows, (colBlocks + 1) / FP4_PER_BYTE, MXSCALE_TAIL_DIM};
108+ // mxScale2 covers the -2 axis: grouped layout keeps one pair gap per group
109+ const int64_t scale2Rows = groups > 0 ? rows / ROW_BLOCK_SIZE + groups :
110+ (rows + ROW_BLOCK_SIZE - 1) / ROW_BLOCK_SIZE;
111+ std::vector<int64_t> scale2Shape{scale2Rows, cols, MXSCALE_TAIL_DIM};
112+ 
113+ auto yOptions = x.options().dtype(yScalarType);
114+ auto scaleOptions = x.options().dtype(at::ScalarType::Float8_e8m0fnu);
115+ at::Tensor y1 = at::empty(yShape, yOptions);
116+ at::Tensor mxScale1 = at::empty(scale1Shape, scaleOptions);
117+ at::Tensor y2 = at::empty(yShape, yOptions);
118+ at::Tensor mxScale2 = at::empty(scale2Shape, scaleOptions);
119+ 
120+ // FP4 y is packed along the last dim; declare packedDim so the wrapper
121+ // restores the logical element count instead of guessing from strides.
122+ TensorWrapper y1Wrapper{y1, yAclType, y1.dim() - 1};
123+ TensorWrapper y2Wrapper{y2, yAclType, y2.dim() - 1};
124+ char* roundModePtr = const_cast<char*>(roundMode.data());
125+ ACLNN_CMD(aclnnSwigluMxQuantWithDualAxis, x, groupIndex, activateLeft, roundModePtr, scaleAlg, dstType,
126+ maxDtypeValue, y1Wrapper, mxScale1, y2Wrapper, mxScale2);
127+ 
128+ return std::make_tuple(std::move(y1), std::move(mxScale1), std::move(y2), std::move(mxScale2));
129+}
130+} // namespace quant
131+} // namespace cann_ops_nn
132+ 
133+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
134+{
135+ m.def("swiglu_mx_quant_with_dual_axis", &cann_ops_nn::quant::swiglu_mx_quant_with_dual_axis,
136+ "SwigluMxQuantWithDualAxis forward on NPU (SwiGLU + dual-axis MX dynamic quantization)");
137+}
@@ -0,0 +1,195 @@
1+# -----------------------------------------------------------------------------------------------------------
2+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+# CANN Open Software License Agreement Version 2.0 (the "License").
5+# Please refer to the License for details. You may not use this file except in compliance with the License.
6+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+# See LICENSE in the root of the software repository for the full text of the License.
9+# -----------------------------------------------------------------------------------------------------------
10+"""Torch registration for the forward-only SwigluMxQuantWithDualAxis operator."""
11+ 
12+import torch
13+from torch.library import impl
14+ 
15+from cann_ops_nn.op_builder import OpBuilder, get_as_library
16+ 
17+BLOCK_SIZE = 32
18+ROW_BLOCK_SIZE = 64
19+MXSCALE_TAIL_DIM = 2
20+FP4_PER_BYTE = 2
21+DST_TYPE_FP8_E5M2 = 35
22+DST_TYPE_FP8_E4M3FN = 36
23+DST_TYPE_FP4_E2M1 = 40
24+DST_TYPE_FP4_E1M2 = 41
25+ 
26+_FP4_DST_TYPES = (DST_TYPE_FP4_E2M1, DST_TYPE_FP4_E1M2)
27+_OUTPUT_DTYPES = {
28+ DST_TYPE_FP8_E5M2: torch.float8_e5m2,
29+ DST_TYPE_FP8_E4M3FN: torch.float8_e4m3fn,
30+ DST_TYPE_FP4_E2M1: torch.uint8, # FP4 packed as uint8
31+ DST_TYPE_FP4_E1M2: torch.uint8,
32+}
33+ 
34+ 
35+def _check_arguments(x, group_index, round_mode, scale_alg, dst_type):
36+ torch._check(
37+ not x.requires_grad, lambda: "swiglu_mx_quant_with_dual_axis is forward-only"
38+ )
39+ torch._check(x.dim() == 2, lambda: f"x must be 2-dimensional, but got {x.dim()}")
40+ torch._check(
41+ x.shape[1] % FP4_PER_BYTE == 0,
42+ lambda: f"x last dim must be even, but got {x.shape[1]}",
43+ )
44+ torch._check(
45+ x.dtype in (torch.float16, torch.bfloat16),
46+ lambda: f"x dtype must be float16 or bfloat16, but got {x.dtype}",
47+ )
48+ torch._check(
49+ dst_type in _OUTPUT_DTYPES,
50+ lambda: (
51+ "dst_type must be 35(FP8_E5M2), 36(FP8_E4M3FN), 40(FP4_E2M1) or 41(FP4_E1M2), "
52+ f"but got {dst_type}"
53+ ),
54+ )
55+ torch._check(
56+ scale_alg in (0, 1),
57+ lambda: f"scale_alg must be 0 (OCP) or 1 (cuBLAS, FP8 only), got {scale_alg}",
58+ )
59+ if dst_type in _FP4_DST_TYPES:
60+ torch._check(
61+ scale_alg == 0,
62+ lambda: f"scale_alg must be 0 (OCP) for FP4 dst_type, got {scale_alg}",
63+ )
64+ torch._check(
65+ round_mode in ("rint", "floor", "round"),
66+ lambda: f"round_mode must be rint, floor or round for FP4 dst_type, got {round_mode}",
67+ )
68+ else:
69+ torch._check(
70+ round_mode == "rint",
71+ lambda: f"round_mode must be rint for FP8 dst_type, got {round_mode}",
72+ )
73+ if group_index is not None:
74+ torch._check(
75+ group_index.dim() == 1,
76+ lambda: f"group_index must be 1-dimensional, got {group_index.dim()}",
77+ )
78+ torch._check(
79+ group_index.dtype in (torch.int32, torch.int64),
80+ lambda: f"group_index dtype must be int32 or int64, got {group_index.dtype}",
81+ )
82+ 
83+ 
84+def _output_shapes(rows, cols, groups, dst_type):
85+ """y1/y2 and the two MX scale shapes (see the op README shape contract)."""
86+ y = [rows, cols // FP4_PER_BYTE if dst_type in _FP4_DST_TYPES else cols]
87+ col_blocks = (cols + BLOCK_SIZE - 1) // BLOCK_SIZE
88+ scale1 = [rows, (col_blocks + 1) // FP4_PER_BYTE, MXSCALE_TAIL_DIM]
89+ # the grouped -2 axis layout keeps one scale pair gap per group
90+ scale2_rows = (
91+ rows // ROW_BLOCK_SIZE + groups
92+ if groups
93+ else (rows + ROW_BLOCK_SIZE - 1) // ROW_BLOCK_SIZE
94+ )
95+ scale2 = [scale2_rows, cols, MXSCALE_TAIL_DIM]
96+ return y, scale1, scale2
97+ 
98+ 
99+class SwigluMxQuantWithDualAxisOpBuilder(OpBuilder):
100+ """SwigluMxQuantWithDualAxis builder (aclnnSwigluMxQuantWithDualAxis).
101+ 
102+ Computes SwiGLU and then quantizes the result along both the -1 and -2 axes
103+ with block-wise MX dynamic quantization, emitting two low-precision tensors
104+ and their shared-exponent scales.
105+ """
106+ 
107+ def __init__(self):
108+ super().__init__("swiglu_mx_quant_with_dual_axis")
109+ 
110+ def sources(self):
111+ return [self.resolve_source("swiglu_mx_quant_with_dual_axis.cpp")]
112+ 
113+ def schema(self) -> str:
114+ return (
115+ "swiglu_mx_quant_with_dual_axis(Tensor x, Tensor? group_index=None, *, "
116+ 'bool activate_left=True, str round_mode="rint", int scale_alg=0, '
117+ "int dst_type=40, float max_dtype_value=0.0) -> (Tensor, Tensor, Tensor, Tensor)"
118+ )
119+ 
120+ def register_meta(self):
121+ @impl(get_as_library(), self.name, "Meta")
122+ def swiglu_mx_quant_with_dual_axis_meta(
123+ x,
124+ group_index=None,
125+ *,
126+ activate_left=True,
127+ round_mode="rint",
128+ scale_alg=0,
129+ dst_type=DST_TYPE_FP4_E2M1,
130+ max_dtype_value=0.0,
131+ ):
132+ _check_arguments(x, group_index, round_mode, scale_alg, dst_type)
133+ y_shape, scale1_shape, scale2_shape = _output_shapes(
134+ x.shape[0],
135+ x.shape[1] // FP4_PER_BYTE,
136+ group_index.numel() if group_index is not None else 0,
137+ dst_type,
138+ )
139+ y_dtype = _OUTPUT_DTYPES[dst_type]
140+ scale_dtype = torch.float8_e8m0fnu
141+ return (
142+ x.new_empty(y_shape, dtype=y_dtype),
143+ x.new_empty(scale1_shape, dtype=scale_dtype),
144+ x.new_empty(y_shape, dtype=y_dtype),
145+ x.new_empty(scale2_shape, dtype=scale_dtype),
146+ )
147+ 
148+ 
149+builder = SwigluMxQuantWithDualAxisOpBuilder()
150+builder._ensure_initialized()
151+ 
152+ 
153+@impl(get_as_library(), builder.name, "PrivateUse1")
154+def _swiglu_mx_quant_with_dual_axis_npu(
155+ x,
156+ group_index=None,
157+ *,
158+ activate_left=True,
159+ round_mode="rint",
160+ scale_alg=0,
161+ dst_type=DST_TYPE_FP4_E2M1,
162+ max_dtype_value=0.0,
163+):
164+ outputs = builder.load().swiglu_mx_quant_with_dual_axis(
165+ x,
166+ group_index,
167+ activate_left,
168+ round_mode,
169+ scale_alg,
170+ dst_type,
171+ max_dtype_value,
172+ )
173+ return tuple(output.detach() for output in outputs)
174+ 
175+ 
176+def swiglu_mx_quant_with_dual_axis(
177+ x,
178+ group_index=None,
179+ *,
180+ activate_left=True,
181+ round_mode="rint",
182+ scale_alg=0,
183+ dst_type=DST_TYPE_FP4_E2M1,
184+ max_dtype_value=0.0,
185+):
186+ """Fused SwiGLU followed by dual-axis (-1 and -2) MX block quantization."""
187+ return torch.ops.cann_ops_nn.swiglu_mx_quant_with_dual_axis.default(
188+ x,
189+ group_index,
190+ activate_left=activate_left,
191+ round_mode=round_mode,
192+ scale_alg=scale_alg,
193+ dst_type=dst_type,
194+ max_dtype_value=max_dtype_value,
195+ )