Merged
test(quant): 补齐 dequant_swiglu_quant/swiglu_mx_quant/dual_axis 的 golden 通路并修正契约偏差 #13332
柴海伦Add on 19 days ago
test(quant): 补齐 dequant_swiglu_quant/swiglu_mx_quant/dual_axis 的 golden 通路并修正契约偏差 #13332
Merged
Total: 6 files changed+3083-959
| @@ -0,0 +1,13 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | + | ||
| 11 | +__all__ = ["swiglu_mx_quant_with_dual_axis"] | ||
| 12 | + | ||
| 13 | +from .swiglu_mx_quant_with_dual_axis import swiglu_mx_quant_with_dual_axis | ||
| @@ -0,0 +1,137 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file swiglu_mx_quant_with_dual_axis.cpp | ||
| 13 | + * \brief Torch binding of the forward-only SwigluMxQuantWithDualAxis operator. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#include <torch/extension.h> | ||
| 17 | +#include "aclnnop/aclnn_swiglu_mx_quant_with_dual_axis.h" | ||
| 18 | +#include "aclnn_common.h" | ||
| 19 | + | ||
| 20 | +namespace cann_ops_nn { | ||
| 21 | +namespace quant { | ||
| 22 | +namespace { | ||
| 23 | +constexpr int64_t BLOCK_SIZE = 32; | ||
| 24 | +constexpr int64_t ROW_BLOCK_SIZE = 64; | ||
| 25 | +constexpr int64_t MXSCALE_TAIL_DIM = 2; | ||
| 26 | +constexpr int64_t FP4_PER_BYTE = 2; | ||
| 27 | +constexpr int64_t DST_TYPE_FP8_E5M2 = 35; | ||
| 28 | +constexpr int64_t DST_TYPE_FP8_E4M3FN = 36; | ||
| 29 | +constexpr int64_t DST_TYPE_FP4_E2M1 = 40; | ||
| 30 | +constexpr int64_t DST_TYPE_FP4_E1M2 = 41; | ||
| 31 | + | ||
| 32 | +bool IsFp4DstType(int64_t dstType) { return dstType == DST_TYPE_FP4_E2M1 || dstType == DST_TYPE_FP4_E1M2; } | ||
| 33 | + | ||
| 34 | +aclDataType GetAclDataTypeFromDstType(int64_t dstType) | ||
| 35 | +{ | ||
| 36 | + switch (dstType) { | ||
| 37 | + case DST_TYPE_FP8_E5M2: | ||
| 38 | + return ACL_FLOAT8_E5M2; | ||
| 39 | + case DST_TYPE_FP8_E4M3FN: | ||
| 40 | + return ACL_FLOAT8_E4M3FN; | ||
| 41 | + case DST_TYPE_FP4_E2M1: | ||
| 42 | + return ACL_FLOAT4_E2M1; | ||
| 43 | + case DST_TYPE_FP4_E1M2: | ||
| 44 | + return ACL_FLOAT4_E1M2; | ||
| 45 | + default: | ||
| 46 | + TORCH_CHECK(false, "dst_type must be 35(FP8_E5M2), 36(FP8_E4M3FN), 40(FP4_E2M1) or 41(FP4_E1M2), but got ", | ||
| 47 | + dstType); | ||
| 48 | + } | ||
| 49 | +} | ||
| 50 | + | ||
| 51 | +at::ScalarType GetScalarTypeFromDstType(int64_t dstType) | ||
| 52 | +{ | ||
| 53 | + switch (dstType) { | ||
| 54 | + case DST_TYPE_FP8_E5M2: | ||
| 55 | + return at::ScalarType::Float8_e5m2; | ||
| 56 | + case DST_TYPE_FP8_E4M3FN: | ||
| 57 | + return at::ScalarType::Float8_e4m3fn; | ||
| 58 | + default: | ||
| 59 | + return at::kByte; // FP4 packed as uint8 | ||
| 60 | + } | ||
| 61 | +} | ||
| 62 | +} // namespace | ||
| 63 | + | ||
| 64 | +std::tuple<at::Tensor, at::Tensor, at::Tensor, at::Tensor> swiglu_mx_quant_with_dual_axis( | ||
| 65 | + const at::Tensor& x, const c10::optional<at::Tensor>& groupIndex, bool activateLeft, c10::string_view roundMode, | ||
| 66 | + int64_t scaleAlg, int64_t dstType, double maxDtypeValue) | ||
| 67 | +{ | ||
| 68 | + TORCH_CHECK(x.device().type() == at::kPrivateUse1, "x must be on NPU device, but got ", x.device()); | ||
| 69 | + TORCH_CHECK(!x.requires_grad(), "swiglu_mx_quant_with_dual_axis is forward-only and does not support autograd"); | ||
| 70 | + TORCH_CHECK(x.dim() == 2, "x must be 2-dimensional, but got ", x.dim()); | ||
| 71 | + TORCH_CHECK(x.size(1) % FP4_PER_BYTE == 0, "x last dim must be even, but got ", x.size(1)); | ||
| 72 | + const auto inputDtype = x.scalar_type(); | ||
| 73 | + TORCH_CHECK(inputDtype == at::kHalf || inputDtype == at::kBFloat16, "x dtype must be float16 or bfloat16, but got ", | ||
| 74 | + inputDtype); | ||
| 75 | + | ||
| 76 | + // NOTE: the checks here mirror register_meta() in | ||
| 77 | + // swiglu_mx_quant_with_dual_axis.py and must stay in lockstep with it | ||
| 78 | + // (same order, same accept/reject semantics); they sit behind the NPU | ||
| 79 | + // device check, so only the Meta copy is reachable without hardware. | ||
| 80 | + const aclDataType yAclType = GetAclDataTypeFromDstType(dstType); | ||
| 81 | + const at::ScalarType yScalarType = GetScalarTypeFromDstType(dstType); | ||
| 82 | + TORCH_CHECK(scaleAlg == 0 || scaleAlg == 1, "scale_alg must be 0 (OCP) or 1 (cuBLAS, FP8 only), but got ", | ||
| 83 | + scaleAlg); | ||
| 84 | + if (IsFp4DstType(dstType)) { | ||
| 85 | + TORCH_CHECK(scaleAlg == 0, "scale_alg must be 0 (OCP) for FP4 dst_type, but got ", scaleAlg); | ||
| 86 | + TORCH_CHECK(roundMode == "rint" || roundMode == "floor" || roundMode == "round", | ||
| 87 | + "round_mode must be rint, floor or round for FP4 dst_type, but got ", roundMode); | ||
| 88 | + } else { | ||
| 89 | + TORCH_CHECK(roundMode == "rint", "round_mode must be rint for FP8 dst_type, but got ", roundMode); | ||
| 90 | + } | ||
| 91 | + | ||
| 92 | + const int64_t rows = x.size(0); | ||
| 93 | + const int64_t cols = x.size(1) / FP4_PER_BYTE; | ||
| 94 | + int64_t groups = 0; | ||
| 95 | + if (groupIndex.has_value() && groupIndex->defined()) { | ||
| 96 | + TORCH_CHECK(groupIndex->device().type() == at::kPrivateUse1, "group_index must be on NPU device"); | ||
| 97 | + TORCH_CHECK(groupIndex->dim() == 1, "group_index must be 1-dimensional, but got ", groupIndex->dim()); | ||
| 98 | + TORCH_CHECK(groupIndex->scalar_type() == at::kLong || groupIndex->scalar_type() == at::kInt, | ||
| 99 | + "group_index dtype must be int32 or int64, but got ", groupIndex->scalar_type()); | ||
| 100 | + groups = groupIndex->numel(); | ||
| 101 | + } | ||
| 102 | + | ||
| 103 | + // y1/y2: [M, N]; FP4 packs two elements per byte along the last dim. | ||
| 104 | + std::vector<int64_t> yShape{rows, IsFp4DstType(dstType) ? cols / FP4_PER_BYTE : cols}; | ||
| 105 | + // mxScale1 pairs the -1 axis blocks: [M, ceil(ceil(N/32)/2), 2] | ||
| 106 | + const int64_t colBlocks = (cols + BLOCK_SIZE - 1) / BLOCK_SIZE; | ||
| 107 | + std::vector<int64_t> scale1Shape{rows, (colBlocks + 1) / FP4_PER_BYTE, MXSCALE_TAIL_DIM}; | ||
| 108 | + // mxScale2 covers the -2 axis: grouped layout keeps one pair gap per group | ||
| 109 | + const int64_t scale2Rows = groups > 0 ? rows / ROW_BLOCK_SIZE + groups : | ||
| 110 | + (rows + ROW_BLOCK_SIZE - 1) / ROW_BLOCK_SIZE; | ||
| 111 | + std::vector<int64_t> scale2Shape{scale2Rows, cols, MXSCALE_TAIL_DIM}; | ||
| 112 | + | ||
| 113 | + auto yOptions = x.options().dtype(yScalarType); | ||
| 114 | + auto scaleOptions = x.options().dtype(at::ScalarType::Float8_e8m0fnu); | ||
| 115 | + at::Tensor y1 = at::empty(yShape, yOptions); | ||
| 116 | + at::Tensor mxScale1 = at::empty(scale1Shape, scaleOptions); | ||
| 117 | + at::Tensor y2 = at::empty(yShape, yOptions); | ||
| 118 | + at::Tensor mxScale2 = at::empty(scale2Shape, scaleOptions); | ||
| 119 | + | ||
| 120 | + // FP4 y is packed along the last dim; declare packedDim so the wrapper | ||
| 121 | + // restores the logical element count instead of guessing from strides. | ||
| 122 | + TensorWrapper y1Wrapper{y1, yAclType, y1.dim() - 1}; | ||
| 123 | + TensorWrapper y2Wrapper{y2, yAclType, y2.dim() - 1}; | ||
| 124 | + char* roundModePtr = const_cast<char*>(roundMode.data()); | ||
| 125 | + ACLNN_CMD(aclnnSwigluMxQuantWithDualAxis, x, groupIndex, activateLeft, roundModePtr, scaleAlg, dstType, | ||
| 126 | + maxDtypeValue, y1Wrapper, mxScale1, y2Wrapper, mxScale2); | ||
| 127 | + | ||
| 128 | + return std::make_tuple(std::move(y1), std::move(mxScale1), std::move(y2), std::move(mxScale2)); | ||
| 129 | +} | ||
| 130 | +} // namespace quant | ||
| 131 | +} // namespace cann_ops_nn | ||
| 132 | + | ||
| 133 | +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) | ||
| 134 | +{ | ||
| 135 | + m.def("swiglu_mx_quant_with_dual_axis", &cann_ops_nn::quant::swiglu_mx_quant_with_dual_axis, | ||
| 136 | + "SwigluMxQuantWithDualAxis forward on NPU (SwiGLU + dual-axis MX dynamic quantization)"); | ||
| 137 | +} | ||
| @@ -0,0 +1,195 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +"""Torch registration for the forward-only SwigluMxQuantWithDualAxis operator.""" | ||
| 11 | + | ||
| 12 | +import torch | ||
| 13 | +from torch.library import impl | ||
| 14 | + | ||
| 15 | +from cann_ops_nn.op_builder import OpBuilder, get_as_library | ||
| 16 | + | ||
| 17 | +BLOCK_SIZE = 32 | ||
| 18 | +ROW_BLOCK_SIZE = 64 | ||
| 19 | +MXSCALE_TAIL_DIM = 2 | ||
| 20 | +FP4_PER_BYTE = 2 | ||
| 21 | +DST_TYPE_FP8_E5M2 = 35 | ||
| 22 | +DST_TYPE_FP8_E4M3FN = 36 | ||
| 23 | +DST_TYPE_FP4_E2M1 = 40 | ||
| 24 | +DST_TYPE_FP4_E1M2 = 41 | ||
| 25 | + | ||
| 26 | +_FP4_DST_TYPES = (DST_TYPE_FP4_E2M1, DST_TYPE_FP4_E1M2) | ||
| 27 | +_OUTPUT_DTYPES = { | ||
| 28 | + DST_TYPE_FP8_E5M2: torch.float8_e5m2, | ||
| 29 | + DST_TYPE_FP8_E4M3FN: torch.float8_e4m3fn, | ||
| 30 | + DST_TYPE_FP4_E2M1: torch.uint8, # FP4 packed as uint8 | ||
| 31 | + DST_TYPE_FP4_E1M2: torch.uint8, | ||
| 32 | +} | ||
| 33 | + | ||
| 34 | + | ||
| 35 | +def _check_arguments(x, group_index, round_mode, scale_alg, dst_type): | ||
| 36 | + torch._check( | ||
| 37 | + not x.requires_grad, lambda: "swiglu_mx_quant_with_dual_axis is forward-only" | ||
| 38 | + ) | ||
| 39 | + torch._check(x.dim() == 2, lambda: f"x must be 2-dimensional, but got {x.dim()}") | ||
| 40 | + torch._check( | ||
| 41 | + x.shape[1] % FP4_PER_BYTE == 0, | ||
| 42 | + lambda: f"x last dim must be even, but got {x.shape[1]}", | ||
| 43 | + ) | ||
| 44 | + torch._check( | ||
| 45 | + x.dtype in (torch.float16, torch.bfloat16), | ||
| 46 | + lambda: f"x dtype must be float16 or bfloat16, but got {x.dtype}", | ||
| 47 | + ) | ||
| 48 | + torch._check( | ||
| 49 | + dst_type in _OUTPUT_DTYPES, | ||
| 50 | + lambda: ( | ||
| 51 | + "dst_type must be 35(FP8_E5M2), 36(FP8_E4M3FN), 40(FP4_E2M1) or 41(FP4_E1M2), " | ||
| 52 | + f"but got {dst_type}" | ||
| 53 | + ), | ||
| 54 | + ) | ||
| 55 | + torch._check( | ||
| 56 | + scale_alg in (0, 1), | ||
| 57 | + lambda: f"scale_alg must be 0 (OCP) or 1 (cuBLAS, FP8 only), got {scale_alg}", | ||
| 58 | + ) | ||
| 59 | + if dst_type in _FP4_DST_TYPES: | ||
| 60 | + torch._check( | ||
| 61 | + scale_alg == 0, | ||
| 62 | + lambda: f"scale_alg must be 0 (OCP) for FP4 dst_type, got {scale_alg}", | ||
| 63 | + ) | ||
| 64 | + torch._check( | ||
| 65 | + round_mode in ("rint", "floor", "round"), | ||
| 66 | + lambda: f"round_mode must be rint, floor or round for FP4 dst_type, got {round_mode}", | ||
| 67 | + ) | ||
| 68 | + else: | ||
| 69 | + torch._check( | ||
| 70 | + round_mode == "rint", | ||
| 71 | + lambda: f"round_mode must be rint for FP8 dst_type, got {round_mode}", | ||
| 72 | + ) | ||
| 73 | + if group_index is not None: | ||
| 74 | + torch._check( | ||
| 75 | + group_index.dim() == 1, | ||
| 76 | + lambda: f"group_index must be 1-dimensional, got {group_index.dim()}", | ||
| 77 | + ) | ||
| 78 | + torch._check( | ||
| 79 | + group_index.dtype in (torch.int32, torch.int64), | ||
| 80 | + lambda: f"group_index dtype must be int32 or int64, got {group_index.dtype}", | ||
| 81 | + ) | ||
| 82 | + | ||
| 83 | + | ||
| 84 | +def _output_shapes(rows, cols, groups, dst_type): | ||
| 85 | + """y1/y2 and the two MX scale shapes (see the op README shape contract).""" | ||
| 86 | + y = [rows, cols // FP4_PER_BYTE if dst_type in _FP4_DST_TYPES else cols] | ||
| 87 | + col_blocks = (cols + BLOCK_SIZE - 1) // BLOCK_SIZE | ||
| 88 | + scale1 = [rows, (col_blocks + 1) // FP4_PER_BYTE, MXSCALE_TAIL_DIM] | ||
| 89 | + # the grouped -2 axis layout keeps one scale pair gap per group | ||
| 90 | + scale2_rows = ( | ||
| 91 | + rows // ROW_BLOCK_SIZE + groups | ||
| 92 | + if groups | ||
| 93 | + else (rows + ROW_BLOCK_SIZE - 1) // ROW_BLOCK_SIZE | ||
| 94 | + ) | ||
| 95 | + scale2 = [scale2_rows, cols, MXSCALE_TAIL_DIM] | ||
| 96 | + return y, scale1, scale2 | ||
| 97 | + | ||
| 98 | + | ||
| 99 | +class SwigluMxQuantWithDualAxisOpBuilder(OpBuilder): | ||
| 100 | + """SwigluMxQuantWithDualAxis builder (aclnnSwigluMxQuantWithDualAxis). | ||
| 101 | + | ||
| 102 | + Computes SwiGLU and then quantizes the result along both the -1 and -2 axes | ||
| 103 | + with block-wise MX dynamic quantization, emitting two low-precision tensors | ||
| 104 | + and their shared-exponent scales. | ||
| 105 | + """ | ||
| 106 | + | ||
| 107 | + def __init__(self): | ||
| 108 | + super().__init__("swiglu_mx_quant_with_dual_axis") | ||
| 109 | + | ||
| 110 | + def sources(self): | ||
| 111 | + return [self.resolve_source("swiglu_mx_quant_with_dual_axis.cpp")] | ||
| 112 | + | ||
| 113 | + def schema(self) -> str: | ||
| 114 | + return ( | ||
| 115 | + "swiglu_mx_quant_with_dual_axis(Tensor x, Tensor? group_index=None, *, " | ||
| 116 | + 'bool activate_left=True, str round_mode="rint", int scale_alg=0, ' | ||
| 117 | + "int dst_type=40, float max_dtype_value=0.0) -> (Tensor, Tensor, Tensor, Tensor)" | ||
| 118 | + ) | ||
| 119 | + | ||
| 120 | + def register_meta(self): | ||
| 121 | + @impl(get_as_library(), self.name, "Meta") | ||
| 122 | + def swiglu_mx_quant_with_dual_axis_meta( | ||
| 123 | + x, | ||
| 124 | + group_index=None, | ||
| 125 | + *, | ||
| 126 | + activate_left=True, | ||
| 127 | + round_mode="rint", | ||
| 128 | + scale_alg=0, | ||
| 129 | + dst_type=DST_TYPE_FP4_E2M1, | ||
| 130 | + max_dtype_value=0.0, | ||
| 131 | + ): | ||
| 132 | + _check_arguments(x, group_index, round_mode, scale_alg, dst_type) | ||
| 133 | + y_shape, scale1_shape, scale2_shape = _output_shapes( | ||
| 134 | + x.shape[0], | ||
| 135 | + x.shape[1] // FP4_PER_BYTE, | ||
| 136 | + group_index.numel() if group_index is not None else 0, | ||
| 137 | + dst_type, | ||
| 138 | + ) | ||
| 139 | + y_dtype = _OUTPUT_DTYPES[dst_type] | ||
| 140 | + scale_dtype = torch.float8_e8m0fnu | ||
| 141 | + return ( | ||
| 142 | + x.new_empty(y_shape, dtype=y_dtype), | ||
| 143 | + x.new_empty(scale1_shape, dtype=scale_dtype), | ||
| 144 | + x.new_empty(y_shape, dtype=y_dtype), | ||
| 145 | + x.new_empty(scale2_shape, dtype=scale_dtype), | ||
| 146 | + ) | ||
| 147 | + | ||
| 148 | + | ||
| 149 | +builder = SwigluMxQuantWithDualAxisOpBuilder() | ||
| 150 | +builder._ensure_initialized() | ||
| 151 | + | ||
| 152 | + | ||
| 153 | +@impl(get_as_library(), builder.name, "PrivateUse1") | ||
| 154 | +def _swiglu_mx_quant_with_dual_axis_npu( | ||
| 155 | + x, | ||
| 156 | + group_index=None, | ||
| 157 | + *, | ||
| 158 | + activate_left=True, | ||
| 159 | + round_mode="rint", | ||
| 160 | + scale_alg=0, | ||
| 161 | + dst_type=DST_TYPE_FP4_E2M1, | ||
| 162 | + max_dtype_value=0.0, | ||
| 163 | +): | ||
| 164 | + outputs = builder.load().swiglu_mx_quant_with_dual_axis( | ||
| 165 | + x, | ||
| 166 | + group_index, | ||
| 167 | + activate_left, | ||
| 168 | + round_mode, | ||
| 169 | + scale_alg, | ||
| 170 | + dst_type, | ||
| 171 | + max_dtype_value, | ||
| 172 | + ) | ||
| 173 | + return tuple(output.detach() for output in outputs) | ||
| 174 | + | ||
| 175 | + | ||
| 176 | +def swiglu_mx_quant_with_dual_axis( | ||
| 177 | + x, | ||
| 178 | + group_index=None, | ||
| 179 | + *, | ||
| 180 | + activate_left=True, | ||
| 181 | + round_mode="rint", | ||
| 182 | + scale_alg=0, | ||
| 183 | + dst_type=DST_TYPE_FP4_E2M1, | ||
| 184 | + max_dtype_value=0.0, | ||
| 185 | +): | ||
| 186 | + """Fused SwiGLU followed by dual-axis (-1 and -2) MX block quantization.""" | ||
| 187 | + return torch.ops.cann_ops_nn.swiglu_mx_quant_with_dual_axis.default( | ||
| 188 | + x, | ||
| 189 | + group_index, | ||
| 190 | + activate_left=activate_left, | ||
| 191 | + round_mode=round_mode, | ||
| 192 | + scale_alg=scale_alg, | ||
| 193 | + dst_type=dst_type, | ||
| 194 | + max_dtype_value=max_dtype_value, | ||
| 195 | + ) | ||