Merged
feat(torch_extension): 新增 HiF4 量化与反量化接口 #13394
F0rty-tw0Add on 19 days ago
feat(torch_extension): 新增 HiF4 量化与反量化接口 #13394
Merged
Total: 18 files changed+1332-4
| @@ -12,10 +12,18 @@ from .mx_to_block_mx_quant import mx_to_block_mx_quant | |||
| 12 | from .dequant_situ_quant import dequant_situ_quant | 12 | from .dequant_situ_quant import dequant_situ_quant |
| 13 | from .situ_mx_quant import situ_mx_quant | 13 | from .situ_mx_quant import situ_mx_quant |
| 14 | from .swiglu_hi_f4_quant import swiglu_hi_f4_quant | 14 | from .swiglu_hi_f4_quant import swiglu_hi_f4_quant |
| 15 | +from .dynamic_hi_f4_quant import dynamic_hi_f4_quant | ||
| 16 | +from .dynamic_block_hi_f4_quant import dynamic_block_hi_f4_quant | ||
| 17 | +from .grouped_dynamic_hi_f4_quant import grouped_dynamic_hi_f4_quant | ||
| 18 | +from .anti_hi_f4_quant import anti_hi_f4_quant | ||
| 15 | 19 | ||
| 16 | __all__ = [ | 20 | __all__ = [ |
| 17 | "mx_to_block_mx_quant", | 21 | "mx_to_block_mx_quant", |
| 18 | "dequant_situ_quant", | 22 | "dequant_situ_quant", |
| 19 | "situ_mx_quant", | 23 | "situ_mx_quant", |
| 20 | "swiglu_hi_f4_quant", | 24 | "swiglu_hi_f4_quant", |
| 25 | + "dynamic_hi_f4_quant", | ||
| 26 | + "dynamic_block_hi_f4_quant", | ||
| 27 | + "grouped_dynamic_hi_f4_quant", | ||
| 28 | + "anti_hi_f4_quant", | ||
| 21 | ] | 29 | ] |
| @@ -0,0 +1,13 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +__all__ = ["anti_hi_f4_quant", "convert_anti_hi_f4_quant"] | ||
| 11 | + | ||
| 12 | +from .anti_hi_f4_quant import anti_hi_f4_quant | ||
| 13 | +from .graph_convert_anti_hi_f4_quant import convert_anti_hi_f4_quant | ||
| @@ -0,0 +1,106 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +import torch | ||
| 11 | +from torch.library import impl | ||
| 12 | + | ||
| 13 | +from cann_ops_nn.op_builder import OpBuilder, get_as_library | ||
| 14 | + | ||
| 15 | +DST_TYPE_FLOAT16 = 1 | ||
| 16 | +DST_TYPE_BFLOAT16 = 27 | ||
| 17 | +FP4_IN_BYTE = 2 | ||
| 18 | +HIF4_BLOCK_SIZE = 64 | ||
| 19 | +X_DIM_NUM_MIN = 2 | ||
| 20 | +X_DIM_NUM_MAX = 7 | ||
| 21 | +SCALE_DIM_NUM_MIN = 3 | ||
| 22 | +SCALE_DIM_NUM_MAX = 8 | ||
| 23 | +SUPPORTED_DST_TYPES = (DST_TYPE_FLOAT16, DST_TYPE_BFLOAT16) | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +def _check_dst_type(dst_type): | ||
| 27 | + if dst_type not in SUPPORTED_DST_TYPES: | ||
| 28 | + raise RuntimeError( | ||
| 29 | + f"dst_type should be one of {SUPPORTED_DST_TYPES} " | ||
| 30 | + f"(1=float16, 27=bfloat16), but got {dst_type}" | ||
| 31 | + ) | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +def _check_rank(x, scale): | ||
| 35 | + if x.dim() < X_DIM_NUM_MIN or x.dim() > X_DIM_NUM_MAX: | ||
| 36 | + raise RuntimeError( | ||
| 37 | + f"x rank should be in [{X_DIM_NUM_MIN}, {X_DIM_NUM_MAX}], but got {x.dim()}" | ||
| 38 | + ) | ||
| 39 | + if scale.dim() < SCALE_DIM_NUM_MIN or scale.dim() > SCALE_DIM_NUM_MAX: | ||
| 40 | + raise RuntimeError( | ||
| 41 | + f"scale rank should be in [{SCALE_DIM_NUM_MIN}, {SCALE_DIM_NUM_MAX}], but got {scale.dim()}" | ||
| 42 | + ) | ||
| 43 | + | ||
| 44 | + | ||
| 45 | +def _check_inputs(x, scale): | ||
| 46 | + _check_rank(x, scale) | ||
| 47 | + if x.dtype != torch.uint8: | ||
| 48 | + raise RuntimeError( | ||
| 49 | + f"x dtype should be uint8 (packed HiFloat4), but got {x.dtype}" | ||
| 50 | + ) | ||
| 51 | + if scale.dtype != torch.float32: | ||
| 52 | + raise RuntimeError( | ||
| 53 | + f"scale dtype should be float32 (packed HiFloat4_SCALE storage), but got {scale.dtype}" | ||
| 54 | + ) | ||
| 55 | + if x.device != scale.device: | ||
| 56 | + raise RuntimeError("x and scale should be on the same device") | ||
| 57 | + if not scale.is_contiguous(): | ||
| 58 | + raise RuntimeError("scale should be contiguous") | ||
| 59 | + if ( | ||
| 60 | + x.size(-1) % (HIF4_BLOCK_SIZE // FP4_IN_BYTE) != 0 | ||
| 61 | + or x.size(-2) % HIF4_BLOCK_SIZE != 0 | ||
| 62 | + ): | ||
| 63 | + raise RuntimeError( | ||
| 64 | + "x logical last and second last dimensions should be multiples of 64" | ||
| 65 | + ) | ||
| 66 | + | ||
| 67 | + | ||
| 68 | +def _y_shape(x): | ||
| 69 | + shape = list(x.shape) | ||
| 70 | + shape[-1] = x.size(x.dim() - 1) * FP4_IN_BYTE | ||
| 71 | + return shape | ||
| 72 | + | ||
| 73 | + | ||
| 74 | +def _y_dtype(dst_type): | ||
| 75 | + return torch.float16 if dst_type == DST_TYPE_FLOAT16 else torch.bfloat16 | ||
| 76 | + | ||
| 77 | + | ||
| 78 | +class AntiHiF4QuantOpBuilder(OpBuilder): | ||
| 79 | + def __init__(self): | ||
| 80 | + super().__init__("anti_hi_f4_quant") | ||
| 81 | + | ||
| 82 | + def sources(self): | ||
| 83 | + return [self.resolve_source("anti_hi_f4_quant.cpp")] | ||
| 84 | + | ||
| 85 | + def schema(self): | ||
| 86 | + return "anti_hi_f4_quant(Tensor x, Tensor scale, *, int dst_type=27) -> Tensor" | ||
| 87 | + | ||
| 88 | + def register_meta(self): | ||
| 89 | + @impl(get_as_library(), self.name, "Meta") | ||
| 90 | + def anti_hi_f4_quant_meta(x, scale, *, dst_type=27): | ||
| 91 | + _check_dst_type(dst_type) | ||
| 92 | + _check_inputs(x, scale) | ||
| 93 | + y = x.new_empty(_y_shape(x), dtype=_y_dtype(dst_type)) | ||
| 94 | + return y | ||
| 95 | + | ||
| 96 | + | ||
| 97 | +builder = AntiHiF4QuantOpBuilder() | ||
| 98 | +builder._ensure_initialized() | ||
| 99 | + | ||
| 100 | + | ||
| 101 | +@impl(get_as_library(), builder.name, "PrivateUse1") | ||
| 102 | +def anti_hi_f4_quant(x, scale, *, dst_type=27): | ||
| 103 | + _check_dst_type(dst_type) | ||
| 104 | + _check_inputs(x, scale) | ||
| 105 | + op_module = builder.load() | ||
| 106 | + return op_module.anti_hi_f4_quant(x, scale, dst_type) | ||
| @@ -0,0 +1,77 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#include <torch/extension.h> | ||
| 11 | +#include "aclnn_common.h" | ||
| 12 | + | ||
| 13 | +namespace cann_ops_nn { | ||
| 14 | +namespace quant { | ||
| 15 | + | ||
| 16 | +constexpr int64_t kDstTypeFloat16 = 1; | ||
| 17 | +constexpr int64_t kDstTypeBFloat16 = 27; | ||
| 18 | +constexpr int64_t kFp4InByte = 2; | ||
| 19 | +constexpr int64_t kHif4BlockSize = 64; | ||
| 20 | +constexpr int64_t kXDimNumMin = 2; | ||
| 21 | +constexpr int64_t kXDimNumMax = 7; | ||
| 22 | +constexpr int64_t kScaleDimNumMin = 3; | ||
| 23 | +constexpr int64_t kScaleDimNumMax = 8; | ||
| 24 | + | ||
| 25 | +void CheckNpuTensor(const at::Tensor& tensor, const char* name) | ||
| 26 | +{ | ||
| 27 | + TORCH_CHECK(tensor.defined(), name, " must be defined"); | ||
| 28 | + TORCH_CHECK(torch_npu::utils::is_npu(tensor), name, " must be on NPU device"); | ||
| 29 | +} | ||
| 30 | + | ||
| 31 | +void CheckDstType(int64_t dstType) | ||
| 32 | +{ | ||
| 33 | + TORCH_CHECK(dstType == kDstTypeFloat16 || dstType == kDstTypeBFloat16, | ||
| 34 | + "dst_type should be 1 (float16) or 27 (bfloat16), but got ", dstType); | ||
| 35 | +} | ||
| 36 | + | ||
| 37 | +c10::SmallVector<int64_t, op_infer::SIZE> GetYShape(const at::Tensor& x) | ||
| 38 | +{ | ||
| 39 | + TORCH_CHECK(x.dim() >= kXDimNumMin, "x rank should be greater than or equal to 2"); | ||
| 40 | + auto shape = op_infer::array_to_small_vector(x.sizes()); | ||
| 41 | + shape[x.dim() - 1] = x.size(x.dim() - 1) * kFp4InByte; | ||
| 42 | + return shape; | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +at::Tensor anti_hi_f4_quant(const at::Tensor& x, const at::Tensor& scale, int64_t dst_type) | ||
| 46 | +{ | ||
| 47 | + CheckNpuTensor(x, "x"); | ||
| 48 | + CheckNpuTensor(scale, "scale"); | ||
| 49 | + CheckDstType(dst_type); | ||
| 50 | + TORCH_CHECK(x.dim() >= kXDimNumMin && x.dim() <= kXDimNumMax, "x rank should be in [2, 7]"); | ||
| 51 | + TORCH_CHECK(scale.dim() >= kScaleDimNumMin && scale.dim() <= kScaleDimNumMax, "scale rank should be in [3, 8]"); | ||
| 52 | + TORCH_CHECK(x.scalar_type() == at::ScalarType::Byte, "x dtype should be uint8 (packed HiFloat4)"); | ||
| 53 | + TORCH_CHECK(scale.scalar_type() == at::ScalarType::Float, | ||
| 54 | + "scale dtype should be float32 (packed HiFloat4_SCALE storage)"); | ||
| 55 | + TORCH_CHECK(x.device() == scale.device(), "x and scale should be on the same device"); | ||
| 56 | + TORCH_CHECK(scale.is_contiguous(), "scale should be contiguous"); | ||
| 57 | + TORCH_CHECK(x.size(-1) % (kHif4BlockSize / kFp4InByte) == 0 && x.size(-2) % kHif4BlockSize == 0, | ||
| 58 | + "x logical last and second last dimensions should be multiples of 64"); | ||
| 59 | + | ||
| 60 | + const auto yShape = GetYShape(x); | ||
| 61 | + const auto yScalarType = (dst_type == kDstTypeFloat16) ? at::ScalarType::Half : at::ScalarType::BFloat16; | ||
| 62 | + at::Tensor y = at::empty(yShape, x.options().dtype(yScalarType)); | ||
| 63 | + | ||
| 64 | + // 输入按实际 acl 数据类型转发:x 为 HIFLOAT4(torch 侧 uint8 打包存储),scale 为 HIFLOAT4_SCALE。 | ||
| 65 | + TensorWrapper xWrapper{x, ACL_HIFLOAT4_NUM}; | ||
| 66 | + TensorWrapper scaleWrapper{scale, ACL_HIFLOAT4_SCALE_NUM}; | ||
| 67 | + ACLNN_CMD(aclnnAntiHiF4Quant, xWrapper, scaleWrapper, dst_type, y); | ||
| 68 | + return y; | ||
| 69 | +} | ||
| 70 | + | ||
| 71 | +} // namespace quant | ||
| 72 | +} // namespace cann_ops_nn | ||
| 73 | + | ||
| 74 | +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) | ||
| 75 | +{ | ||
| 76 | + m.def("anti_hi_f4_quant", &cann_ops_nn::quant::anti_hi_f4_quant, "AntiHiF4Quant on NPU"); | ||
| 77 | +} | ||
| @@ -0,0 +1,113 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +# GE Converter for Graph Mode | ||
| 11 | +try: | ||
| 12 | + import torch | ||
| 13 | + | ||
| 14 | + from torchair._ge_concrete_graph import ge_apis as ge | ||
| 15 | + from torchair.ge import attr | ||
| 16 | + from torchair.ge._ge_graph import ( | ||
| 17 | + DataType, | ||
| 18 | + Tensor, | ||
| 19 | + TensorSpec, | ||
| 20 | + _ge_dtype_to_ge_proto_dtype, | ||
| 21 | + ) | ||
| 22 | + from torchair._ge_concrete_graph.compat_ir import ge_op, IrDef | ||
| 23 | + from torchair._ge_concrete_graph.fx2ge_converter import ( | ||
| 24 | + register_fx_node_ge_converter, | ||
| 25 | + ) | ||
| 26 | + | ||
| 27 | + _TORCHAIR_AVAILABLE = True | ||
| 28 | + | ||
| 29 | +except ImportError: | ||
| 30 | + _TORCHAIR_AVAILABLE = False | ||
| 31 | + | ||
| 32 | +DST_TYPE_FLOAT16 = 1 | ||
| 33 | +DST_TYPE_BFLOAT16 = 27 | ||
| 34 | +X_DIM_NUM_MIN = 2 | ||
| 35 | +X_DIM_NUM_MAX = 7 | ||
| 36 | +SCALE_DIM_NUM_MIN = 3 | ||
| 37 | +SCALE_DIM_NUM_MAX = 8 | ||
| 38 | + | ||
| 39 | + | ||
| 40 | +def _check_dst_type(dst_type): | ||
| 41 | + if dst_type not in (DST_TYPE_FLOAT16, DST_TYPE_BFLOAT16): | ||
| 42 | + raise RuntimeError( | ||
| 43 | + f"dst_type should be 1 (float16) or 27 (bfloat16), but got {dst_type}" | ||
| 44 | + ) | ||
| 45 | + | ||
| 46 | + | ||
| 47 | +def _dst_ge_dtype(dst_type): | ||
| 48 | + if dst_type == DST_TYPE_FLOAT16: | ||
| 49 | + return DataType.DT_FLOAT16 | ||
| 50 | + return DataType.DT_BF16 | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +def _check_rank(x, scale): | ||
| 54 | + if x.rank < X_DIM_NUM_MIN or x.rank > X_DIM_NUM_MAX: | ||
| 55 | + raise RuntimeError( | ||
| 56 | + f"x rank should be in [{X_DIM_NUM_MIN}, {X_DIM_NUM_MAX}], but got {x.rank}" | ||
| 57 | + ) | ||
| 58 | + if scale.rank < SCALE_DIM_NUM_MIN or scale.rank > SCALE_DIM_NUM_MAX: | ||
| 59 | + raise RuntimeError( | ||
| 60 | + f"scale rank should be in [{SCALE_DIM_NUM_MIN}, {SCALE_DIM_NUM_MAX}], but got {scale.rank}" | ||
| 61 | + ) | ||
| 62 | + | ||
| 63 | + | ||
| 64 | +def _unpack_uint8_to_hif4(x): | ||
| 65 | + bit_shape = [1] * x.rank | ||
| 66 | + bit_shape[-1] = 2 | ||
| 67 | + mul_x2 = ge.Cast(ge.Const(bit_shape), dst_type=DataType.DT_INT32) | ||
| 68 | + x_shape_uint8 = ge.Shape(x) | ||
| 69 | + x_shape_hif4 = ge.Mul(x_shape_uint8, mul_x2) | ||
| 70 | + x_hif4 = ge.Bitcast(x, type=DataType.DT_HIFLOAT4) | ||
| 71 | + return ge.Reshape(x_hif4, x_shape_hif4) | ||
| 72 | + | ||
| 73 | + | ||
| 74 | +if _TORCHAIR_AVAILABLE: | ||
| 75 | + | ||
| 76 | + @register_fx_node_ge_converter(torch.ops.cann_ops_nn.anti_hi_f4_quant.default) | ||
| 77 | + def convert_anti_hi_f4_quant( | ||
| 78 | + x: Tensor, | ||
| 79 | + scale: Tensor, | ||
| 80 | + *, | ||
| 81 | + dst_type: int = 27, | ||
| 82 | + meta_outputs: TensorSpec = None, | ||
| 83 | + ): | ||
| 84 | + _check_dst_type(dst_type) | ||
| 85 | + _check_rank(x, scale) | ||
| 86 | + if x.dtype != DataType.DT_UINT8: | ||
| 87 | + raise RuntimeError("x dtype should be uint8 (packed HiFloat4)") | ||
| 88 | + if scale.dtype != DataType.DT_FLOAT: | ||
| 89 | + raise RuntimeError( | ||
| 90 | + "scale dtype should be float32 (packed HiFloat4_SCALE storage)" | ||
| 91 | + ) | ||
| 92 | + x_hif4 = _unpack_uint8_to_hif4(x) | ||
| 93 | + # Both storage types occupy 32 bits, so this preserves shape and payload. | ||
| 94 | + scale_hif4 = ge.Bitcast(scale, type=DataType.DT_HIFLOAT4_SCALE) | ||
| 95 | + y = ge_op( | ||
| 96 | + op_type="AntiHiF4Quant", | ||
| 97 | + inputs={"x": x_hif4, "scale": scale_hif4}, | ||
| 98 | + attrs={"dst_type": attr.Int(dst_type)}, | ||
| 99 | + outputs=["y"], | ||
| 100 | + ir=IrDef("AntiHiF4Quant") | ||
| 101 | + .input("x", "DT_HIFLOAT4") | ||
| 102 | + .input("scale", "DT_HIFLOAT4_SCALE") | ||
| 103 | + .attr("dst_type", attr.Int(DST_TYPE_BFLOAT16)) | ||
| 104 | + .output("y", "DT_FLOAT16, DT_BF16"), | ||
| 105 | + ) | ||
| 106 | + | ||
| 107 | + y.desc.dtype = _ge_dtype_to_ge_proto_dtype(_dst_ge_dtype(dst_type)) | ||
| 108 | + return y | ||
| 109 | + | ||
| 110 | +else: | ||
| 111 | + | ||
| 112 | + def convert_anti_hi_f4_quant(*args, **kwargs): | ||
| 113 | + raise RuntimeError("AntiHiF4Quant graph converter: torchair is not available.") | ||
| @@ -0,0 +1,13 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +__all__ = ["dynamic_block_hi_f4_quant", "convert_dynamic_block_hi_f4_quant"] | ||
| 11 | + | ||
| 12 | +from .dynamic_block_hi_f4_quant import dynamic_block_hi_f4_quant | ||
| 13 | +from .graph_convert_dynamic_block_hi_f4_quant import convert_dynamic_block_hi_f4_quant | ||
| @@ -0,0 +1,111 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#include <torch/extension.h> | ||
| 11 | +#include "aclnn_common.h" | ||
| 12 | + | ||
| 13 | +namespace cann_ops_nn { | ||
| 14 | +namespace quant { | ||
| 15 | + | ||
| 16 | +constexpr int64_t kHif4BlockSize = 64; | ||
| 17 | +constexpr int64_t kFp4InByte = 2; | ||
| 18 | +constexpr int64_t kScaleTrailingDim = 1; | ||
| 19 | + | ||
| 20 | +void CheckNpuTensor(const at::Tensor& tensor, const char* name) | ||
| 21 | +{ | ||
| 22 | + TORCH_CHECK(tensor.defined(), name, " must be defined"); | ||
| 23 | + TORCH_CHECK(torch_npu::utils::is_npu(tensor), name, " must be on NPU device"); | ||
| 24 | +} | ||
| 25 | + | ||
| 26 | +int64_t CeilDiv(int64_t value, int64_t factor) | ||
| 27 | +{ | ||
| 28 | + TORCH_CHECK(factor > 0, "factor must be positive"); | ||
| 29 | + return (value + factor - 1) / factor; | ||
| 30 | +} | ||
| 31 | + | ||
| 32 | +void CheckRoundMode(const std::string& roundMode) | ||
| 33 | +{ | ||
| 34 | + TORCH_CHECK(roundMode == "rint" || roundMode == "trunc", "round_mode should be one of 'rint' or 'trunc', but got ", | ||
| 35 | + roundMode); | ||
| 36 | +} | ||
| 37 | + | ||
| 38 | +// scale_alg: 0-标准阈值(4/2), 1-放宽阈值(3.75/1.875) | ||
| 39 | +constexpr int64_t kScaleAlgDefault = 0; | ||
| 40 | +constexpr int64_t kScaleAlgRelaxed = 1; | ||
| 41 | + | ||
| 42 | +void ValidateScaleAlg(int64_t scaleAlg) | ||
| 43 | +{ | ||
| 44 | + TORCH_CHECK(scaleAlg == kScaleAlgDefault || scaleAlg == kScaleAlgRelaxed, "scale_alg should be 0 or 1, but got ", | ||
| 45 | + scaleAlg); | ||
| 46 | +} | ||
| 47 | + | ||
| 48 | +c10::SmallVector<int64_t, op_infer::SIZE> GetYShape(const at::Tensor& x) | ||
| 49 | +{ | ||
| 50 | + TORCH_CHECK(x.dim() >= 2, "x rank should be greater than or equal to 2"); | ||
| 51 | + auto shape = op_infer::array_to_small_vector(x.sizes()); | ||
| 52 | + shape[x.dim() - 1] = CeilDiv(x.size(x.dim() - 1), kFp4InByte); | ||
| 53 | + return shape; | ||
| 54 | +} | ||
| 55 | + | ||
| 56 | +c10::SmallVector<int64_t, op_infer::SIZE> GetScale1Shape(const at::Tensor& x) | ||
| 57 | +{ | ||
| 58 | + TORCH_CHECK(x.dim() >= 2, "x rank should be greater than or equal to 2"); | ||
| 59 | + auto shape = op_infer::array_to_small_vector(x.sizes()); | ||
| 60 | + shape[x.dim() - 1] = CeilDiv(x.size(x.dim() - 1), kHif4BlockSize); | ||
| 61 | + shape.emplace_back(kScaleTrailingDim); | ||
| 62 | + return shape; | ||
| 63 | +} | ||
| 64 | + | ||
| 65 | +c10::SmallVector<int64_t, op_infer::SIZE> GetScale2Shape(const at::Tensor& x) | ||
| 66 | +{ | ||
| 67 | + TORCH_CHECK(x.dim() >= 2, "x rank should be greater than or equal to 2"); | ||
| 68 | + auto shape = op_infer::array_to_small_vector(x.sizes()); | ||
| 69 | + shape[x.dim() - 2] = CeilDiv(x.size(x.dim() - 2), kHif4BlockSize); | ||
| 70 | + shape.emplace_back(kScaleTrailingDim); | ||
| 71 | + return shape; | ||
| 72 | +} | ||
| 73 | + | ||
| 74 | +std::tuple<at::Tensor, at::Tensor, at::Tensor> dynamic_block_hi_f4_quant(const at::Tensor& x, | ||
| 75 | + const std::string& round_mode, | ||
| 76 | + const int64_t scale_alg) | ||
| 77 | +{ | ||
| 78 | + CheckNpuTensor(x, "x"); | ||
| 79 | + CheckRoundMode(round_mode); | ||
| 80 | + ValidateScaleAlg(scale_alg); | ||
| 81 | + TORCH_CHECK(x.dim() >= 2 && x.dim() <= 7, "x rank should be in [2, 7]"); | ||
| 82 | + TORCH_CHECK(x.size(-1) % kHif4BlockSize == 0 && x.size(-2) % kHif4BlockSize == 0, | ||
| 83 | + "x last and second last dimensions should be multiples of 64"); | ||
| 84 | + TORCH_CHECK(x.scalar_type() == at::ScalarType::Half || x.scalar_type() == at::ScalarType::BFloat16, | ||
| 85 | + "x dtype should be float16 or bfloat16"); | ||
| 86 | + | ||
| 87 | + const auto yShape = GetYShape(x); | ||
| 88 | + at::Tensor y = at::empty(yShape, x.options().dtype(at::ScalarType::Byte)); | ||
| 89 | + | ||
| 90 | + const auto scale1Shape = GetScale1Shape(x); | ||
| 91 | + const auto scale2Shape = GetScale2Shape(x); | ||
| 92 | + at::Tensor yScale1 = at::empty(scale1Shape, x.options().dtype(at::ScalarType::Float)); | ||
| 93 | + at::Tensor yScale2 = at::empty(scale2Shape, x.options().dtype(at::ScalarType::Float)); | ||
| 94 | + | ||
| 95 | + // 输出按实际 acl 数据类型转发:y 为 HIFLOAT4,scale1/scale2 为 HIFLOAT4_SCALE。 | ||
| 96 | + // y 显式指定 packedDim=最后一维:acl 侧按元素级 shape(最后一维×2,与 x 一致)声明, | ||
| 97 | + // 满足算子 tiling 的 y.shape == x.shape 校验;torch 侧 y 保持 packed(D/2)视图。 | ||
| 98 | + TensorWrapper yWrapper{y, ACL_HIFLOAT4_NUM, y.dim() - 1}; | ||
| 99 | + TensorWrapper yScale1Wrapper{yScale1, ACL_HIFLOAT4_SCALE_NUM}; | ||
| 100 | + TensorWrapper yScale2Wrapper{yScale2, ACL_HIFLOAT4_SCALE_NUM}; | ||
| 101 | + ACLNN_CMD(aclnnDynamicBlockHiF4Quant, x, round_mode, scale_alg, yWrapper, yScale1Wrapper, yScale2Wrapper); | ||
| 102 | + return std::make_tuple(y, yScale1, yScale2); | ||
| 103 | +} | ||
| 104 | + | ||
| 105 | +} // namespace quant | ||
| 106 | +} // namespace cann_ops_nn | ||
| 107 | + | ||
| 108 | +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) | ||
| 109 | +{ | ||
| 110 | + m.def("dynamic_block_hi_f4_quant", &cann_ops_nn::quant::dynamic_block_hi_f4_quant, "DynamicBlockHiF4Quant on NPU"); | ||
| 111 | +} | ||
| @@ -0,0 +1,109 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +import torch | ||
| 11 | +from torch.library import impl | ||
| 12 | + | ||
| 13 | +from cann_ops_nn.op_builder import OpBuilder, get_as_library | ||
| 14 | + | ||
| 15 | +HIF4_BLOCK_SIZE = 64 | ||
| 16 | +FP4_IN_BYTE = 2 | ||
| 17 | +SUPPORTED_ROUND_MODES = ("rint", "trunc") | ||
| 18 | +DEFAULT_SCALE_ALG = 0 | ||
| 19 | +SCALE_ALG_CHOICES = (0, 1) | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +def _ceil_div(value, factor): | ||
| 23 | + return (value + factor - 1) // factor | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +def _check_x(x): | ||
| 27 | + if x.dim() < 2 or x.dim() > 7: | ||
| 28 | + raise RuntimeError(f"x rank should be in [2, 7], but got {x.dim()}") | ||
| 29 | + if x.dtype not in (torch.float16, torch.bfloat16): | ||
| 30 | + raise RuntimeError(f"x dtype should be float16 or bfloat16, but got {x.dtype}") | ||
| 31 | + if x.size(-1) % HIF4_BLOCK_SIZE != 0 or x.size(-2) % HIF4_BLOCK_SIZE != 0: | ||
| 32 | + raise RuntimeError( | ||
| 33 | + "x last and second last dimensions should be multiples of 64" | ||
| 34 | + ) | ||
| 35 | + | ||
| 36 | + | ||
| 37 | +def _check_round_mode(round_mode): | ||
| 38 | + if round_mode not in SUPPORTED_ROUND_MODES: | ||
| 39 | + raise RuntimeError( | ||
| 40 | + f"round_mode should be one of {SUPPORTED_ROUND_MODES}, but got {round_mode}" | ||
| 41 | + ) | ||
| 42 | + | ||
| 43 | + | ||
| 44 | +def _validate_scale_alg(scale_alg): | ||
| 45 | + """scale_alg: 0-标准阈值(4/2), 1-放宽阈值(3.75/1.875)""" | ||
| 46 | + if scale_alg not in SCALE_ALG_CHOICES: | ||
| 47 | + raise RuntimeError( | ||
| 48 | + f"scale_alg must be one of {SCALE_ALG_CHOICES}, but got {scale_alg}" | ||
| 49 | + ) | ||
| 50 | + | ||
| 51 | + | ||
| 52 | +def _y_shape(x): | ||
| 53 | + shape = list(x.shape) | ||
| 54 | + shape[-1] = _ceil_div(x.size(x.dim() - 1), FP4_IN_BYTE) | ||
| 55 | + return shape | ||
| 56 | + | ||
| 57 | + | ||
| 58 | +def _scale1_shape(x): | ||
| 59 | + shape = list(x.shape) | ||
| 60 | + shape[-1] = _ceil_div(x.size(x.dim() - 1), HIF4_BLOCK_SIZE) | ||
| 61 | + shape.append(1) | ||
| 62 | + return shape | ||
| 63 | + | ||
| 64 | + | ||
| 65 | +def _scale2_shape(x): | ||
| 66 | + shape = list(x.shape) | ||
| 67 | + shape[-2] = _ceil_div(x.size(x.dim() - 2), HIF4_BLOCK_SIZE) | ||
| 68 | + shape.append(1) | ||
| 69 | + return shape | ||
| 70 | + | ||
| 71 | + | ||
| 72 | +class DynamicBlockHiF4QuantOpBuilder(OpBuilder): | ||
| 73 | + def __init__(self): | ||
| 74 | + super().__init__("dynamic_block_hi_f4_quant") | ||
| 75 | + | ||
| 76 | + def sources(self): | ||
| 77 | + return [self.resolve_source("dynamic_block_hi_f4_quant.cpp")] | ||
| 78 | + | ||
| 79 | + def schema(self): | ||
| 80 | + return ( | ||
| 81 | + "dynamic_block_hi_f4_quant(Tensor x, *, str round_mode='rint', " | ||
| 82 | + "int scale_alg=0) -> (Tensor, Tensor, Tensor)" | ||
| 83 | + ) | ||
| 84 | + | ||
| 85 | + def register_meta(self): | ||
| 86 | + @impl(get_as_library(), self.name, "Meta") | ||
| 87 | + def dynamic_block_hi_f4_quant_meta( | ||
| 88 | + x, *, round_mode="rint", scale_alg=DEFAULT_SCALE_ALG | ||
| 89 | + ): | ||
| 90 | + _check_x(x) | ||
| 91 | + _check_round_mode(round_mode) | ||
| 92 | + _validate_scale_alg(scale_alg) | ||
| 93 | + y = x.new_empty(_y_shape(x), dtype=torch.uint8) | ||
| 94 | + y_scale1 = x.new_empty(_scale1_shape(x), dtype=torch.float32) | ||
| 95 | + y_scale2 = x.new_empty(_scale2_shape(x), dtype=torch.float32) | ||
| 96 | + return y, y_scale1, y_scale2 | ||
| 97 | + | ||
| 98 | + | ||
| 99 | +builder = DynamicBlockHiF4QuantOpBuilder() | ||
| 100 | +builder._ensure_initialized() | ||
| 101 | + | ||
| 102 | + | ||
| 103 | +@impl(get_as_library(), builder.name, "PrivateUse1") | ||
| 104 | +def dynamic_block_hi_f4_quant(x, *, round_mode="rint", scale_alg=DEFAULT_SCALE_ALG): | ||
| 105 | + _check_x(x) | ||
| 106 | + _check_round_mode(round_mode) | ||
| 107 | + _validate_scale_alg(scale_alg) | ||
| 108 | + op_module = builder.load() | ||
| 109 | + return op_module.dynamic_block_hi_f4_quant(x, round_mode, scale_alg) | ||
| @@ -0,0 +1,131 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +# GE Converter for Graph Mode | ||
| 11 | +try: | ||
| 12 | + import torch | ||
| 13 | + | ||
| 14 | + from torchair._ge_concrete_graph import ge_apis as ge | ||
| 15 | + from torchair.ge import attr | ||
| 16 | + from torchair.ge._ge_graph import ( | ||
| 17 | + DataType, | ||
| 18 | + Tensor, | ||
| 19 | + TensorSpec, | ||
| 20 | + _ge_dtype_to_ge_proto_dtype, | ||
| 21 | + ) | ||
| 22 | + from torchair._ge_concrete_graph.compat_ir import ge_op, IrDef | ||
| 23 | + from torchair._ge_concrete_graph.fx2ge_converter import ( | ||
| 24 | + register_fx_node_ge_converter, | ||
| 25 | + ) | ||
| 26 | + | ||
| 27 | + _TORCHAIR_AVAILABLE = True | ||
| 28 | + | ||
| 29 | +except ImportError: | ||
| 30 | + _TORCHAIR_AVAILABLE = False | ||
| 31 | + | ||
| 32 | +HIF4_BLOCK_SIZE = 64 | ||
| 33 | +SUPPORTED_ROUND_MODES = ("rint", "trunc") | ||
| 34 | +DEFAULT_SCALE_ALG = 0 | ||
| 35 | +SCALE_ALG_CHOICES = (0, 1) | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +def _ceil_div(value, factor): | ||
| 39 | + return (value + factor - 1) // factor | ||
| 40 | + | ||
| 41 | + | ||
| 42 | +def _check_round_mode(round_mode): | ||
| 43 | + if round_mode not in SUPPORTED_ROUND_MODES: | ||
| 44 | + raise RuntimeError( | ||
| 45 | + f"round_mode should be one of {SUPPORTED_ROUND_MODES}, but got {round_mode}" | ||
| 46 | + ) | ||
| 47 | + | ||
| 48 | + | ||
| 49 | +def _validate_scale_alg(scale_alg): | ||
| 50 | + """scale_alg: 0-标准阈值(4/2), 1-放宽阈值(3.75/1.875)""" | ||
| 51 | + if scale_alg not in SCALE_ALG_CHOICES: | ||
| 52 | + raise RuntimeError( | ||
| 53 | + f"scale_alg must be one of {SCALE_ALG_CHOICES}, but got {scale_alg}" | ||
| 54 | + ) | ||
| 55 | + | ||
| 56 | + | ||
| 57 | +def _scale1_shape(x): | ||
| 58 | + shape = list(x.shape) | ||
| 59 | + shape[-1] = _ceil_div(shape[-1], HIF4_BLOCK_SIZE) | ||
| 60 | + shape.append(1) | ||
| 61 | + return shape | ||
| 62 | + | ||
| 63 | + | ||
| 64 | +def _scale2_shape(x): | ||
| 65 | + shape = list(x.shape) | ||
| 66 | + shape[-2] = _ceil_div(shape[-2], HIF4_BLOCK_SIZE) | ||
| 67 | + shape.append(1) | ||
| 68 | + return shape | ||
| 69 | + | ||
| 70 | + | ||
| 71 | +def _pack_hif4_to_uint8(x, y): | ||
| 72 | + bit_shape = [1] * x.rank | ||
| 73 | + bit_shape[-1] = 2 | ||
| 74 | + div_x2 = ge.Cast(ge.Const(bit_shape), dst_type=DataType.DT_INT32) | ||
| 75 | + y_shape_hif4 = ge.Shape(y) | ||
| 76 | + y_shape_uint8 = ge.Div(y_shape_hif4, div_x2) | ||
| 77 | + y_shape_hif4_2bit = ge.ConcatV2( | ||
| 78 | + [y_shape_uint8, ge.Cast(ge.Const([2]), dst_type=DataType.DT_INT32)], | ||
| 79 | + concat_dim=0, | ||
| 80 | + N=2, | ||
| 81 | + ) | ||
| 82 | + y_uint8 = ge.Bitcast(ge.Reshape(y, y_shape_hif4_2bit), type=DataType.DT_UINT8) | ||
| 83 | + return ge.Reshape(y_uint8, y_shape_uint8) | ||
| 84 | + | ||
| 85 | + | ||
| 86 | +if _TORCHAIR_AVAILABLE: | ||
| 87 | + | ||
| 88 | + @register_fx_node_ge_converter( | ||
| 89 | + torch.ops.cann_ops_nn.dynamic_block_hi_f4_quant.default | ||
| 90 | + ) | ||
| 91 | + def convert_dynamic_block_hi_f4_quant( | ||
| 92 | + x: Tensor, | ||
| 93 | + *, | ||
| 94 | + round_mode: str = "rint", | ||
| 95 | + scale_alg: int = DEFAULT_SCALE_ALG, | ||
| 96 | + meta_outputs: TensorSpec = None, | ||
| 97 | + ): | ||
| 98 | + _check_round_mode(round_mode) | ||
| 99 | + _validate_scale_alg(scale_alg) | ||
| 100 | + y, y_scale1, y_scale2 = ge_op( | ||
| 101 | + op_type="DynamicBlockHiF4Quant", | ||
| 102 | + inputs={"x": x}, | ||
| 103 | + attrs={ | ||
| 104 | + "round_mode": attr.Str(round_mode), | ||
| 105 | + "scale_alg": attr.Int(scale_alg), | ||
| 106 | + }, | ||
| 107 | + outputs=["y", "scale1", "scale2"], | ||
| 108 | + ir=IrDef("DynamicBlockHiF4Quant") | ||
| 109 | + .input("x", "DT_FLOAT16, DT_BF16") | ||
| 110 | + .attr("round_mode", attr.Str("rint")) | ||
| 111 | + .attr("scale_alg", attr.Int(DEFAULT_SCALE_ALG)) | ||
| 112 | + .output("y", "DT_HIFLOAT4") | ||
| 113 | + .output("scale1", "DT_HIFLOAT4_SCALE") | ||
| 114 | + .output("scale2", "DT_HIFLOAT4_SCALE"), | ||
| 115 | + ) | ||
| 116 | + | ||
| 117 | + y.desc.dtype = _ge_dtype_to_ge_proto_dtype(DataType.DT_HIFLOAT4) | ||
| 118 | + y = _pack_hif4_to_uint8(x, y) | ||
| 119 | + y_scale1.desc.dtype = _ge_dtype_to_ge_proto_dtype(DataType.DT_HIFLOAT4_SCALE) | ||
| 120 | + y_scale2.desc.dtype = _ge_dtype_to_ge_proto_dtype(DataType.DT_HIFLOAT4_SCALE) | ||
| 121 | + # Preserve both packed 32-bit scale payloads in torch-visible float32 carriers. | ||
| 122 | + y_scale1 = ge.Bitcast(y_scale1, type=DataType.DT_FLOAT) | ||
| 123 | + y_scale2 = ge.Bitcast(y_scale2, type=DataType.DT_FLOAT) | ||
| 124 | + return y, y_scale1, y_scale2 | ||
| 125 | + | ||
| 126 | +else: | ||
| 127 | + | ||
| 128 | + def convert_dynamic_block_hi_f4_quant(*args, **kwargs): | ||
| 129 | + raise RuntimeError( | ||
| 130 | + "DynamicBlockHiF4Quant graph converter: torchair is not available." | ||
| 131 | + ) | ||
| @@ -0,0 +1,13 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +__all__ = ["dynamic_hi_f4_quant", "convert_dynamic_hi_f4_quant"] | ||
| 11 | + | ||
| 12 | +from .dynamic_hi_f4_quant import dynamic_hi_f4_quant | ||
| 13 | +from .graph_convert_dynamic_hi_f4_quant import convert_dynamic_hi_f4_quant | ||
| @@ -0,0 +1,100 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#include <torch/extension.h> | ||
| 11 | +#include "aclnn_common.h" | ||
| 12 | + | ||
| 13 | +namespace cann_ops_nn { | ||
| 14 | +namespace quant { | ||
| 15 | + | ||
| 16 | +constexpr int64_t kHif4BlockSize = 64; | ||
| 17 | +constexpr int64_t kFp4InByte = 2; | ||
| 18 | +constexpr int64_t kScaleTrailingDim = 1; | ||
| 19 | +// scale_alg: 0-标准阈值(4/2), 1-放宽阈值(3.75/1.875) | ||
| 20 | +constexpr int64_t kScaleAlgDefault = 0; | ||
| 21 | +constexpr int64_t kScaleAlgRelaxed = 1; | ||
| 22 | + | ||
| 23 | +void CheckNpuTensor(const at::Tensor& tensor, const char* name) | ||
| 24 | +{ | ||
| 25 | + TORCH_CHECK(tensor.defined(), name, " must be defined"); | ||
| 26 | + TORCH_CHECK(torch_npu::utils::is_npu(tensor), name, " must be on NPU device"); | ||
| 27 | +} | ||
| 28 | + | ||
| 29 | +int64_t CeilDiv(int64_t value, int64_t factor) | ||
| 30 | +{ | ||
| 31 | + TORCH_CHECK(factor > 0, "factor must be positive"); | ||
| 32 | + return (value + factor - 1) / factor; | ||
| 33 | +} | ||
| 34 | + | ||
| 35 | +void CheckAxis(const at::Tensor& x, int64_t axis) | ||
| 36 | +{ | ||
| 37 | + TORCH_CHECK(axis >= -1 * x.dim() && axis < x.dim(), "axis is out of input dimension range [", -1 * x.dim(), ", ", | ||
| 38 | + x.dim(), "), but got ", axis); | ||
| 39 | + const int64_t axisIdx = axis < 0 ? axis + x.dim() : axis; | ||
| 40 | + TORCH_CHECK(axisIdx == x.dim() - 1 || axisIdx == x.dim() - 2, | ||
| 41 | + "axis should select the last or second last dimension"); | ||
| 42 | + TORCH_CHECK(x.size(-1) % kHif4BlockSize == 0, "x last dimension should be a multiple of 64"); | ||
| 43 | + if (axisIdx == x.dim() - 2) { | ||
| 44 | + TORCH_CHECK(x.size(-2) % kHif4BlockSize == 0, "x second last dimension should be a multiple of 64"); | ||
| 45 | + } | ||
| 46 | +} | ||
| 47 | + | ||
| 48 | +void ValidateScaleAlg(int64_t scaleAlg) | ||
| 49 | +{ | ||
| 50 | + TORCH_CHECK(scaleAlg == kScaleAlgDefault || scaleAlg == kScaleAlgRelaxed, "scale_alg should be 0 or 1, but got ", | ||
| 51 | + scaleAlg); | ||
| 52 | +} | ||
| 53 | + | ||
| 54 | +c10::SmallVector<int64_t, op_infer::SIZE> GetYShape(const at::Tensor& x) | ||
| 55 | +{ | ||
| 56 | + TORCH_CHECK(x.dim() >= 2, "x rank should be greater than or equal to 2"); | ||
| 57 | + auto shape = op_infer::array_to_small_vector(x.sizes()); | ||
| 58 | + shape[x.dim() - 1] = CeilDiv(x.size(x.dim() - 1), kFp4InByte); | ||
| 59 | + return shape; | ||
| 60 | +} | ||
| 61 | + | ||
| 62 | +c10::SmallVector<int64_t, op_infer::SIZE> GetScaleShape(const at::Tensor& x, int64_t axis) | ||
| 63 | +{ | ||
| 64 | + TORCH_CHECK(x.dim() >= 2, "x rank should be greater than or equal to 2"); | ||
| 65 | + const int64_t axisIdx = axis < 0 ? axis + x.dim() : axis; | ||
| 66 | + auto shape = op_infer::array_to_small_vector(x.sizes()); | ||
| 67 | + shape[axisIdx] = CeilDiv(x.size(axisIdx), kHif4BlockSize); | ||
| 68 | + shape.emplace_back(kScaleTrailingDim); | ||
| 69 | + return shape; | ||
| 70 | +} | ||
| 71 | + | ||
| 72 | +std::tuple<at::Tensor, at::Tensor> dynamic_hi_f4_quant(const at::Tensor& x, int64_t axis, const int64_t scale_alg) | ||
| 73 | +{ | ||
| 74 | + CheckNpuTensor(x, "x"); | ||
| 75 | + TORCH_CHECK(x.dim() >= 2 && x.dim() <= 7, "x rank should be in [2, 7]"); | ||
| 76 | + CheckAxis(x, axis); | ||
| 77 | + ValidateScaleAlg(scale_alg); | ||
| 78 | + TORCH_CHECK(x.scalar_type() == at::ScalarType::Half || x.scalar_type() == at::ScalarType::BFloat16, | ||
| 79 | + "x dtype should be float16 or bfloat16"); | ||
| 80 | + | ||
| 81 | + const auto yShape = GetYShape(x); | ||
| 82 | + at::Tensor y = at::empty(yShape, x.options().dtype(at::ScalarType::Byte)); | ||
| 83 | + | ||
| 84 | + const auto scaleShape = GetScaleShape(x, axis); | ||
| 85 | + at::Tensor yScale = at::empty(scaleShape, x.options().dtype(at::ScalarType::Float)); | ||
| 86 | + | ||
| 87 | + // 输出按实际 acl 数据类型转发:y 为 HIFLOAT4,scale 为 HIFLOAT4_SCALE。 | ||
| 88 | + TensorWrapper yWrapper{y, ACL_HIFLOAT4_NUM}; | ||
| 89 | + TensorWrapper yScaleWrapper{yScale, ACL_HIFLOAT4_SCALE_NUM}; | ||
| 90 | + ACLNN_CMD(aclnnDynamicHiF4Quant, x, axis, scale_alg, yWrapper, yScaleWrapper); | ||
| 91 | + return std::make_tuple(y, yScale); | ||
| 92 | +} | ||
| 93 | + | ||
| 94 | +} // namespace quant | ||
| 95 | +} // namespace cann_ops_nn | ||
| 96 | + | ||
| 97 | +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) | ||
| 98 | +{ | ||
| 99 | + m.def("dynamic_hi_f4_quant", &cann_ops_nn::quant::dynamic_hi_f4_quant, "DynamicHiF4Quant on NPU"); | ||
| 100 | +} | ||
| @@ -0,0 +1,97 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +import torch | ||
| 11 | +from torch.library import impl | ||
| 12 | + | ||
| 13 | +from cann_ops_nn.op_builder import OpBuilder, get_as_library | ||
| 14 | + | ||
| 15 | +HIF4_BLOCK_SIZE = 64 | ||
| 16 | +FP4_IN_BYTE = 2 | ||
| 17 | +DEFAULT_SCALE_ALG = 0 | ||
| 18 | +SCALE_ALG_CHOICES = (0, 1) | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +def _ceil_div(value, factor): | ||
| 22 | + return (value + factor - 1) // factor | ||
| 23 | + | ||
| 24 | + | ||
| 25 | +def _check_x(x): | ||
| 26 | + if x.dim() < 2 or x.dim() > 7: | ||
| 27 | + raise RuntimeError(f"x rank should be in [2, 7], but got {x.dim()}") | ||
| 28 | + if x.dtype not in (torch.float16, torch.bfloat16): | ||
| 29 | + raise RuntimeError(f"x dtype should be float16 or bfloat16, but got {x.dtype}") | ||
| 30 | + if x.size(-1) % HIF4_BLOCK_SIZE != 0: | ||
| 31 | + raise RuntimeError("x last dimension should be a multiple of 64") | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +def _check_axis(x, axis): | ||
| 35 | + if axis < -x.dim() or axis >= x.dim(): | ||
| 36 | + raise RuntimeError( | ||
| 37 | + f"axis is out of input dimension range [{-x.dim()}, {x.dim()}), but got {axis}" | ||
| 38 | + ) | ||
| 39 | + axis_idx = axis % x.dim() | ||
| 40 | + if axis_idx not in (x.dim() - 1, x.dim() - 2): | ||
| 41 | + raise RuntimeError("axis should select the last or second last dimension") | ||
| 42 | + if axis_idx == x.dim() - 2 and x.size(-2) % HIF4_BLOCK_SIZE != 0: | ||
| 43 | + raise RuntimeError("x second last dimension should be a multiple of 64") | ||
| 44 | + | ||
| 45 | + | ||
| 46 | +def _validate_scale_alg(scale_alg): | ||
| 47 | + """scale_alg: 0-标准阈值(4/2), 1-放宽阈值(3.75/1.875)""" | ||
| 48 | + if scale_alg not in SCALE_ALG_CHOICES: | ||
| 49 | + raise RuntimeError( | ||
| 50 | + f"scale_alg must be one of {SCALE_ALG_CHOICES}, but got {scale_alg}" | ||
| 51 | + ) | ||
| 52 | + | ||
| 53 | + | ||
| 54 | +def _y_shape(x): | ||
| 55 | + shape = list(x.shape) | ||
| 56 | + shape[-1] = _ceil_div(x.size(x.dim() - 1), FP4_IN_BYTE) | ||
| 57 | + return shape | ||
| 58 | + | ||
| 59 | + | ||
| 60 | +def _scale_shape(x, axis): | ||
| 61 | + shape = list(x.shape) | ||
| 62 | + shape[axis] = _ceil_div(x.size(axis), HIF4_BLOCK_SIZE) | ||
| 63 | + return shape + [1] | ||
| 64 | + | ||
| 65 | + | ||
| 66 | +class DynamicHiF4QuantOpBuilder(OpBuilder): | ||
| 67 | + def __init__(self): | ||
| 68 | + super().__init__("dynamic_hi_f4_quant") | ||
| 69 | + | ||
| 70 | + def sources(self): | ||
| 71 | + return [self.resolve_source("dynamic_hi_f4_quant.cpp")] | ||
| 72 | + | ||
| 73 | + def schema(self): | ||
| 74 | + return "dynamic_hi_f4_quant(Tensor x, *, int axis=-1, int scale_alg=0) -> (Tensor, Tensor)" | ||
| 75 | + | ||
| 76 | + def register_meta(self): | ||
| 77 | + @impl(get_as_library(), self.name, "Meta") | ||
| 78 | + def dynamic_hi_f4_quant_meta(x, *, axis=-1, scale_alg=DEFAULT_SCALE_ALG): | ||
| 79 | + _check_x(x) | ||
| 80 | + _check_axis(x, axis) | ||
| 81 | + _validate_scale_alg(scale_alg) | ||
| 82 | + y = x.new_empty(_y_shape(x), dtype=torch.uint8) | ||
| 83 | + y_scale = x.new_empty(_scale_shape(x, axis), dtype=torch.float32) | ||
| 84 | + return y, y_scale | ||
| 85 | + | ||
| 86 | + | ||
| 87 | +builder = DynamicHiF4QuantOpBuilder() | ||
| 88 | +builder._ensure_initialized() | ||
| 89 | + | ||
| 90 | + | ||
| 91 | +@impl(get_as_library(), builder.name, "PrivateUse1") | ||
| 92 | +def dynamic_hi_f4_quant(x, *, axis=-1, scale_alg=DEFAULT_SCALE_ALG): | ||
| 93 | + _check_x(x) | ||
| 94 | + _check_axis(x, axis) | ||
| 95 | + _validate_scale_alg(scale_alg) | ||
| 96 | + op_module = builder.load() | ||
| 97 | + return op_module.dynamic_hi_f4_quant(x, axis, scale_alg) | ||
| @@ -0,0 +1,106 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +# GE Converter for Graph Mode | ||
| 11 | +try: | ||
| 12 | + import torch | ||
| 13 | + | ||
| 14 | + from torchair._ge_concrete_graph import ge_apis as ge | ||
| 15 | + from torchair.ge import attr | ||
| 16 | + from torchair.ge._ge_graph import ( | ||
| 17 | + DataType, | ||
| 18 | + Tensor, | ||
| 19 | + TensorSpec, | ||
| 20 | + _ge_dtype_to_ge_proto_dtype, | ||
| 21 | + ) | ||
| 22 | + from torchair._ge_concrete_graph.compat_ir import ge_op, IrDef | ||
| 23 | + from torchair._ge_concrete_graph.fx2ge_converter import ( | ||
| 24 | + register_fx_node_ge_converter, | ||
| 25 | + ) | ||
| 26 | + | ||
| 27 | + _TORCHAIR_AVAILABLE = True | ||
| 28 | + | ||
| 29 | +except ImportError: | ||
| 30 | + _TORCHAIR_AVAILABLE = False | ||
| 31 | + | ||
| 32 | +DEFAULT_SCALE_ALG = 0 | ||
| 33 | +SCALE_ALG_CHOICES = (0, 1) | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +def _check_axis(x, axis): | ||
| 37 | + if axis < -x.rank or axis >= x.rank: | ||
| 38 | + raise RuntimeError( | ||
| 39 | + f"axis is out of input dimension range [{-x.rank}, {x.rank}), but got {axis}" | ||
| 40 | + ) | ||
| 41 | + | ||
| 42 | + | ||
| 43 | +def _validate_scale_alg(scale_alg): | ||
| 44 | + """scale_alg: 0-标准阈值(4/2), 1-放宽阈值(3.75/1.875)""" | ||
| 45 | + if scale_alg not in SCALE_ALG_CHOICES: | ||
| 46 | + raise RuntimeError( | ||
| 47 | + f"scale_alg must be one of {SCALE_ALG_CHOICES}, but got {scale_alg}" | ||
| 48 | + ) | ||
| 49 | + | ||
| 50 | + | ||
| 51 | +def _pack_hif4_to_uint8(x, y): | ||
| 52 | + bit_shape = [1] * x.rank | ||
| 53 | + bit_shape[-1] = 2 | ||
| 54 | + div_x2 = ge.Cast(ge.Const(bit_shape), dst_type=DataType.DT_INT32) | ||
| 55 | + y_shape_hif4 = ge.Shape(y) | ||
| 56 | + y_shape_uint8 = ge.Div(y_shape_hif4, div_x2) | ||
| 57 | + y_shape_hif4_2bit = ge.ConcatV2( | ||
| 58 | + [y_shape_uint8, ge.Cast(ge.Const([2]), dst_type=DataType.DT_INT32)], | ||
| 59 | + concat_dim=0, | ||
| 60 | + N=2, | ||
| 61 | + ) | ||
| 62 | + y_uint8 = ge.Bitcast(ge.Reshape(y, y_shape_hif4_2bit), type=DataType.DT_UINT8) | ||
| 63 | + return ge.Reshape(y_uint8, y_shape_uint8) | ||
| 64 | + | ||
| 65 | + | ||
| 66 | +if _TORCHAIR_AVAILABLE: | ||
| 67 | + | ||
| 68 | + @register_fx_node_ge_converter(torch.ops.cann_ops_nn.dynamic_hi_f4_quant.default) | ||
| 69 | + def convert_dynamic_hi_f4_quant( | ||
| 70 | + x: Tensor, | ||
| 71 | + *, | ||
| 72 | + axis: int = -1, | ||
| 73 | + scale_alg: int = DEFAULT_SCALE_ALG, | ||
| 74 | + meta_outputs: TensorSpec = None, | ||
| 75 | + ): | ||
| 76 | + _check_axis(x, axis) | ||
| 77 | + _validate_scale_alg(scale_alg) | ||
| 78 | + y, scale = ge_op( | ||
| 79 | + op_type="DynamicHiF4Quant", | ||
| 80 | + inputs={"x": x}, | ||
| 81 | + attrs={ | ||
| 82 | + "axis": attr.Int(axis), | ||
| 83 | + "scale_alg": attr.Int(scale_alg), | ||
| 84 | + }, | ||
| 85 | + outputs=["y", "scale"], | ||
| 86 | + ir=IrDef("DynamicHiF4Quant") | ||
| 87 | + .input("x", "DT_FLOAT16, DT_BF16") | ||
| 88 | + .attr("axis", attr.Int(-1)) | ||
| 89 | + .attr("scale_alg", attr.Int(DEFAULT_SCALE_ALG)) | ||
| 90 | + .output("y", "DT_HIFLOAT4") | ||
| 91 | + .output("scale", "DT_HIFLOAT4_SCALE"), | ||
| 92 | + ) | ||
| 93 | + | ||
| 94 | + y.desc.dtype = _ge_dtype_to_ge_proto_dtype(DataType.DT_HIFLOAT4) | ||
| 95 | + y = _pack_hif4_to_uint8(x, y) | ||
| 96 | + scale.desc.dtype = _ge_dtype_to_ge_proto_dtype(DataType.DT_HIFLOAT4_SCALE) | ||
| 97 | + # Preserve the packed 32-bit scale payload in the torch-visible float32 carrier. | ||
| 98 | + scale = ge.Bitcast(scale, type=DataType.DT_FLOAT) | ||
| 99 | + return y, scale | ||
| 100 | + | ||
| 101 | +else: | ||
| 102 | + | ||
| 103 | + def convert_dynamic_hi_f4_quant(*args, **kwargs): | ||
| 104 | + raise RuntimeError( | ||
| 105 | + "DynamicHiF4Quant graph converter: torchair is not available." | ||
| 106 | + ) | ||
| @@ -0,0 +1,15 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +__all__ = ["grouped_dynamic_hi_f4_quant", "convert_grouped_dynamic_hi_f4_quant"] | ||
| 11 | + | ||
| 12 | +from .grouped_dynamic_hi_f4_quant import grouped_dynamic_hi_f4_quant | ||
| 13 | +from .graph_convert_grouped_dynamic_hi_f4_quant import ( | ||
| 14 | + convert_grouped_dynamic_hi_f4_quant, | ||
| 15 | +) | ||
| @@ -0,0 +1,101 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#include <torch/extension.h> | ||
| 11 | +#include "aclnn_common.h" | ||
| 12 | + | ||
| 13 | +namespace cann_ops_nn { | ||
| 14 | +namespace quant { | ||
| 15 | + | ||
| 16 | +constexpr int64_t kHif4BlockSize = 64; | ||
| 17 | +constexpr int64_t kFp4InByte = 2; | ||
| 18 | +constexpr int64_t kScaleTrailingDim = 1; | ||
| 19 | +constexpr int64_t kXDimNum = 2; | ||
| 20 | +constexpr int64_t kGroupIndexDimNum = 1; | ||
| 21 | +constexpr int64_t kScaleAlgDefault = 0; | ||
| 22 | +constexpr int64_t kScaleAlgRelaxed = 1; | ||
| 23 | + | ||
| 24 | +void CheckNpuTensor(const at::Tensor& tensor, const char* name) | ||
| 25 | +{ | ||
| 26 | + TORCH_CHECK(tensor.defined(), name, " must be defined"); | ||
| 27 | + TORCH_CHECK(torch_npu::utils::is_npu(tensor), name, " must be on NPU device"); | ||
| 28 | +} | ||
| 29 | + | ||
| 30 | +int64_t CeilDiv(int64_t value, int64_t factor) | ||
| 31 | +{ | ||
| 32 | + TORCH_CHECK(factor > 0, "factor must be positive"); | ||
| 33 | + return (value + factor - 1) / factor; | ||
| 34 | +} | ||
| 35 | + | ||
| 36 | +void CheckGroupIndex(const at::Tensor& groupIndex) | ||
| 37 | +{ | ||
| 38 | + CheckNpuTensor(groupIndex, "group_index"); | ||
| 39 | + TORCH_CHECK(groupIndex.dim() == kGroupIndexDimNum, "group_index rank should be 1"); | ||
| 40 | + TORCH_CHECK(groupIndex.numel() > 0, "group_index does not support empty tensor"); | ||
| 41 | + TORCH_CHECK(groupIndex.scalar_type() == at::ScalarType::Int, "group_index dtype should be int32"); | ||
| 42 | +} | ||
| 43 | + | ||
| 44 | +void ValidateScaleAlg(int64_t scaleAlg) | ||
| 45 | +{ | ||
| 46 | + TORCH_CHECK(scaleAlg == kScaleAlgDefault || scaleAlg == kScaleAlgRelaxed, "scale_alg should be 0 or 1, but got ", | ||
| 47 | + scaleAlg); | ||
| 48 | +} | ||
| 49 | + | ||
| 50 | +c10::SmallVector<int64_t, op_infer::SIZE> GetYShape(const at::Tensor& x) | ||
| 51 | +{ | ||
| 52 | + TORCH_CHECK(x.dim() == kXDimNum, "x rank should be 2"); | ||
| 53 | + auto shape = op_infer::array_to_small_vector(x.sizes()); | ||
| 54 | + shape[x.dim() - 1] = CeilDiv(x.size(x.dim() - 1), kFp4InByte); | ||
| 55 | + return shape; | ||
| 56 | +} | ||
| 57 | + | ||
| 58 | +c10::SmallVector<int64_t, op_infer::SIZE> GetScaleShape(const at::Tensor& x, const at::Tensor& groupIndex) | ||
| 59 | +{ | ||
| 60 | + TORCH_CHECK(x.dim() == kXDimNum, "x rank should be 2"); | ||
| 61 | + c10::SmallVector<int64_t, op_infer::SIZE> shape; | ||
| 62 | + // Match the reserved group layout used by InferShape, Tiling and kernel output offsets. | ||
| 63 | + shape.emplace_back(x.size(0) / kHif4BlockSize + groupIndex.size(0)); | ||
| 64 | + shape.emplace_back(x.size(1)); | ||
| 65 | + shape.emplace_back(kScaleTrailingDim); | ||
| 66 | + return shape; | ||
| 67 | +} | ||
| 68 | + | ||
| 69 | +std::tuple<at::Tensor, at::Tensor> grouped_dynamic_hi_f4_quant(const at::Tensor& x, const at::Tensor& group_index, | ||
| 70 | + const int64_t scale_alg) | ||
| 71 | +{ | ||
| 72 | + CheckNpuTensor(x, "x"); | ||
| 73 | + CheckGroupIndex(group_index); | ||
| 74 | + ValidateScaleAlg(scale_alg); | ||
| 75 | + TORCH_CHECK(x.dim() == kXDimNum, "x rank should be 2"); | ||
| 76 | + TORCH_CHECK(x.size(-1) % kHif4BlockSize == 0, "x last dimension should be a multiple of 64"); | ||
| 77 | + TORCH_CHECK(x.device() == group_index.device(), "x and group_index should be on the same device"); | ||
| 78 | + TORCH_CHECK(x.scalar_type() == at::ScalarType::Half || x.scalar_type() == at::ScalarType::BFloat16, | ||
| 79 | + "x dtype should be float16 or bfloat16"); | ||
| 80 | + | ||
| 81 | + const auto yShape = GetYShape(x); | ||
| 82 | + at::Tensor y = at::empty(yShape, x.options().dtype(at::ScalarType::Byte)); | ||
| 83 | + | ||
| 84 | + const auto scaleShape = GetScaleShape(x, group_index); | ||
| 85 | + at::Tensor yScale = at::empty(scaleShape, x.options().dtype(at::ScalarType::Float)); | ||
| 86 | + | ||
| 87 | + // 输出按实际 acl 数据类型转发:y 为 HIFLOAT4,scale 为 HIFLOAT4_SCALE。 | ||
| 88 | + TensorWrapper yWrapper{y, ACL_HIFLOAT4_NUM}; | ||
| 89 | + TensorWrapper yScaleWrapper{yScale, ACL_HIFLOAT4_SCALE_NUM}; | ||
| 90 | + ACLNN_CMD(aclnnGroupedDynamicHiF4Quant, x, group_index, scale_alg, yWrapper, yScaleWrapper); | ||
| 91 | + return std::make_tuple(y, yScale); | ||
| 92 | +} | ||
| 93 | + | ||
| 94 | +} // namespace quant | ||
| 95 | +} // namespace cann_ops_nn | ||
| 96 | + | ||
| 97 | +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) | ||
| 98 | +{ | ||
| 99 | + m.def("grouped_dynamic_hi_f4_quant", &cann_ops_nn::quant::grouped_dynamic_hi_f4_quant, | ||
| 100 | + "GroupedDynamicHiF4Quant on NPU"); | ||
| 101 | +} | ||
Aquant/grouped_dynamic_hi_f4_quant/torch_extension/graph_convert_grouped_dynamic_hi_f4_quant.py+110-0
| @@ -0,0 +1,110 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +# GE Converter for Graph Mode | ||
| 11 | +try: | ||
| 12 | + import torch | ||
| 13 | + | ||
| 14 | + from torchair._ge_concrete_graph import ge_apis as ge | ||
| 15 | + from torchair.ge import attr | ||
| 16 | + from torchair.ge._ge_graph import ( | ||
| 17 | + DataType, | ||
| 18 | + Tensor, | ||
| 19 | + TensorSpec, | ||
| 20 | + _ge_dtype_to_ge_proto_dtype, | ||
| 21 | + ) | ||
| 22 | + from torchair._ge_concrete_graph.compat_ir import ge_op, IrDef | ||
| 23 | + from torchair._ge_concrete_graph.fx2ge_converter import ( | ||
| 24 | + register_fx_node_ge_converter, | ||
| 25 | + ) | ||
| 26 | + | ||
| 27 | + _TORCHAIR_AVAILABLE = True | ||
| 28 | + | ||
| 29 | +except ImportError: | ||
| 30 | + _TORCHAIR_AVAILABLE = False | ||
| 31 | + | ||
| 32 | +X_DIM_NUM = 2 | ||
| 33 | +GROUP_INDEX_DIM_NUM = 1 | ||
| 34 | +DEFAULT_SCALE_ALG = 0 | ||
| 35 | +SCALE_ALG_CHOICES = (0, 1) | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +def _check_rank(x, group_index): | ||
| 39 | + if x.rank != X_DIM_NUM: | ||
| 40 | + raise RuntimeError(f"x rank should be {X_DIM_NUM}, but got {x.rank}") | ||
| 41 | + if group_index.rank != GROUP_INDEX_DIM_NUM: | ||
| 42 | + raise RuntimeError( | ||
| 43 | + f"group_index rank should be {GROUP_INDEX_DIM_NUM}, but got {group_index.rank}" | ||
| 44 | + ) | ||
| 45 | + | ||
| 46 | + | ||
| 47 | +def _validate_scale_alg(scale_alg): | ||
| 48 | + """scale_alg: 0-标准阈值(4/2), 1-放宽阈值(3.75/1.875)""" | ||
| 49 | + if scale_alg not in SCALE_ALG_CHOICES: | ||
| 50 | + raise RuntimeError( | ||
| 51 | + f"scale_alg must be one of {SCALE_ALG_CHOICES}, but got {scale_alg}" | ||
| 52 | + ) | ||
| 53 | + | ||
| 54 | + | ||
| 55 | +def _pack_hif4_to_uint8(x, y): | ||
| 56 | + bit_shape = [1] * x.rank | ||
| 57 | + bit_shape[-1] = 2 | ||
| 58 | + div_x2 = ge.Cast(ge.Const(bit_shape), dst_type=DataType.DT_INT32) | ||
| 59 | + y_shape_hif4 = ge.Shape(y) | ||
| 60 | + y_shape_uint8 = ge.Div(y_shape_hif4, div_x2) | ||
| 61 | + y_shape_hif4_2bit = ge.ConcatV2( | ||
| 62 | + [y_shape_uint8, ge.Cast(ge.Const([2]), dst_type=DataType.DT_INT32)], | ||
| 63 | + concat_dim=0, | ||
| 64 | + N=2, | ||
| 65 | + ) | ||
| 66 | + y_uint8 = ge.Bitcast(ge.Reshape(y, y_shape_hif4_2bit), type=DataType.DT_UINT8) | ||
| 67 | + return ge.Reshape(y_uint8, y_shape_uint8) | ||
| 68 | + | ||
| 69 | + | ||
| 70 | +if _TORCHAIR_AVAILABLE: | ||
| 71 | + | ||
| 72 | + @register_fx_node_ge_converter( | ||
| 73 | + torch.ops.cann_ops_nn.grouped_dynamic_hi_f4_quant.default | ||
| 74 | + ) | ||
| 75 | + def convert_grouped_dynamic_hi_f4_quant( | ||
| 76 | + x: Tensor, | ||
| 77 | + group_index: Tensor, | ||
| 78 | + scale_alg: int = DEFAULT_SCALE_ALG, | ||
| 79 | + meta_outputs: TensorSpec = None, | ||
| 80 | + ): | ||
| 81 | + _check_rank(x, group_index) | ||
| 82 | + _validate_scale_alg(scale_alg) | ||
| 83 | + y, scale = ge_op( | ||
| 84 | + op_type="GroupedDynamicHiF4Quant", | ||
| 85 | + inputs={"x": x, "group_index": group_index}, | ||
| 86 | + attrs={ | ||
| 87 | + "scale_alg": attr.Int(scale_alg), | ||
| 88 | + }, | ||
| 89 | + outputs=["y", "scale"], | ||
| 90 | + ir=IrDef("GroupedDynamicHiF4Quant") | ||
| 91 | + .input("x", "DT_FLOAT16, DT_BF16") | ||
| 92 | + .input("group_index", "DT_INT32") | ||
| 93 | + .attr("scale_alg", attr.Int(DEFAULT_SCALE_ALG)) | ||
| 94 | + .output("y", "DT_HIFLOAT4") | ||
| 95 | + .output("scale", "DT_HIFLOAT4_SCALE"), | ||
| 96 | + ) | ||
| 97 | + | ||
| 98 | + y.desc.dtype = _ge_dtype_to_ge_proto_dtype(DataType.DT_HIFLOAT4) | ||
| 99 | + y = _pack_hif4_to_uint8(x, y) | ||
| 100 | + scale.desc.dtype = _ge_dtype_to_ge_proto_dtype(DataType.DT_HIFLOAT4_SCALE) | ||
| 101 | + # Preserve the packed 32-bit scale payload in the torch-visible float32 carrier. | ||
| 102 | + scale = ge.Bitcast(scale, type=DataType.DT_FLOAT) | ||
| 103 | + return y, scale | ||
| 104 | + | ||
| 105 | +else: | ||
| 106 | + | ||
| 107 | + def convert_grouped_dynamic_hi_f4_quant(*args, **kwargs): | ||
| 108 | + raise RuntimeError( | ||
| 109 | + "GroupedDynamicHiF4Quant graph converter: torchair is not available." | ||
| 110 | + ) | ||
| @@ -0,0 +1,103 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | +import torch | ||
| 11 | +from torch.library import impl | ||
| 12 | + | ||
| 13 | +from cann_ops_nn.op_builder import OpBuilder, get_as_library | ||
| 14 | + | ||
| 15 | +HIF4_BLOCK_SIZE = 64 | ||
| 16 | +FP4_IN_BYTE = 2 | ||
| 17 | +X_DIM_NUM = 2 | ||
| 18 | +GROUP_INDEX_DTYPE = torch.int32 | ||
| 19 | +DEFAULT_SCALE_ALG = 0 | ||
| 20 | +SCALE_ALG_CHOICES = (0, 1) | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +def _ceil_div(value, factor): | ||
| 24 | + return (value + factor - 1) // factor | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +def _check_x(x): | ||
| 28 | + if x.dim() != X_DIM_NUM: | ||
| 29 | + raise RuntimeError(f"x rank should be {X_DIM_NUM}, but got {x.dim()}") | ||
| 30 | + if x.dtype not in (torch.float16, torch.bfloat16): | ||
| 31 | + raise RuntimeError(f"x dtype should be float16 or bfloat16, but got {x.dtype}") | ||
| 32 | + if x.size(-1) % HIF4_BLOCK_SIZE != 0: | ||
| 33 | + raise RuntimeError("x last dimension should be a multiple of 64") | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +def _check_group_index(group_index): | ||
| 37 | + if group_index.dim() != 1: | ||
| 38 | + raise RuntimeError(f"group_index rank should be 1, but got {group_index.dim()}") | ||
| 39 | + if group_index.numel() == 0: | ||
| 40 | + raise RuntimeError("group_index does not support empty tensor") | ||
| 41 | + if group_index.dtype != GROUP_INDEX_DTYPE: | ||
| 42 | + raise RuntimeError( | ||
| 43 | + f"group_index dtype should be int32, but got {group_index.dtype}" | ||
| 44 | + ) | ||
| 45 | + | ||
| 46 | + | ||
| 47 | +def _validate_scale_alg(scale_alg): | ||
| 48 | + """scale_alg: 0-标准阈值(4/2), 1-放宽阈值(3.75/1.875)""" | ||
| 49 | + if scale_alg not in SCALE_ALG_CHOICES: | ||
| 50 | + raise RuntimeError( | ||
| 51 | + f"scale_alg must be one of {SCALE_ALG_CHOICES}, but got {scale_alg}" | ||
| 52 | + ) | ||
| 53 | + | ||
| 54 | + | ||
| 55 | +def _y_shape(x): | ||
| 56 | + shape = list(x.shape) | ||
| 57 | + shape[-1] = _ceil_div(x.size(x.dim() - 1), FP4_IN_BYTE) | ||
| 58 | + return shape | ||
| 59 | + | ||
| 60 | + | ||
| 61 | +def _scale_shape(x, group_index): | ||
| 62 | + # Match the reserved group layout used by the operator, not the sum of group block counts. | ||
| 63 | + return [x.size(0) // HIF4_BLOCK_SIZE + group_index.size(0), x.size(1), 1] | ||
| 64 | + | ||
| 65 | + | ||
| 66 | +class GroupedDynamicHiF4QuantOpBuilder(OpBuilder): | ||
| 67 | + def __init__(self): | ||
| 68 | + super().__init__("grouped_dynamic_hi_f4_quant") | ||
| 69 | + | ||
| 70 | + def sources(self): | ||
| 71 | + return [self.resolve_source("grouped_dynamic_hi_f4_quant.cpp")] | ||
| 72 | + | ||
| 73 | + def schema(self): | ||
| 74 | + return "grouped_dynamic_hi_f4_quant(Tensor x, Tensor group_index, int scale_alg=0) -> (Tensor, Tensor)" | ||
| 75 | + | ||
| 76 | + def register_meta(self): | ||
| 77 | + @impl(get_as_library(), self.name, "Meta") | ||
| 78 | + def grouped_dynamic_hi_f4_quant_meta( | ||
| 79 | + x, group_index, scale_alg=DEFAULT_SCALE_ALG | ||
| 80 | + ): | ||
| 81 | + _check_x(x) | ||
| 82 | + _check_group_index(group_index) | ||
| 83 | + if x.device != group_index.device: | ||
| 84 | + raise RuntimeError("x and group_index should be on the same device") | ||
| 85 | + _validate_scale_alg(scale_alg) | ||
| 86 | + y = x.new_empty(_y_shape(x), dtype=torch.uint8) | ||
| 87 | + y_scale = x.new_empty(_scale_shape(x, group_index), dtype=torch.float32) | ||
| 88 | + return y, y_scale | ||
| 89 | + | ||
| 90 | + | ||
| 91 | +builder = GroupedDynamicHiF4QuantOpBuilder() | ||
| 92 | +builder._ensure_initialized() | ||
| 93 | + | ||
| 94 | + | ||
| 95 | +@impl(get_as_library(), builder.name, "PrivateUse1") | ||
| 96 | +def grouped_dynamic_hi_f4_quant(x, group_index, scale_alg=DEFAULT_SCALE_ALG): | ||
| 97 | + _check_x(x) | ||
| 98 | + _check_group_index(group_index) | ||
| 99 | + if x.device != group_index.device: | ||
| 100 | + raise RuntimeError("x and group_index should be on the same device") | ||
| 101 | + _validate_scale_alg(scale_alg) | ||
| 102 | + op_module = builder.load() | ||
| 103 | + return op_module.grouped_dynamic_hi_f4_quant(x, group_index, scale_alg) | ||
| @@ -134,6 +134,10 @@ constexpr aclDataType kATenScalarTypeToAclDataTypeTable[static_cast<int64_t>(at: | |||
| 134 | #undef DEFINE_ENUM | 134 | #undef DEFINE_ENUM |
| 135 | }; | 135 | }; |
| 136 | 136 | ||
| 137 | +// 旧版 CANN 头文件未提供 HiF4 枚举,使用对应的 aclDataType 数值以兼容这些头文件。 | ||
| 138 | +constexpr aclDataType ACL_HIFLOAT4_NUM = static_cast<aclDataType>(42); | ||
| 139 | +constexpr aclDataType ACL_HIFLOAT4_SCALE_NUM = static_cast<aclDataType>(43); | ||
| 140 | + | ||
| 137 | #define ENUM_OFFSET(new_name, old_name) new_name = static_cast<int>(old_name) + g_toAclOffset, | 141 | #define ENUM_OFFSET(new_name, old_name) new_name = static_cast<int>(old_name) + g_toAclOffset, |
| 138 | 142 | ||
| 139 | enum class DType { | 143 | enum class DType { |
| @@ -148,6 +152,8 @@ enum class DType { | |||
| 148 | ENUM_OFFSET(FLOAT8_E4M3FN, ACL_FLOAT8_E4M3FN) ENUM_OFFSET(FLOAT8_E8M0, ACL_FLOAT8_E8M0) | 152 | ENUM_OFFSET(FLOAT8_E4M3FN, ACL_FLOAT8_E4M3FN) ENUM_OFFSET(FLOAT8_E8M0, ACL_FLOAT8_E8M0) |
| 149 | ENUM_OFFSET(FLOAT6_E3M2, ACL_FLOAT6_E3M2) ENUM_OFFSET(FLOAT6_E2M3, ACL_FLOAT6_E2M3) | 153 | ENUM_OFFSET(FLOAT6_E3M2, ACL_FLOAT6_E3M2) ENUM_OFFSET(FLOAT6_E2M3, ACL_FLOAT6_E2M3) |
| 150 | ENUM_OFFSET(FLOAT4_E2M1, ACL_FLOAT4_E2M1) ENUM_OFFSET(FLOAT4_E1M2, ACL_FLOAT4_E1M2) | 154 | ENUM_OFFSET(FLOAT4_E2M1, ACL_FLOAT4_E2M1) ENUM_OFFSET(FLOAT4_E1M2, ACL_FLOAT4_E1M2) |
| 155 | + ENUM_OFFSET(HIFLOAT4, ACL_HIFLOAT4_NUM) | ||
| 156 | + ENUM_OFFSET(HIFLOAT4_SCALE, ACL_HIFLOAT4_SCALE_NUM) | ||
| 151 | }; | 157 | }; |
| 152 | 158 | ||
| 153 | namespace op_infer { | 159 | namespace op_infer { |
| @@ -179,10 +185,6 @@ typedef struct { | |||
| 179 | const at::Tensor& tensor; | 185 | const at::Tensor& tensor; |
| 180 | } StorageShapeTensor; | 186 | } StorageShapeTensor; |
| 181 | 187 | ||
| 182 | -// HIFLOAT4(4-bit浮点,aclDataType=42)由目标CANN版本定义,旧版本acl头文件未提供该枚举, | ||
| 183 | -// 以数值常量表达以保证向前兼容;与op_plugin_for_lingqu基线中ACL_HIFLOAT4的4-bit处理行为保持一致 | ||
| 184 | -constexpr aclDataType ACL_HIFLOAT4_NUM = static_cast<aclDataType>(42); | ||
| 185 | - | ||
| 186 | inline bool Is4BitDtype(const aclDataType acl_data_type) | 188 | inline bool Is4BitDtype(const aclDataType acl_data_type) |
| 187 | { | 189 | { |
| 188 | return acl_data_type == ACL_FLOAT4_E2M1 || acl_data_type == ACL_FLOAT4_E1M2 || acl_data_type == ACL_INT4 || | 190 | return acl_data_type == ACL_FLOAT4_E2M1 || acl_data_type == ACL_FLOAT4_E1M2 || acl_data_type == ACL_INT4 || |
这个是多算子使用? 不放到算子目录下吗