Star929
1852
代码
代码
Issues250
Pull Requests351
Actions
项目讨论
Wiki
分析
Star929
1852
已合并
fix(qbmm): 迁移公共校验安全整改到 9.1.0 #7027
zhaoyingchao2创建于 7月4日
fix(qbmm): 迁移公共校验安全整改到 9.1.0 #7027
已合并
zhaoyingchao2创建于 7月4日
从zhaoyingchao2/ops-nnfix/9.1.0-qbmm-common-check合入到cann/ops-nn9.1.0
共 4 个文件变更+239-179
Mmatmul/quant_batch_matmul_v3/op_api/aclnn_quant_matmul_v4.cpp+8-170
@@ -30,133 +30,17 @@
30#include "aclnn_kernels/reshape.h"30#include "aclnn_kernels/reshape.h"
31#include "util/math_util.h"31#include "util/math_util.h"
32#include "quant_matmul_checker.h"32#include "quant_matmul_checker.h"
33+#include "quant_matmul_v4_common.h"
33 34 
34using namespace op;35using namespace op;
36+using namespace quant_matmul_v4;
35using Ops::NN::SwapLastTwoDimValue;37using Ops::NN::SwapLastTwoDimValue;
36using Ops::NN::BoolToString;38using Ops::NN::BoolToString;
37using Ops::NN::FormatString;39using Ops::NN::FormatString;
38using Ops::NN::IsTransposeLastTwoDims;40using Ops::NN::IsTransposeLastTwoDims;
39using Ops::Base::CeilDiv;41using Ops::Base::CeilDiv;
40-using TupleTensor = std::tuple<const aclTensor *, const aclTensor *, const aclTensor *>;
41-using TupleOptional = std::tuple<const aclTensor *, const aclTensor *, const aclTensor *, const aclTensor *,
42- const aclTensor *, const int64_t &>;
43-using TupleInput = std::tuple<const aclTensor *, const aclTensor *>;
44-using TupleQuant = std::tuple<const aclTensor *, const aclTensor *, const aclTensor *, const aclTensor *,
45- const aclTensor *, const aclTensor *, const aclTensor *, const int64_t &, const int64_t &>;
46-using TupleAttr = std::tuple<bool, bool>;
47 42 
48namespace {43namespace {
49-static constexpr int INDEX_X1_IN_MANDTORY_TUPLE = 0;
50-static constexpr int INDEX_X2_IN_MANDTORY_TUPLE = 1;
51-static constexpr int INDEX_SCALE_IN_MANDTORY_TUPLE = 2;
52-static constexpr int INDEX_OFFSET_IN_OPTIONAL_TUPLE = 0;
53-static constexpr int INDEX_PERTOKEN_IN_OPTIONAL_TUPLE = 1;
54-static constexpr int INDEX_BIAS_IN_OPTIONAL_TUPLE = 2;
55-static constexpr int INDEX_Y_SCALE_IN_OPTIONAL_TUPLE = 3;
56-static constexpr int INDEX_Y_OFFSET_IN_OPTIONAL_TUPLE = 4;
57-static constexpr int INDEX_GROUP_SIZE_IN_OPTIONAL_TUPLE = 5;
58-static constexpr int INDEX_OUT_IN_TUPLE = 2;
59-static constexpr int INDEX_ISA4W4_IN_BOOL_TUPLE = 2;
60-static constexpr size_t LAST_SECOND_DIM_INDEX = 2;
61- 
62-static constexpr int MIN_DIM_NUM_ND = 2;
63-static constexpr int MAX_DIM_NUM_ND = 6;
64-static constexpr int MIN_DIM_NUM_NZ = 4;
65-static constexpr int MAX_DIM_NUM_NZ = 8;
66-static constexpr int PENULTIMATE_DIM = 2;
67-static constexpr int NZ_K1_INDEX = 3;
68-static constexpr int NZ_K1_INDEX_TRANS = 4;
69-static constexpr int NZ_STORAGE_PENULTIMATE_DIM = 16;
70-static constexpr int NZ_STORAGE_LAST_DIM = 32;
71-static constexpr int64_t NZ_K0_VALUE_BMM_BLOCK_NUM = 16;
72-static constexpr int64_t NZ_K0_VALUE_INT32_TRANS = 8;
73-static constexpr int64_t NZ_K0_VALUE_INT8_TRANS = 32;
74-static constexpr int64_t NZ_K0_VALUE_INT4_TRANS = 64;
75-static constexpr int64_t OUTPUT_INFER_FAIL = -1L;
76-static constexpr int64_t LAST_AXIS_LIMIT = 65535;
77-static constexpr int X2_FIXED_DIM_NUM_A4W4 = 2;
78-static constexpr int64_t INT4_NUMS_IN_INT8 = 2;
79-static constexpr int64_t INT4_NUMS_IN_INT32 = 8;
80-static constexpr int64_t INNER_SIZE_MULTIPLE = 64;
81-static constexpr int64_t K_VALUE = 3696;
82-static constexpr int64_t N_VALUE = 8192;
83-static constexpr int64_t M_RANGE1_LEFT = 128;
84-static constexpr int64_t M_RANGE1_RIGHT = 512;
85-static constexpr int32_t CORE_NUM_20 = 20;
86-static constexpr int64_t SUPPORTED_GROUP_SIZE = 32;
87-static constexpr uint64_t B4_PER_B32 = 8UL;
88-static constexpr int64_t SUPPORTED_TCG_A8W4_K_ALIGN_NUM = 32;
89-static constexpr int64_t SUPPORTED_MX_A8W4_K_ALIGN_NUM = 8;
90-static constexpr int64_t SUPPORTED_N_ALIGN_NUM = 8;
91-static constexpr size_t MAX_DIM_VALUE = 2;
92-static constexpr size_t MX_SCALE_DIM_VALUE = 3;
93-static constexpr uint64_t GROUP_M_OFFSET = 32;
94-static constexpr uint64_t GROUP_N_OFFSET = 16;
95-static constexpr uint64_t GROUP_MNK_BIT_SIZE = 0xFFFF;
96-static constexpr size_t MX_SCALE_MAX_DIM = 3;
97-static constexpr size_t MX_SCALE_DIM_NUM = 3;
98-static constexpr int64_t MAX_SHAPE_SIZE_A8W4_INT = 29576;
99-static constexpr int64_t PPMATMUL_PRIORITY_M = 1024;
100-static constexpr int64_t NO_BATCH_DIM_SUM = 2;
101- 
102-static const std::initializer_list<op::DataType> IN_TYPE_SUPPORT_LIST = {op::DataType::DT_INT4,
103- op::DataType::DT_INT8};
104-static const std::initializer_list<op::DataType> INT4_TYPE_SUPPORT_LIST = {op::DataType::DT_INT4,
105- op::DataType::DT_INT32};
106-static const std::initializer_list<op::DataType> OUT_TYPE_SUPPORT_LIST = {op::DataType::DT_INT8,
107- op::DataType::DT_FLOAT16,
108- op::DataType::DT_BF16,
109- op::DataType::DT_INT32};
110-static const std::initializer_list<op::DataType> SCALE_TYPE_SUPPORT_LIST = {op::DataType::DT_UINT64,
111- op::DataType::DT_BF16,
112- op::DataType::DT_INT64,
113- op::DataType::DT_FLOAT};
114-static const std::initializer_list<op::DataType> BIAS_TYPE_SUPPORT_LIST = {op::DataType::DT_INT32,
115- op::DataType::DT_BF16,
116- op::DataType::DT_FLOAT16,
117- op::DataType::DT_FLOAT};
118-static const std::initializer_list<op::DataType> Y_SCALE_SUPPORT_LIST = {op::DataType::DT_UINT64};
119- 
120-static inline bool isA8W4Float(const aclTensor* x1, const aclTensor* x2)
121-{
122- return x1->GetDataType() == op::DataType::DT_FLOAT8_E4M3FN &&
123- (x2->GetDataType() == op::DataType::DT_FLOAT || x2->GetDataType() == op::DataType::DT_FLOAT4_E2M1);
124-}
125- 
126-static inline bool isA8W4Int(const aclTensor* x1, const aclTensor* x2)
127-{
128- return x1->GetDataType() == op::DataType::DT_INT8 &&
129- (x2->GetDataType() == op::DataType::DT_INT4 || x2->GetDataType() == op::DataType::DT_INT32);
130-}
131- 
132-static inline bool isMx(const aclTensor* scale)
133-{
134- return scale->GetDataType() == op::DataType::DT_FLOAT8_E8M0;
135-}
136- 
137-static inline bool isA8W4Msd(const aclTensor* x1, const aclTensor* x2, const aclTensor* scale,
138- const aclTensor* pertokenScale)
139-{
140- if (x1->GetDataType() != op::DataType::DT_INT8) {
141- return false;
142- }
143- 
144- if (std::find(INT4_TYPE_SUPPORT_LIST.begin(), INT4_TYPE_SUPPORT_LIST.end(),
145- x2->GetDataType()) == INT4_TYPE_SUPPORT_LIST.end()) {
146- return false;
147- }
148- 
149- if (scale->GetDataType() != op::DataType::DT_UINT64) {
150- return false;
151- }
152- 
153- if (pertokenScale == nullptr || pertokenScale->GetDataType() != op::DataType::DT_FLOAT) {
154- return false;
155- }
156- 
157- return true;
158-}
159- 
160static inline bool CheckNotNull(TupleTensor mandatoryTensors, const aclTensor *out) {44static inline bool CheckNotNull(TupleTensor mandatoryTensors, const aclTensor *out) {
161 auto x1 = std::get<INDEX_X1_IN_MANDTORY_TUPLE>(mandatoryTensors);45 auto x1 = std::get<INDEX_X1_IN_MANDTORY_TUPLE>(mandatoryTensors);
162 auto x2 = std::get<INDEX_X2_IN_MANDTORY_TUPLE>(mandatoryTensors);46 auto x2 = std::get<INDEX_X2_IN_MANDTORY_TUPLE>(mandatoryTensors);
@@ -667,23 +551,6 @@ static inline bool MaxDimCheck(int64_t x1DimNum, int64_t x2DimNum, const op::Sha
667 return true;551 return true;
668}552}
669 553 
670-static inline int64_t SelectNzK0Value(op::DataType dataType, const bool isA8W4Float) {
671- switch (dataType) {
672- case op::DataType::DT_INT4:
673- return NZ_K0_VALUE_INT4_TRANS;
674- case op::DataType::DT_FLOAT4_E2M1:
675- if (isA8W4Float) {
676- return NZ_K0_VALUE_INT8_TRANS;
677- } else {
678- return NZ_K0_VALUE_INT4_TRANS;
679- }
680- case op::DataType::DT_INT32:
681- return NZ_K0_VALUE_INT32_TRANS;
682- default:
683- return NZ_K0_VALUE_INT8_TRANS;
684- }
685-}
686- 
687static inline bool CheckShapeForWeightNz(const aclTensor *x1, const aclTensor *x2, bool transposeX1, bool transposeX2) {554static inline bool CheckShapeForWeightNz(const aclTensor *x1, const aclTensor *x2, bool transposeX1, bool transposeX2) {
688 const op::Shape x1Shape = x1->GetViewShape();555 const op::Shape x1Shape = x1->GetViewShape();
689 const op::Shape x2Shape = x2->GetStorageShape();556 const op::Shape x2Shape = x2->GetStorageShape();
@@ -703,17 +570,6 @@ static inline bool CheckShapeForWeightNz(const aclTensor *x1, const aclTensor *x
703 return true;570 return true;
704}571}
705 572 
706-template <typename T>
707-static inline bool IsAligned(T num, T factor)
708-{
709- if (factor == 0) {
710- OP_LOGE(ACLNN_ERR_PARAM_INVALID,
711- "The divisor cannot be zero.");
712- return false;
713- }
714- return num > 0 && num % factor == 0;
715-}
716- 
717static inline bool CheckShapeInt4(const aclTensor *x1, const aclTensor *x2, bool transposeX1, bool transposeX2,573static inline bool CheckShapeInt4(const aclTensor *x1, const aclTensor *x2, bool transposeX1, bool transposeX2,
718 const aclTensor *bias)574 const aclTensor *bias)
719{575{
@@ -818,19 +674,6 @@ static inline bool CheckEmptyTensor(TupleTensor mandatoryTensors) {
818 return true;674 return true;
819}675}
820 676 
821-static inline bool IsMicroScaling(const aclTensor *x1Scale, const aclTensor *x2Scale) {
822- if (x1Scale == nullptr) {
823- return false;
824- }
825- return x1Scale->GetDataType() == op::DataType::DT_FLOAT8_E8M0 &&
826- x2Scale->GetDataType() == op::DataType::DT_FLOAT8_E8M0;
827-}
828- 
829-static inline bool IsTCG(const aclTensor *x1Scale, const aclTensor *x2Scale) {
830- return x1Scale == nullptr && x2Scale != nullptr &&
831- (x2Scale->GetDataType() == op::DataType::DT_BF16 || x2Scale->GetDataType() == op::DataType::DT_FLOAT16);
832-}
833- 
834static inline bool CheckA8W4FloatQuantType(const aclTensor *x1, const aclTensor *x2,677static inline bool CheckA8W4FloatQuantType(const aclTensor *x1, const aclTensor *x2,
835 const aclTensor *perTokenScale, const aclTensor *scale) {678 const aclTensor *perTokenScale, const aclTensor *scale) {
836 if (isA8W4Float(x1, x2)) {679 if (isA8W4Float(x1, x2)) {
@@ -990,10 +833,11 @@ static inline bool CheckA8W4ScaleX2Shape(
990 if (IsMicroScaling(x1Scale, x2Scale)) {833 if (IsMicroScaling(x1Scale, x2Scale)) {
991 // 2： x2Scale形状：（n, groupDimK / 2, 2）834 // 2： x2Scale形状：（n, groupDimK / 2, 2）
992 if (x2ScaleNDim != groupDimN || x2ScaleGroupDim != CeilDiv(groupDimK, x2ScaleReshapeFactor) ||835 if (x2ScaleNDim != groupDimN || x2ScaleGroupDim != CeilDiv(groupDimK, x2ScaleReshapeFactor) ||
993- x2Scale->GetViewShape().GetDim(2) != 2) {836+ x2Scale->GetViewShape().GetDim(MX_SCALE_LAST_DIM_INDEX) != MX_SCALE_LAST_DIM) {
994 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(837 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
995 "aclnnQuantMatmulWeightNzGetWorkspaceSize", "x2Scale",838 "aclnnQuantMatmulWeightNzGetWorkspaceSize", "x2Scale",
996- FormatString("%ld, %ld, %ld", x2ScaleNDim, x2ScaleGroupDim, x2Scale->GetViewShape().GetDim(2)).c_str(),839+ FormatString("%ld, %ld, %ld", x2ScaleNDim, x2ScaleGroupDim,
840+ x2Scale->GetViewShape().GetDim(MX_SCALE_LAST_DIM_INDEX)).c_str(),
997 FormatString("the shape of x2Scale must be [%ld, %ld, 2]", groupDimN,841 FormatString("the shape of x2Scale must be [%ld, %ld, 2]", groupDimN,
998 CeilDiv(groupDimK, x2ScaleReshapeFactor)).c_str());842 CeilDiv(groupDimK, x2ScaleReshapeFactor)).c_str());
999 return false;843 return false;
@@ -1066,7 +910,7 @@ static inline bool CheckA8W4X1X2Shape(int64_t x1KDim, int64_t x2KDim, int64_t x2
1066 if (isMx && (x1KDim % SUPPORTED_MX_A8W4_K_ALIGN_NUM != 0)) { // Mx量化k方向8对齐910 if (isMx && (x1KDim % SUPPORTED_MX_A8W4_K_ALIGN_NUM != 0)) { // Mx量化k方向8对齐
1067 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(911 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(
1068 "aclnnQuantMatmulWeightNzGetWorkspaceSize", "x1", std::to_string(x1KDim).c_str(),912 "aclnnQuantMatmulWeightNzGetWorkspaceSize", "x1", std::to_string(x1KDim).c_str(),
1069- FormatString("the k dimension of x1 must be aligned to %ld for MX quantization", 913+ FormatString("the k dimension of x1 must be aligned to %ld for MX quantization",
1070 SUPPORTED_MX_A8W4_K_ALIGN_NUM).c_str());914 SUPPORTED_MX_A8W4_K_ALIGN_NUM).c_str());
1071 return false;915 return false;
1072 }916 }
@@ -1168,12 +1012,6 @@ static aclnnStatus CheckParamsDAV3510(TupleTensor mandatoryTensors, TupleOptiona
1168 return qmmV3Checker.CheckParams();1012 return qmmV3Checker.CheckParams();
1169}1013}
1170 1014 
1171-static bool IsFormatNZ(const aclTensor* tensor) {
1172- return ge::GetPrimaryFormat(tensor->GetStorageFormat()) == op::Format::FORMAT_FRACTAL_NZ ||
1173- ge::GetPrimaryFormat(tensor->GetStorageFormat()) == op::Format::FORMAT_FRACTAL_NZ_C0_4 ||
1174- ge::GetPrimaryFormat(tensor->GetStorageFormat()) == op::Format::FORMAT_FRACTAL_NZ_C0_32;
1175-}
1176- 
1177static aclnnStatus CheckWeightNzParamsDAV3510(const aclTensor *x1, const aclTensor *x2, const aclTensor *out)1015static aclnnStatus CheckWeightNzParamsDAV3510(const aclTensor *x1, const aclTensor *x2, const aclTensor *out)
1178{1016{
1179 if (op::GetCurrentPlatformInfo().GetCurNpuArch() != NpuArch::DAV_3510) {1017 if (op::GetCurrentPlatformInfo().GetCurNpuArch() != NpuArch::DAV_3510) {
@@ -1524,7 +1362,7 @@ static inline bool CheckInputAttrExistence(const TupleAttr &boolsTrans, const Tu
1524 "aclnnQuantMatmulWeightNzGetWorkspaceSize", "transposeX2", transposeX2 ? "true" : "false",1362 "aclnnQuantMatmulWeightNzGetWorkspaceSize", "transposeX2", transposeX2 ? "true" : "false",
1525 "in A8W4 scenario with NZ format, when the quantization mode is t-cg, transposeX2 must be false");1363 "in A8W4 scenario with NZ format, when the quantization mode is t-cg, transposeX2 must be false");
1526 return false;1364 return false;
1527- } else if (IsMicroScaling(x1Scale, x2Scale) && !transposeX2) { 1365+ } else if (IsMicroScaling(x1Scale, x2Scale) && !transposeX2) {
1528 // A8W4 scenario with mx quant mode1366 // A8W4 scenario with mx quant mode
1529 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(1367 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(
1530 "aclnnQuantMatmulWeightNzGetWorkspaceSize", "transposeX2", transposeX2 ? "true" : "false",1368 "aclnnQuantMatmulWeightNzGetWorkspaceSize", "transposeX2", transposeX2 ? "true" : "false",
@@ -1548,7 +1386,7 @@ static inline bool CheckDimRangeA8W4(const TupleTensor& mandatoryTensors, const
1548 auto bias = std::get<INDEX_BIAS_IN_OPTIONAL_TUPLE>(optionalTensors);1386 auto bias = std::get<INDEX_BIAS_IN_OPTIONAL_TUPLE>(optionalTensors);
1549 1387 
1550 if (x1->GetViewShape().GetDimNum() != MAX_DIM_VALUE) {1388 if (x1->GetViewShape().GetDimNum() != MAX_DIM_VALUE) {
1551- OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON("aclnnQuantMatmulWeightNzGetWorkspaceSize", "x1", 1389+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON("aclnnQuantMatmulWeightNzGetWorkspaceSize", "x1",
1552 FormatString("%zuD", x1->GetViewShape().GetDimNum()).c_str(), "the shape dim of x1 must be 2");1390 FormatString("%zuD", x1->GetViewShape().GetDimNum()).c_str(), "the shape dim of x1 must be 2");
1553 return false;1391 return false;
1554 }1392 }
Amatmul/quant_batch_matmul_v3/op_api/quant_matmul_v4_common.h+214-0
@@ -0,0 +1,214 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+#ifndef QUANT_MATMUL_V4_COMMON_H
11+#define QUANT_MATMUL_V4_COMMON_H
12+ 
13+#include <algorithm>
14+#include <cstdint>
15+#include <cstddef>
16+#include <initializer_list>
17+#include <tuple>
18+ 
19+#include "opdev/op_executor.h"
20+#include "opdev/op_log.h"
21+#include "opdev/platform.h"
22+#include "matmul/common/op_host/op_api/matmul_util.h"
23+#include "opdev/common_types.h"
24+ 
25+namespace quant_matmul_v4 {
26+// Constants
27+static constexpr int MX_SCALE_LAST_DIM_INDEX = 2;
28+static constexpr int MX_SCALE_LAST_DIM = 2;
29+ 
30+static constexpr int INDEX_X1_IN_MANDTORY_TUPLE = 0;
31+static constexpr int INDEX_X2_IN_MANDTORY_TUPLE = 1;
32+static constexpr int INDEX_SCALE_IN_MANDTORY_TUPLE = 2;
33+static constexpr int INDEX_OFFSET_IN_OPTIONAL_TUPLE = 0;
34+static constexpr int INDEX_PERTOKEN_IN_OPTIONAL_TUPLE = 1;
35+static constexpr int INDEX_BIAS_IN_OPTIONAL_TUPLE = 2;
36+static constexpr int INDEX_Y_SCALE_IN_OPTIONAL_TUPLE = 3;
37+static constexpr int INDEX_Y_OFFSET_IN_OPTIONAL_TUPLE = 4;
38+static constexpr int INDEX_GROUP_SIZE_IN_OPTIONAL_TUPLE = 5;
39+static constexpr int INDEX_OUT_IN_TUPLE = 2;
40+static constexpr int INDEX_ISA4W4_IN_BOOL_TUPLE = 2;
41+static constexpr size_t LAST_SECOND_DIM_INDEX = 2;
42+ 
43+static constexpr int MIN_DIM_NUM_ND = 2;
44+static constexpr int MAX_DIM_NUM_ND = 6;
45+static constexpr int MIN_DIM_NUM_NZ = 4;
46+static constexpr int MAX_DIM_NUM_NZ = 8;
47+static constexpr int PENULTIMATE_DIM = 2;
48+static constexpr int NZ_K1_INDEX = 3;
49+static constexpr int NZ_K1_INDEX_TRANS = 4;
50+static constexpr int NZ_STORAGE_PENULTIMATE_DIM = 16;
51+static constexpr int NZ_STORAGE_LAST_DIM = 32;
52+static constexpr int64_t NZ_K0_VALUE_BMM_BLOCK_NUM = 16;
53+static constexpr int64_t NZ_K0_VALUE_INT32_TRANS = 8;
54+static constexpr int64_t NZ_K0_VALUE_INT8_TRANS = 32;
55+static constexpr int64_t NZ_K0_VALUE_INT4_TRANS = 64;
56+static constexpr int64_t OUTPUT_INFER_FAIL = -1L;
57+static constexpr int64_t LAST_AXIS_LIMIT = 65535;
58+static constexpr int X2_FIXED_DIM_NUM_A4W4 = 2;
59+static constexpr int64_t INT4_NUMS_IN_INT8 = 2;
60+static constexpr int64_t INT4_NUMS_IN_INT32 = 8;
61+static constexpr int64_t INNER_SIZE_MULTIPLE = 64;
62+static constexpr int64_t K_VALUE = 3696;
63+static constexpr int64_t N_VALUE = 8192;
64+static constexpr int64_t M_RANGE1_LEFT = 128;
65+static constexpr int64_t M_RANGE1_RIGHT = 512;
66+static constexpr int32_t CORE_NUM_20 = 20;
67+static constexpr int64_t SUPPORTED_GROUP_SIZE = 32;
68+static constexpr uint64_t B4_PER_B32 = 8UL;
69+static constexpr int64_t SUPPORTED_TCG_A8W4_K_ALIGN_NUM = 32;
70+static constexpr int64_t SUPPORTED_MX_A8W4_K_ALIGN_NUM = 8;
71+static constexpr int64_t SUPPORTED_N_ALIGN_NUM = 8;
72+static constexpr size_t MAX_DIM_VALUE = 2;
73+static constexpr size_t MX_SCALE_DIM_VALUE = 3;
74+static constexpr uint64_t GROUP_M_OFFSET = 32;
75+static constexpr uint64_t GROUP_N_OFFSET = 16;
76+static constexpr uint64_t GROUP_MNK_BIT_SIZE = 0xFFFF;
77+static constexpr size_t MX_SCALE_MAX_DIM = 3;
78+static constexpr size_t MX_SCALE_DIM_NUM = 3;
79+static constexpr int64_t MAX_SHAPE_SIZE_A8W4_INT = 29576;
80+static constexpr int64_t PPMATMUL_PRIORITY_M = 1024;
81+static constexpr int64_t NO_BATCH_DIM_SUM = 2;
82+ 
83+static const std::initializer_list<op::DataType> IN_TYPE_SUPPORT_LIST = {op::DataType::DT_INT4,
84+ op::DataType::DT_INT8};
85+static const std::initializer_list<op::DataType> INT4_TYPE_SUPPORT_LIST = {op::DataType::DT_INT4,
86+ op::DataType::DT_INT32};
87+static const std::initializer_list<op::DataType> OUT_TYPE_SUPPORT_LIST = {op::DataType::DT_INT8,
88+ op::DataType::DT_FLOAT16,
89+ op::DataType::DT_BF16,
90+ op::DataType::DT_INT32};
91+static const std::initializer_list<op::DataType> SCALE_TYPE_SUPPORT_LIST = {op::DataType::DT_UINT64,
92+ op::DataType::DT_BF16,
93+ op::DataType::DT_INT64,
94+ op::DataType::DT_FLOAT};
95+static const std::initializer_list<op::DataType> BIAS_TYPE_SUPPORT_LIST = {op::DataType::DT_INT32,
96+ op::DataType::DT_BF16,
97+ op::DataType::DT_FLOAT16,
98+ op::DataType::DT_FLOAT};
99+static const std::initializer_list<op::DataType> Y_SCALE_SUPPORT_LIST = {op::DataType::DT_UINT64};
100+ 
101+using TupleTensor = std::tuple<const aclTensor *, const aclTensor *, const aclTensor *>;
102+using TupleOptional = std::tuple<const aclTensor *, const aclTensor *, const aclTensor *, const aclTensor *,
103+ const aclTensor *, const int64_t &>;
104+using TupleInput = std::tuple<const aclTensor *, const aclTensor *>;
105+using TupleQuant = std::tuple<const aclTensor *, const aclTensor *, const aclTensor *, const aclTensor *,
106+ const aclTensor *, const aclTensor *, const aclTensor *, const int64_t &, const int64_t &>;
107+using TupleAttr = std::tuple<bool, bool>;
108+ 
109+static inline bool isA8W4Float(const aclTensor* x1, const aclTensor* x2)
110+{
111+ if (x1 == nullptr || x2 == nullptr) {
112+ return false;
113+ }
114+ return x1->GetDataType() == op::DataType::DT_FLOAT8_E4M3FN &&
115+ (x2->GetDataType() == op::DataType::DT_FLOAT || x2->GetDataType() == op::DataType::DT_FLOAT4_E2M1);
116+}
117+ 
118+static inline bool isA8W4Int(const aclTensor* x1, const aclTensor* x2)
119+{
120+ if (x1 == nullptr || x2 == nullptr) {
121+ return false;
122+ }
123+ return x1->GetDataType() == op::DataType::DT_INT8 &&
124+ (x2->GetDataType() == op::DataType::DT_INT4 || x2->GetDataType() == op::DataType::DT_INT32);
125+}
126+ 
127+static inline bool isMx(const aclTensor* scale)
128+{
129+ if (scale == nullptr) {
130+ return false;
131+ }
132+ return scale->GetDataType() == op::DataType::DT_FLOAT8_E8M0;
133+}
134+ 
135+static inline bool isA8W4Msd(const aclTensor* x1, const aclTensor* x2, const aclTensor* scale,
136+ const aclTensor* pertokenScale)
137+{
138+ if (x1 == nullptr || x2 == nullptr || scale == nullptr) {
139+ return false;
140+ }
141+ if (x1->GetDataType() != op::DataType::DT_INT8) {
142+ return false;
143+ }
144+ 
145+ if (std::find(INT4_TYPE_SUPPORT_LIST.begin(), INT4_TYPE_SUPPORT_LIST.end(),
146+ x2->GetDataType()) == INT4_TYPE_SUPPORT_LIST.end()) {
147+ return false;
148+ }
149+ 
150+ if (scale->GetDataType() != op::DataType::DT_UINT64) {
151+ return false;
152+ }
153+ 
154+ if (pertokenScale == nullptr || pertokenScale->GetDataType() != op::DataType::DT_FLOAT) {
155+ return false;
156+ }
157+ 
158+ return true;
159+}
160+ 
161+static inline bool IsMicroScaling(const aclTensor *x1Scale, const aclTensor *x2Scale) {
162+ if (x1Scale == nullptr) {
163+ return false;
164+ }
165+ if (x2Scale == nullptr) {
166+ return false;
167+ }
168+ return x1Scale->GetDataType() == op::DataType::DT_FLOAT8_E8M0 &&
169+ x2Scale->GetDataType() == op::DataType::DT_FLOAT8_E8M0;
170+}
171+ 
172+static inline bool IsTCG(const aclTensor *x1Scale, const aclTensor *x2Scale) {
173+ return x1Scale == nullptr && x2Scale != nullptr &&
174+ (x2Scale->GetDataType() == op::DataType::DT_BF16 || x2Scale->GetDataType() == op::DataType::DT_FLOAT16);
175+}
176+ 
177+static inline int64_t SelectNzK0Value(op::DataType dataType, const bool isA8W4Float) {
178+ switch (dataType) {
179+ case op::DataType::DT_INT4:
180+ return NZ_K0_VALUE_INT4_TRANS;
181+ case op::DataType::DT_FLOAT4_E2M1:
182+ if (isA8W4Float) {
183+ return NZ_K0_VALUE_INT8_TRANS;
184+ } else {
185+ return NZ_K0_VALUE_INT4_TRANS;
186+ }
187+ case op::DataType::DT_INT32:
188+ return NZ_K0_VALUE_INT32_TRANS;
189+ default:
190+ return NZ_K0_VALUE_INT8_TRANS;
191+ }
192+}
193+ 
194+template <typename T>
195+static inline bool IsAligned(T num, T factor)
196+{
197+ if (factor == 0) {
198+ OP_LOGE(ACLNN_ERR_PARAM_INVALID,
199+ "The divisor cannot be zero.");
200+ return false;
201+ }
202+ return num > 0 && num % factor == 0;
203+}
204+ 
205+static inline bool IsFormatNZ(const aclTensor* tensor) {
206+ return tensor != nullptr &&
207+ (ge::GetPrimaryFormat(tensor->GetStorageFormat()) == op::Format::FORMAT_FRACTAL_NZ ||
208+ ge::GetPrimaryFormat(tensor->GetStorageFormat()) == op::Format::FORMAT_FRACTAL_NZ_C0_4 ||
209+ ge::GetPrimaryFormat(tensor->GetStorageFormat()) == op::Format::FORMAT_FRACTAL_NZ_C0_32);
210+}
211+ 
212+} // namespace quant_matmul_v4
213+ 
214+#endif // QUANT_MATMUL_V4_COMMON_H
Mmatmul/quant_batch_matmul_v3/op_host/op_tiling/arch35/adaptive_sliding_window_mx_basic_api_tiling.cpp+1-1
@@ -85,7 +85,7 @@ const void* AdaptiveSlidingWindowMXBasicAPITiling::GetTilingData() const
85 return &tilingData_;85 return &tilingData_;
86}86}
87 87 
88-uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetApiLevel(NpuArch) const88+uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetApiLevel(NpuArch npuArch) const
89{89{
90 return IsTensorapiCapable() ? static_cast<uint64_t>(QMMApiLevel::BLAZE_LEVEL) :90 return IsTensorapiCapable() ? static_cast<uint64_t>(QMMApiLevel::BLAZE_LEVEL) :
91 static_cast<uint64_t>(QMMApiLevel::BASIC_LEVEL);91 static_cast<uint64_t>(QMMApiLevel::BASIC_LEVEL);
Mmatmul/quant_batch_matmul_v3/op_kernel/arch35/quant_batch_matmul_v3_apt_tiling_key.h+16-8
@@ -24,6 +24,14 @@
24#endif24#endif
25#endif25#endif
26 26 
27+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 5102)
28+#define IS_BLAZE false
29+#elif ASC_DEVKIT_MAJOR >= 9 && ASC_DEVKIT_MINOR > 0
30+#define IS_BLAZE true
31+#else
32+#define IS_BLAZE false
33+#endif
34+ 
27#if !(defined(__NPU_ARCH__) && __NPU_ARCH__ == 5102) && \35#if !(defined(__NPU_ARCH__) && __NPU_ARCH__ == 5102) && \
28 defined(ORIG_DTYPE_X1) && defined(ORIG_DTYPE_X2) && defined(DT_INT4) && \36 defined(ORIG_DTYPE_X1) && defined(ORIG_DTYPE_X2) && defined(DT_INT4) && \
29 ORIG_DTYPE_X1 == DT_INT4 && ORIG_DTYPE_X2 == DT_INT437 ORIG_DTYPE_X1 == DT_INT4 && ORIG_DTYPE_X2 == DT_INT4
@@ -49,7 +57,7 @@ namespace QuantBatchMatmulV3Arch35TilingKey {
49#endif57#endif
50 58 
51#if defined(__CCE_AICORE__) && defined(ORIG_DTYPE_X1) && defined(ORIG_DTYPE_X2) && defined(ORIG_DTYPE_SCALE) && \59#if defined(__CCE_AICORE__) && defined(ORIG_DTYPE_X1) && defined(ORIG_DTYPE_X2) && defined(ORIG_DTYPE_SCALE) && \
52- defined(FORMAT_X2) && \60+ defined(DT_FLOAT8_E4M3FN) && defined(DT_FLOAT8_E5M2) && defined(DT_FLOAT4_E2M1) && defined(DT_FLOAT8_E8M0) && \
53 (ORIG_DTYPE_X1 == DT_FLOAT8_E4M3FN || ORIG_DTYPE_X1 == DT_FLOAT8_E5M2 || \61 (ORIG_DTYPE_X1 == DT_FLOAT8_E4M3FN || ORIG_DTYPE_X1 == DT_FLOAT8_E5M2 || \
54 ORIG_DTYPE_X1 == DT_FLOAT4_E2M1) && \62 ORIG_DTYPE_X1 == DT_FLOAT4_E2M1) && \
55 (ORIG_DTYPE_X2 == DT_FLOAT8_E4M3FN || ORIG_DTYPE_X2 == DT_FLOAT8_E5M2 || \63 (ORIG_DTYPE_X2 == DT_FLOAT8_E4M3FN || ORIG_DTYPE_X2 == DT_FLOAT8_E5M2 || \
@@ -121,9 +129,9 @@ ASCENDC_TPL_SEL(
121 ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),129 ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),
122 ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),130 ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),
123 ASCENDC_TPL_UINT_SEL(BIASMODE, ASCENDC_TPL_UI_LIST, TPL_EXCLUDE_FROM_TEMPLATE), // Bias Mode 只有0131 ASCENDC_TPL_UINT_SEL(BIASMODE, ASCENDC_TPL_UI_LIST, TPL_EXCLUDE_FROM_TEMPLATE), // Bias Mode 只有0
124- ASCENDC_TPL_UINT_SEL(KERNELTYPE, 132+ ASCENDC_TPL_UINT_SEL(KERNELTYPE,
125 ASCENDC_TPL_UI_LIST,133 ASCENDC_TPL_UI_LIST,
126- TPL_VEC_EPILOGUE_WITH_MMAPI, 134+ TPL_VEC_EPILOGUE_WITH_MMAPI,
127 TPL_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI),135 TPL_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI),
128 ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_HIGH)),136 ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_HIGH)),
129#endif137#endif
@@ -133,8 +141,8 @@ ASCENDC_TPL_SEL(
133 ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),141 ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),
134 ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),142 ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),
135 ASCENDC_TPL_UINT_SEL(BIASMODE, ASCENDC_TPL_UI_LIST, TPL_EXCLUDE_FROM_TEMPLATE), // Bias Mode 只有0143 ASCENDC_TPL_UINT_SEL(BIASMODE, ASCENDC_TPL_UI_LIST, TPL_EXCLUDE_FROM_TEMPLATE), // Bias Mode 只有0
136- ASCENDC_TPL_UINT_SEL(KERNELTYPE, 144+ ASCENDC_TPL_UINT_SEL(KERNELTYPE,
137- ASCENDC_TPL_UI_LIST, 145+ ASCENDC_TPL_UI_LIST,
138 TPL_VEC_EPILOGUE_WITH_CUSTOM_MM),146 TPL_VEC_EPILOGUE_WITH_CUSTOM_MM),
139 ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BASIC)),147 ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BASIC)),
140#endif148#endif
@@ -196,9 +204,9 @@ ASCENDC_TPL_SEL(
196 ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),204 ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),
197 ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),205 ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),
198 ASCENDC_TPL_UINT_SEL(BIASMODE, ASCENDC_TPL_UI_LIST, TPL_EXCLUDE_FROM_TEMPLATE),206 ASCENDC_TPL_UINT_SEL(BIASMODE, ASCENDC_TPL_UI_LIST, TPL_EXCLUDE_FROM_TEMPLATE),
199- ASCENDC_TPL_UINT_SEL(KERNELTYPE, 207+ ASCENDC_TPL_UINT_SEL(KERNELTYPE,
200- ASCENDC_TPL_UI_LIST, 208+ ASCENDC_TPL_UI_LIST,
201- TPL_NO_VEC_EPILOGUE_WITH_MMAPI, 209+ TPL_NO_VEC_EPILOGUE_WITH_MMAPI,
202 TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI),210 TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI),
203 ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_HIGH)),211 ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_HIGH)),
204#endif212#endif