Star1354
2089
代码
代码
Issues791
Pull Requests663
Actions
项目讨论
Wiki
分析
Star1354
2089
已合并
新增QBMMV3 A4W4 pertoken场景支持weight nz特性 #1187
zhang-junming21创建于 1月29日
新增QBMMV3 A4W4 pertoken场景支持weight nz特性 #1187
已合并
zhang-junming21创建于 1月29日
从hid54649903/ops-nnqbmmv4_nz_test合入到cann/ops-nnmaster
共 13 个文件变更+1435-66
Amatmul/quant_batch_matmul_v3/examples/test_aclnn_quant_matmul_weight_nz_at2_at3_int4.cpp+281-0
@@ -0,0 +1,281 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include <iostream>
12+#include <memory>
13+#include <vector>
14+ 
15+#include "acl/acl.h"
16+#include "aclnnop/aclnn_quant_matmul_weight_nz.h"
17+#include "aclnnop/aclnn_weight_quant_batch_matmul_v2.h"
18+ 
19+#define CHECK_RET(cond, return_expr) \
20+ do { \
21+ if (!(cond)) { \
22+ return_expr; \
23+ } \
24+ } while (0)
25+ 
26+#define CHECK_FREE_RET(cond, return_expr) \
27+ do { \
28+ if (!(cond)) { \
29+ Finalize(deviceId, stream); \
30+ return_expr; \
31+ } \
32+ } while (0)
33+ 
34+#define LOG_PRINT(message, ...) \
35+ do { \
36+ printf(message, ##__VA_ARGS__); \
37+ } while (0)
38+ 
39+#define CEIL_DIV(x, y) ((((x) + (y)) - 1) / (y))
40+#define CEIL_ALIGN(x, y) ((((x) + (y)) - 1) / (y) * (y))
41+ 
42+int64_t GetShapeSize(const std::vector<int64_t> &shape)
43+{
44+ int64_t shapeSize = 1;
45+ for (auto i : shape) {
46+ shapeSize *= i;
47+ }
48+ return shapeSize;
49+}
50+ 
51+int Init(int32_t deviceId, aclrtStream *stream)
52+{
53+ // 固定写法，AscendCL初始化
54+ auto ret = aclInit(nullptr);
55+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclInit failed. ERROR: %d\n", ret); return ret);
56+ ret = aclrtSetDevice(deviceId);
57+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret); return ret);
58+ ret = aclrtCreateStream(stream);
59+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret); return ret);
60+ return 0;
61+}
62+ 
63+template <typename T>
64+int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr,
65+ aclDataType dataType, aclTensor **tensor)
66+{
67+ auto size = GetShapeSize(shape) * sizeof(T);
68+ // 调用aclrtMalloc申请device侧内存
69+ auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST);
70+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtMalloc failed. ERROR: %d\n", ret); return ret);
71+ // 调用aclrtMemcpy将host侧数据拷贝到device侧内存上
72+ ret = aclrtMemcpy(*deviceAddr, size, hostData.data(), size, ACL_MEMCPY_HOST_TO_DEVICE);
73+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtMemcpy failed. ERROR: %d\n", ret); return ret);
74+ 
75+ // 计算连续tensor的strides
76+ std::vector<int64_t> strides(shape.size(), 1);
77+ for (int64_t i = shape.size() - 2; i >= 0; i--) {
78+ strides[i] = shape[i + 1] * strides[i + 1];
79+ }
80+ 
81+ // 调用aclCreateTensor接口创建aclTensor
82+ *tensor = aclCreateTensor(shape.data(), shape.size(), dataType, strides.data(), 0, aclFormat::ACL_FORMAT_ND,
83+ shape.data(), shape.size(), *deviceAddr);
84+ return 0;
85+}
86+ 
87+void Finalize(int32_t deviceId, aclrtStream stream)
88+{
89+ aclrtDestroyStream(stream);
90+ aclrtResetDevice(deviceId);
91+ aclFinalize();
92+}
93+ 
94+template <typename T>
95+int CreateAclTensorInt4(const std::vector<T>& hostData, const std::vector<int64_t>& shape, void** deviceAddr,
96+ aclDataType dataType, aclTensor** tensor, aclFormat format) {
97+ auto size = hostData.size() * sizeof(T);
98+ // 调用aclrtMalloc申请device侧内存
99+ auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST);
100+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtMalloc failed. ERROR: %d\n", ret); return ret);
101+ // 调用aclrtMemcpy将host侧数据拷贝到device侧内存上
102+ ret = aclrtMemcpy(*deviceAddr, size, hostData.data(), size, ACL_MEMCPY_HOST_TO_DEVICE);
103+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtMemcpy failed. ERROR: %d\n", ret); return ret);
104+ 
105+ // 计算连续tensor的strides
106+ std::vector<int64_t> strides(shape.size(), 1);
107+ for (int64_t i = shape.size() - 2; i >= 0; i--) {
108+ strides[i] = shape[i + 1] * strides[i + 1];
109+ }
110+ 
111+ // 调用aclCreateTensor接口创建aclTensor
112+ if (format == aclFormat::ACL_FORMAT_ND) {
113+ *tensor = aclCreateTensor(shape.data(), shape.size(), dataType, strides.data(), 0, aclFormat::ACL_FORMAT_ND,
114+ shape.data(), shape.size(), *deviceAddr);
115+ } else {
116+ std::vector<int64_t> nzShape;
117+ if (dataType == aclDataType::ACL_INT4) {
118+ nzShape = {CEIL_DIV(shape[1], 64), CEIL_DIV(shape[0], 16), 16, 64};
119+ } else {
120+ nzShape = {CEIL_DIV(shape[1], 8), CEIL_DIV(shape[0], 16), 16, 8};
121+ }
122+ *tensor = aclCreateTensor(shape.data(), shape.size(), dataType, strides.data(), 0,
123+ aclFormat::ACL_FORMAT_FRACTAL_NZ, nzShape.data(), nzShape.size(), *deviceAddr);
124+ }
125+ 
126+ return 0;
127+}
128+ 
129+int aclnnQuantMatmulWeightNzTest(int32_t deviceId, aclrtStream &stream)
130+{
131+ int64_t msize = 6256;
132+ int64_t ksize = 5120;
133+ int64_t nsize = 15360;
134+ int64_t biasBool = 0;
135+ int64_t pertokenScaleBool = 1;
136+ std::cout << "\n===== Matrix Parameters =====" << std::endl;
137+ std::cout << "Matrix m size: " << msize << std::endl;
138+ std::cout << "Kernel k size: " << ksize << std::endl;
139+ std::cout << "Matrix n size: " << nsize << std::endl;
140+ std::cout << "Bias enabled: " << (biasBool ? "true" : "false") << std::endl;
141+ std::cout << "=============================\n" << std::endl;
142+ 
143+ bool transposeX1 = false;
144+ bool transposeX2 = true;
145+ 
146+ auto ret = Init(deviceId, &stream);
147+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("Init acl failed. ERROR: %d\n", ret); return ret);
148+ 
149+ // 2. 构造输入与输出，需要根据API的接口自定义构造
150+ std::vector<int64_t> x1Shape = {msize, ksize};
151+ 
152+ int64_t weightDim0 = ksize;
153+ int64_t weightDim1 = nsize;
154+ if (transposeX2) {
155+ weightDim0 = nsize;
156+ weightDim1 = ksize;
157+ }
158+ std::vector<int64_t> x2Shape = {weightDim0, weightDim1};
159+ std::vector<int64_t> pertokenScaleShape = {msize};
160+ std::vector<int64_t> scaleShape = {nsize};
161+ std::vector<int64_t> biasShape = {nsize};
162+ std::vector<int64_t> outShape = {msize, nsize};
163+ void *x1DeviceAddr = nullptr;
164+ void *x2DeviceAddr = nullptr;
165+ void* weightInt4PackDeviceAddr = nullptr;
166+ void *pertokenScaleDeviceAddr = nullptr;
167+ void *scaleDeviceAddr = nullptr;
168+ void *biasDeviceAddr = nullptr;
169+ void *outDeviceAddr = nullptr;
170+ aclTensor *x1 = nullptr;
171+ aclTensor *x2 = nullptr;
172+ aclTensor* weightInt4Pack = nullptr;
173+ aclTensor *pertokenScale = nullptr;
174+ aclTensor *scale = nullptr;
175+ aclTensor *bias = nullptr;
176+ aclTensor *out = nullptr;
177+ std::vector<int8_t> x1HostData(GetShapeSize(x1Shape), 1);
178+ std::vector<int32_t> x2HostData(GetShapeSize(x2Shape), 1);
179+ std::vector<float> scaleHostData(GetShapeSize(scaleShape), 1);
180+ std::vector<float> pertokenScaleHostData(GetShapeSize(pertokenScaleShape), 1);
181+ std::vector<uint16_t> biasHostData(GetShapeSize(biasShape), 1);
182+ std::vector<uint16_t> outHostData(GetShapeSize(outShape), 1); // 实际上是float16半精度方式
183+ 
184+ // 创建x1 aclTensor
185+ ret = CreateAclTensorInt4(x1HostData, x1Shape, &x1DeviceAddr, aclDataType::ACL_INT4, &x1, aclFormat::ACL_FORMAT_ND);
186+ std::unique_ptr<aclTensor, aclnnStatus (*)(const aclTensor *)> x1TensorPtr(x1, aclDestroyTensor);
187+ std::unique_ptr<void, aclError (*)(void *)> x1DeviceAddrPtr(x1DeviceAddr, aclrtFree);
188+ CHECK_RET(ret == ACL_SUCCESS, return ret);
189+ 
190+ // 创建x2 aclTensor
191+ ret = CreateAclTensorInt4(x2HostData, x2Shape, &x2DeviceAddr, aclDataType::ACL_INT4, &x2, aclFormat::ACL_FORMAT_FRACTAL_NZ);
192+ std::unique_ptr<aclTensor, aclnnStatus (*)(const aclTensor *)> x2TensorPtr(x2, aclDestroyTensor);
193+ std::unique_ptr<void, aclError (*)(void *)> x2DeviceAddrPtr(x2DeviceAddr, aclrtFree);
194+ CHECK_RET(ret == ACL_SUCCESS, return ret);
195+ 
196+ // 创建pertokenScale aclTensor
197+ ret = CreateAclTensor(pertokenScaleHostData, pertokenScaleShape, &pertokenScaleDeviceAddr, aclDataType::ACL_FLOAT, &pertokenScale);
198+ std::unique_ptr<aclTensor, aclnnStatus (*)(const aclTensor *)> pertokenScaleTensorPtr(pertokenScale, aclDestroyTensor);
199+ std::unique_ptr<void, aclError (*)(void *)> pertokenScaleDeviceAddrPtr(pertokenScaleDeviceAddr, aclrtFree);
200+ CHECK_RET(ret == ACL_SUCCESS, return ret);
201+ if (pertokenScaleBool == 0) {
202+ pertokenScale = nullptr;
203+ }
204+
205+ // 创建scale aclTensor
206+ ret = CreateAclTensor(scaleHostData, scaleShape, &scaleDeviceAddr, aclDataType::ACL_FLOAT, &scale);
207+ std::unique_ptr<aclTensor, aclnnStatus (*)(const aclTensor *)> scaleTensorPtr(scale, aclDestroyTensor);
208+ std::unique_ptr<void, aclError (*)(void *)> scaleDeviceAddrPtr(scaleDeviceAddr, aclrtFree);
209+ CHECK_RET(ret == ACL_SUCCESS, return ret);
210+ // 创建bias aclTensor
211+ ret = CreateAclTensor(biasHostData, biasShape, &biasDeviceAddr, aclDataType::ACL_BF16, &bias);
212+ std::unique_ptr<aclTensor, aclnnStatus (*)(const aclTensor *)> biasTensorPtr(bias, aclDestroyTensor);
213+ std::unique_ptr<void, aclError (*)(void *)> biasDeviceAddrPtr(biasDeviceAddr, aclrtFree);
214+ CHECK_RET(ret == ACL_SUCCESS, return ret);
215+ if (biasBool == 0) {
216+ bias = nullptr;
217+ }
218+ 
219+ // 创建out aclTensor
220+ ret = CreateAclTensor(outHostData, outShape, &outDeviceAddr, aclDataType::ACL_BF16, &out);
221+ std::unique_ptr<aclTensor, aclnnStatus (*)(const aclTensor *)> outTensorPtr(out, aclDestroyTensor);
222+ std::unique_ptr<void, aclError (*)(void *)> outDeviceAddrPtr(outDeviceAddr, aclrtFree);
223+ CHECK_RET(ret == ACL_SUCCESS, return ret);
224+ 
225+ // 3. 调用CANN算子库API，需要修改为具体的Api名称
226+ uint64_t workspaceSize = 0;
227+ aclOpExecutor *executor;
228+ void *workspaceAddr = nullptr;
229+ 
230+ // 调用aclnnQuantMatmulWeightNz第一段接口
231+ workspaceSize = 0;
232+ ret = aclnnQuantMatmulWeightNzGetWorkspaceSize(x1, x2, pertokenScale, scale, nullptr, nullptr, nullptr, nullptr,
233+ bias, transposeX1, transposeX2, 0, out, &workspaceSize,
234+ &executor);
235+ 
236+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclnnQuantMatmulWeightNzGetWorkspaceSize failed. ERROR: %d\n", ret);
237+ return ret);
238+ // 根据第一段接口计算出的workspaceSize申请device内存
239+ 
240+ std::unique_ptr<void, aclError (*)(void *)> workspaceAddrPtrNZ(nullptr, aclrtFree);
241+ if (workspaceSize > 0) {
242+ ret = aclrtMalloc(&workspaceAddr, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST);
243+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("allocate workspace failed. ERROR: %d\n", ret); return ret);
244+ workspaceAddrPtrNZ.reset(workspaceAddr);
245+ }
246+ // 调用aclnnQuantMatmulWeightNz第二段接口
247+ ret = aclnnQuantMatmulWeightNz(workspaceAddr, workspaceSize, executor, stream);
248+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclnnQuantMatmulWeightNz failed. ERROR: %d\n", ret); return ret);
249+ 
250+ // 4. （固定写法）同步等待任务执行结束
251+ ret = aclrtSynchronizeStream(stream);
252+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtSynchronizeStream failed. ERROR: %d\n", ret); return ret);
253+ 
254+ // 5. 获取输出的值，将device侧内存上的结果拷贝至host侧，需要根据具体API的接口定义修改
255+ auto size = GetShapeSize(outShape);
256+ std::vector<uint16_t> resultData(
257+ size, 0); // C语言中无法直接打印fp16的数据，需要用uint16读出来，自行通过二进制转成fp16
258+ ret = aclrtMemcpy(resultData.data(), resultData.size() * sizeof(resultData[0]), outDeviceAddr,
259+ size * sizeof(resultData[0]), ACL_MEMCPY_DEVICE_TO_HOST);
260+ CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("copy result from device to host failed. ERROR: %d\n", ret);
261+ return ret);
262+ int64_t numPrint = 10;
263+ for (int64_t i = 0; i < numPrint; i++) {
264+ LOG_PRINT("result[%ld] is: %u\n", i, resultData[i]);
265+ }
266+ return ACL_SUCCESS;
267+}
268+ 
269+int main()
270+{
271+ // 1. （固定写法）device/stream初始化，参考AscendCL对外接口列表
272+ // 根据自己的实际device填写deviceId
273+ int32_t deviceId = 0;
274+ aclrtStream stream;
275+ auto ret = aclnnQuantMatmulWeightNzTest(deviceId, stream);
276+ CHECK_FREE_RET(ret == ACL_SUCCESS, LOG_PRINT("aclnnQuantMatmulWeightNzTest failed. ERROR: %d\n", ret);
277+ return ret);
278+ 
279+ Finalize(deviceId, stream);
280+ return 0;
281+}
Mmatmul/quant_batch_matmul_v3/op_api/aclnn_quant_matmul_v4.cpp+20-19
@@ -62,8 +62,9 @@ static const int NZ_K1_INDEX = 3;
62static const int NZ_K1_INDEX_TRANS = 4;62static const int NZ_K1_INDEX_TRANS = 4;
63static const int NZ_STORAGE_PENULTIMATE_DIM = 16;63static const int NZ_STORAGE_PENULTIMATE_DIM = 16;
64static const int NZ_STORAGE_LAST_DIM = 32;64static const int NZ_STORAGE_LAST_DIM = 32;
65-static const int64_t NZ_K0_VALUE_INT8 = 16;65+static const int64_t NZ_K0_VALUE_BMM_BLOCK_NUM = 16;
66static const int64_t NZ_K0_VALUE_INT8_TRANS = 32;66static const int64_t NZ_K0_VALUE_INT8_TRANS = 32;
67+static const int64_t NZ_K0_VALUE_INT4_TRANS = 64;
67static constexpr int64_t OUTPUT_INFER_FAIL = -1L;68static constexpr int64_t OUTPUT_INFER_FAIL = -1L;
68static const int64_t LAST_AXIS_LIMIT = 65535;69static const int64_t LAST_AXIS_LIMIT = 65535;
69static const int X2_FIXED_DIM_NUM_A4W4 = 2;70static const int X2_FIXED_DIM_NUM_A4W4 = 2;
@@ -356,12 +357,12 @@ static inline bool CheckDtypeValid(TupleTensor mandatoryTensors, TupleOptional o
356static inline bool CheckFormatInt4(const aclTensor *x1, const aclTensor *x2) {357static inline bool CheckFormatInt4(const aclTensor *x1, const aclTensor *x2) {
357 if (x1->GetStorageFormat() != op::Format::FORMAT_ND) {358 if (x1->GetStorageFormat() != op::Format::FORMAT_ND) {
358 OP_LOGE(ACLNN_ERR_PARAM_INVALID, "x1 only support ND format in a4w4 scenario, but now is %s.",359 OP_LOGE(ACLNN_ERR_PARAM_INVALID, "x1 only support ND format in a4w4 scenario, but now is %s.",
359- op::ToString(x1->GetStorageFormat()).GetString());360+ op::ToString(x1->GetStorageFormat()).GetString());
360 return false;361 return false;
361 }362 }
362- if (x2->GetStorageFormat() != op::Format::FORMAT_ND) {363+ if (x2->GetStorageFormat() != op::Format::FORMAT_ND && x2->GetStorageFormat() != op::Format::FORMAT_FRACTAL_NZ) {
363- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "x2 only support ND in a4w4 scenario, but now is %s.",364+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "x2 only support ND/NZ in a4w4 scenario, but now is %s.",
364- op::ToString(x2->GetStorageFormat()).GetString());365+ op::ToString(x2->GetStorageFormat()).GetString());
365 return false;366 return false;
366 }367 }
367 return true;368 return true;
@@ -566,12 +567,13 @@ static inline bool CheckShapeForWeightNz(const aclTensor *x1, const aclTensor *x
566 auto x2DimNum = x2->GetStorageShape().GetDimNum();567 auto x2DimNum = x2->GetStorageShape().GetDimNum();
567 int64_t x1KDim = transposeX1 ? x1Shape[x1DimNum - PENULTIMATE_DIM] : x1Shape[x1DimNum - 1];568 int64_t x1KDim = transposeX1 ? x1Shape[x1DimNum - PENULTIMATE_DIM] : x1Shape[x1DimNum - 1];
568 int64_t x2K1Dim = transposeX2 ? x2Shape[x2DimNum - NZ_K1_INDEX_TRANS] : x2Shape[x2DimNum - NZ_K1_INDEX];569 int64_t x2K1Dim = transposeX2 ? x2Shape[x2DimNum - NZ_K1_INDEX_TRANS] : x2Shape[x2DimNum - NZ_K1_INDEX];
569- int64_t aligneValue = transposeX2 ? NZ_K0_VALUE_INT8_TRANS : NZ_K0_VALUE_INT8;570+ int64_t nz_k0_value_trans = x1->GetDataType() == op::DataType::DT_INT8 ? NZ_K0_VALUE_INT8_TRANS : NZ_K0_VALUE_INT4_TRANS;
CANN-robot
CANN-robotCANN-robot1月29日

拼写错误: 第562行变量名'aligneValue'拼写错误，应为'alignedValue'或'alignmentValue'，以更准确地表达其含义（对齐值）。

问题类型: 拼写错误 文件路径: matmul/quant_batch_matmul_v3/op_api/aclnn_quant_matmul_v4.cpp 行号: 561 问题代码:

    int64_t nz_k0_value_trans = x1->GetDataType() == op::DataType::DT_INT8 ? NZ_K0_VALUE_INT8_TRANS : NZ_K0_VALUE_INT4_TRANS;
 	int64_t aligneValue = transposeX2 ? nz_k0_value_trans : NZ_K0_VALUE_BMM_BLOCK_NUM;

修改建议:

将变量名'aligneValue'改为'alignedValue'或'alignmentValue'，以保持代码的一致性和可读性。修改为：'int64_t alignedValue = transposeX2 ? nz_k0_value_trans : NZ_K0_VALUE_BMM_BLOCK_NUM;'

此评论由代码审查工具自动生成

likedislike
570- int64_t alignedX1K = ((x1KDim + aligneValue - 1) / aligneValue) * aligneValue;571+ int64_t roundValue = transposeX2 ? nz_k0_value_trans : NZ_K0_VALUE_BMM_BLOCK_NUM;
571- if (alignedX1K != x2K1Dim * aligneValue) {572+ int64_t x1KDimRound = ((x1KDim + roundValue - 1) / roundValue) * roundValue;
573+ if (x1KDimRound != x2K1Dim * roundValue) {
572 OP_LOGE(ACLNN_ERR_PARAM_INVALID,574 OP_LOGE(ACLNN_ERR_PARAM_INVALID,
573- "AlignedK1 value %ld is not matched with k1 value times aligneValue, which is %ld.",575+ "AlignedK1 value %ld is not matched with k1 value times roundValue, which is %ld.",
574- alignedX1K, x2K1Dim * aligneValue);576+ x1KDimRound, x2K1Dim * roundValue);
575 return false;577 return false;
576 }578 }
577 return true;579 return true;
@@ -591,10 +593,7 @@ static inline bool IsAligned(T num, T factor)
591static inline bool CheckShapeInt4(const aclTensor *x1, const aclTensor *x2, bool transposeX1, bool transposeX2,593static inline bool CheckShapeInt4(const aclTensor *x1, const aclTensor *x2, bool transposeX1, bool transposeX2,
592 const aclTensor *bias)594 const aclTensor *bias)
593{595{
594- int64_t x1KDim;596+ int64_t x1KDim, x1MDim, x2KDim, x2NDim;
595- int64_t x1MDim;
596- int64_t x2KDim;
597- int64_t x2NDim;
598 std::tie(x1KDim, x1MDim, x2KDim, x2NDim) = GetX1X2DimValue(x1, x2, transposeX1, transposeX2);597 std::tie(x1KDim, x1MDim, x2KDim, x2NDim) = GetX1X2DimValue(x1, x2, transposeX1, transposeX2);
599 if (!IsAligned<int64_t>(x1KDim, INT4_NUMS_IN_INT8)) {598 if (!IsAligned<int64_t>(x1KDim, INT4_NUMS_IN_INT8)) {
600 OP_LOGE(ACLNN_ERR_PARAM_INVALID,599 OP_LOGE(ACLNN_ERR_PARAM_INVALID,
@@ -1638,12 +1637,14 @@ static op::Shape GetWeightNzShape(const aclTensor *input, bool transpose)
1638{1637{
1639 size_t viewDimNum = input->GetViewShape().GetDimNum();1638 size_t viewDimNum = input->GetViewShape().GetDimNum();
1640 int64_t k = transpose ? input->GetViewShape().GetDim(viewDimNum - 1)1639 int64_t k = transpose ? input->GetViewShape().GetDim(viewDimNum - 1)
1641- : input->GetViewShape().GetDim(viewDimNum - LAST_SECOND_DIM_INDEX);1640+ : input->GetViewShape().GetDim(viewDimNum - LAST_SECOND_DIM_INDEX);
1642 int64_t n = transpose ? input->GetViewShape().GetDim(viewDimNum - LAST_SECOND_DIM_INDEX)1641 int64_t n = transpose ? input->GetViewShape().GetDim(viewDimNum - LAST_SECOND_DIM_INDEX)
1643- : input->GetViewShape().GetDim(viewDimNum - 1);1642+ : input->GetViewShape().GetDim(viewDimNum - 1);
1644 1643 
1645- int64_t k1 = transpose ? CeilDiv(k, NZ_K0_VALUE_INT8_TRANS) : CeilDiv(k, NZ_K0_VALUE_INT8);1644+ int64_t nz_k0_value_trans =
1646- int64_t n1 = transpose ? CeilDiv(n, NZ_K0_VALUE_INT8) : CeilDiv(n, NZ_K0_VALUE_INT8_TRANS);1645+ input->GetDataType() == op::DataType::DT_INT8 ? NZ_K0_VALUE_INT8_TRANS : NZ_K0_VALUE_INT4_TRANS;
1646+ int64_t k1 = transpose ? CeilDiv(k, nz_k0_value_trans) : CeilDiv(k, NZ_K0_VALUE_BMM_BLOCK_NUM);
ztj0209
ztj0209ztj02092月3日

建议：定义变量里，对于已经向上对齐的变量 最好和初始变量从名字上分开 K1 or K1_round

likedislike
zhang-junming21
zhang-junming21
2月3日 评论：
1647+ int64_t n1 = transpose ? CeilDiv(n, NZ_K0_VALUE_BMM_BLOCK_NUM) : CeilDiv(n, nz_k0_value_trans);
1647 1648 
1648 op::Shape weightNzShape;1649 op::Shape weightNzShape;
1649 for (size_t i = 0; i < viewDimNum - LAST_SECOND_DIM_INDEX; i++) {1650 for (size_t i = 0; i < viewDimNum - LAST_SECOND_DIM_INDEX; i++) {
@@ -1657,7 +1658,7 @@ static op::Shape GetWeightNzShape(const aclTensor *input, bool transpose)
1657 weightNzShape.AppendDim(k1);1658 weightNzShape.AppendDim(k1);
1658 }1659 }
1659 weightNzShape.AppendDim(NZ_STORAGE_PENULTIMATE_DIM);1660 weightNzShape.AppendDim(NZ_STORAGE_PENULTIMATE_DIM);
ztj0209
ztj0209ztj02092月3日

问题：NZ_STORAGE_PENULTIMATE_DIM和 nz_k0_value_trans的区别从名字上很难分清楚，为什么nz_k0_value_trans需要根据数据类型做区分，但是NZ_STORAGE_PENULTIMATE_DIM不需要区分数据类型

likedislike
zhang-junming21
zhang-junming21
2月3日 评论：
1660- weightNzShape.AppendDim(NZ_STORAGE_LAST_DIM);1661+ weightNzShape.AppendDim(nz_k0_value_trans);
1661 return weightNzShape;1662 return weightNzShape;
1662}1663}
1663 1664 
Mmatmul/quant_batch_matmul_v3/op_host/config/ascend910_93/quant_batch_matmul_v3_binary.json+428-0
@@ -2568,6 +2568,434 @@
2568 "value": null2568 "value": null
2569 }2569 }
2570 ]2570 ]
2571+ },
2572+ {
2573+ "bin_filename": "QuantBatchMatmulV3_ND_NZ_int4_int4_fp16",
2574+ "inputs": [
2575+ {
2576+ "name": "x1",
2577+ "index": 0,
2578+ "dtype": "int4",
2579+ "format": "ND",
2580+ "paramType": "required",
2581+ "shape": [
2582+ -2
2583+ ],
2584+ "format_match_mode": "FormatDefault"
2585+ },
2586+ {
2587+ "name": "x2",
2588+ "index": 1,
2589+ "dtype": "int4",
2590+ "format": "FRACTAL_NZ",
2591+ "paramType": "required",
2592+ "shape": [
2593+ -2
2594+ ],
2595+ "format_match_mode": "FormatDefault"
2596+ },
2597+ {
2598+ "name": "scale",
2599+ "index": 2,
2600+ "dtype": "uint64",
2601+ "format": "ND",
2602+ "paramType": "required",
2603+ "shape": [
2604+ -2
2605+ ],
2606+ "format_match_mode": "FormatDefault",
2607+ "dtype_match_mode": "DtypeByte"
2608+ },
2609+ {
2610+ "name": "offset",
2611+ "index": 3,
2612+ "dtype": "float32",
2613+ "format": "ND",
2614+ "paramType": "optional",
2615+ "shape": [
2616+ -2
2617+ ],
2618+ "format_match_mode": "FormatDefault"
2619+ },
2620+ {
2621+ "name": "bias",
2622+ "index": 4,
2623+ "dtype": "int32",
2624+ "format": "ND",
2625+ "paramType": "optional",
2626+ "shape": [
2627+ -2
2628+ ],
2629+ "format_match_mode": "FormatDefault"
2630+ },
2631+ {
2632+ "name": "pertoken_scale",
2633+ "index": 5,
2634+ "dtype": "float32",
2635+ "format": "ND",
2636+ "paramType": "optional",
2637+ "shape": [
2638+ -2
2639+ ],
2640+ "format_match_mode": "FormatDefault"
2641+ }
2642+ ],
2643+ "outputs": [
2644+ {
2645+ "name": "y",
2646+ "index": 0,
2647+ "dtype": "float16",
2648+ "format": "ND",
2649+ "paramType": "required",
2650+ "shape": [
2651+ -2
2652+ ],
2653+ "format_match_mode": "FormatDefault"
2654+ }
2655+ ],
2656+ "attrs": [
2657+ {
2658+ "name": "dtype",
2659+ "dtype": "int",
2660+ "value": 1
2661+ },
2662+ {
2663+ "name": "transpose_x1",
2664+ "dtype": "bool",
2665+ "value": null
2666+ },
2667+ {
2668+ "name": "transpose_x2",
2669+ "dtype": "bool",
2670+ "value": null
2671+ },
2672+ {
2673+ "name": "group_size",
2674+ "dtype": "int",
2675+ "value": null
2676+ }
2677+ ]
2678+ },
2679+ {
2680+ "bin_filename": "QuantBatchMatmulV3_ND_NZ_int4_int4_fp32_fp16",
2681+ "inputs": [
2682+ {
2683+ "name": "x1",
2684+ "index": 0,
2685+ "dtype": "int4",
2686+ "format": "ND",
2687+ "paramType": "required",
2688+ "shape": [
2689+ -2
2690+ ],
2691+ "format_match_mode": "FormatDefault"
2692+ },
2693+ {
2694+ "name": "x2",
2695+ "index": 1,
2696+ "dtype": "int4",
2697+ "format": "FRACTAL_NZ",
2698+ "paramType": "required",
2699+ "shape": [
2700+ -2
2701+ ],
2702+ "format_match_mode": "FormatDefault"
2703+ },
2704+ {
2705+ "name": "scale",
2706+ "index": 2,
2707+ "dtype": "float32",
2708+ "format": "ND",
2709+ "paramType": "required",
2710+ "shape": [
2711+ -2
2712+ ],
2713+ "format_match_mode": "FormatDefault",
2714+ "dtype_match_mode": "DtypeByte"
2715+ },
2716+ {
2717+ "name": "offset",
2718+ "index": 3,
2719+ "dtype": "float32",
2720+ "format": "ND",
2721+ "paramType": "optional",
2722+ "shape": [
2723+ -2
2724+ ],
2725+ "format_match_mode": "FormatDefault"
2726+ },
2727+ {
2728+ "name": "bias",
2729+ "index": 4,
2730+ "dtype": "int32",
2731+ "format": "ND",
2732+ "paramType": "optional",
2733+ "shape": [
2734+ -2
2735+ ],
2736+ "format_match_mode": "FormatDefault"
2737+ },
2738+ {
2739+ "name": "pertoken_scale",
2740+ "index": 5,
2741+ "dtype": "float32",
2742+ "format": "ND",
2743+ "paramType": "optional",
2744+ "shape": [
2745+ -2
2746+ ],
2747+ "format_match_mode": "FormatDefault"
2748+ }
2749+ ],
2750+ "outputs": [
2751+ {
2752+ "name": "y",
2753+ "index": 0,
2754+ "dtype": "float16",
2755+ "format": "ND",
2756+ "paramType": "required",
2757+ "shape": [
2758+ -2
2759+ ],
2760+ "format_match_mode": "FormatDefault"
2761+ }
2762+ ],
2763+ "attrs": [
2764+ {
2765+ "name": "dtype",
2766+ "dtype": "int",
2767+ "value": 1
2768+ },
2769+ {
2770+ "name": "transpose_x1",
2771+ "dtype": "bool",
2772+ "value": null
2773+ },
2774+ {
2775+ "name": "transpose_x2",
2776+ "dtype": "bool",
2777+ "value": null
2778+ },
2779+ {
2780+ "name": "group_size",
2781+ "dtype": "int",
2782+ "value": null
2783+ }
2784+ ]
2785+ },
2786+ {
2787+ "bin_filename": "QuantBatchMatmulV3_ND_NZ_int4_int4_bf16",
2788+ "inputs": [
2789+ {
2790+ "name": "x1",
2791+ "index": 0,
2792+ "dtype": "int4",
2793+ "format": "ND",
2794+ "paramType": "required",
2795+ "shape": [
2796+ -2
2797+ ],
2798+ "format_match_mode": "FormatDefault"
2799+ },
2800+ {
2801+ "name": "x2",
2802+ "index": 1,
2803+ "dtype": "int4",
2804+ "format": "FRACTAL_NZ",
2805+ "paramType": "required",
2806+ "shape": [
2807+ -2
2808+ ],
2809+ "format_match_mode": "FormatDefault"
2810+ },
2811+ {
2812+ "name": "scale",
2813+ "index": 2,
2814+ "dtype": "bfloat16",
2815+ "format": "ND",
2816+ "paramType": "required",
2817+ "shape": [
2818+ -2
2819+ ],
2820+ "format_match_mode": "FormatDefault",
2821+ "dtype_match_mode": "DtypeByte"
2822+ },
2823+ {
2824+ "name": "offset",
2825+ "index": 3,
2826+ "dtype": "float32",
2827+ "format": "ND",
2828+ "paramType": "optional",
2829+ "shape": [
2830+ -2
2831+ ],
2832+ "format_match_mode": "FormatDefault"
2833+ },
2834+ {
2835+ "name": "bias",
2836+ "index": 4,
2837+ "dtype": "bfloat16",
2838+ "format": "ND",
2839+ "paramType": "optional",
2840+ "shape": [
2841+ -2
2842+ ],
2843+ "format_match_mode": "FormatDefault"
2844+ },
2845+ {
2846+ "name": "pertoken_scale",
2847+ "index": 5,
2848+ "dtype": "float32",
2849+ "format": "ND",
2850+ "paramType": "optional",
2851+ "shape": [
2852+ -2
2853+ ],
2854+ "format_match_mode": "FormatDefault"
2855+ }
2856+ ],
2857+ "outputs": [
2858+ {
2859+ "name": "y",
2860+ "index": 0,
2861+ "dtype": "bfloat16",
2862+ "format": "ND",
2863+ "paramType": "required",
2864+ "shape": [
2865+ -2
2866+ ],
2867+ "format_match_mode": "FormatDefault"
2868+ }
2869+ ],
2870+ "attrs": [
2871+ {
2872+ "name": "dtype",
2873+ "dtype": "int",
2874+ "value": 27
2875+ },
2876+ {
2877+ "name": "transpose_x1",
2878+ "dtype": "bool",
2879+ "value": null
2880+ },
2881+ {
2882+ "name": "transpose_x2",
2883+ "dtype": "bool",
2884+ "value": null
2885+ },
2886+ {
2887+ "name": "group_size",
2888+ "dtype": "int",
2889+ "value": null
2890+ }
2891+ ]
2892+ },
2893+ {
2894+ "bin_filename": "QuantBatchMatmulV3_ND_NZ_int4_int4_fp32_bf16",
2895+ "inputs": [
2896+ {
ztj0209
ztj0209ztj02092月3日

问题：会新增NZ format的.o，超过5%需要评审基线合入，确定.o的大小

likedislike
zhang-junming21
zhang-junming21
2月3日 评论：
ztj0209
ztj0209
2月3日 评论：
zhang-junming21
zhang-junming21
2月3日 评论：
2897+ "name": "x1",
2898+ "index": 0,
2899+ "dtype": "int4",
2900+ "format": "ND",
2901+ "paramType": "required",
2902+ "shape": [
2903+ -2
2904+ ],
2905+ "format_match_mode": "FormatDefault"
2906+ },
2907+ {
2908+ "name": "x2",
2909+ "index": 1,
2910+ "dtype": "int4",
2911+ "format": "FRACTAL_NZ",
2912+ "paramType": "required",
2913+ "shape": [
2914+ -2
2915+ ],
2916+ "format_match_mode": "FormatDefault"
2917+ },
2918+ {
2919+ "name": "scale",
2920+ "index": 2,
2921+ "dtype": "float32",
2922+ "format": "ND",
2923+ "paramType": "required",
2924+ "shape": [
2925+ -2
2926+ ],
2927+ "format_match_mode": "FormatDefault",
2928+ "dtype_match_mode": "DtypeByte"
2929+ },
2930+ {
2931+ "name": "offset",
2932+ "index": 3,
2933+ "dtype": "float32",
2934+ "format": "ND",
2935+ "paramType": "optional",
2936+ "shape": [
2937+ -2
2938+ ],
2939+ "format_match_mode": "FormatDefault"
2940+ },
2941+ {
2942+ "name": "bias",
2943+ "index": 4,
2944+ "dtype": "bfloat16",
2945+ "format": "ND",
2946+ "paramType": "optional",
2947+ "shape": [
2948+ -2
2949+ ],
2950+ "format_match_mode": "FormatDefault"
2951+ },
2952+ {
2953+ "name": "pertoken_scale",
2954+ "index": 5,
2955+ "dtype": "float32",
2956+ "format": "ND",
2957+ "paramType": "optional",
2958+ "shape": [
2959+ -2
2960+ ],
2961+ "format_match_mode": "FormatDefault"
2962+ }
2963+ ],
2964+ "outputs": [
2965+ {
2966+ "name": "y",
2967+ "index": 0,
2968+ "dtype": "bfloat16",
2969+ "format": "ND",
2970+ "paramType": "required",
2971+ "shape": [
2972+ -2
2973+ ],
2974+ "format_match_mode": "FormatDefault"
2975+ }
2976+ ],
2977+ "attrs": [
2978+ {
2979+ "name": "dtype",
2980+ "dtype": "int",
2981+ "value": 27
2982+ },
2983+ {
2984+ "name": "transpose_x1",
2985+ "dtype": "bool",
2986+ "value": null
2987+ },
2988+ {
2989+ "name": "transpose_x2",
2990+ "dtype": "bool",
2991+ "value": null
2992+ },
2993+ {
2994+ "name": "group_size",
2995+ "dtype": "int",
2996+ "value": null
2997+ }
2998+ ]
2571 }2999 }
2572 ]3000 ]
2573}3001}
Mmatmul/quant_batch_matmul_v3/op_host/config/ascend910b/quant_batch_matmul_v3_binary.json+428-0
@@ -2568,6 +2568,434 @@
2568 "value": null2568 "value": null
2569 }2569 }
2570 ]2570 ]
2571+ },
2572+ {
ztj0209
ztj0209ztj02092月6日

问题：fp16 out 的数据类型的配置有问题，bias配置成了BF16，这个是非法输入 修改：fp16 out 的数据类型 配套的 bias 支持 的是 不传/ int32 ，需要改正

likedislike
2573+ "bin_filename": "QuantBatchMatmulV3_ND_NZ_int4_int4_fp16",
2574+ "inputs": [
2575+ {
2576+ "name": "x1",
2577+ "index": 0,
2578+ "dtype": "int4",
2579+ "format": "ND",
2580+ "paramType": "required",
2581+ "shape": [
2582+ -2
2583+ ],
2584+ "format_match_mode": "FormatDefault"
2585+ },
2586+ {
2587+ "name": "x2",
2588+ "index": 1,
2589+ "dtype": "int4",
2590+ "format": "FRACTAL_NZ",
2591+ "paramType": "required",
2592+ "shape": [
2593+ -2
2594+ ],
2595+ "format_match_mode": "FormatDefault"
2596+ },
2597+ {
2598+ "name": "scale",
2599+ "index": 2,
2600+ "dtype": "uint64",
2601+ "format": "ND",
2602+ "paramType": "required",
2603+ "shape": [
2604+ -2
2605+ ],
2606+ "format_match_mode": "FormatDefault",
2607+ "dtype_match_mode": "DtypeByte"
2608+ },
2609+ {
2610+ "name": "offset",
2611+ "index": 3,
2612+ "dtype": "float32",
2613+ "format": "ND",
2614+ "paramType": "optional",
2615+ "shape": [
2616+ -2
2617+ ],
2618+ "format_match_mode": "FormatDefault"
2619+ },
2620+ {
2621+ "name": "bias",
2622+ "index": 4,
2623+ "dtype": "int32",
2624+ "format": "ND",
2625+ "paramType": "optional",
2626+ "shape": [
2627+ -2
2628+ ],
2629+ "format_match_mode": "FormatDefault"
2630+ },
2631+ {
2632+ "name": "pertoken_scale",
2633+ "index": 5,
2634+ "dtype": "float32",
2635+ "format": "ND",
2636+ "paramType": "optional",
2637+ "shape": [
2638+ -2
2639+ ],
2640+ "format_match_mode": "FormatDefault"
2641+ }
2642+ ],
2643+ "outputs": [
2644+ {
2645+ "name": "y",
2646+ "index": 0,
2647+ "dtype": "float16",
2648+ "format": "ND",
2649+ "paramType": "required",
2650+ "shape": [
2651+ -2
2652+ ],
2653+ "format_match_mode": "FormatDefault"
2654+ }
2655+ ],
2656+ "attrs": [
2657+ {
2658+ "name": "dtype",
2659+ "dtype": "int",
2660+ "value": 1
2661+ },
2662+ {
2663+ "name": "transpose_x1",
2664+ "dtype": "bool",
2665+ "value": null
2666+ },
2667+ {
2668+ "name": "transpose_x2",
2669+ "dtype": "bool",
2670+ "value": null
2671+ },
2672+ {
2673+ "name": "group_size",
2674+ "dtype": "int",
2675+ "value": null
2676+ }
2677+ ]
2678+ },
2679+ {
2680+ "bin_filename": "QuantBatchMatmulV3_ND_NZ_int4_int4_fp32_fp16",
2681+ "inputs": [
2682+ {
2683+ "name": "x1",
2684+ "index": 0,
2685+ "dtype": "int4",
2686+ "format": "ND",
2687+ "paramType": "required",
2688+ "shape": [
2689+ -2
2690+ ],
2691+ "format_match_mode": "FormatDefault"
2692+ },
2693+ {
2694+ "name": "x2",
2695+ "index": 1,
2696+ "dtype": "int4",
2697+ "format": "FRACTAL_NZ",
2698+ "paramType": "required",
2699+ "shape": [
2700+ -2
2701+ ],
2702+ "format_match_mode": "FormatDefault"
2703+ },
2704+ {
2705+ "name": "scale",
2706+ "index": 2,
2707+ "dtype": "float32",
2708+ "format": "ND",
2709+ "paramType": "required",
2710+ "shape": [
2711+ -2
2712+ ],
2713+ "format_match_mode": "FormatDefault",
2714+ "dtype_match_mode": "DtypeByte"
2715+ },
2716+ {
2717+ "name": "offset",
2718+ "index": 3,
2719+ "dtype": "float32",
2720+ "format": "ND",
2721+ "paramType": "optional",
2722+ "shape": [
2723+ -2
2724+ ],
2725+ "format_match_mode": "FormatDefault"
2726+ },
2727+ {
2728+ "name": "bias",
2729+ "index": 4,
2730+ "dtype": "int32",
2731+ "format": "ND",
2732+ "paramType": "optional",
2733+ "shape": [
2734+ -2
2735+ ],
2736+ "format_match_mode": "FormatDefault"
2737+ },
2738+ {
2739+ "name": "pertoken_scale",
2740+ "index": 5,
2741+ "dtype": "float32",
2742+ "format": "ND",
2743+ "paramType": "optional",
2744+ "shape": [
2745+ -2
2746+ ],
2747+ "format_match_mode": "FormatDefault"
2748+ }
2749+ ],
2750+ "outputs": [
2751+ {
2752+ "name": "y",
2753+ "index": 0,
2754+ "dtype": "float16",
2755+ "format": "ND",
2756+ "paramType": "required",
2757+ "shape": [
2758+ -2
2759+ ],
2760+ "format_match_mode": "FormatDefault"
2761+ }
2762+ ],
2763+ "attrs": [
2764+ {
2765+ "name": "dtype",
2766+ "dtype": "int",
2767+ "value": 1
2768+ },
2769+ {
2770+ "name": "transpose_x1",
2771+ "dtype": "bool",
2772+ "value": null
2773+ },
2774+ {
2775+ "name": "transpose_x2",
2776+ "dtype": "bool",
2777+ "value": null
2778+ },
2779+ {
2780+ "name": "group_size",
2781+ "dtype": "int",
2782+ "value": null
2783+ }
2784+ ]
2785+ },
2786+ {
2787+ "bin_filename": "QuantBatchMatmulV3_ND_NZ_int4_int4_bf16",
2788+ "inputs": [
2789+ {
2790+ "name": "x1",
2791+ "index": 0,
2792+ "dtype": "int4",
2793+ "format": "ND",
2794+ "paramType": "required",
2795+ "shape": [
2796+ -2
2797+ ],
2798+ "format_match_mode": "FormatDefault"
2799+ },
2800+ {
2801+ "name": "x2",
2802+ "index": 1,
2803+ "dtype": "int4",
2804+ "format": "FRACTAL_NZ",
2805+ "paramType": "required",
2806+ "shape": [
2807+ -2
2808+ ],
2809+ "format_match_mode": "FormatDefault"
2810+ },
2811+ {
2812+ "name": "scale",
2813+ "index": 2,
2814+ "dtype": "bfloat16",
2815+ "format": "ND",
2816+ "paramType": "required",
2817+ "shape": [
2818+ -2
2819+ ],
2820+ "format_match_mode": "FormatDefault",
2821+ "dtype_match_mode": "DtypeByte"
2822+ },
2823+ {
2824+ "name": "offset",
2825+ "index": 3,
2826+ "dtype": "float32",
2827+ "format": "ND",
2828+ "paramType": "optional",
2829+ "shape": [
2830+ -2
2831+ ],
2832+ "format_match_mode": "FormatDefault"
2833+ },
2834+ {
2835+ "name": "bias",
2836+ "index": 4,
2837+ "dtype": "bfloat16",
2838+ "format": "ND",
2839+ "paramType": "optional",
2840+ "shape": [
2841+ -2
2842+ ],
2843+ "format_match_mode": "FormatDefault"
2844+ },
2845+ {
2846+ "name": "pertoken_scale",
2847+ "index": 5,
2848+ "dtype": "float32",
2849+ "format": "ND",
2850+ "paramType": "optional",
2851+ "shape": [
2852+ -2
2853+ ],
2854+ "format_match_mode": "FormatDefault"
2855+ }
2856+ ],
2857+ "outputs": [
2858+ {
2859+ "name": "y",
2860+ "index": 0,
2861+ "dtype": "bfloat16",
2862+ "format": "ND",
2863+ "paramType": "required",
2864+ "shape": [
2865+ -2
2866+ ],
2867+ "format_match_mode": "FormatDefault"
2868+ }
2869+ ],
2870+ "attrs": [
2871+ {
2872+ "name": "dtype",
2873+ "dtype": "int",
2874+ "value": 27
2875+ },
2876+ {
2877+ "name": "transpose_x1",
2878+ "dtype": "bool",
2879+ "value": null
2880+ },
2881+ {
2882+ "name": "transpose_x2",
2883+ "dtype": "bool",
2884+ "value": null
2885+ },
2886+ {
2887+ "name": "group_size",
2888+ "dtype": "int",
2889+ "value": null
2890+ }
2891+ ]
2892+ },
2893+ {
2894+ "bin_filename": "QuantBatchMatmulV3_ND_NZ_int4_int4_fp32_bf16",
2895+ "inputs": [
2896+ {
2897+ "name": "x1",
2898+ "index": 0,
2899+ "dtype": "int4",
2900+ "format": "ND",
2901+ "paramType": "required",
2902+ "shape": [
2903+ -2
2904+ ],
2905+ "format_match_mode": "FormatDefault"
2906+ },
2907+ {
2908+ "name": "x2",
2909+ "index": 1,
2910+ "dtype": "int4",
2911+ "format": "FRACTAL_NZ",
2912+ "paramType": "required",
2913+ "shape": [
2914+ -2
2915+ ],
2916+ "format_match_mode": "FormatDefault"
2917+ },
2918+ {
2919+ "name": "scale",
2920+ "index": 2,
2921+ "dtype": "float32",
2922+ "format": "ND",
2923+ "paramType": "required",
2924+ "shape": [
2925+ -2
2926+ ],
2927+ "format_match_mode": "FormatDefault",
2928+ "dtype_match_mode": "DtypeByte"
2929+ },
2930+ {
2931+ "name": "offset",
2932+ "index": 3,
2933+ "dtype": "float32",
2934+ "format": "ND",
2935+ "paramType": "optional",
2936+ "shape": [
2937+ -2
2938+ ],
2939+ "format_match_mode": "FormatDefault"
2940+ },
2941+ {
2942+ "name": "bias",
2943+ "index": 4,
2944+ "dtype": "bfloat16",
2945+ "format": "ND",
2946+ "paramType": "optional",
2947+ "shape": [
2948+ -2
2949+ ],
2950+ "format_match_mode": "FormatDefault"
2951+ },
2952+ {
2953+ "name": "pertoken_scale",
2954+ "index": 5,
2955+ "dtype": "float32",
2956+ "format": "ND",
2957+ "paramType": "optional",
2958+ "shape": [
2959+ -2
2960+ ],
2961+ "format_match_mode": "FormatDefault"
2962+ }
2963+ ],
2964+ "outputs": [
2965+ {
2966+ "name": "y",
2967+ "index": 0,
2968+ "dtype": "bfloat16",
2969+ "format": "ND",
2970+ "paramType": "required",
2971+ "shape": [
2972+ -2
2973+ ],
2974+ "format_match_mode": "FormatDefault"
2975+ }
2976+ ],
2977+ "attrs": [
2978+ {
2979+ "name": "dtype",
2980+ "dtype": "int",
2981+ "value": 27
2982+ },
2983+ {
2984+ "name": "transpose_x1",
2985+ "dtype": "bool",
2986+ "value": null
2987+ },
2988+ {
2989+ "name": "transpose_x2",
2990+ "dtype": "bool",
2991+ "value": null
2992+ },
2993+ {
2994+ "name": "group_size",
2995+ "dtype": "int",
2996+ "value": null
2997+ }
2998+ ]
2571 }2999 }
2572 ]3000 ]
2573}3001}
Mmatmul/quant_batch_matmul_v3/op_host/op_tiling/quant_batch_matmul_v3_tiling_base.cpp+38-28
@@ -347,10 +347,12 @@ the dimension of x2 must be 2."), return false);
347bool QuantBatchMatmulV3TilingBase::CheckStorageShape4WeightNz(const gert::Shape &x2StorageShape,347bool QuantBatchMatmulV3TilingBase::CheckStorageShape4WeightNz(const gert::Shape &x2StorageShape,
348 size_t &x2StorageDim) const348 size_t &x2StorageDim) const
349{349{
350+ std::set<uint64_t> weightStorageLastDims = {WEIGHTNZ_STORAGE_LAST_DIM_32, WEIGHTNZ_STORAGE_LAST_DIM_64};
350 OP_TILING_CHECK(351 OP_TILING_CHECK(
351- x2StorageShape[x2StorageDim - LAST_FIRST_DIM_INDEX] != WEIGHTNZ_STORAGE_LAST_DIM,352+ weightStorageLastDims.find(x2StorageShape[x2StorageDim - LAST_FIRST_DIM_INDEX]) == weightStorageLastDims.end(),
352 CUBE_INNER_ERR_REPORT(inputParams_.opName,353 CUBE_INNER_ERR_REPORT(inputParams_.opName,
353- "When format of x2 is FRACTAL_NZ, the last first dim of x2 should be 32, actual is %lld.",354+ "When format of x2 is FRACTAL_NZ, the last first dim of x2 should be 32 for int8 \
355+ and 64 for int4, actual is %lld.",
354 x2StorageShape[x2StorageDim - LAST_FIRST_DIM_INDEX]),356 x2StorageShape[x2StorageDim - LAST_FIRST_DIM_INDEX]),
355 return false);357 return false);
356 OP_TILING_CHECK(x2StorageShape[x2StorageDim - LAST_SECOND_DIM_INDEX] != WEIGHTNZ_STORAGE_PENULTIMATE_DIM,358 OP_TILING_CHECK(x2StorageShape[x2StorageDim - LAST_SECOND_DIM_INDEX] != WEIGHTNZ_STORAGE_PENULTIMATE_DIM,
@@ -362,39 +364,47 @@ bool QuantBatchMatmulV3TilingBase::CheckStorageShape4WeightNz(const gert::Shape
362 auto x2StorageLastThirdDim = static_cast<uint64_t>(x2StorageShape[x2StorageDim - LAST_THIRD_DIM_INDEX]);364 auto x2StorageLastThirdDim = static_cast<uint64_t>(x2StorageShape[x2StorageDim - LAST_THIRD_DIM_INDEX]);
363 auto x2StorageLastFourthDim = static_cast<uint64_t>(x2StorageShape[x2StorageDim - LAST_FOURTH_DIM_INDEX]);365 auto x2StorageLastFourthDim = static_cast<uint64_t>(x2StorageShape[x2StorageDim - LAST_FOURTH_DIM_INDEX]);
364 if (!inputParams_.transB) {366 if (!inputParams_.transB) {
365- OP_TILING_CHECK(x2StorageLastThirdDim != ops::CeilDiv(inputParams_.kSize, WEIGHTNZ_STORAGE_PENULTIMATE_DIM),367+ OP_TILING_CHECK(!CheckStorageShape4WeightNzDimAlignment(inputParams_.kSize, inputParams_.nSize,
366- CUBE_INNER_ERR_REPORT(inputParams_.opName,368+ x2StorageLastThirdDim, x2StorageLastFourthDim),
367- "When format of x2 is FRACTAL_NZ, the last third dim of x2 should be \
368-ceil(k/16) = %lu, actual is %lu.",
369- ops::CeilDiv(inputParams_.kSize, WEIGHTNZ_STORAGE_PENULTIMATE_DIM),
370- x2StorageLastThirdDim),
371- return false);
372- OP_TILING_CHECK(
373- x2StorageLastFourthDim != ops::CeilDiv(inputParams_.nSize, WEIGHTNZ_STORAGE_LAST_DIM),
374 CUBE_INNER_ERR_REPORT(inputParams_.opName,369 CUBE_INNER_ERR_REPORT(inputParams_.opName,
375- "When format of x2 is FRACTAL_NZ, the last fourth dim of x2 should be ceil(n/32) \370+ "Dim check does not pass, please read the error report above"),
376-= %lu, actual is %lu.",371+ return false;
377- ops::CeilDiv(inputParams_.nSize, WEIGHTNZ_STORAGE_LAST_DIM), x2StorageLastFourthDim),372+ );
378- return false);
379 } else {373 } else {
380- OP_TILING_CHECK(x2StorageLastThirdDim != ops::CeilDiv(inputParams_.nSize, WEIGHTNZ_STORAGE_PENULTIMATE_DIM),374+ OP_TILING_CHECK(!CheckStorageShape4WeightNzDimAlignment(inputParams_.nSize, inputParams_.kSize,
381- CUBE_INNER_ERR_REPORT(inputParams_.opName,375+ x2StorageLastThirdDim, x2StorageLastFourthDim),
382- "When format of x2 is FRACTAL_NZ, the last third dim of x2 should be \
383-ceil(n/16) = %lu, actual is %lu.",
384- ops::CeilDiv(inputParams_.nSize, WEIGHTNZ_STORAGE_PENULTIMATE_DIM),
385- x2StorageLastThirdDim),
386- return false);
387- OP_TILING_CHECK(
388- x2StorageLastFourthDim != ops::CeilDiv(inputParams_.kSize, WEIGHTNZ_STORAGE_LAST_DIM),
389 CUBE_INNER_ERR_REPORT(inputParams_.opName,376 CUBE_INNER_ERR_REPORT(inputParams_.opName,
390- "When format of x2 is FRACTAL_NZ, the last fourth dim of x2 should be ceil(k/32) \377+ "Dim check does not pass, please read the error report above"),
391-= %lu, actual is %lu.",378+ return false;
392- ops::CeilDiv(inputParams_.kSize, WEIGHTNZ_STORAGE_LAST_DIM), x2StorageLastFourthDim),379+ );
393- return false);
394 }380 }
395 return true;381 return true;
396}382}
397 383 
384+bool QuantBatchMatmulV3TilingBase::CheckStorageShape4WeightNzDimAlignment(uint64_t innerDim, uint64_t outerDim,
385+ uint64_t x2StorageLastThirdDim, uint64_t x2StorageLastFourthDim) const
386+{
387+ bool isNotLastThirdDimAligned = (x2StorageLastThirdDim != ops::CeilDiv(innerDim, WEIGHTNZ_STORAGE_PENULTIMATE_DIM));
388+ bool isNotLastFourthDimAligned = (x2StorageLastFourthDim != ops::CeilDiv(outerDim, WEIGHTNZ_STORAGE_LAST_DIM_32)) &&
389+ (x2StorageLastFourthDim != ops::CeilDiv(outerDim, WEIGHTNZ_STORAGE_LAST_DIM_64));
390+ OP_TILING_CHECK(isNotLastThirdDimAligned,
391+ CUBE_INNER_ERR_REPORT(inputParams_.opName,
392+ "When format of x2 is FRACTAL_NZ, the last third dim of x2 should be ceil(k/16) = %lu, actual is %lu.",
393+ ops::CeilDiv(innerDim, WEIGHTNZ_STORAGE_PENULTIMATE_DIM),
394+ x2StorageLastThirdDim),
395+ return false);
396+ OP_TILING_CHECK(
397+ isNotLastFourthDimAligned,
398+ CUBE_INNER_ERR_REPORT(inputParams_.opName,
399+ "When format of x2 is FRACTAL_NZ, the last fourth dim of x2 should be ceil(n/32) \
400+ = %lu for int8, or ceil(n/64) = %lu for int4, actual is %lu.",
401+ ops::CeilDiv(outerDim, WEIGHTNZ_STORAGE_LAST_DIM_32),
402+ ops::CeilDiv(outerDim, WEIGHTNZ_STORAGE_LAST_DIM_64),
403+ x2StorageLastFourthDim),
404+ return false);
405+ return true;
406+}
407+ 
398bool QuantBatchMatmulV3TilingBase::CheckShape4WeightNz() const408bool QuantBatchMatmulV3TilingBase::CheckShape4WeightNz() const
399{409{
400 OP_TILING_CHECK(1 == inputParams_.kSize || 1 == inputParams_.nSize,410 OP_TILING_CHECK(1 == inputParams_.kSize || 1 == inputParams_.nSize,
Mmatmul/quant_batch_matmul_v3/op_host/op_tiling/quant_batch_matmul_v3_tiling_base.h+5-1
@@ -16,6 +16,7 @@
16#ifndef QUANT_BATCH_MATMUL_V3_TILING_BASE_H16#ifndef QUANT_BATCH_MATMUL_V3_TILING_BASE_H
17#define QUANT_BATCH_MATMUL_V3_TILING_BASE_H17#define QUANT_BATCH_MATMUL_V3_TILING_BASE_H
18#include <memory>18#include <memory>
19+#include <set>
19#include "register/tilingdata_base.h"20#include "register/tilingdata_base.h"
20#include "tiling/tiling_api.h"21#include "tiling/tiling_api.h"
21#include "quant_batch_matmul_v3_compile_info.h"22#include "quant_batch_matmul_v3_compile_info.h"
@@ -183,6 +184,8 @@ protected:
183 bool CheckShapeInRangeForMandtoryInputs(size_t x1ShapeLen, size_t x2ShapeLen) const;184 bool CheckShapeInRangeForMandtoryInputs(size_t x1ShapeLen, size_t x2ShapeLen) const;
184 bool CheckShape4WeightNz() const;185 bool CheckShape4WeightNz() const;
185 bool CheckStorageShape4WeightNz(const gert::Shape &x2StorageShape, size_t &x2StorageDim) const;186 bool CheckStorageShape4WeightNz(const gert::Shape &x2StorageShape, size_t &x2StorageDim) const;
187+ bool CheckStorageShape4WeightNzDimAlignment(uint64_t innerDim, uint64_t outerDim, uint64_t x2StorageLastThirdDim,
188+ uint64_t x2StorageLastFourthDim) const;
186 void SetTransAttr(QuantBatchMatmulV3Trans &trans) const;189 void SetTransAttr(QuantBatchMatmulV3Trans &trans) const;
187 virtual bool SetPlatformInfoForTiling();190 virtual bool SetPlatformInfoForTiling();
188 191 
@@ -238,7 +241,8 @@ protected:
238 static constexpr size_t LAST_FOURTH_DIM_INDEX = 4;241 static constexpr size_t LAST_FOURTH_DIM_INDEX = 4;
239 static constexpr size_t WEIGHTNZ_STORAGE_MIN_DIM = 4;242 static constexpr size_t WEIGHTNZ_STORAGE_MIN_DIM = 4;
240 static constexpr size_t WEIGHTNZ_STORAGE_MAX_DIM = 8;243 static constexpr size_t WEIGHTNZ_STORAGE_MAX_DIM = 8;
241- static constexpr uint64_t WEIGHTNZ_STORAGE_LAST_DIM = 32UL;244+ static constexpr uint64_t WEIGHTNZ_STORAGE_LAST_DIM_32 = 32UL;
245+ static constexpr uint64_t WEIGHTNZ_STORAGE_LAST_DIM_64 = 64UL;
242 static constexpr uint64_t WEIGHTNZ_STORAGE_PENULTIMATE_DIM = 16UL;246 static constexpr uint64_t WEIGHTNZ_STORAGE_PENULTIMATE_DIM = 16UL;
243 247 
244 // 新增数据成员请注意：如果是在GetShapeAttrsInfo函数过程中获取的，请放到QuantBatchMatmulInfo结构体中，或者保证在DoOpTiling赋值248 // 新增数据成员请注意：如果是在GetShapeAttrsInfo函数过程中获取的，请放到QuantBatchMatmulInfo结构体中，或者保证在DoOpTiling赋值
Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_bf16_basic.h+2-2
@@ -42,7 +42,7 @@ public:
42 offsetWorkspaceC_ = BUFFER_NUM * blockIdx * baseM_ * baseN_;42 offsetWorkspaceC_ = BUFFER_NUM * blockIdx * baseM_ * baseN_;
43 43 
44 block_.Init(tilingData);44 block_.Init(tilingData);
45- update_.template Init<x1Format, x2Format, aTrans, bTrans>(&tilingData->matmulTiling, block_.params_);45+ update_.template Init<x1Format, x2Format, aTrans, bTrans, x1Type, x2Type>(&tilingData->matmulTiling, block_.params_);
46 loop_ = 0; // all_gather_quant_batch_mat_mul.h循环调Init和Process，管理CV同步的计数器每次都要清零46 loop_ = 0; // all_gather_quant_batch_mat_mul.h循环调Init和Process，管理CV同步的计数器每次都要清零
47 }47 }
48 48 
@@ -151,7 +151,7 @@ private:
151 {151 {
152 for (uint64_t j = 0; j < block_.realRound_; j++) {152 for (uint64_t j = 0; j < block_.realRound_; j++) {
153 // 更新此次基本块的大小和输入输出地址153 // 更新此次基本块的大小和输入输出地址
154- update_.template UpdateBlockParamsAndCalcGmOffset<x1Format, x2Format, aTrans, bTrans>(154+ update_.template UpdateBlockParamsAndCalcGmOffset<x1Format, x2Format, aTrans, bTrans, x1Type, x2Type>(
155 block_.params_, offset_, mTileIndex, nTileIndex);155 block_.params_, offset_, mTileIndex, nTileIndex);
156 156 
157 offsetWorkspaceC_ = pingOffsetC + pongSwitch * baseM_ * baseN_;157 offsetWorkspaceC_ = pingOffsetC + pongSwitch * baseM_ * baseN_;
Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_cube_basic.h+2-2
@@ -61,7 +61,7 @@ __aicore__ inline void QuantBatchMatmulV3BaseKernel<TemplateBasicValue>::Init(
61 const QuantBatchMatmulV3TilingData *__restrict tilingData, TPipe *tPipe)61 const QuantBatchMatmulV3TilingData *__restrict tilingData, TPipe *tPipe)
62{62{
63 block_.Init(tilingData);63 block_.Init(tilingData);
64- update_.template Init<x1Format, x2Format, aTrans, bTrans>(&tilingData->matmulTiling, block_.params_);64+ update_.template Init<x1Format, x2Format, aTrans, bTrans, x1Type, x2Type>(&tilingData->matmulTiling, block_.params_);
65 pipe_ = tPipe;65 pipe_ = tPipe;
66 isPerTensor_ = tilingData->params.isPerTensor;66 isPerTensor_ = tilingData->params.isPerTensor;
67 InitInputs(x1, x2, scale, bias, y);67 InitInputs(x1, x2, scale, bias, y);
@@ -120,7 +120,7 @@ __aicore__ inline void QuantBatchMatmulV3BaseKernel<TemplateBasicValue>::OneTile
120{120{
121 for (uint64_t j = 0; j < block_.realRound_; j++) {121 for (uint64_t j = 0; j < block_.realRound_; j++) {
122 // 更新此次基本块的大小和输入输出地址122 // 更新此次基本块的大小和输入输出地址
123- update_.template UpdateBlockParamsAndCalcGmOffset<x1Format, x2Format, aTrans, bTrans>(block_.params_, offset_,123+ update_.template UpdateBlockParamsAndCalcGmOffset<x1Format, x2Format, aTrans, bTrans, x1Type, x2Type>(block_.params_, offset_,
124 mTileIndex, nTileIndex);124 mTileIndex, nTileIndex);
125 125 
126 MMCompute();126 MMCompute();
Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_pertoken_basic.h+2-2
@@ -58,7 +58,7 @@ public:
58 offsetWorkspaceC_ = BUFFER_NUM * blockIdx_ * baseM_ * baseN_;58 offsetWorkspaceC_ = BUFFER_NUM * blockIdx_ * baseM_ * baseN_;
59 59 
60 block_.Init(tilingData);60 block_.Init(tilingData);
61- update_.template Init<x1Format, x2Format, aTrans, bTrans>(&tilingData->matmulTiling, block_.params_);61+ update_.template Init<x1Format, x2Format, aTrans, bTrans, x1Type, x2Type>(&tilingData->matmulTiling, block_.params_);
62 loop_ = 0; // all_gather_quant_batch_mat_mul.h循环调Init和Process，管理CV同步的计数器每次都要清零62 loop_ = 0; // all_gather_quant_batch_mat_mul.h循环调Init和Process，管理CV同步的计数器每次都要清零
63 }63 }
64 64 
@@ -190,7 +190,7 @@ private:
190 {190 {
191 for (uint64_t j = 0; j < block_.realRound_; j++) {191 for (uint64_t j = 0; j < block_.realRound_; j++) {
192 // 更新此次基本块的大小和输入输出地址192 // 更新此次基本块的大小和输入输出地址
193- update_.template UpdateBlockParamsAndCalcGmOffset<x1Format, x2Format, aTrans, bTrans>(193+ update_.template UpdateBlockParamsAndCalcGmOffset<x1Format, x2Format, aTrans, bTrans, x1Type, x2Type>(
194 block_.params_, offset_, mTileIndex, nTileIndex);194 block_.params_, offset_, mTileIndex, nTileIndex);
195 195 
196 UpdateBatchOffset(batchIndex, offset_);196 UpdateBatchOffset(batchIndex, offset_);
Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_update.h+11-11
@@ -31,13 +31,13 @@ struct QBmmUpdateInfo {
31class QuantBatchMatmulV3Update {31class QuantBatchMatmulV3Update {
32public:32public:
33 __aicore__ inline QuantBatchMatmulV3Update() {}33 __aicore__ inline QuantBatchMatmulV3Update() {}
34- template <int x1Format, int x2Format, bool aTrans, bool bTrans>34+ template <int x1Format, int x2Format, bool aTrans, bool bTrans, typename x1Type=int8_t, typename x2Type=int8_t>
35 __aicore__ inline void Init(const TCubeTiling *mmTiling, const QBmmBaseBlockArgs &params);35 __aicore__ inline void Init(const TCubeTiling *mmTiling, const QBmmBaseBlockArgs &params);
36- template <int x1Format, int x2Format, bool aTrans, bool bTrans>36+ template <int x1Format, int x2Format, bool aTrans, bool bTrans, typename x1Type=int8_t, typename x2Type=int8_t>
37 __aicore__ inline void UpdateBlockParamsAndCalcGmOffset(QBmmBaseBlockArgs &params, QBmmBlockOffset &offset,37 __aicore__ inline void UpdateBlockParamsAndCalcGmOffset(QBmmBaseBlockArgs &params, QBmmBlockOffset &offset,
38 uint64_t mTileIndex, uint64_t nTileIndex);38 uint64_t mTileIndex, uint64_t nTileIndex);
39 __aicore__ inline void UpdateBlockParams(QBmmBaseBlockArgs &params, uint64_t mTileIndex, uint64_t nTileIndex);39 __aicore__ inline void UpdateBlockParams(QBmmBaseBlockArgs &params, uint64_t mTileIndex, uint64_t nTileIndex);
40- template <int x1Format, int x2Format, bool aTrans, bool bTrans>40+ template <int x1Format, int x2Format, bool aTrans, bool bTrans, typename x1Type=int8_t, typename x2Type=int8_t>
41 __aicore__ inline void CalcGMOffset(QBmmBaseBlockArgs &params, QBmmBlockOffset &offset);41 __aicore__ inline void CalcGMOffset(QBmmBaseBlockArgs &params, QBmmBlockOffset &offset);
42 42 
43private:43private:
@@ -45,7 +45,7 @@ private:
45 const TCubeTiling *mmTiling_;45 const TCubeTiling *mmTiling_;
46};46};
47 47 
48-template <int x1Format, int x2Format, bool aTrans, bool bTrans>48+template <int x1Format, int x2Format, bool aTrans, bool bTrans, typename x1Type, typename x2Type>
49__aicore__ inline void QuantBatchMatmulV3Update::Init(const TCubeTiling *mmTiling, const QBmmBaseBlockArgs &params)49__aicore__ inline void QuantBatchMatmulV3Update::Init(const TCubeTiling *mmTiling, const QBmmBaseBlockArgs &params)
50{50{
51 mmTiling_ = mmTiling;51 mmTiling_ = mmTiling;
@@ -56,23 +56,23 @@ __aicore__ inline void QuantBatchMatmulV3Update::Init(const TCubeTiling *mmTilin
56 if constexpr (aTrans) { // (k, m)56 if constexpr (aTrans) { // (k, m)
57 info_.alignedKaSize = DequantBmm::Align(mmTiling_->Ka, BMM_BLOCK_NUM);57 info_.alignedKaSize = DequantBmm::Align(mmTiling_->Ka, BMM_BLOCK_NUM);
58 } else { // (m, k)58 } else { // (m, k)
59- info_.alignedKaSize = DequantBmm::Align(mmTiling_->Ka, K0_INT8);59+ info_.alignedKaSize = DequantBmm::Align(mmTiling_->Ka, DequantBmm::GetC0Size<x1Type>());
60 }60 }
61 if constexpr (bTrans) { // (n, k)61 if constexpr (bTrans) { // (n, k)
62- info_.alignedKbSize = DequantBmm::Align(mmTiling_->Kb, K0_INT8);62+ info_.alignedKbSize = DequantBmm::Align(mmTiling_->Kb, DequantBmm::GetC0Size<x2Type>());
63 } else { // (k, n)63 } else { // (k, n)
64 info_.alignedKbSize = DequantBmm::Align(mmTiling_->Kb, BMM_BLOCK_NUM);64 info_.alignedKbSize = DequantBmm::Align(mmTiling_->Kb, BMM_BLOCK_NUM);
65 }65 }
66}66}
67 67 
68-template <int x1Format, int x2Format, bool aTrans, bool bTrans>68+template <int x1Format, int x2Format, bool aTrans, bool bTrans, typename x1Type, typename x2Type>
69__aicore__ inline void QuantBatchMatmulV3Update::UpdateBlockParamsAndCalcGmOffset(QBmmBaseBlockArgs &params,69__aicore__ inline void QuantBatchMatmulV3Update::UpdateBlockParamsAndCalcGmOffset(QBmmBaseBlockArgs &params,
70 QBmmBlockOffset &offset,70 QBmmBlockOffset &offset,
71 uint64_t mTileIndex,71 uint64_t mTileIndex,
72 uint64_t nTileIndex)72 uint64_t nTileIndex)
73{73{
74 UpdateBlockParams(params, mTileIndex, nTileIndex);74 UpdateBlockParams(params, mTileIndex, nTileIndex);
75- CalcGMOffset<x1Format, x2Format, aTrans, bTrans>(params, offset);75+ CalcGMOffset<x1Format, x2Format, aTrans, bTrans, x1Type, x2Type>(params, offset);
76}76}
77 77 
78__aicore__ inline void QuantBatchMatmulV3Update::UpdateBlockParams(QBmmBaseBlockArgs &params, uint64_t mTileIndex,78__aicore__ inline void QuantBatchMatmulV3Update::UpdateBlockParams(QBmmBaseBlockArgs &params, uint64_t mTileIndex,
@@ -94,7 +94,7 @@ __aicore__ inline void QuantBatchMatmulV3Update::UpdateBlockParams(QBmmBaseBlock
94 }94 }
95}95}
96 96 
97-template <int x1Format, int x2Format, bool aTrans, bool bTrans>97+template <int x1Format, int x2Format, bool aTrans, bool bTrans, typename x1Type, typename x2Type>
98__aicore__ inline void QuantBatchMatmulV3Update::CalcGMOffset(QBmmBaseBlockArgs &params, QBmmBlockOffset &offset)98__aicore__ inline void QuantBatchMatmulV3Update::CalcGMOffset(QBmmBaseBlockArgs &params, QBmmBlockOffset &offset)
99{99{
100 uint64_t mCntIndex = params.index / params.nCntUse;100 uint64_t mCntIndex = params.index / params.nCntUse;
@@ -112,7 +112,7 @@ __aicore__ inline void QuantBatchMatmulV3Update::CalcGMOffset(QBmmBaseBlockArgs
112 if constexpr (aTrans) { // (m1, k1, k0, m0)112 if constexpr (aTrans) { // (m1, k1, k0, m0)
113 offset.offsetA = mOffset * info_.alignedKaSize;113 offset.offsetA = mOffset * info_.alignedKaSize;
114 } else { // (k1, m1, m0, k0)114 } else { // (k1, m1, m0, k0)
115- offset.offsetA = mOffset * K0_INT8;115+ offset.offsetA = mOffset * DequantBmm::GetC0Size<x1Type>();
116 }116 }
117 }117 }
118 // 前面的n都能保证是singleCoreN(baseN)的倍数，n尾块只会出现在n轴最后118 // 前面的n都能保证是singleCoreN(baseN)的倍数，n尾块只会出现在n轴最后
@@ -125,7 +125,7 @@ __aicore__ inline void QuantBatchMatmulV3Update::CalcGMOffset(QBmmBaseBlockArgs
125 }125 }
126 } else if constexpr (DequantBmm::GetFormat(x2Format) == CubeFormat::NZ) {126 } else if constexpr (DequantBmm::GetFormat(x2Format) == CubeFormat::NZ) {
127 if constexpr (bTrans) { // (k1, n1, n0, k0)127 if constexpr (bTrans) { // (k1, n1, n0, k0)
128- offset.offsetB = nOffset * K0_INT8;128+ offset.offsetB = nOffset * DequantBmm::GetC0Size<x2Type>();
129 } else { // (n1, k1, k0, n0)129 } else { // (n1, k1, k0, n0)
130 offset.offsetB = nOffset * info_.alignedKbSize;130 offset.offsetB = nOffset * info_.alignedKbSize;
131 }131 }
Mmatmul/quant_batch_matmul_v3/tests/ut/op_api/test_aclnn_quant_matmul_v4.cpp+1-1
@@ -1422,7 +1422,7 @@ TEST_F(l2_QuantBatchMatmulV4_test, ascend910B2_test_case_70)
1422 OUTPUT(out_desc));1422 OUTPUT(out_desc));
1423 uint64_t workspace_size = 0;1423 uint64_t workspace_size = 0;
1424 aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);1424 aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
1425- EXPECT_EQ(aclRet, ACLNN_ERR_PARAM_INVALID);1425+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
1426}1426}
1427 1427 
1428TEST_F(l2_QuantBatchMatmulV4_test, ascend910B2_test_case_71)1428TEST_F(l2_QuantBatchMatmulV4_test, ascend910B2_test_case_71)
Mmatmul/quant_batch_matmul_v3/tests/ut/op_api/test_aclnn_quant_matmul_weight_nz.cpp+205-0
@@ -270,6 +270,20 @@ TEST_F(l2_QuantBatchMatmulWeightNz_test, ascend950_test_a8w4_mx_0)
270 EXPECT_EQ(aclRet, ACLNN_SUCCESS);270 EXPECT_EQ(aclRet, ACLNN_SUCCESS);
271}271}
272 272 
273+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_01_bf16)
274+{
275+ TensorDesc x1_desc = TensorDesc({6256, 5120}, ACL_INT4, ACL_FORMAT_ND);
276+ TensorDesc x2_desc = TensorDesc({15360, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 960, 16, 32});
277+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
278+ TensorDesc x2Scale_desc = TensorDesc({15360}, ACL_FLOAT, ACL_FORMAT_ND);
279+ TensorDesc out_desc = TensorDesc({6256, 15360}, ACL_BF16, ACL_FORMAT_ND);
280+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, nullptr, false, true, 0),
281+ OUTPUT(out_desc));
282+ uint64_t workspace_size = 0;
283+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
284+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
285+}
286+ 
273TEST_F(l2_QuantBatchMatmulWeightNz_test, ascend950_test_a8w4_mx_1)287TEST_F(l2_QuantBatchMatmulWeightNz_test, ascend950_test_a8w4_mx_1)
274{288{
275 op::NpuArchManager archManager(NpuArch::DAV_3510);289 op::NpuArchManager archManager(NpuArch::DAV_3510);
@@ -284,4 +298,195 @@ TEST_F(l2_QuantBatchMatmulWeightNz_test, ascend950_test_a8w4_mx_1)
284 uint64_t workspace_size = 0;298 uint64_t workspace_size = 0;
285 aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);299 aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
286 EXPECT_EQ(aclRet, ACLNN_SUCCESS);300 EXPECT_EQ(aclRet, ACLNN_SUCCESS);
301+}
302+ 
303+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_01_fp16)
304+{
305+ TensorDesc x1_desc = TensorDesc({6256, 5120}, ACL_INT4, ACL_FORMAT_ND);
306+ TensorDesc x2_desc = TensorDesc({15360, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 960, 16, 32});
307+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
308+ TensorDesc x2Scale_desc = TensorDesc({15360}, ACL_FLOAT, ACL_FORMAT_ND);
309+ TensorDesc out_desc = TensorDesc({6256, 15360}, ACL_FLOAT16, ACL_FORMAT_ND);
310+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, nullptr, false, true, 0),
311+ OUTPUT(out_desc));
312+ uint64_t workspace_size = 0;
313+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
314+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
315+}
316+ 
317+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_02)
318+{
319+ TensorDesc x1_desc = TensorDesc({124, 5120}, ACL_INT4, ACL_FORMAT_ND);
320+ TensorDesc x2_desc = TensorDesc({15360, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 960, 16, 32});
321+ TensorDesc x1Scale_desc = TensorDesc({124}, ACL_FLOAT, ACL_FORMAT_ND);
322+ TensorDesc x2Scale_desc = TensorDesc({15360}, ACL_FLOAT, ACL_FORMAT_ND);
323+ TensorDesc out_desc = TensorDesc({124, 15360}, ACL_BF16, ACL_FORMAT_ND);
324+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, nullptr, false, true, 0),
325+ OUTPUT(out_desc));
326+ uint64_t workspace_size = 0;
327+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
328+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
329+}
330+ 
331+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_03_bf16)
332+{
333+ TensorDesc x1_desc = TensorDesc({6256, 5120}, ACL_INT4, ACL_FORMAT_ND);
334+ TensorDesc x2_desc = TensorDesc({5120, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 320, 16, 32});
335+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
336+ TensorDesc x2Scale_desc = TensorDesc({5120}, ACL_FLOAT, ACL_FORMAT_ND);
337+ TensorDesc bias_desc = TensorDesc({5120}, ACL_BF16, ACL_FORMAT_ND);
338+ TensorDesc out_desc = TensorDesc({6256, 5120}, ACL_BF16, ACL_FORMAT_ND);
339+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, bias_desc, false, true, 0),
340+ OUTPUT(out_desc));
341+ uint64_t workspace_size = 0;
342+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
343+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
344+}
345+ 
346+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_03_fp16)
347+{
348+ TensorDesc x1_desc = TensorDesc({6256, 5120}, ACL_INT4, ACL_FORMAT_ND);
349+ TensorDesc x2_desc = TensorDesc({5120, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 320, 16, 32});
350+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
351+ TensorDesc x2Scale_desc = TensorDesc({5120}, ACL_FLOAT, ACL_FORMAT_ND);
352+ TensorDesc out_desc = TensorDesc({6256, 5120}, ACL_FLOAT16, ACL_FORMAT_ND);
353+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, nullptr, false, true, 0),
354+ OUTPUT(out_desc));
355+ uint64_t workspace_size = 0;
356+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
357+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
358+}
359+ 
360+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_04)
361+{
362+ TensorDesc x1_desc = TensorDesc({124, 5120}, ACL_INT4, ACL_FORMAT_ND);
363+ TensorDesc x2_desc = TensorDesc({5120, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 320, 16, 32});
364+ TensorDesc x1Scale_desc = TensorDesc({124}, ACL_FLOAT, ACL_FORMAT_ND);
365+ TensorDesc x2Scale_desc = TensorDesc({5120}, ACL_FLOAT, ACL_FORMAT_ND);
366+ TensorDesc bias_desc = TensorDesc({5120}, ACL_BF16, ACL_FORMAT_ND);
367+ TensorDesc out_desc = TensorDesc({124, 5120}, ACL_BF16, ACL_FORMAT_ND);
368+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, bias_desc, false, true, 0),
369+ OUTPUT(out_desc));
370+ uint64_t workspace_size = 0;
371+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
372+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
373+}
374+ 
375+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_05)
376+{
377+ TensorDesc x1_desc = TensorDesc({6256, 5120}, ACL_INT4, ACL_FORMAT_ND);
378+ TensorDesc x2_desc = TensorDesc({20480, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 1280, 16, 32});
379+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
380+ TensorDesc x2Scale_desc = TensorDesc({20480}, ACL_FLOAT, ACL_FORMAT_ND);
381+ TensorDesc bias_desc = TensorDesc({20480}, ACL_BF16, ACL_FORMAT_ND);
382+ TensorDesc out_desc = TensorDesc({6256, 20480}, ACL_BF16, ACL_FORMAT_ND);
383+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, bias_desc, false, true, 0),
384+ OUTPUT(out_desc));
385+ uint64_t workspace_size = 0;
386+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
387+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
388+}
389+ 
390+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_06)
391+{
392+ TensorDesc x1_desc = TensorDesc({6256, 20480}, ACL_INT4, ACL_FORMAT_ND);
393+ TensorDesc x2_desc = TensorDesc({5120, 20480}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {640, 320, 16, 32});
394+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
395+ TensorDesc x2Scale_desc = TensorDesc({5120}, ACL_FLOAT, ACL_FORMAT_ND);
396+ TensorDesc bias_desc = TensorDesc({5120}, ACL_BF16, ACL_FORMAT_ND);
397+ TensorDesc out_desc = TensorDesc({6256, 5120}, ACL_BF16, ACL_FORMAT_ND);
398+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, bias_desc, false, true, 0),
399+ OUTPUT(out_desc));
400+ uint64_t workspace_size = 0;
401+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
402+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
403+}
404+ 
405+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_07)
406+{
407+ TensorDesc x1_desc = TensorDesc({124, 5120}, ACL_INT4, ACL_FORMAT_ND);
408+ TensorDesc x2_desc = TensorDesc({20480, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 1280, 16, 32});
409+ TensorDesc x1Scale_desc = TensorDesc({124}, ACL_FLOAT, ACL_FORMAT_ND);
410+ TensorDesc x2Scale_desc = TensorDesc({20480}, ACL_FLOAT, ACL_FORMAT_ND);
411+ TensorDesc bias_desc = TensorDesc({20480}, ACL_BF16, ACL_FORMAT_ND);
412+ TensorDesc out_desc = TensorDesc({124, 20480}, ACL_BF16, ACL_FORMAT_ND);
413+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, bias_desc, false, true, 0),
414+ OUTPUT(out_desc));
415+ uint64_t workspace_size = 0;
416+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
417+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
418+}
419+ 
420+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_08)
421+{
422+ TensorDesc x1_desc = TensorDesc({124, 20480}, ACL_INT4, ACL_FORMAT_ND);
423+ TensorDesc x2_desc = TensorDesc({5120, 20480}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {640, 320, 16, 32});
424+ TensorDesc x1Scale_desc = TensorDesc({124}, ACL_FLOAT, ACL_FORMAT_ND);
425+ TensorDesc x2Scale_desc = TensorDesc({5120}, ACL_FLOAT, ACL_FORMAT_ND);
426+ TensorDesc bias_desc = TensorDesc({5120}, ACL_BF16, ACL_FORMAT_ND);
427+ TensorDesc out_desc = TensorDesc({124, 5120}, ACL_BF16, ACL_FORMAT_ND);
428+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, bias_desc, false, true, 0),
429+ OUTPUT(out_desc));
430+ uint64_t workspace_size = 0;
431+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
432+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
433+}
434+ 
435+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_09)
436+{
437+ TensorDesc x1_desc = TensorDesc({6256, 5120}, ACL_INT4, ACL_FORMAT_ND);
438+ TensorDesc x2_desc = TensorDesc({15360, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 960, 16, 32});
439+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
440+ TensorDesc x2Scale_desc = TensorDesc({15360}, ACL_FLOAT, ACL_FORMAT_ND);
441+ TensorDesc out_desc = TensorDesc({6256, 15360}, ACL_FLOAT16, ACL_FORMAT_ND);
442+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, nullptr, false, true, 0),
443+ OUTPUT(out_desc));
444+ uint64_t workspace_size = 0;
445+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
446+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
447+}
448+ 
449+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_10)
450+{
451+ TensorDesc x1_desc = TensorDesc({6256, 5120}, ACL_INT4, ACL_FORMAT_ND);
452+ TensorDesc x2_desc = TensorDesc({5120, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 320, 16, 32});
453+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
454+ TensorDesc x2Scale_desc = TensorDesc({5120}, ACL_FLOAT, ACL_FORMAT_ND);
455+ TensorDesc bias_desc = TensorDesc({5120}, ACL_BF16, ACL_FORMAT_ND);
456+ TensorDesc out_desc = TensorDesc({6256, 5120}, ACL_BF16, ACL_FORMAT_ND);
457+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, bias_desc, false, true, 0),
458+ OUTPUT(out_desc));
459+ uint64_t workspace_size = 0;
460+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
461+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
462+}
463+ 
464+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_11)
465+{
466+ TensorDesc x1_desc = TensorDesc({6256, 5120}, ACL_INT4, ACL_FORMAT_ND);
467+ TensorDesc x2_desc = TensorDesc({20480, 5120}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {160, 1280, 16, 32});
468+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
469+ TensorDesc x2Scale_desc = TensorDesc({20480}, ACL_FLOAT, ACL_FORMAT_ND);
470+ TensorDesc bias_desc = TensorDesc({20480}, ACL_BF16, ACL_FORMAT_ND);
471+ TensorDesc out_desc = TensorDesc({6256, 20480}, ACL_BF16, ACL_FORMAT_ND);
472+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, bias_desc, false, true, 0),
473+ OUTPUT(out_desc));
474+ uint64_t workspace_size = 0;
475+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
476+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
477+}
478+ 
479+TEST_F(l2_QuantBatchMatmulWeightNz_test, a4w4_weight_nz_case_12)
480+{
481+ TensorDesc x1_desc = TensorDesc({6256, 20480}, ACL_INT4, ACL_FORMAT_ND);
482+ TensorDesc x2_desc = TensorDesc({5120, 20480}, ACL_INT4, ACL_FORMAT_FRACTAL_NZ, {}, 0, {640, 320, 16, 32});
483+ TensorDesc x1Scale_desc = TensorDesc({6256}, ACL_FLOAT, ACL_FORMAT_ND);
484+ TensorDesc x2Scale_desc = TensorDesc({5120}, ACL_FLOAT, ACL_FORMAT_ND);
485+ TensorDesc bias_desc = TensorDesc({5120}, ACL_BF16, ACL_FORMAT_ND);
486+ TensorDesc out_desc = TensorDesc({6256, 5120}, ACL_BF16, ACL_FORMAT_ND);
487+ auto ut = OP_API_UT(aclnnQuantMatmulWeightNz, INPUT(x1_desc, x2_desc, x1Scale_desc, x2Scale_desc, nullptr, nullptr, nullptr, nullptr, bias_desc, false, true, 0),
488+ OUTPUT(out_desc));
489+ uint64_t workspace_size = 0;
490+ aclnnStatus aclRet = ut.TestGetWorkspaceSize(&workspace_size);
491+ EXPECT_EQ(aclRet, ACLNN_SUCCESS);
287}492}
Mmatmul/quant_batch_matmul_v3/tests/ut/op_host/test_quant_batch_matmul_v3.csv+12-0
@@ -59,6 +59,18 @@ Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/56,,fanhua,,2,2
59Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/57,,fanhua,,2,2,2,0,0,0,1024,8192,3072,0,1,0,1,0,1,INT8,INT8,BF16,FLOAT,INT32,FLOAT16,ND,NZ,FALSE,24,0,,FALSE,59Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/57,,fanhua,,2,2,2,0,0,0,1024,8192,3072,0,1,0,1,0,1,INT8,INT8,BF16,FLOAT,INT32,FLOAT16,ND,NZ,FALSE,24,0,,FALSE,
60Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/58,,fanhua,,2,2,2,0,0,0,1024,8192,3072,0,1,0,1,1,1,INT8,INT8,BF16,FLOAT,INT32,FLOAT16,ND,ND,FALSE,24,0,,FALSE,60Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/58,,fanhua,,2,2,2,0,0,0,1024,8192,3072,0,1,0,1,1,1,INT8,INT8,BF16,FLOAT,INT32,FLOAT16,ND,ND,FALSE,24,0,,FALSE,
61Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/59,,fanhua,,2,2,2,0,0,0,24,8192,1280,0,1,1,0,0,1,INT8,INT8,FLOAT,FLOAT,FLOAT,FLOAT16,ND,ND,TRUE,20,24,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 24 64 0 32 64 0 0 0 0 0 0 0 20 24 1280 8192 8192 32 64 8192 32 64 512 16 4 1 1 1 32768 1 0 393216 8192 0 1 1 1 1 16 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,61Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/59,,fanhua,,2,2,2,0,0,0,24,8192,1280,0,1,1,0,0,1,INT8,INT8,FLOAT,FLOAT,FLOAT,FLOAT16,ND,ND,TRUE,20,24,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 24 64 0 32 64 0 0 0 0 0 0 0 20 24 1280 8192 8192 32 64 8192 32 64 512 16 4 1 1 1 32768 1 0 393216 8192 0 1 1 1 1 16 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,
62+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/0,,int4weightnz,,2,2,2,1,1,1,6256,5120,15360,0,1,0,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 20 256 40960 0 0 3 0 0 1 0 0 0 24 6256 15360 5120 5120 128 256 5120 128 256 256 12 4 1 1 0 0 0 0 360448 131072 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 6 33 11 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case0
63+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/1,,int4weightnz,,2,2,2,1,1,1,124,5120,15360,0,1,0,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 20 256 40960 0 0 3 0 0 1 0 0 0 24 124 15360 5120 5120 128 256 5120 128 256 256 12 4 1 1 0 0 0 0 458752 131072 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 60 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case1
64+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/2,,int4weightnz,,2,2,2,1,1,1,6256,5120,5120,0,1,1,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 18 256 36864 0 0 27 0 0 0 0 0 0 24 6256 5120 5120 5120 128 256 5120 128 256 256 12 4 1 1 1 0 0 0 319936 114688 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 49 20 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case2
65+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/3,,int4weightnz,,2,2,2,1,1,1,124,5120,5120,0,1,1,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,20,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 18 256 36864 0 0 27 0 0 1 0 0 0 20 124 5120 5120 5120 128 256 5120 128 256 256 12 4 1 1 1 0 0 0 442816 114688 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 20 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case3
66+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/4,,int4weightnz,,2,2,2,1,1,1,6256,5120,20480,0,1,1,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 18 256 36864 0 0 27 0 0 0 0 0 0 24 6256 20480 5120 5120 128 256 5120 128 256 256 12 4 1 1 1 0 0 0 360960 131072 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 5 33 16 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case4
67+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/5,,int4weightnz,,2,2,2,1,1,1,6256,20480,5120,0,1,1,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 18 256 36864 0 0 27 0 0 0 0 0 0 24 6256 5120 20480 20480 128 256 20480 128 256 256 12 4 1 1 1 0 0 0 491968 114688 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 3 33 8 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case5
68+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/6,,int4weightnz,,2,2,2,1,1,1,124,5120,20480,0,1,1,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 18 256 36864 0 0 27 0 0 0 0 0 0 24 124 20480 5120 5120 128 256 5120 128 256 256 12 4 1 1 1 0 0 0 459264 131072 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 80 2 1 0 0 0 0 0 0,FALSE,int4 weightnz case6
69+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/7,,int4weightnz,,2,2,2,1,1,1,124,20480,5120,0,1,1,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,20,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 18 256 36864 0 0 27 0 0 1 0 0 0 20 124 5120 20480 20480 128 256 20480 128 256 256 12 4 1 1 1 0 0 0 491968 114688 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 20 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case7
70+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/8,,int4weightnz,,2,2,2,1,1,1,6256,5120,15360,0,1,0,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 20 256 40960 0 0 3 0 0 1 0 0 0 24 6256 15360 5120 5120 128 256 5120 128 256 256 12 4 1 1 0 0 0 0 360448 131072 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 6 33 11 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case8
71+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/9,,int4weightnz,,2,2,2,1,1,1,6256,5120,5120,0,1,1,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 18 256 36864 0 0 27 0 0 0 0 0 0 24 6256 5120 5120 5120 128 256 5120 128 256 256 12 4 1 1 1 0 0 0 319936 114688 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 49 20 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case9
72+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/10,,int4weightnz,,2,2,2,1,1,1,6256,5120,20480,0,1,1,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 18 256 36864 0 0 27 0 0 0 0 0 0 24 6256 20480 5120 5120 128 256 5120 128 256 256 12 4 1 1 1 0 0 0 360960 131072 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 5 33 16 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case10
73+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/11,,int4weightnz,,2,2,2,1,1,1,6256,20480,5120,0,1,1,0,1,1,INT4,INT4,FLOAT,FLOAT,BF16,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 18 256 36864 0 0 27 0 0 0 0 0 0 24 6256 5120 20480 20480 128 256 20480 128 256 256 12 4 1 1 1 0 0 0 491968 114688 0 1 1 1 1 6 2 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 3 33 8 1 1 0 0 0 0 0 0,FALSE,int4 weightnz case11
62Ascend910B4,QUANTMM910B4/TestQuantBatchMatmulV3Tiling.generalTest/4,quant_batch_matmul_v3_int32,splitk,,2,2,2,0,0,0,256,29568,8192,0,0,0,0,0,0,INT8,INT8,FLOAT,FLOAT,INT32,INT32,ND,NZ,TRUE,20,32,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 20 256 8192 29568 29568 256 4096 1024 128 128 256 8 8 2 1 0 65536 1 0 524288 65536 0 1 1 1 1 4 4 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,TRUE,74Ascend910B4,QUANTMM910B4/TestQuantBatchMatmulV3Tiling.generalTest/4,quant_batch_matmul_v3_int32,splitk,,2,2,2,0,0,0,256,29568,8192,0,0,0,0,0,0,INT8,INT8,FLOAT,FLOAT,INT32,INT32,ND,NZ,TRUE,20,32,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 20 256 8192 29568 29568 256 4096 1024 128 128 256 8 8 2 1 0 65536 1 0 524288 65536 0 1 1 1 1 4 4 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,TRUE,
63Ascend910B4,QUANTMM910B4/TestQuantBatchMatmulV3Tiling.generalTest/5,quant_batch_matmul_v3_int32,splitk,,2,2,2,0,0,0,512,29568,8192,0,0,1,0,1,1,INT8,INT8,BF16,FLOAT,INT32,INT32,ND,NZ,TRUE,20,33,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 20 512 8192 29568 29568 256 8192 1024 128 128 256 8 8 2 1 0 65536 1 0 524288 65536 0 1 1 1 1 4 4 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,TRUE,75Ascend910B4,QUANTMM910B4/TestQuantBatchMatmulV3Tiling.generalTest/5,quant_batch_matmul_v3_int32,splitk,,2,2,2,0,0,0,512,29568,8192,0,0,1,0,1,1,INT8,INT8,BF16,FLOAT,INT32,INT32,ND,NZ,TRUE,20,33,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 20 512 8192 29568 29568 256 8192 1024 128 128 256 8 8 2 1 0 65536 1 0 524288 65536 0 1 1 1 1 4 4 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,TRUE,
64Ascend910B4,QUANTMM910B4/TestQuantBatchMatmulV3Tiling.generalTest/6,quant_batch_matmul_v3_int32,splitk,,2,2,2,0,0,0,768,29568,8192,0,0,0,1,0,1,INT8,INT8,FLOAT,FLOAT,INT32,INT32,ND,NZ,TRUE,20,34,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 20 768 8192 29568 29568 256 4096 1024 128 128 256 8 8 2 1 0 65536 1 0 524288 65536 0 1 1 1 1 4 4 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,TRUE,76Ascend910B4,QUANTMM910B4/TestQuantBatchMatmulV3Tiling.generalTest/6,quant_batch_matmul_v3_int32,splitk,,2,2,2,0,0,0,768,29568,8192,0,0,0,1,0,1,INT8,INT8,FLOAT,FLOAT,INT32,INT32,ND,NZ,TRUE,20,34,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 20 768 8192 29568 29568 256 4096 1024 128 128 256 8 8 2 1 0 65536 1 0 524288 65536 0 1 1 1 1 4 4 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,TRUE,