Merged
feat(quant_batch_matmul_v3): 全量支持 Ascend960 HIF4 Blaze 与 batch 场景 #12939
wangzitaoAdd on 25 days ago
feat(quant_batch_matmul_v3): 全量支持 Ascend960 HIF4 Blaze 与 batch 场景 #12939
Merged
wangzitaoAdd on 25 days ago
Total: 21 files changed+553-147
@@ -303,9 +303,16 @@ __aicore__ inline void QuantMmBatchMX<CMCT_QBMM_MX_KERNEL_FUN_TEM_PARAMS>::AddBa
303 Get<QuantBatchMatmul::IDX_BIAS_OFFSET>(blockOffset_) += batchCOffset_ * params.problemShape.n;303 Get<QuantBatchMatmul::IDX_BIAS_OFFSET>(blockOffset_) += batchCOffset_ * params.problemShape.n;
304 }304 }
305 // x1Scale/x2Scale batch offset. e8m0 is 1 byte and never packed, so no sizeShift here.305 // x1Scale/x2Scale batch offset. e8m0 is 1 byte and never packed, so no sizeShift here.
306- // Per-batch element count = M(or N) * ceil(K/64) * 2, matching the MX scale stride in GetQuantOffset.306+ // MX per-batch element count = M(or N) * ceil(K/64) * 2, matching the MX scale stride in GetQuantOffset.
307- // hif4 scale has no batch dim (3D only), so no batch offset is applied for hif4.307+ // HIF4 per-batch element count = M(or N) * ceil(K/64), since the scale last dim is 1.
308- if constexpr (!BlockMmad::IS_HIF4) {308+ if constexpr (BlockMmad::IS_HIF4) {
309+ Get<QuantBatchMatmul::IDX_X1SCALE_OFFSET>(
310+ blockOffset_) += batchAOffset_ * params.problemShape.m *
311+ Cmct::Gemm::CeilDiv(params.problemShape.k, static_cast<int64_t>(MXFP_DIVISOR_SIZE));
312+ Get<QuantBatchMatmul::IDX_X2SCALE_OFFSET>(
313+ blockOffset_) += batchBOffset_ * params.problemShape.n *
314+ Cmct::Gemm::CeilDiv(params.problemShape.k, static_cast<int64_t>(MXFP_DIVISOR_SIZE));
315+ } else {
309 Get<QuantBatchMatmul::IDX_X1SCALE_OFFSET>(316 Get<QuantBatchMatmul::IDX_X1SCALE_OFFSET>(
310 blockOffset_) += batchAOffset_ * params.problemShape.m *317 blockOffset_) += batchAOffset_ * params.problemShape.m *
311 Cmct::Gemm::CeilDiv(params.problemShape.k, static_cast<int64_t>(MXFP_DIVISOR_SIZE)) *318 Cmct::Gemm::CeilDiv(params.problemShape.k, static_cast<int64_t>(MXFP_DIVISOR_SIZE)) *
@@ -12,9 +12,6 @@
12 * \file quant_batch_matmul_inplace_add_mx_basic_api_tiling.cpp12 * \file quant_batch_matmul_inplace_add_mx_basic_api_tiling.cpp
13 * \brief13 * \brief
14 */14 */
15-#include <string>
16-#include <vector>
17-#include "base/err_msg.h"
18#include <vector>15#include <vector>
19#include "quant_batch_matmul_inplace_add_mx_basic_api_tiling.h"16#include "quant_batch_matmul_inplace_add_mx_basic_api_tiling.h"
20#include "op_host/tiling_templates_registry.h"17#include "op_host/tiling_templates_registry.h"
@@ -63,19 +60,6 @@ ge::graphStatus QuantBatchMatmulInplaceAddMXBasicAPITiling::DoLibApiTiling()
63 if (ret != ge::GRAPH_SUCCESS) {60 if (ret != ge::GRAPH_SUCCESS) {
64 return ret;61 return ret;
65 }62 }
panzhija
panzhijapanzhija15 days ago

没有 ut

likedislike
wangzitao
wangzitao
12 days ago Comments:
likedislike
66- // TE HIF4新方案合入并且适配后放开,如果未命中QBMMV3中BLAZE tiling
67- // 基类不会生成 withoutBatch 数据,直接报错避免下发全零 tiling。
68- if (inputParams_.isHif4PerGroup && basicTiling_.baseK % (qmmv3_tiling_const::MXFP_DIVISOR_SIZE * 2UL) != 0UL) {
69- {
70- const std::string _msg = "When the dtype of x1 and x2 is HIFLOAT4, baseK[" +
71- std::to_string(basicTiling_.baseK) + "] must be an even multiple of 64.";
72- OP_LOGE_WITHOUT_REPORT(inputParams_.opName, "%s", _msg.c_str());
73- const std::vector<const char*> _msgKey = {"message"};
74- const std::vector<const char*> _msgValue = {_msg.c_str()};
75- REPORT_PREDEFINED_ERR_MSG("EZ1001", _msgKey, _msgValue);
76- }
77- return ge::GRAPH_FAILED;
78- }
79 UpdateTilingData();63 UpdateTilingData();
80 return ge::GRAPH_SUCCESS;64 return ge::GRAPH_SUCCESS;
81}65}
@@ -11,7 +11,6 @@
11#include <gtest/gtest.h>11#include <gtest/gtest.h>
12#include <stdlib.h>12#include <stdlib.h>
13 13 
14-#include <algorithm>
15#include <cctype>14#include <cctype>
16#include <fstream>15#include <fstream>
17#include <iostream>16#include <iostream>
@@ -19,17 +18,12 @@
19#include <string>18#include <string>
20#include <vector>19#include <vector>
21 20 
22-#include "log/log.h"
23- 
24#include "exe_graph/runtime/storage_format.h"21#include "exe_graph/runtime/storage_format.h"
25#include "exe_graph/runtime/storage_shape.h"22#include "exe_graph/runtime/storage_shape.h"
26-#include "exe_graph/runtime/tiling_parse_context.h"
27#include "kernel_run_context_facker.h"23#include "kernel_run_context_facker.h"
28#include "platform/platform_infos_def.h"24#include "platform/platform_infos_def.h"
29#include "register/op_impl_registry.h"25#include "register/op_impl_registry.h"
30-#include "op_host/tiling_templates_registry.h"
31#include "test_cube_util.h"26#include "test_cube_util.h"
32-#include "ut_op_util.h"
33#include "../../../op_host/op_tiling/quant_batch_matmul_inplace_add_tiling.h"27#include "../../../op_host/op_tiling/quant_batch_matmul_inplace_add_tiling.h"
34#include "../../../op_kernel/arch35/quant_batch_matmul_inplace_add_tiling_key.h"28#include "../../../op_kernel/arch35/quant_batch_matmul_inplace_add_tiling_key.h"
35#include "../../../../common/op_host/math_util_nn.h"29#include "../../../../common/op_host/math_util_nn.h"
@@ -75,6 +69,18 @@ struct QuantBatchMatmulInplaceAddTilingCsvLoadResult {
75 vector<string> errors;69 vector<string> errors;
76};70};
77 71 
72+static ge::DataType ParseInplaceAddDtype(const string& value)
73+{
74+ const string dtype = Trim(value);
75+ if (dtype == "HIFLOAT4" || dtype == "DT_HIFLOAT4") {
76+ return Ops::NN::DT_HIFLOAT4;
77+ }
78+ if (dtype == "HIFLOAT4_SCALE" || dtype == "DT_HIFLOAT4_SCALE") {
79+ return Ops::NN::DT_HIFLOAT4_SCALE;
80+ }
81+ return ParseDtype(value);
82+}
83+ 
78static QuantBatchMatmulInplaceAddTilingCsvLoadResult LoadParams()84static QuantBatchMatmulInplaceAddTilingCsvLoadResult LoadParams()
79{85{
80 QuantBatchMatmulInplaceAddTilingCsvLoadResult result;86 QuantBatchMatmulInplaceAddTilingCsvLoadResult result;
@@ -125,13 +131,13 @@ static QuantBatchMatmulInplaceAddTilingCsvLoadResult LoadParams()
125 param.groupSize = ParseInt64OrDefault(cols[idx++], 0);131 param.groupSize = ParseInt64OrDefault(cols[idx++], 0);
126 param.x1Format = ParseFormat(cols[idx++]);132 param.x1Format = ParseFormat(cols[idx++]);
127 param.x2Format = ParseFormat(cols[idx++]);133 param.x2Format = ParseFormat(cols[idx++]);
128- param.x1Dtype = ParseDtype(cols[idx++]);134+ param.x1Dtype = ParseInplaceAddDtype(cols[idx++]);
129- param.x2Dtype = ParseDtype(cols[idx++]);135+ param.x2Dtype = ParseInplaceAddDtype(cols[idx++]);
130 param.hasX1Scale = ParseBool(cols[idx++]);136 param.hasX1Scale = ParseBool(cols[idx++]);
131- param.x1ScaleDtype = ParseDtype(cols[idx++]);137+ param.x1ScaleDtype = ParseInplaceAddDtype(cols[idx++]);
132 param.hasX2Scale = ParseBool(cols[idx++]);138 param.hasX2Scale = ParseBool(cols[idx++]);
133- param.x2ScaleDtype = ParseDtype(cols[idx++]);139+ param.x2ScaleDtype = ParseInplaceAddDtype(cols[idx++]);
134- param.yDtype = ParseDtype(cols[idx++]);140+ param.yDtype = ParseInplaceAddDtype(cols[idx++]);
135 param.aicNum = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0));141 param.aicNum = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0));
136 param.aivNum = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0));142 param.aivNum = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0));
137 param.expectBlockDim = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0));143 param.expectBlockDim = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0));
@@ -241,8 +247,13 @@ static void TestOneParamCase(const QuantBatchMatmulInplaceAddTilingTestParam& pa
241 }247 }
242 yInputShape.MutableOriginShape() = gert::Shape({param.m, param.n});248 yInputShape.MutableOriginShape() = gert::Shape({param.m, param.n});
243 yInputShape.MutableStorageShape() = gert::Shape({param.m, param.n});249 yInputShape.MutableStorageShape() = gert::Shape({param.m, param.n});
244- if (param.x1Dtype == ge::DT_HIFLOAT8 && param.x2Dtype == ge::DT_HIFLOAT8 && param.x1ScaleDtype == ge::DT_FLOAT &&250+ if (param.x1Dtype == Ops::NN::DT_HIFLOAT4 && param.x2Dtype == Ops::NN::DT_HIFLOAT4) {
245- param.x2ScaleDtype == ge::DT_FLOAT) {251+ x1ScaleShape.MutableOriginShape() = gert::Shape({k1, param.m, 1});
252+ x1ScaleShape.MutableStorageShape() = gert::Shape({k1, param.m, 1});
253+ x2ScaleShape.MutableOriginShape() = gert::Shape({k1, param.n, 1});
254+ x2ScaleShape.MutableStorageShape() = gert::Shape({k1, param.n, 1});
255+ } else if (param.x1Dtype == ge::DT_HIFLOAT8 && param.x2Dtype == ge::DT_HIFLOAT8 &&
256+ param.x1ScaleDtype == ge::DT_FLOAT && param.x2ScaleDtype == ge::DT_FLOAT) {
246 x1ScaleShape.MutableOriginShape() = gert::Shape({1});257 x1ScaleShape.MutableOriginShape() = gert::Shape({1});
247 x1ScaleShape.MutableStorageShape() = gert::Shape({1});258 x1ScaleShape.MutableStorageShape() = gert::Shape({1});
248 x2ScaleShape.MutableOriginShape() = gert::Shape({1});259 x2ScaleShape.MutableOriginShape() = gert::Shape({1});
@@ -312,7 +323,7 @@ static void TestOneParamCase(const QuantBatchMatmulInplaceAddTilingTestParam& pa
312 {"Short_SoC_version", param.socVersion},323 {"Short_SoC_version", param.socVersion},
313 {"SoC_version", param.socVersion},324 {"SoC_version", param.socVersion},
314 };325 };
315- const map<string, string> soc2Arch = {{"Dav3510", "3510"}};326+ const map<string, string> soc2Arch = {{"Dav3510", "3510"}, {"Dav9201", "9201"}};
316 auto soc2ArchIter = soc2Arch.find(param.socVersion);327 auto soc2ArchIter = soc2Arch.find(param.socVersion);
317 if (soc2ArchIter != soc2Arch.end()) {328 if (soc2ArchIter != soc2Arch.end()) {
318 socVersionInfos.clear();329 socVersionInfos.clear();
@@ -4,6 +4,7 @@ general,mx_2,Dav3510,1024,512,1024,true,false,32,ND,ND,FP8-E5M2,FP8-E5M2,true,FP
4general,mx_3,Dav3510,128,512,128,true,false,32,ND,ND,FP8-E5M2,FP8-E5M2,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,16,GRAPH_SUCCESS,14general,mx_3,Dav3510,128,512,128,true,false,32,ND,ND,FP8-E5M2,FP8-E5M2,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,16,GRAPH_SUCCESS,1
5general,mx_4,Dav3510,1024,512,1024,true,false,32,ND,ND,FP8-E4M3,FP8-E4M3,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,15general,mx_4,Dav3510,1024,512,1024,true,false,32,ND,ND,FP8-E4M3,FP8-E4M3,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,1
6general,hif8_tt,Dav3510,128,512,128,true,false,0,ND,ND,HIFLOAT8,HIFLOAT8,true,FP32,true,FP32,FP32,32,64,16,GRAPH_SUCCESS,16general,hif8_tt,Dav3510,128,512,128,true,false,0,ND,ND,HIFLOAT8,HIFLOAT8,true,FP32,true,FP32,FP32,32,64,16,GRAPH_SUCCESS,1
7+general,hif4_m268_k160_n3680,Dav9201,268,160,3680,true,false,4295032896,ND,ND,HIFLOAT4,HIFLOAT4,true,HIFLOAT4_SCALE,true,HIFLOAT4_SCALE,FP32,32,64,32,GRAPH_SUCCESS,17
7general,mxfp4_test1,Dav3510,128,512,128,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,4,GRAPH_SUCCESS,18general,mxfp4_test1,Dav3510,128,512,128,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,4,GRAPH_SUCCESS,1
8general,mxfp4_a_full_load,Dav3510,128,512,16384,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,179general,mxfp4_a_full_load,Dav3510,128,512,16384,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,17
9general,mxfp4_test2,Dav3510,1024,512,1024,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,110general,mxfp4_test2,Dav3510,1024,512,1024,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,1
@@ -44,6 +44,7 @@ static const int64_t MIN_DIM_NUM_ND = 2;
44static const size_t MX_SCALE_DIM = 3;44static const size_t MX_SCALE_DIM = 3;
45static const size_t MX_SCALE_MIN_DIM = 3;45static const size_t MX_SCALE_MIN_DIM = 3;
46static const size_t HIFP4_SCALE_DIM = 3;46static const size_t HIFP4_SCALE_DIM = 3;
47+static const size_t HIFP4_SCALE_MAX_DIM = 7;
47static const size_t PENULTIMATE_DIM = 2;48static const size_t PENULTIMATE_DIM = 2;
48static const size_t BATCH_TAILENDER_DIM = 3;49static const size_t BATCH_TAILENDER_DIM = 3;
49static const size_t NZ_K1_INDEX = 3;50static const size_t NZ_K1_INDEX = 3;
@@ -541,21 +542,27 @@ bool QuantMatmulChecker::CheckDimValueMicroScaling() const
541 542 
542bool QuantMatmulChecker::CheckDimValueHif4() const543bool QuantMatmulChecker::CheckDimValueHif4() const
543{544{
544- auto x1ScaleMDim = transposeX1_ ? 1 : 0;545+ size_t x1ScaleBatchDimNum = x1Scale_->GetViewShape().GetDimNum() - HIFP4_SCALE_DIM;
545- auto x1ScaleKDim = transposeX1_ ? 0 : 1;546+ size_t x2ScaleBatchDimNum = x2Scale_->GetViewShape().GetDimNum() - HIFP4_SCALE_DIM;
546- auto x2ScaleNDim = transposeX2_ ? 0 : 1;547+ auto x1ScaleMDimIndex = x1ScaleBatchDimNum + (transposeX1_ ? 1 : 0);
547- auto x2ScaleKDim = transposeX2_ ? 1 : 0;548+ auto x1ScaleKDimIndex = x1ScaleBatchDimNum + (transposeX1_ ? 0 : 1);
548- 549+ auto x2ScaleNDimIndex = x2ScaleBatchDimNum + (transposeX2_ ? 0 : 1);
549- bool x1ScaleShapeHasOne = x1Scale_->GetViewShape().GetDim(0) == 1 || x1Scale_->GetViewShape().GetDim(1) == 1;550+ auto x2ScaleKDimIndex = x2ScaleBatchDimNum + (transposeX2_ ? 1 : 0);
550- bool x2ScaleShapeHasOne = x2Scale_->GetViewShape().GetDim(0) == 1 || x2Scale_->GetViewShape().GetDim(1) == 1;551+ auto x1ScaleMDim = x1Scale_->GetViewShape().GetDim(x1ScaleMDimIndex);
551- if ((!x1ScaleShapeHasOne && x1Scale_->GetViewShape().GetDim(x1ScaleMDim) != x1MDim_) ||552+ auto x1ScaleKDim = x1Scale_->GetViewShape().GetDim(x1ScaleKDimIndex);
552- (!x2ScaleShapeHasOne && x2Scale_->GetViewShape().GetDim(x2ScaleNDim) != x2NDim_)) {553+ auto x2ScaleNDim = x2Scale_->GetViewShape().GetDim(x2ScaleNDimIndex);
554+ auto x2ScaleKDim = x2Scale_->GetViewShape().GetDim(x2ScaleKDimIndex);
555+ bool x1ScaleHasOne = (x1ScaleMDim == 1 && x1ScaleKDim == x1MDim_) || (x1ScaleMDim == x1MDim_ && x1ScaleKDim == 1) ||
556+ (x1ScaleMDim == 1 && x1ScaleKDim == CeilDiv(x1KDim_, HIFP4_GROUP_SIZE)) ||
557+ (x1ScaleMDim == CeilDiv(x1KDim_, HIFP4_GROUP_SIZE) && x1ScaleKDim == 1);
558+ bool x2ScaleHasOne = (x2ScaleNDim == 1 && x2ScaleKDim == x2NDim_) || (x2ScaleNDim == x2NDim_ && x2ScaleKDim == 1) ||
559+ (x2ScaleNDim == 1 && x2ScaleKDim == CeilDiv(x2KDim_, HIFP4_GROUP_SIZE)) ||
560+ (x2ScaleNDim == CeilDiv(x2KDim_, HIFP4_GROUP_SIZE) && x2ScaleKDim == 1);
561+ if ((!x1ScaleHasOne && x1ScaleMDim != x1MDim_) || (!x2ScaleHasOne && x2ScaleNDim != x2NDim_)) {
553 OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(562 OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
554 apiName_,563 apiName_,
555 FormatString("x1 M, %s M, x2 N, %s N", GetX1ScaleName().c_str(), GetX2ScaleName().c_str()).c_str(),564 FormatString("x1 M, %s M, x2 N, %s N", GetX1ScaleName().c_str(), GetX2ScaleName().c_str()).c_str(),
556- FormatString("%ld, %ld, %ld, %ld", x1MDim_, x1Scale_->GetViewShape().GetDim(x1ScaleMDim), x2NDim_,565+ FormatString("%ld, %ld, %ld, %ld", x1MDim_, x1ScaleMDim, x2NDim_, x2ScaleNDim).c_str(),
557- x2Scale_->GetViewShape().GetDim(x2ScaleNDim))
558- .c_str(),
559 FormatString("when the quantization mode is hif4, the M dimension of x1 and %s must be equal, and the N "566 FormatString("when the quantization mode is hif4, the M dimension of x1 and %s must be equal, and the N "
560 "dimension of x2 and %s must be equal",567 "dimension of x2 and %s must be equal",
561 GetX1ScaleName().c_str(), GetX2ScaleName().c_str())568 GetX1ScaleName().c_str(), GetX2ScaleName().c_str())
@@ -563,10 +570,10 @@ bool QuantMatmulChecker::CheckDimValueHif4() const
563 return false;570 return false;
564 }571 }
565 572 
566- if (!x1ScaleShapeHasOne && CeilDiv(x1KDim_, HIFP4_GROUP_SIZE) != x1Scale_->GetViewShape().GetDim(x1ScaleKDim)) {573+ if (!x1ScaleHasOne && CeilDiv(x1KDim_, HIFP4_GROUP_SIZE) != x1ScaleKDim) {
567 OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(574 OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
568 apiName_, FormatString("x1 K, %s K", GetX1ScaleName().c_str()).c_str(),575 apiName_, FormatString("x1 K, %s K", GetX1ScaleName().c_str()).c_str(),
569- FormatString("%ld, %ld", x1KDim_, x1Scale_->GetViewShape().GetDim(x1ScaleKDim)).c_str(),576+ FormatString("%ld, %ld", x1KDim_, x1ScaleKDim).c_str(),
570 FormatString("when the quantization mode is hif4, the K dimension of %s must be equal to the K dimension "577 FormatString("when the quantization mode is hif4, the K dimension of %s must be equal to the K dimension "
571 "of x1 ceildivided by 64",578 "of x1 ceildivided by 64",
572 GetX1ScaleName().c_str())579 GetX1ScaleName().c_str())
@@ -574,10 +581,10 @@ bool QuantMatmulChecker::CheckDimValueHif4() const
574 return false;581 return false;
575 }582 }
576 583 
577- if (!x2ScaleShapeHasOne && CeilDiv(x2KDim_, HIFP4_GROUP_SIZE) != x2Scale_->GetViewShape().GetDim(x2ScaleKDim)) {584+ if (!x2ScaleHasOne && CeilDiv(x2KDim_, HIFP4_GROUP_SIZE) != x2ScaleKDim) {
578 OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(585 OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
579 apiName_, FormatString("x2 K, %s K", GetX2ScaleName().c_str()).c_str(),586 apiName_, FormatString("x2 K, %s K", GetX2ScaleName().c_str()).c_str(),
580- FormatString("%ld, %ld", x2KDim_, x2Scale_->GetViewShape().GetDim(x2ScaleKDim)).c_str(),587+ FormatString("%ld, %ld", x2KDim_, x2ScaleKDim).c_str(),
581 FormatString("when the quantization mode is hif4, the K dimension of %s must be equal to the K dimension "588 FormatString("when the quantization mode is hif4, the K dimension of %s must be equal to the K dimension "
582 "of x2 ceildivided by 64",589 "of x2 ceildivided by 64",
583 GetX2ScaleName().c_str())590 GetX2ScaleName().c_str())
@@ -1211,15 +1218,106 @@ bool QuantMatmulChecker::CheckMxScaleDimRange(size_t x1ScaleDim, size_t x2ScaleD
1211 return true;1218 return true;
1212}1219}
1213 1220 
1221+bool QuantMatmulChecker::CheckHif4ScaleMinDim(size_t x1ScaleDim, size_t x2ScaleDim) const
1222+{
1223+ if (x1ScaleDim < HIFP4_SCALE_DIM) {
1224+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
1225+ apiName_, GetX1ScaleName().c_str(), FormatString("%zuD", x1ScaleDim).c_str(),
1226+ FormatString("when the quantization mode is hif4, the shape dim of %s must be at least %zuD",
1227+ GetX1ScaleName().c_str(), HIFP4_SCALE_DIM)
1228+ .c_str());
1229+ return false;
1230+ }
1231+ if (x2ScaleDim < HIFP4_SCALE_DIM) {
1232+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
1233+ apiName_, GetX2ScaleName().c_str(), FormatString("%zuD", x2ScaleDim).c_str(),
1234+ FormatString("when the quantization mode is hif4, the shape dim of %s must be at least %zuD",
1235+ GetX2ScaleName().c_str(), HIFP4_SCALE_DIM)
1236+ .c_str());
1237+ return false;
1238+ }
1239+ return true;
1240+}
1241+ 
1242+bool QuantMatmulChecker::CheckHif4ScaleMaxDim(size_t x1ScaleDim, size_t x2ScaleDim) const
1243+{
1244+ if (x1ScaleDim > HIFP4_SCALE_MAX_DIM) {
1245+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
1246+ apiName_, GetX1ScaleName().c_str(), FormatString("%zuD", x1ScaleDim).c_str(),
1247+ FormatString("when the quantization mode is hif4, the shape dim of %s must be at most %zuD",
1248+ GetX1ScaleName().c_str(), HIFP4_SCALE_MAX_DIM)
1249+ .c_str());
1250+ return false;
1251+ }
1252+ if (x2ScaleDim > HIFP4_SCALE_MAX_DIM) {
1253+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
1254+ apiName_, GetX2ScaleName().c_str(), FormatString("%zuD", x2ScaleDim).c_str(),
1255+ FormatString("when the quantization mode is hif4, the shape dim of %s must be at most %zuD",
1256+ GetX2ScaleName().c_str(), HIFP4_SCALE_MAX_DIM)
1257+ .c_str());
1258+ return false;
1259+ }
1260+ return true;
1261+}
1262+ 
1263+bool QuantMatmulChecker::CheckHif4ScaleBatchDimMatch(size_t x1ScaleDim, size_t x2ScaleDim) const
1264+{
1265+ // batch dim count alignment: scale removes hif4 fixed 3 dims, x removes last 2 dims
1266+ size_t x1DimNum = x1_->GetViewShape().GetDimNum();
1267+ size_t x2DimNum = x2_->GetViewShape().GetDimNum();
1268+ size_t x1BatchDimNum = x1ScaleDim - HIFP4_SCALE_DIM;
1269+ size_t x2BatchDimNum = x2ScaleDim - HIFP4_SCALE_DIM;
1270+ size_t x1InputBatchDimNum = (x1DimNum > MIN_DIM_NUM_ND) ? (x1DimNum - MIN_DIM_NUM_ND) : 0;
1271+ size_t x2InputBatchDimNum = (x2DimNum > MIN_DIM_NUM_ND) ? (x2DimNum - MIN_DIM_NUM_ND) : 0;
1272+ if (x1BatchDimNum != x1InputBatchDimNum) {
1273+ OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
1274+ apiName_, "pertokenBatchDimNum, x1BatchDimNum",
1275+ FormatString("%zu, %zu", x1BatchDimNum, x1InputBatchDimNum).c_str(),
1276+ "when the quantization mode is hif4, the batch dimension num of x1Scale must be equal to that of x1");
1277+ return false;
1278+ }
1279+ if (x2BatchDimNum != x2InputBatchDimNum) {
1280+ OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
1281+ apiName_, "scaleBatchDimNum, x2BatchDimNum",
1282+ FormatString("%zu, %zu", x2BatchDimNum, x2InputBatchDimNum).c_str(),
1283+ "when the quantization mode is hif4, the batch dimension num of x2Scale must be equal to that of x2");
1284+ return false;
1285+ }
1286+ // per-batch dim value match
1287+ const auto& x1Shape = x1_->GetViewShape();
1288+ const auto& x2Shape = x2_->GetViewShape();
1289+ const auto& x1ScaleShape = x1Scale_->GetViewShape();
1290+ const auto& x2ScaleShape = x2Scale_->GetViewShape();
1291+ for (size_t i = 0; i < x1BatchDimNum; ++i) {
1292+ if (x1ScaleShape.GetDim(i) != x1Shape.GetDim(i)) {
1293+ OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
1294+ apiName_, "dimIndex, x1Batch, pertokenBatch",
1295+ FormatString("%zu, %ld, %ld", i, x1Shape.GetDim(i), x1ScaleShape.GetDim(i)).c_str(),
1296+ "when the quantization mode is hif4, the batch dimension of x1Scale must be equal to that of x1");
1297+ return false;
1298+ }
1299+ }
1300+ for (size_t i = 0; i < x2BatchDimNum; ++i) {
1301+ if (x2ScaleShape.GetDim(i) != x2Shape.GetDim(i)) {
1302+ OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
1303+ apiName_, "dimIndex, x2Batch, scaleBatch",
1304+ FormatString("%zu, %ld, %ld", i, x2Shape.GetDim(i), x2ScaleShape.GetDim(i)).c_str(),
1305+ "when the quantization mode is hif4, the batch dimension of x2Scale must be equal to that of x2");
1306+ return false;
1307+ }
1308+ }
1309+ return true;
1310+}
1311+ 
1214bool QuantMatmulChecker::CheckHif4ScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const1312bool QuantMatmulChecker::CheckHif4ScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const
panzhija
panzhijapanzhija15 days ago

hif4 也算 mx 量化吧,为什么不合并到 CheckMxScale 相关的函数中呢

likedislike
wangzitao
wangzitao
15 days ago Comments:
likedislike
1215{1313{
1216- if (x1ScaleDim != HIFP4_SCALE_DIM || x2ScaleDim != HIFP4_SCALE_DIM) {1314+ if (!CheckHif4ScaleMinDim(x1ScaleDim, x2ScaleDim)) {
1217- OP_LOGE_FOR_INVALID_SHAPEDIMS_WITH_REASON(1315+ return false;
1218- apiName_, FormatString("%s, %s", GetX1ScaleName().c_str(), GetX2ScaleName().c_str()).c_str(),1316+ }
1219- FormatString("%zuD, %zuD", x1ScaleDim, x2ScaleDim).c_str(),1317+ if (!CheckHif4ScaleMaxDim(x1ScaleDim, x2ScaleDim)) {
1220- FormatString("when the quantization mode is hif4, the shape dims of %s and %s must be %zu",1318+ return false;
1221- GetX1ScaleName().c_str(), GetX2ScaleName().c_str(), HIFP4_SCALE_DIM)1319+ }
1222- .c_str());1320+ if (!CheckHif4ScaleBatchDimMatch(x1ScaleDim, x2ScaleDim)) {
1223 return false;1321 return false;
1224 }1322 }
1225 return true;1323 return true;
@@ -74,6 +74,9 @@ private:
74 bool CheckHif4ScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const;74 bool CheckHif4ScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const;
75 bool CheckMxScaleMinDim(size_t x1ScaleDim, size_t x2ScaleDim) const;75 bool CheckMxScaleMinDim(size_t x1ScaleDim, size_t x2ScaleDim) const;
76 bool CheckMxScaleBatchDimMatch(size_t x1ScaleDim, size_t x2ScaleDim) const;76 bool CheckMxScaleBatchDimMatch(size_t x1ScaleDim, size_t x2ScaleDim) const;
77+ bool CheckHif4ScaleMinDim(size_t x1ScaleDim, size_t x2ScaleDim) const;
78+ bool CheckHif4ScaleMaxDim(size_t x1ScaleDim, size_t x2ScaleDim) const;
79+ bool CheckHif4ScaleBatchDimMatch(size_t x1ScaleDim, size_t x2ScaleDim) const;
77 bool CheckNormalScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const;80 bool CheckNormalScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const;
78 bool CheckGroupSize() const;81 bool CheckGroupSize() const;
79 bool CheckOutShape(bool twoDimMatmulCaseFlag, const std::vector<int64_t>& batchRecord) const;82 bool CheckOutShape(bool twoDimMatmulCaseFlag, const std::vector<int64_t>& batchRecord) const;
@@ -17,9 +17,7 @@
17 17 
18#include <algorithm>18#include <algorithm>
19 19 
20-#include "common/op_host/op_tiling/tiling_type_mm.h"
21#include "log/log.h"20#include "log/log.h"
22-#include "error_util.h"
23#include "op_host/tiling_templates_registry.h"21#include "op_host/tiling_templates_registry.h"
24#include "base_block_calculator.h"22#include "base_block_calculator.h"
25#include "l1_tiling_data_calculator.h"23#include "l1_tiling_data_calculator.h"
@@ -107,18 +105,7 @@ void AdaptiveSlidingWindowMXBasicAPITiling::ResetTilingData()
107bool AdaptiveSlidingWindowMXBasicAPITiling::IsWithoutBatchTilingData() const105bool AdaptiveSlidingWindowMXBasicAPITiling::IsWithoutBatchTilingData() const
108{106{
109 return IsTensorApiEnabled() && inputParams_.batchC == 1UL &&107 return IsTensorApiEnabled() && inputParams_.batchC == 1UL &&
110- (compileInfo_.npuArch == Ops::Base::DAV_3510 || compileInfo_.npuArch == Ops::Base::DAV_9201) &&108+ (compileInfo_.npuArch == Ops::Base::DAV_3510 || compileInfo_.npuArch == NpuArch::DAV_9201);
111- IsHif4BlazeCapable();
112-}
113- 
114-bool AdaptiveSlidingWindowMXBasicAPITiling::IsHif4BlazeCapable() const
115-{
116- // 9202 上 hif4 内核只实现 CMCT(basic) 路径,不消费 WithoutBatchTilingData,禁止走 blaze/without-batch
117- if (inputParams_.isHif4PerGroup && compileInfo_.npuArch != Ops::Base::DAV_9201) {
118- return false;
119- }
120- // tensor-api crd2idx 对 hif4_scale 暂不支持奇数坐标,baseK 需为 64 的偶数倍
121- return !(inputParams_.isHif4PerGroup && basicTiling_.baseK % (qmmv3_tiling_const::MXFP_DIVISOR_SIZE * 2UL) != 0UL);
122}109}
123 110 
124void AdaptiveSlidingWindowMXBasicAPITiling::AdjustScaleFactorForL0CPingpong(uint32_t& scaleFactor, uint32_t step,111void AdaptiveSlidingWindowMXBasicAPITiling::AdjustScaleFactorForL0CPingpong(uint32_t& scaleFactor, uint32_t step,
@@ -148,10 +135,9 @@ const void* AdaptiveSlidingWindowMXBasicAPITiling::GetTilingData() const
148 135 
149uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetApiLevel(NpuArch npuArch) const136uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetApiLevel(NpuArch npuArch) const
150{137{
151- const bool supportsBlaze = npuArch == Ops::Base::DAV_3510 || npuArch == Ops::Base::DAV_9201;138+ const bool supportsBlaze = npuArch == Ops::Base::DAV_3510 || npuArch == NpuArch::DAV_9201;
152- return IsTensorApiEnabled() && supportsBlaze && IsHif4BlazeCapable() ?139+ return IsTensorApiEnabled() && supportsBlaze ? static_cast<uint64_t>(QMMApiLevel::BLAZE_LEVEL) :
153- static_cast<uint64_t>(QMMApiLevel::BLAZE_LEVEL) :140+ static_cast<uint64_t>(QMMApiLevel::BASIC_LEVEL);
154- static_cast<uint64_t>(QMMApiLevel::BASIC_LEVEL);
155}141}
156 142 
157uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetBatchMode() const143uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetBatchMode() const
@@ -72,7 +72,6 @@ private:
72 void SetWithoutBatchTilingData();72 void SetWithoutBatchTilingData();
73 void NormalizeSingleRoundTailSplitBasicBlock();73 void NormalizeSingleRoundTailSplitBasicBlock();
74 void AdjustScaleFactorForL0CPingpong(uint32_t& scaleFactor, uint32_t step, uint32_t baseK) const;74 void AdjustScaleFactorForL0CPingpong(uint32_t& scaleFactor, uint32_t step, uint32_t baseK) const;
75- bool IsHif4BlazeCapable() const;
76 75 
77protected:76protected:
78 void ResetTilingData() override;77 void ResetTilingData() override;
@@ -20,7 +20,6 @@
20#include "op_host/tiling_templates_registry.h"20#include "op_host/tiling_templates_registry.h"
21#include "error_util.h"21#include "error_util.h"
22#include "log/log.h"22#include "log/log.h"
23-#include "op_cache_tiling.h"
24#include "quant_batch_matmul_v3_checker_arch_9201.h"23#include "quant_batch_matmul_v3_checker_arch_9201.h"
25#include "quant_batch_matmul_v3_tiling_strategy.h"24#include "quant_batch_matmul_v3_tiling_strategy.h"
26#include "base_block_calculator.h"25#include "base_block_calculator.h"
@@ -68,7 +67,7 @@ QBMMV3StreamKTiling960::QBMMV3StreamKTiling960(gert::TilingContext* context,
68bool QBMMV3StreamKTiling960::IsCapable()67bool QBMMV3StreamKTiling960::IsCapable()
69{68{
70 // 量化场景分流:9201/9202 可走 SK 的量化场景一致(MX per-group + 非MX per-tensor + HIF4 per-group),均走 Blaze。69 // 量化场景分流:9201/9202 可走 SK 的量化场景一致(MX per-group + 非MX per-tensor + HIF4 per-group),均走 Blaze。
71- bool isQuantCapable = inputParams_.isMxPerGroup || IsPertensorStreamKInput();70+ bool isQuantCapable = inputParams_.isMxPerGroup || inputParams_.isHif4PerGroup || IsPertensorStreamKInput();
panzhijazhaoyingchao2
panzhijapanzhija15 days ago
Outdated

还是一个疑问,为什么 isHif4PerGroup 和 isMxPerGroup 要分离?

likedislike
wangzitao
wangzitao
15 days ago Comments:
likedislike
zhaoyingchao2zhaoyingchao215 days ago
Outdated

[跨仓兼容性检视] 9202 HIF4 StreamK 的 SDK 支持尚未闭合

这里把 isHif4PerGroup 纳入 9201/9202 的 StreamK 候选,但 IsTensorapiCapable() 只能证明 TensorAPI 可用,不能证明目标架构已支持 HIF4。

本次核验的公开 asc-devkit master 9ef8932c95c553f0ed89915ab5e8e178897f41c5 中:

  • impl/tensor_api/utils/constant_impl.h 的 is_data_type / is_b4_type 仅在 __NPU_ARCH__ == 9201 时纳入 hifloat4x2_t;
  • impl/tensor_api/tensor/layout_pattern.h 的相关 NN/scale 布局,仅在 9201 分支允许 c0=1,9202 落入要求 c0=2 的静态断言;
  • Blaze HIF4 路径使用 GetScaleC0() == 1,#564 又依赖 SDK 的 half-element/C0 推导,因此不能仅凭 TensorAPI 已启用就开放 9202 路由。

建议明确提供支持 9202 HIF4 的 SDK 版本,并补充 9202 目标编译与数值验证;在该前提未满足前,将 HIF4 StreamK 暂限于 9201,并同步约束 Host 路由、tiling key 与 Kernel 分发。

以上结论限定于所核验的公开 SDK 版本;若目标使用已有 9202 支持的其他 SDK,请提供对应版本/实现和验证结果。相关 Device 分支有存量代码,本意见针对本次开放 Host 可达路径,并不将全部底层限制描述为本次新增回归。

依据:constant_impl.h、layout_pattern.h。

likedislike
72 if (compileInfo_.aicNum != compileInfo_.aivNum || !isQuantCapable || inputParams_.batchC > 1) {71 if (compileInfo_.aicNum != compileInfo_.aivNum || !isQuantCapable || inputParams_.batchC > 1) {
73 return false;72 return false;
74 }73 }
@@ -356,6 +355,7 @@ ge::graphStatus QBMMV3StreamKTiling960::PostTiling()
356 return ge::GRAPH_PARAM_INVALID;355 return ge::GRAPH_PARAM_INVALID;
357 }356 }
358 context_->SetBlockDim(tilingData_.streamKTiling.usedCoreNum);357 context_->SetBlockDim(tilingData_.streamKTiling.usedCoreNum);
358+ context_->SetScheduleMode(1); // StreamK kernel uses cross-core synchronization and requires batch mode.
359 context_->GetRawTilingData()->SetDataSize(tilingDataSize_);359 context_->GetRawTilingData()->SetDataSize(tilingDataSize_);
360 size_t* workspaces = context_->GetWorkspaceSizes(1); // set workspace360 size_t* workspaces = context_->GetWorkspaceSizes(1); // set workspace
361 OPS_CHECK_NULL_WITH_CONTEXT(context_, workspaces);361 OPS_CHECK_NULL_WITH_CONTEXT(context_, workspaces);
@@ -10,9 +10,10 @@
10 10 
11/*!11/*!
12 * \file qbmm_streamk_tiling_960.h12 * \file qbmm_streamk_tiling_960.h
13- * \brief StreamK tiling implementation for DAV_9201/DAV_9202 QuantBatchMatmulV3,13+ * \brief StreamK tiling implementation for Ascend960DT/PR(DAV_9201/9202) QuantBatchMatmulV3,
14- * DAV_9201所有量化走Blaze(复用QbmmMxStreamKBasicApiKernel),14+ * 960DT/960PR(9201/9202)的StreamK量化场景统一走Blaze(复用QbmmMxStreamKBasicApiKernel),
15- * DAV_9202走CMCT BasicAPI。tiling data复用QuantBatchMatmulV3StreamKBasicAPITilingData15+ * 非StreamK场景由ASW MX BasicAPI模板分流(9201 Blaze,9202 CMCT)。
16+ * tiling data复用QuantBatchMatmulV3StreamKBasicAPITilingData
16 * (与3510 Blaze StreamK共用布局),按SocVersion注册调度。17 * (与3510 Blaze StreamK共用布局),按SocVersion注册调度。
17 */18 */
18#ifndef QBMM_STREAMK_TILING_960_H19#ifndef QBMM_STREAMK_TILING_960_H
@@ -14,7 +14,6 @@
14 */14 */
15 15 
16#include "common/inc/op_host/op_const_def_nn.h"16#include "common/inc/op_host/op_const_def_nn.h"
17-#include "common/op_host/op_tiling/tiling_type_mm.h"
18#include "log/log.h"17#include "log/log.h"
19#include "matmul/common/op_host/log_format_util.h"18#include "matmul/common/op_host/log_format_util.h"
20#include "quant_batch_matmul_v3_checker.h"19#include "quant_batch_matmul_v3_checker.h"
@@ -1192,7 +1191,7 @@ bool QuantBatchMatmulV3Checker::CheckShape(const std::vector<gert::Shape*>& mand
1192 !CheckInputValidInMxPerGroupMode(scaleShape, pertokenShape, x1Shape, x2Shape, dimValueOfMKN))) {1191 !CheckInputValidInMxPerGroupMode(scaleShape, pertokenShape, x1Shape, x2Shape, dimValueOfMKN))) {
1193 return false;1192 return false;
1194 }1193 }
1195- if (!CheckExtraShapeValid(scaleShape, pertokenShape, dimValueOfMKN)) {1194+ if (!CheckExtraShapeValid(scaleShape, pertokenShape, x1Shape, x2Shape, dimValueOfMKN)) {
1196 return false;1195 return false;
1197 }1196 }
1198 if (!ExtraInputCheck()) {1197 if (!ExtraInputCheck()) {
@@ -1232,8 +1231,11 @@ bool QuantBatchMatmulV3Checker::CheckHif4ScaleDtypeWithPertoken() const { return
1232 1231 
1233bool QuantBatchMatmulV3Checker::CheckExtraOutputDtype() const { return true; }1232bool QuantBatchMatmulV3Checker::CheckExtraOutputDtype() const { return true; }
1234 1233 
1235-bool QuantBatchMatmulV3Checker::CheckExtraShapeValid(const gert::Shape&, const gert::StorageShape*,1234+bool QuantBatchMatmulV3Checker::CheckExtraShapeValid([[maybe_unused]] const gert::Shape& scaleShape,
1236- const std::vector<int64_t>&) const1235+ [[maybe_unused]] const gert::StorageShape* pertokenShape,
1236+ [[maybe_unused]] const gert::Shape& x1Shape,
1237+ [[maybe_unused]] const gert::Shape& x2Shape,
1238+ [[maybe_unused]] const std::vector<int64_t>& dimValueOfMKN) const
1237{1239{
1238 return true;1240 return true;
1239}1241}
@@ -83,6 +83,7 @@ protected:
83 virtual bool CheckHif4ScaleDtypeWithPertoken() const;83 virtual bool CheckHif4ScaleDtypeWithPertoken() const;
84 virtual bool CheckExtraOutputDtype() const;84 virtual bool CheckExtraOutputDtype() const;
85 virtual bool CheckExtraShapeValid(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape,85 virtual bool CheckExtraShapeValid(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape,
86+ const gert::Shape& x1Shape, const gert::Shape& x2Shape,
86 const std::vector<int64_t>& dimValueOfMKN) const;87 const std::vector<int64_t>& dimValueOfMKN) const;
87};88};
88} // namespace optiling89} // namespace optiling
@@ -14,18 +14,21 @@
14 */14 */
15 15 
16#include "common/inc/op_host/op_const_def_nn.h"16#include "common/inc/op_host/op_const_def_nn.h"
17-#include "common/op_host/op_tiling/tiling_type_mm.h"
18#include "error_util.h"17#include "error_util.h"
19#include "log/log.h"18#include "log/log.h"
19+#include "matmul/common/op_host/log_format_util.h"
20#include "quant_batch_matmul_v3_checker_arch_9201.h"20#include "quant_batch_matmul_v3_checker_arch_9201.h"
21 21 
22namespace {22namespace {
23constexpr uint64_t HIF4_GROUP_SIZE = 64UL;23constexpr uint64_t HIF4_GROUP_SIZE = 64UL;
24constexpr size_t SCALE_THREE_DIM = 3;24constexpr size_t SCALE_THREE_DIM = 3;
25+constexpr size_t DIM_NUM_TWO = 2;
25} // namespace26} // namespace
26 27 
27namespace optiling {28namespace optiling {
28 29 
30+using Ops::NN::FormatString;
31+ 
29const std::vector<ge::DataType>& QuantBatchMatmulV3CheckerArch9201::GetLegalInputDtypes() const32const std::vector<ge::DataType>& QuantBatchMatmulV3CheckerArch9201::GetLegalInputDtypes() const
30{33{
31 static const std::vector<ge::DataType> legalInputDtypes = {ge::DT_INT8, ge::DT_FLOAT8_E4M3FN,34 static const std::vector<ge::DataType> legalInputDtypes = {ge::DT_INT8, ge::DT_FLOAT8_E4M3FN,
@@ -112,22 +115,26 @@ bool QuantBatchMatmulV3CheckerArch9201::CheckExtraOutputDtype() const
112 115 
113bool QuantBatchMatmulV3CheckerArch9201::CheckExtraShapeValid(const gert::Shape& scaleShape,116bool QuantBatchMatmulV3CheckerArch9201::CheckExtraShapeValid(const gert::Shape& scaleShape,
114 const gert::StorageShape* pertokenShape,117 const gert::StorageShape* pertokenShape,
118+ const gert::Shape& x1Shape, const gert::Shape& x2Shape,
115 const std::vector<int64_t>& dimValueOfMKN) const119 const std::vector<int64_t>& dimValueOfMKN) const
116{120{
117- return CheckInputValidInHif4PerGroupMode(scaleShape, pertokenShape, dimValueOfMKN);121+ return CheckInputValidInHif4PerGroupMode(scaleShape, pertokenShape, x1Shape, x2Shape, dimValueOfMKN);
118}122}
119 123 
120bool QuantBatchMatmulV3CheckerArch9201::Hif4PertokenScaleShapeCheck(const gert::StorageShape* pertokenShape) const124bool QuantBatchMatmulV3CheckerArch9201::Hif4PertokenScaleShapeCheck(const gert::StorageShape* pertokenShape) const
121{125{
122 auto& pertoken = pertokenShape->GetStorageShape();126 auto& pertoken = pertokenShape->GetStorageShape();
123 auto pertokenShapeLen = pertoken.GetDimNum();127 auto pertokenShapeLen = pertoken.GetDimNum();
124- OP_TILING_CHECK(pertokenShapeLen != SCALE_THREE_DIM,128+ OP_TILING_CHECK(pertokenShapeLen < SCALE_THREE_DIM,
125- CUBE_INNER_ERR_REPORT(inputParams_.opName,129+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
126- "In hif4 quantification, pertokenScale's dimension must be 3, but it is %zu.",130+ inputParams_.opName, "pertokenScale", FormatString("%zuD", pertokenShapeLen).c_str(),
127- pertokenShapeLen),131+ FormatString("in hif4 quantification, the shape dim of pertokenScale must be at least %zuD",
132+ SCALE_THREE_DIM)
133+ .c_str()),
128 return false);134 return false);
129- auto mDimIdx = inputParams_.transA ? 1 : 0;135+ size_t batchDimNum = pertokenShapeLen - SCALE_THREE_DIM;
130- auto kDimIdx = inputParams_.transA ? 0 : 1;136+ auto mDimIdx = batchDimNum + (inputParams_.transA ? 1 : 0);
137+ auto kDimIdx = batchDimNum + (inputParams_.transA ? 0 : 1);
131 OP_TILING_CHECK(static_cast<uint64_t>(pertoken.GetDim(pertokenShapeLen - 1)) != 1,138 OP_TILING_CHECK(static_cast<uint64_t>(pertoken.GetDim(pertokenShapeLen - 1)) != 1,
132 CUBE_INNER_ERR_REPORT(inputParams_.opName, "In hif4 quantification, pertokenScale's last dimension \139 CUBE_INNER_ERR_REPORT(inputParams_.opName, "In hif4 quantification, pertokenScale's last dimension \
133should be equal to 1, but actual is [%ld].",140should be equal to 1, but actual is [%ld].",
@@ -146,7 +153,7 @@ should be equal to 1, but actual is [%ld].",
146 static_cast<uint64_t>(pertoken.GetDim(kDimIdx)) != ops::CeilDiv(inputParams_.kSize, HIF4_GROUP_SIZE),153 static_cast<uint64_t>(pertoken.GetDim(kDimIdx)) != ops::CeilDiv(inputParams_.kSize, HIF4_GROUP_SIZE),
147 CUBE_INNER_ERR_REPORT(154 CUBE_INNER_ERR_REPORT(
148 inputParams_.opName,155 inputParams_.opName,
149- "In hif4 quantification, when x1's transposition is %s, pertokenScale's dimension %d shape \156+ "In hif4 quantification, when x1's transposition is %s, pertokenScale's dimension %zu shape \
150should be equal to x1 k dimension size[%lu] ceil div 64, but actual is [%ld].",157should be equal to x1 k dimension size[%lu] ceil div 64, but actual is [%ld].",
151 inputParams_.transA ? "true" : "false", kDimIdx, inputParams_.kSize, pertoken.GetDim(kDimIdx)),158 inputParams_.transA ? "true" : "false", kDimIdx, inputParams_.kSize, pertoken.GetDim(kDimIdx)),
152 return false);159 return false);
@@ -154,7 +161,7 @@ should be equal to x1 k dimension size[%lu] ceil div 64, but actual is [%ld].",
154 static_cast<uint64_t>(pertoken.GetDim(mDimIdx)) != inputParams_.mSize,161 static_cast<uint64_t>(pertoken.GetDim(mDimIdx)) != inputParams_.mSize,
155 CUBE_INNER_ERR_REPORT(162 CUBE_INNER_ERR_REPORT(
156 inputParams_.opName,163 inputParams_.opName,
157- "In hif4 quantification, when x1's transposition is %s, pertokenScale's dimension %d shape \164+ "In hif4 quantification, when x1's transposition is %s, pertokenScale's dimension %zu shape \
158should be equal to x1 m dimension size[%lu], but actual is [%ld].",165should be equal to x1 m dimension size[%lu], but actual is [%ld].",
159 inputParams_.transA ? "true" : "false", mDimIdx, inputParams_.mSize, pertoken.GetDim(mDimIdx)),166 inputParams_.transA ? "true" : "false", mDimIdx, inputParams_.mSize, pertoken.GetDim(mDimIdx)),
160 return false);167 return false);
@@ -166,12 +173,15 @@ bool QuantBatchMatmulV3CheckerArch9201::Hif4ScaleShapeCheck(const gert::Shape& s
166{173{
167 auto scaleShapeLen = scaleShape.GetDimNum();174 auto scaleShapeLen = scaleShape.GetDimNum();
168 OP_TILING_CHECK(175 OP_TILING_CHECK(
169- scaleShapeLen != SCALE_THREE_DIM,176+ scaleShapeLen < SCALE_THREE_DIM,
170- CUBE_INNER_ERR_REPORT(inputParams_.opName,177+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
171- "In hif4 quantification, scale's dimension must be 3, but it is %zu.", scaleShapeLen),178+ inputParams_.opName, "scale", FormatString("%zuD", scaleShapeLen).c_str(),
179+ FormatString("in hif4 quantification, the shape dim of scale must be at least %zuD", SCALE_THREE_DIM)
180+ .c_str()),
172 return false);181 return false);
173- auto kDimIdx = inputParams_.transB ? 1 : 0;182+ size_t batchDimNum = scaleShapeLen - SCALE_THREE_DIM;
174- auto nDimIdx = inputParams_.transB ? 0 : 1;183+ auto kDimIdx = batchDimNum + (inputParams_.transB ? 1 : 0);
184+ auto nDimIdx = batchDimNum + (inputParams_.transB ? 0 : 1);
175 OP_TILING_CHECK(static_cast<uint64_t>(scaleShape.GetDim(scaleShapeLen - 1)) != 1,185 OP_TILING_CHECK(static_cast<uint64_t>(scaleShape.GetDim(scaleShapeLen - 1)) != 1,
176 CUBE_INNER_ERR_REPORT(inputParams_.opName,186 CUBE_INNER_ERR_REPORT(inputParams_.opName,
177 "In hif4 quantification, scale's dimension 2 shape should be equal to 1, but \187 "In hif4 quantification, scale's dimension 2 shape should be equal to 1, but \
@@ -191,14 +201,14 @@ actual is [%ld].",
191 static_cast<uint64_t>(scaleShape.GetDim(kDimIdx)) != ops::CeilDiv(inputParams_.kSize, HIF4_GROUP_SIZE),201 static_cast<uint64_t>(scaleShape.GetDim(kDimIdx)) != ops::CeilDiv(inputParams_.kSize, HIF4_GROUP_SIZE),
192 CUBE_INNER_ERR_REPORT(202 CUBE_INNER_ERR_REPORT(
193 inputParams_.opName,203 inputParams_.opName,
194- "In hif4 quantification, when x2's transposition is %s, scale's dimension %d shape should \204+ "In hif4 quantification, when x2's transposition is %s, scale's dimension %zu shape should \
195be equal to x2 k dimension size[%lu] ceil div 64, but actual is [%ld].",205be equal to x2 k dimension size[%lu] ceil div 64, but actual is [%ld].",
196 inputParams_.transB ? "true" : "false", kDimIdx, inputParams_.kSize, scaleShape.GetDim(kDimIdx)),206 inputParams_.transB ? "true" : "false", kDimIdx, inputParams_.kSize, scaleShape.GetDim(kDimIdx)),
197 return false);207 return false);
198 OP_TILING_CHECK(208 OP_TILING_CHECK(
199 static_cast<uint64_t>(scaleShape.GetDim(nDimIdx)) != inputParams_.nSize,209 static_cast<uint64_t>(scaleShape.GetDim(nDimIdx)) != inputParams_.nSize,
200 CUBE_INNER_ERR_REPORT(210 CUBE_INNER_ERR_REPORT(
201- inputParams_.opName, "In hif4 quantification, when x2's transposition is %s, scale's dimension %d \211+ inputParams_.opName, "In hif4 quantification, when x2's transposition is %s, scale's dimension %zu \
202shape should be equal to x2 n dimension size[%lu], but actual is [%ld].",212shape should be equal to x2 n dimension size[%lu], but actual is [%ld].",
203 inputParams_.transB ? "true" : "false", nDimIdx, inputParams_.nSize, scaleShape.GetDim(nDimIdx)),213 inputParams_.transB ? "true" : "false", nDimIdx, inputParams_.nSize, scaleShape.GetDim(nDimIdx)),
204 return false);214 return false);
@@ -206,9 +216,51 @@ shape should be equal to x2 n dimension size[%lu], but actual is [%ld].",
206 return true;216 return true;
207}217}
208 218 
219+bool QuantBatchMatmulV3CheckerArch9201::CheckBatchValidInHif4PerGroupMode(const gert::Shape& scaleShape,
220+ const gert::Shape& pertoken,
221+ const gert::Shape& x1Shape,
222+ const gert::Shape& x2Shape) const
223+{
224+ auto x1ShapeLen = x1Shape.GetDimNum();
225+ auto x2ShapeLen = x2Shape.GetDimNum();
226+ auto pertokenBatchLen = pertoken.GetDimNum() - SCALE_THREE_DIM;
227+ auto scaleBatchLen = scaleShape.GetDimNum() - SCALE_THREE_DIM;
228+ auto x1BatchLen = (x1ShapeLen > DIM_NUM_TWO) ? (x1ShapeLen - DIM_NUM_TWO) : 0;
229+ auto x2BatchLen = (x2ShapeLen > DIM_NUM_TWO) ? (x2ShapeLen - DIM_NUM_TWO) : 0;
230+ OP_TILING_CHECK(scaleBatchLen != x2BatchLen,
231+ OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
232+ inputParams_.opName, "scaleBatchDimNum, x2BatchDimNum",
233+ FormatString("%zu, %zu", scaleBatchLen, x2BatchLen).c_str(),
234+ "in hif4 quantification, the batch dimension num of scale must be equal to that of x2"),
235+ return false);
236+ OP_TILING_CHECK(pertokenBatchLen != x1BatchLen,
237+ OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
238+ inputParams_.opName, "pertokenBatchDimNum, x1BatchDimNum",
239+ FormatString("%zu, %zu", pertokenBatchLen, x1BatchLen).c_str(),
240+ "in hif4 quantification, the batch dimension num of pertokenScale must be equal to that of x1"),
241+ return false);
242+ for (size_t i = 0; i < x2BatchLen; ++i) {
243+ OP_TILING_CHECK(scaleShape.GetDim(i) != x2Shape.GetDim(i),
244+ OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
245+ inputParams_.opName, "dimIndex, x2Batch, scaleBatch",
246+ FormatString("%zu, %ld, %ld", i, x2Shape.GetDim(i), scaleShape.GetDim(i)).c_str(),
247+ "in hif4 quantification, the batch dimension of scale must be equal to that of x2"),
248+ return false);
249+ }
250+ for (size_t i = 0; i < x1BatchLen; ++i) {
251+ OP_TILING_CHECK(pertoken.GetDim(i) != x1Shape.GetDim(i),
252+ OP_LOGE_FOR_INVALID_VALUES_WITH_REASON(
253+ inputParams_.opName, "dimIndex, x1Batch, pertokenBatch",
254+ FormatString("%zu, %ld, %ld", i, x1Shape.GetDim(i), pertoken.GetDim(i)).c_str(),
255+ "in hif4 quantification, the batch dimension of pertokenScale must be equal to that of x1"),
256+ return false);
257+ }
258+ return true;
259+}
260+ 
209bool QuantBatchMatmulV3CheckerArch9201::CheckInputValidInHif4PerGroupMode(261bool QuantBatchMatmulV3CheckerArch9201::CheckInputValidInHif4PerGroupMode(
210- const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape,262+ const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape, const gert::Shape& x1Shape,
211- const std::vector<int64_t>& dimValueOfMKN) const263+ const gert::Shape& x2Shape, const std::vector<int64_t>& dimValueOfMKN) const
212{264{
213 if (!inputParams_.isHif4PerGroup) {265 if (!inputParams_.isHif4PerGroup) {
214 return true;266 return true;
@@ -232,6 +284,9 @@ groupSize = groupSizeK | groupSizeN << 16 | groupSizeM << 32.",
232 CUBE_INNER_ERR_REPORT(inputParams_.opName, "The perTokenScale shape check failed."), return false);284 CUBE_INNER_ERR_REPORT(inputParams_.opName, "The perTokenScale shape check failed."), return false);
233 OP_TILING_CHECK(!Hif4ScaleShapeCheck(scaleShape),285 OP_TILING_CHECK(!Hif4ScaleShapeCheck(scaleShape),
234 CUBE_INNER_ERR_REPORT(inputParams_.opName, "The scale shape check failed."), return false);286 CUBE_INNER_ERR_REPORT(inputParams_.opName, "The scale shape check failed."), return false);
287+ if (!CheckBatchValidInHif4PerGroupMode(scaleShape, pertokenShape->GetStorageShape(), x1Shape, x2Shape)) {
288+ return false;
289+ }
235 // hif4 场景下 K 需大于 2(与 aclnn/torch 接口约束一致,K<=2 时无法判断是否转置)290 // hif4 场景下 K 需大于 2(与 aclnn/torch 接口约束一致,K<=2 时无法判断是否转置)
236 OP_TILING_CHECK(291 OP_TILING_CHECK(
237 inputParams_.kSize <= 2UL,292 inputParams_.kSize <= 2UL,
@@ -36,12 +36,16 @@ protected:
36 bool CheckHif4ScaleDtypeWithPertoken() const override;36 bool CheckHif4ScaleDtypeWithPertoken() const override;
37 bool CheckExtraOutputDtype() const override;37 bool CheckExtraOutputDtype() const override;
38 bool CheckExtraShapeValid(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape,38 bool CheckExtraShapeValid(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape,
39+ const gert::Shape& x1Shape, const gert::Shape& x2Shape,
39 const std::vector<int64_t>& dimValueOfMKN) const override;40 const std::vector<int64_t>& dimValueOfMKN) const override;
40 41 
41private:42private:
42 bool Hif4PertokenScaleShapeCheck(const gert::StorageShape* pertokenShape) const;43 bool Hif4PertokenScaleShapeCheck(const gert::StorageShape* pertokenShape) const;
43 bool Hif4ScaleShapeCheck(const gert::Shape& scaleShape) const;44 bool Hif4ScaleShapeCheck(const gert::Shape& scaleShape) const;
45+ bool CheckBatchValidInHif4PerGroupMode(const gert::Shape& scaleShape, const gert::Shape& pertoken,
46+ const gert::Shape& x1Shape, const gert::Shape& x2Shape) const;
44 bool CheckInputValidInHif4PerGroupMode(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape,47 bool CheckInputValidInHif4PerGroupMode(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape,
48+ const gert::Shape& x1Shape, const gert::Shape& x2Shape,
45 const std::vector<int64_t>& dimValueOfMKN) const;49 const std::vector<int64_t>& dimValueOfMKN) const;
46};50};
47} // namespace optiling51} // namespace optiling
@@ -800,14 +800,20 @@ bool QuantBatchMatmulV3TilingBase::AnalyzeHif4GroupInfo(const gert::Shape& scale
800 if (inputParams_.groupSizeK == 0UL) {800 if (inputParams_.groupSizeK == 0UL) {
801 inputParams_.groupSizeK = hif4GroupSize;801 inputParams_.groupSizeK = hif4GroupSize;
802 }802 }
803- if (pertokenShapeLen != hif4ScaleDimNum || scaleShapeLen != hif4ScaleDimNum) {803+ if (pertokenShapeLen < hif4ScaleDimNum || scaleShapeLen < hif4ScaleDimNum) {
804 // scale 维度非法时由后续 checker 报错804 // scale 维度非法时由后续 checker 报错
805 return true;805 return true;
806 }806 }
807+ const size_t pertokenBatchDimNum = pertokenShapeLen - hif4ScaleDimNum;
808+ const size_t scaleBatchDimNum = scaleShapeLen - hif4ScaleDimNum;
809+ const size_t pertokenMDimIdx = pertokenBatchDimNum + (inputParams_.transA ? 1 : 0);
810+ const size_t pertokenKDimIdx = pertokenBatchDimNum + (inputParams_.transA ? 0 : 1);
811+ const size_t scaleKDimIdx = scaleBatchDimNum + (inputParams_.transB ? 1 : 0);
812+ const size_t scaleNDimIdx = scaleBatchDimNum + (inputParams_.transB ? 0 : 1);
807 uint64_t expectedScaleK = ops::CeilDiv(inputParams_.kSize, hif4GroupSize);813 uint64_t expectedScaleK = ops::CeilDiv(inputParams_.kSize, hif4GroupSize);
808 // 存在 1 维广播时 K 维校验交由 checker 判定,此处只校验非广播场景的 K 维覆盖 ceil(K / 64)814 // 存在 1 维广播时 K 维校验交由 checker 判定,此处只校验非广播场景的 K 维覆盖 ceil(K / 64)
809- if (pertoken.GetDim(0) != 1 && pertoken.GetDim(1) != 1) {815+ if (pertoken.GetDim(pertokenMDimIdx) != 1 && pertoken.GetDim(pertokenKDimIdx) != 1) {
810- uint64_t pertokenKSize = static_cast<uint64_t>(pertoken.GetDim(inputParams_.transA ? 0 : 1));816+ uint64_t pertokenKSize = static_cast<uint64_t>(pertoken.GetDim(pertokenKDimIdx));
811 OP_TILING_CHECK(817 OP_TILING_CHECK(
812 pertokenKSize != expectedScaleK,818 pertokenKSize != expectedScaleK,
813 CUBE_INNER_ERR_REPORT(inputParams_.opName,819 CUBE_INNER_ERR_REPORT(inputParams_.opName,
@@ -816,8 +822,8 @@ bool QuantBatchMatmulV3TilingBase::AnalyzeHif4GroupInfo(const gert::Shape& scale
816 pertokenKSize, inputParams_.kSize, expectedScaleK),822 pertokenKSize, inputParams_.kSize, expectedScaleK),
817 return false);823 return false);
818 }824 }
819- if (scaleShape.GetDim(0) != 1 && scaleShape.GetDim(1) != 1) {825+ if (scaleShape.GetDim(scaleKDimIdx) != 1 && scaleShape.GetDim(scaleNDimIdx) != 1) {
820- uint64_t scaleKSize = static_cast<uint64_t>(scaleShape.GetDim(inputParams_.transB ? 1 : 0));826+ uint64_t scaleKSize = static_cast<uint64_t>(scaleShape.GetDim(scaleKDimIdx));
821 OP_TILING_CHECK(827 OP_TILING_CHECK(
822 scaleKSize != expectedScaleK,828 scaleKSize != expectedScaleK,
823 CUBE_INNER_ERR_REPORT(inputParams_.opName,829 CUBE_INNER_ERR_REPORT(inputParams_.opName,
@@ -448,7 +448,8 @@ UT_STATIC __global__ __aicore__ void quant_batch_matmul_v3(GM_ADDR x1, GM_ADDR x
448 448 
449#if QBMMV3_IS_HIF4_DTYPE_TPL && defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)449#if QBMMV3_IS_HIF4_DTYPE_TPL && defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)
450 if constexpr (DequantBmm::IsHif4Type<DTYPE_X1, DTYPE_SCALE>()) {450 if constexpr (DequantBmm::IsHif4Type<DTYPE_X1, DTYPE_SCALE>()) {
451- // hif4: 9202 全部走 CMCT;9201 按 api level 分流 Blaze/CMCT451+ // hif4: 9201 ASW/StreamK 均走 Blaze(BASIC 仅兜底 tensorapi 不可用);
452+ // 9202 仅 StreamK 走 Blaze,其余 ASW 走 CMCT
452#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201) && IS_BLAZE453#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201) && IS_BLAZE
453 if constexpr (TPL_BATCHMODE == TPL_WITHOUT_BATCH && TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI &&454 if constexpr (TPL_BATCHMODE == TPL_WITHOUT_BATCH && TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI &&
454 TPL_APILEVEL == TPL_API_LEVEL_BLAZE) {455 TPL_APILEVEL == TPL_API_LEVEL_BLAZE) {
@@ -458,28 +459,25 @@ UT_STATIC __global__ __aicore__ void quant_batch_matmul_v3(GM_ADDR x1, GM_ADDR x
458 TPL_APILEVEL == TPL_API_LEVEL_BLAZE) {459 TPL_APILEVEL == TPL_API_LEVEL_BLAZE) {
459 QUANT_BMMV3_MX_WITHOUT_BATCH_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn,460 QUANT_BMMV3_MX_WITHOUT_BATCH_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn,
460 Blaze::Gemm::A_FULL_LOAD_MODE);461 Blaze::Gemm::A_FULL_LOAD_MODE);
462+ } else if constexpr (TPL_BATCHMODE == TPL_WITH_BATCH && TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI &&
463+ TPL_APILEVEL == TPL_API_LEVEL_BLAZE) {
464+ QUANT_BMMV3_MX_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn, 0);
atomgit-bot
atomgit-botatomgit-bot25 days ago

🟡 Medium Priority

改动点:kernel 新增 TPL_WITH_BATCH + TPL_NO_VEC_EPILOGUE_WITH_MMAPI + TPL_API_LEVEL_BLAZE 的 hif4 入口(quant_batch_matmul_v3.cpp:457-459),并在 quant_batch_matmul_v3_apt_tiling_key.h:372-377 新注册该 key;PR 说明“未进行设备侧数值验证”。

问题:这条路由由三方隐式约定构成——host 侧 AdaptiveSlidingWindowMXBasicAPITiling::GetBatchMode()/GetKernelType()/GetApiLevel()(adaptive_sliding_window_mx_basic_api_tiling.cpp:141-165)→ tiling key 注册表(apt_tiling_key.h:372-377)→ kernel 分支(quant_batch_matmul_v3.cpp:457-459)。kernel 侧 if constexpr 链没有兜底分支,未命中的 key 会被编译成“空 kernel”(不写 y),失败模式是静默输出未初始化。当前 host 侧能生成该 key 完全依赖 UpdateAFullLoadStatus() 中 inputParams_.batchC == 1 这一分散在两处(非自适应分支 536 行、自适应分支 546 行)的隐式不变量。

建议:在 op_host tiling UT 中新增上述 3 个组合的 tiling key 断言(hif4 9201 batch>1 / batchC==1、hif4 9202),确保 host 下发的 key 与 apt_tiling_key.h 的注册集合、kernel 的 if-constexpr 分支三者一致;若框架允许,可在 kernel 尾部加静态断言/兜底分支替代“静默空 kernel”。

likedislike
Incorrect?
461 }465 }
462#endif466#endif
463-#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)467+ // 9202 ASW / 9201 tensorapi fallback
464- if constexpr (TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI && TPL_APILEVEL == TPL_API_LEVEL_BASIC) {468+ if constexpr (TPL_BATCHMODE == TPL_WITH_BATCH && TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI &&
panzhija
panzhijapanzhija15 days ago

[正确性][必须修复] 🔴 补齐 CMCT 的 HIF4 batch scale 偏移

本 PR 的 checker 已允许 HIF4 的 batch scale,9202 的 batch>1 输入会进入这里,9201 的 TensorAPI fallback 也受影响。例如 transA=transB=false,x1=[2,16,256]、x2=[2,256,32]、pertokenScale=[2,16,4,1]、scale=[2,4,32,1] 时,matmul/common/cmct/kernel/kernel_qbmm_mx.h:305-317 的 AddBatchOffset() 仍以 if constexpr (!BlockMmad::IS_HIF4) 跳过两路 scale 偏移:第二个 batch 的 A/B 已前移,而 scale 仍使用第一个 batch,batch 间 scale 不同时结果错误。固定 Head 原函数的 CPU 隔离复现得到第二个 batch 的 scale 偏移为 0/0,正确值应为 64/128 个 scale 元素。需要为 HIF4 补上按 batchAOffset_ * M * ceil(K/64) / batchBOffset_ * N * ceil(K/64) 计算的偏移,并在现有 CSV 中覆盖各 batch scale 不同及 batch 广播场景。

likedislike
wangzitao
wangzitao
12 days ago Comments:
likedislike
469+ TPL_APILEVEL == TPL_API_LEVEL_BASIC) {
465 QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign, 0);470 QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign, 0);
466- } else if constexpr (TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI &&471+ } else if constexpr (TPL_BATCHMODE == TPL_WITH_BATCH &&
472+ TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI &&
467 TPL_APILEVEL == TPL_API_LEVEL_BASIC) {473 TPL_APILEVEL == TPL_API_LEVEL_BASIC) {
468 QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign,474 QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign,
469 Cmct::Gemm::A_FULL_LOAD_MODE);475 Cmct::Gemm::A_FULL_LOAD_MODE);
470 }476 }
471-#else477+#if IS_BLAZE
472- if constexpr (TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI) {
473- QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign, 0);
474- } else if constexpr (TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI) {
475- QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign,
476- Cmct::Gemm::A_FULL_LOAD_MODE);
477- }
478-#endif
479-#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202) && IS_BLAZE
480 if constexpr (TPL_BATCHMODE == TPL_WITHOUT_BATCH && TPL_KERNELTYPE == TPL_VEC_EPILOGUE_STREAMK_WITH_MMAPI &&478 if constexpr (TPL_BATCHMODE == TPL_WITHOUT_BATCH && TPL_KERNELTYPE == TPL_VEC_EPILOGUE_STREAMK_WITH_MMAPI &&
481 TPL_APILEVEL == TPL_API_LEVEL_BLAZE) {479 TPL_APILEVEL == TPL_API_LEVEL_BLAZE) {
482- // hif4 streamk, blaze(9201/9202)480+ // hif4 streamk: 9201/9202 均走 Blaze
483 QUANT_BMMV3_MX_STREAMK_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn, 0);481 QUANT_BMMV3_MX_STREAMK_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn, 0);
484 }482 }
485#endif483#endif
@@ -185,7 +185,7 @@ namespace QuantBatchMatmulV3Arch35TilingKey {
185 185 
186// Kernel type 11 is shared by the MX StreamK path, the non-MX per-tensor StreamK path and the HIF4 StreamK path.186// Kernel type 11 is shared by the MX StreamK path, the non-MX per-tensor StreamK path and the HIF4 StreamK path.
187#define SUPPORT_STREAMK_TILING_KEY \187#define SUPPORT_STREAMK_TILING_KEY \
188- (SUPPORT_MX_WITHOUT_BATCH_TILING_KEY || SUPPORT_NON_MX_STREAMK_TILING_KEY || QBMMV3_IS_HIF4_DTYPE_TPL)188+ (SUPPORT_MX_WITHOUT_BATCH_TILING_KEY || SUPPORT_NON_MX_STREAMK_TILING_KEY || (IS_BLAZE && QBMMV3_IS_HIF4_DTYPE_TPL))
189 189 
190// Batch Mode190// Batch Mode
191#define TPL_WITH_BATCH 0191#define TPL_WITH_BATCH 0
@@ -361,20 +361,28 @@ ASCENDC_TPL_SEL(
361#endif361#endif
362#if ((!defined(__CCE_AICORE__)) || \362#if ((!defined(__CCE_AICORE__)) || \
363 (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202) && QBMMV3_IS_HIF4_DTYPE_TPL))363 (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202) && QBMMV3_IS_HIF4_DTYPE_TPL))
364- ASCENDC_TPL_ARGS_SEL( // hif4 sliding-window: kernel type {0, 1} * batch mode {0, 1} * api level {1, 2}364+#if ((!defined(__CCE_AICORE__)) || (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201) && IS_BLAZE))
365+ ASCENDC_TPL_ARGS_SEL( // hif4 9201 blaze without batch: kernel type {0, 1} * api level {2}
365 ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),366 ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),
366 ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),367 ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),
367 ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITHOUT_BATCH),368 ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITHOUT_BATCH),
368 ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI,369 ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI,
369 TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI),370 TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI),
370- ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BASIC, TPL_API_LEVEL_BLAZE)),371+ ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BLAZE)),
371- ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY),372+ ASCENDC_TPL_ARGS_SEL( // hif4 9201 blaze with batch: kernel type {0} (A full load is disabled for batch > 1)
372- ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),373+ ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),
373- ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),374+ ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),
374- ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITH_BATCH),375+ ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITH_BATCH),
375- ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI,376+ ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI),
376- TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI),377+ ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BLAZE)),
377- ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BASIC)),378+#endif
379+ ASCENDC_TPL_ARGS_SEL( // hif4 CMCT basic (ASW): 9202 and 9201 tensorapi fallback
380+ ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1),
381+ ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1),
382+ ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITH_BATCH),
383+ ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI,
384+ TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI),
385+ ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BASIC)),
378#endif386#endif
379 387 
380#if !defined(__CCE_AICORE__) || \388#if !defined(__CCE_AICORE__) || \
@@ -22,6 +22,20 @@ Hif4RejectsGroupN,DAV_9201,16 256,HIFLOAT4,256 32,HIFLOAT4,16 4 1,HIFLOAT4_SCALE
22Hif4RejectsGroupK,DAV_9201,16 256,HIFLOAT4,256 32,HIFLOAT4,16 4 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,16 32,ACL_BF16,false,false,4295032864,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false22Hif4RejectsGroupK,DAV_9201,16 256,HIFLOAT4,256 32,HIFLOAT4,16 4 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,16 32,ACL_BF16,false,false,4295032864,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
23Hif4RejectsOddInnerAxis,DAV_9201,256 15,HIFLOAT4,256 32,HIFLOAT4,4 15 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,15 32,ACL_BF16,true,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false23Hif4RejectsOddInnerAxis,DAV_9201,256 15,HIFLOAT4,256 32,HIFLOAT4,4 15 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,15 32,ACL_BF16,true,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
24Hif4RejectsKTooSmall,DAV_9201,16 2,HIFLOAT4,2 32,HIFLOAT4,16 1 1,HIFLOAT4_SCALE,1 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false24Hif4RejectsKTooSmall,DAV_9201,16 2,HIFLOAT4,2 32,HIFLOAT4,16 1 1,HIFLOAT4_SCALE,1 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
25+Hif4BatchScale4D,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false
26+Hif4BatchScale4DTrans,DAV_9201,2 256 16,HIFLOAT4,2 32 256,HIFLOAT4,2 4 16 1,HIFLOAT4_SCALE,2 32 4 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,true,true,4295032896,5,check,ACLNN_SUCCESS,true,0,false
27+Hif4BatchScale5D,DAV_9201,2 3 16 256,HIFLOAT4,2 3 256 32,HIFLOAT4,2 3 16 4 1,HIFLOAT4_SCALE,2 3 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false
28+Hif4BatchScale6D,DAV_9201,2 3 4 16 256,HIFLOAT4,2 3 4 256 32,HIFLOAT4,2 3 4 16 4 1,HIFLOAT4_SCALE,2 3 4 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 4 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false
29+Hif4BatchScale7D,DAV_9201,2 3 4 5 16 256,HIFLOAT4,2 3 4 5 256 32,HIFLOAT4,2 3 4 5 16 4 1,HIFLOAT4_SCALE,2 3 4 5 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 4 5 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false
30+Hif4BatchMixedScaleRanks,DAV_9201,2 3 16 256,HIFLOAT4,256 32,HIFLOAT4,2 3 16 4 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false
31+Hif4ScaleBatchOnly,DAV_9201,16 256,HIFLOAT4,2 256 32,HIFLOAT4,16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false
32+Hif4RejectsScaleOver7D,DAV_9201,2 3 4 5 6 16 256,HIFLOAT4,2 3 4 5 6 256 32,HIFLOAT4,2 3 4 5 6 16 4 1,HIFLOAT4_SCALE,2 3 4 5 6 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 4 5 6 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
33+Hif4RejectsPertokenBatchCount,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 3 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
34+Hif4RejectsPertokenInputBatchCount,DAV_9201,16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
35+Hif4RejectsScaleBatchCount,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,2 3 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
36+Hif4RejectsScaleInputBatchCount,DAV_9201,2 16 256,HIFLOAT4,256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
37+Hif4RejectsPertokenBatchValue,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,3 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
38+Hif4RejectsScaleBatchValue,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,3 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false
25PergroupValidDAV_3510,DAV_3510,16 1024,ACL_INT4,256 1024,ACL_INT4,16 1,ACL_FLOAT,4 256,ACL_FLOAT,4 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_SUCCESS,true,0,true39PergroupValidDAV_3510,DAV_3510,16 1024,ACL_INT4,256 1024,ACL_INT4,16 1,ACL_FLOAT,4 256,ACL_FLOAT,4 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_SUCCESS,true,0,true
26PergroupValidDAV_9201,DAV_9201,16 1024,ACL_INT4,256 1024,ACL_INT4,16 1,ACL_FLOAT,4 256,ACL_FLOAT,4 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_SUCCESS,true,0,true40PergroupValidDAV_9201,DAV_9201,16 1024,ACL_INT4,256 1024,ACL_INT4,16 1,ACL_FLOAT,4 256,ACL_FLOAT,4 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_SUCCESS,true,0,true
27PergroupRejectsKAlignment,DAV_3510,16 512,ACL_INT4,256 512,ACL_INT4,16 1,ACL_FLOAT,2 256,ACL_FLOAT,2 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_ERR_PARAM_INVALID,true,0,true41PergroupRejectsKAlignment,DAV_3510,16 512,ACL_INT4,256 512,ACL_INT4,16 1,ACL_FLOAT,2 256,ACL_FLOAT,2 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_ERR_PARAM_INVALID,true,0,true
@@ -14,7 +14,6 @@
14 */14 */
15 15 
16#include <array>16#include <array>
17-#include <cmath>
18#include <limits>17#include <limits>
19#include <map>18#include <map>
20#include <memory>19#include <memory>
@@ -25,10 +24,12 @@
25#define private public24#define private public
26#define protected public25#define protected public
27#include "../test_quant_batch_matmul_v3_tiling.h"26#include "../test_quant_batch_matmul_v3_tiling.h"
27+#include "../../../../op_kernel/arch35/quant_batch_matmul_v3_apt_tiling_key.h"
28#include "../../../../op_host/op_tiling/arch35/qbmm_streamk_tiling_960.h"28#include "../../../../op_host/op_tiling/arch35/qbmm_streamk_tiling_960.h"
29#include "../../../../op_host/op_tiling/arch35/quant_batch_matmul_v3_checker_arch_9201.h"29#include "../../../../op_host/op_tiling/arch35/quant_batch_matmul_v3_checker_arch_9201.h"
30#include "../../../../op_host/op_tiling/arch35/tiling_cost_evaluator.h"30#include "../../../../op_host/op_tiling/arch35/tiling_cost_evaluator.h"
31#include "../../../../op_host/op_tiling/arch35/l1_tiling_data_calculator.h"31#include "../../../../op_host/op_tiling/arch35/l1_tiling_data_calculator.h"
32+#include "../../../../op_host/op_tiling/arch35/adaptive_sliding_window_mx_basic_api_tiling.h"
32#include "../../../../op_host/op_tiling/arch35/adaptive_sliding_window_perblock_basic_api_tiling.h"33#include "../../../../op_host/op_tiling/arch35/adaptive_sliding_window_perblock_basic_api_tiling.h"
33#include "../../../../op_host/op_tiling/arch35/quant_batch_matmul_v3_tiling_util.h"34#include "../../../../op_host/op_tiling/arch35/quant_batch_matmul_v3_tiling_util.h"
34#undef protected35#undef protected
@@ -712,9 +713,11 @@ TEST(QuantBatchMatmulV3Checker960, Hif4GroupValidationRejectsMissingScaleAndWron
712 auto input = MakeHif4Info();713 auto input = MakeHif4Info();
713 const gert::Shape scale({3, 48, 1});714 const gert::Shape scale({3, 48, 1});
714 const gert::StorageShape token({16, 3, 1}, {16, 3, 1});715 const gert::StorageShape token({16, 3, 1}, {16, 3, 1});
716+ const gert::Shape x1Shape({16, 130});
717+ const gert::Shape x2Shape({130, 48});
715 QuantBatchMatmulV3CheckerArch9201 valid(nullptr, input);718 QuantBatchMatmulV3CheckerArch9201 valid(nullptr, input);
716- EXPECT_TRUE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, {130, 16, 130, 48}));719+ EXPECT_TRUE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {130, 16, 130, 48}));
717- EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, nullptr, {130, 16, 130, 48}));720+ EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, nullptr, x1Shape, x2Shape, {130, 16, 130, 48}));
718 for (int axis = 0; axis < 3; ++axis) {721 for (int axis = 0; axis < 3; ++axis) {
719 auto invalid = input;722 auto invalid = input;
720 if (axis == 0) {723 if (axis == 0) {
@@ -727,10 +730,150 @@ TEST(QuantBatchMatmulV3Checker960, Hif4GroupValidationRejectsMissingScaleAndWron
727 invalid.groupSizeK = 32;730 invalid.groupSizeK = 32;
728 }731 }
729 QuantBatchMatmulV3CheckerArch9201 checker(nullptr, invalid);732 QuantBatchMatmulV3CheckerArch9201 checker(nullptr, invalid);
730- EXPECT_FALSE(checker.CheckInputValidInHif4PerGroupMode(scale, &token, {130, 16, 130, 48}));733+ EXPECT_FALSE(checker.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {130, 16, 130, 48}));
734+ }
735+ EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {129, 16, 130, 48}));
736+ EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {130, 16, 129, 48}));
737+}
738+ 
739+gert::Shape MakeHif4BatchShape(const std::vector<int64_t>& batch, int64_t dim0, int64_t dim1)
740+{
741+ gert::Shape shape;
742+ for (const auto dim : batch) {
743+ shape.AppendDim(dim);
744+ }
745+ shape.AppendDim(dim0);
746+ shape.AppendDim(dim1);
747+ shape.AppendDim(1);
748+ return shape;
749+}
750+ 
751+gert::Shape MakeHif4InputShape(const std::vector<int64_t>& batch, int64_t dim0, int64_t dim1)
752+{
753+ gert::Shape shape;
754+ for (const auto dim : batch) {
755+ shape.AppendDim(dim);
756+ }
757+ shape.AppendDim(dim0);
758+ shape.AppendDim(dim1);
759+ return shape;
760+}
761+ 
762+TEST(QuantBatchMatmulV3Checker960, Hif4ScaleShapesAcceptBatchDimsUpToSevenD)
763+{
764+ auto input = MakeHif4Info();
765+ // Scale/pertoken ranks 4D~7D carry one to four leading batch dims.
766+ const std::vector<std::vector<int64_t>> batchDims{{2}, {2, 3}, {2, 3, 4}, {2, 3, 4, 5}};
767+ for (bool transpose : {false, true}) {
768+ SCOPED_TRACE(transpose);
769+ input.transA = input.transB = transpose;
770+ QuantBatchMatmulV3CheckerArch9201 checker(nullptr, input);
771+ for (const auto& batch : batchDims) {
772+ gert::Shape scale = MakeHif4BatchShape(batch, transpose ? 48 : 3, transpose ? 3 : 48);
773+ gert::Shape token = MakeHif4BatchShape(batch, transpose ? 3 : 16, transpose ? 16 : 3);
774+ auto tokenStorage = MakeStorageShape(token);
775+ EXPECT_TRUE(checker.Hif4ScaleShapeCheck(scale));
776+ EXPECT_TRUE(checker.Hif4PertokenScaleShapeCheck(&tokenStorage));
777+ gert::Shape x1Shape = MakeHif4InputShape(batch, transpose ? 130 : 16, transpose ? 16 : 130);
778+ gert::Shape x2Shape = MakeHif4InputShape(batch, transpose ? 48 : 130, transpose ? 130 : 48);
779+ EXPECT_TRUE(
780+ checker.CheckInputValidInHif4PerGroupMode(scale, &tokenStorage, x1Shape, x2Shape, {130, 16, 130, 48}));
781+ }
782+ }
783+}
784+ 
785+TEST(QuantBatchMatmulV3Checker960, Hif4BatchValidationRejectsMismatchedBatchDims)
786+{
787+ auto input = MakeHif4Info();
788+ const gert::Shape scale({2, 3, 48, 1});
789+ const gert::StorageShape token({2, 16, 3, 1}, {2, 16, 3, 1});
790+ const gert::Shape x1Shape({2, 16, 130});
791+ const gert::Shape x2Shape({2, 130, 48});
792+ QuantBatchMatmulV3CheckerArch9201 checker(nullptr, input);
793+ EXPECT_TRUE(checker.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {130, 16, 130, 48}));
794+ 
795+ const gert::StorageShape tokenExtraBatch({2, 2, 16, 3, 1}, {2, 2, 16, 3, 1});
796+ EXPECT_FALSE(
797+ checker.CheckInputValidInHif4PerGroupMode(scale, &tokenExtraBatch, x1Shape, x2Shape, {130, 16, 130, 48}));
798+ const gert::StorageShape tokenWrongBatch({3, 16, 3, 1}, {3, 16, 3, 1});
799+ EXPECT_FALSE(
800+ checker.CheckInputValidInHif4PerGroupMode(scale, &tokenWrongBatch, x1Shape, x2Shape, {130, 16, 130, 48}));
801+ 
802+ const gert::Shape scaleExtraBatch({2, 2, 3, 48, 1});
803+ EXPECT_FALSE(
804+ checker.CheckInputValidInHif4PerGroupMode(scaleExtraBatch, &token, x1Shape, x2Shape, {130, 16, 130, 48}));
805+ const gert::Shape scaleWrongBatch({3, 3, 48, 1});
806+ EXPECT_FALSE(
807+ checker.CheckInputValidInHif4PerGroupMode(scaleWrongBatch, &token, x1Shape, x2Shape, {130, 16, 130, 48}));
808+}
809+ 
810+class TestableMxBasicApiTiling : public AdaptiveSlidingWindowMXBasicAPITiling {
811+public:
812+ TestableMxBasicApiTiling() : AdaptiveSlidingWindowMXBasicAPITiling(nullptr) {}
813+ bool IsTensorApiEnabled() const override { return tensorApiEnabled; }
814+ bool tensorApiEnabled = true;
815+};
816+ 
817+TEST(QuantBatchMatmulV3Checker960, Hif4BatchUsesBlazeBatchTilingKeyOn9201)
818+{
819+ using namespace QuantBatchMatmulV3Arch35TilingKey;
820+ for (bool transpose : {false, true}) {
821+ SCOPED_TRACE(transpose);
822+ const auto trans = static_cast<uint64_t>(transpose);
823+ TestableMxBasicApiTiling tiling;
824+ auto& input = tiling.inputParams_;
825+ input.Reset();
826+ input.opName = "QuantBatchMatmulV3Hif4BatchTiling";
827+ input.aDtype = input.bDtype = Ops::NN::DT_HIFLOAT4;
828+ input.scaleDtype = input.perTokenScaleDtype = Ops::NN::DT_HIFLOAT4_SCALE;
829+ input.mSize = 268;
830+ input.nSize = 3680;
831+ input.kSize = 160;
832+ input.batchA = input.batchB = input.batchC = 2;
833+ input.batchA1 = input.batchB1 = input.batchC1 = 2;
834+ input.groupSizeM = input.groupSizeN = 1;
835+ input.groupSizeK = 64;
836+ input.isHif4PerGroup = true;
837+ input.transA = input.transB = transpose;
838+ tiling.compileInfo_.npuArch = NpuArch::DAV_9201;
839+ tiling.isAFullLoad_ = false;
840+ // baseK=64 used to force the HIF4 CMCT fallback; the API level must no longer depend on it.
841+ tiling.basicTiling_.baseK = static_cast<uint32_t>(qmmv3_tiling_const::MXFP_DIVISOR_SIZE);
842+ 
843+ EXPECT_FALSE(tiling.IsWithoutBatchTilingData());
844+ EXPECT_EQ(tiling.GetBatchMode(), static_cast<uint64_t>(TPL_WITH_BATCH));
845+ EXPECT_EQ(tiling.GetApiLevel(NpuArch::DAV_9201), static_cast<uint64_t>(TPL_API_LEVEL_BLAZE));
846+ EXPECT_EQ(tiling.GetKernelType(), static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI));
847+ EXPECT_EQ(tiling.GetTilingKey(), GET_TPL_TILING_KEY(trans, trans, static_cast<uint64_t>(TPL_WITH_BATCH),
848+ static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI),
849+ static_cast<uint64_t>(TPL_API_LEVEL_BLAZE)));
850+ 
851+ // 9202 keeps using the CMCT Basic path for the same batched HIF4 shape.
852+ tiling.compileInfo_.npuArch = NpuArch::DAV_9202;
853+ EXPECT_EQ(tiling.GetApiLevel(NpuArch::DAV_9202), static_cast<uint64_t>(TPL_API_LEVEL_BASIC));
854+ EXPECT_EQ(tiling.GetTilingKey(), GET_TPL_TILING_KEY(trans, trans, static_cast<uint64_t>(TPL_WITH_BATCH),
855+ static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI),
856+ static_cast<uint64_t>(TPL_API_LEVEL_BASIC)));
857+ 
858+ // TensorAPI unavailable on 9201 falls back to CMCT Basic with the same batch mode.
859+ tiling.compileInfo_.npuArch = NpuArch::DAV_9201;
860+ tiling.tensorApiEnabled = false;
861+ EXPECT_FALSE(tiling.IsWithoutBatchTilingData());
862+ EXPECT_EQ(tiling.GetApiLevel(NpuArch::DAV_9201), static_cast<uint64_t>(TPL_API_LEVEL_BASIC));
863+ EXPECT_EQ(tiling.GetTilingKey(), GET_TPL_TILING_KEY(trans, trans, static_cast<uint64_t>(TPL_WITH_BATCH),
864+ static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI),
865+ static_cast<uint64_t>(TPL_API_LEVEL_BASIC)));
866+ 
867+ // batchC == 1 still selects the WithoutBatch Blaze tiling data and key.
868+ tiling.tensorApiEnabled = true;
869+ input.batchA = input.batchB = input.batchC = 1;
870+ input.batchA1 = input.batchB1 = input.batchC1 = 1;
871+ EXPECT_TRUE(tiling.IsWithoutBatchTilingData());
872+ EXPECT_EQ(tiling.GetBatchMode(), static_cast<uint64_t>(TPL_WITHOUT_BATCH));
873+ EXPECT_EQ(tiling.GetTilingKey(), GET_TPL_TILING_KEY(trans, trans, static_cast<uint64_t>(TPL_WITHOUT_BATCH),
874+ static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI),
875+ static_cast<uint64_t>(TPL_API_LEVEL_BLAZE)));
731 }876 }
732- EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, {129, 16, 130, 48}));
733- EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, {130, 16, 129, 48}));
734}877}
735 878 
736class QuantBatchMatmulV3StreamK960 : public testing::Test {879class QuantBatchMatmulV3StreamK960 : public testing::Test {
@@ -801,6 +944,26 @@ TEST_F(QuantBatchMatmulV3StreamK960, CapabilityRequiresMatchingCoresAndSingleBat
801 EXPECT_FALSE(tiling->IsCapable());944 EXPECT_FALSE(tiling->IsCapable());
802}945}
803 946 
947+TEST_F(QuantBatchMatmulV3StreamK960, CapabilityAcceptsHif4PerGroupWithoutBatch)
948+{
949+ auto& input = tiling->inputParams_;
950+ // HIF4 per-group is the only enabled quant mode here, so IsCapable goes through the HIF4 branch.
951+ input.isPerTensor = false;
952+ input.isHif4PerGroup = true;
953+ EXPECT_FALSE(tiling->IsPertensorStreamKInput());
954+ EXPECT_EQ(tiling->compileInfo_.aicNum, tiling->compileInfo_.aivNum);
955+ EXPECT_EQ(input.batchC, 1U);
956+ EXPECT_EQ(tiling->IsCapable(), IsTensorapiCapable());
957+ tiling->compileInfo_.aivNum = 64;
958+ EXPECT_FALSE(tiling->IsCapable());
959+ tiling->compileInfo_.aivNum = 32;
960+ input.batchC = 2;
961+ EXPECT_FALSE(tiling->IsCapable());
962+ input.batchC = 1;
963+ input.isHif4PerGroup = false;
964+ EXPECT_FALSE(tiling->IsCapable());
965+}
966+ 
804TEST_F(QuantBatchMatmulV3StreamK960, FloatingBiasMustMatchScaleAndAccumulationDomain)967TEST_F(QuantBatchMatmulV3StreamK960, FloatingBiasMustMatchScaleAndAccumulationDomain)
805{968{
806 auto& input = tiling->inputParams_;969 auto& input = tiling->inputParams_;
@@ -857,6 +1020,7 @@ TEST_F(QuantBatchMatmulV3StreamK960, WorkspaceAndSerializedDataMatchKernelContra
857 ASSERT_EQ(tiling->PostTiling(), ge::GRAPH_SUCCESS);1020 ASSERT_EQ(tiling->PostTiling(), ge::GRAPH_SUCCESS);
858 auto* context = holder.GetContext<gert::TilingContext>();1021 auto* context = holder.GetContext<gert::TilingContext>();
859 EXPECT_EQ(context->GetBlockDim(), 24);1022 EXPECT_EQ(context->GetBlockDim(), 24);
1023+ EXPECT_EQ(context->GetScheduleMode(), 1U);
860 EXPECT_EQ(context->GetRawTilingData()->GetDataSize(), sizeof(tiling->tilingData_));1024 EXPECT_EQ(context->GetRawTilingData()->GetDataSize(), sizeof(tiling->tilingData_));
861 EXPECT_EQ(context->GetWorkspaceSizes(1)[0], tiling->workspaceSize_);1025 EXPECT_EQ(context->GetWorkspaceSizes(1)[0], tiling->workspaceSize_);
862 const auto* data = reinterpret_cast<const DequantBmm::QuantBatchMatmulV3StreamKBasicAPITilingData*>(1026 const auto* data = reinterpret_cast<const DequantBmm::QuantBatchMatmulV3StreamKBasicAPITilingData*>(
@@ -565,6 +565,13 @@ Dav9201,Dav9201_hif4_group_00,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,0
565Dav9201,Dav9201_hif4_group_01,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,0,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131092,,FALSE,hif4_group_01,0,565Dav9201,Dav9201_hif4_group_01,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,0,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131092,,FALSE,hif4_group_01,0,
566Dav9201,Dav9201_hif4_group_10,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,1,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131089,,FALSE,hif4_group_10,0,566Dav9201,Dav9201_hif4_group_10,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,1,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131089,,FALSE,hif4_group_10,0,
567Dav9201,Dav9201_hif4_group_11,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,1,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131093,,FALSE,hif4_group_11,0,567Dav9201,Dav9201_hif4_group_11,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,1,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131093,,FALSE,hif4_group_11,0,
568+Dav9201,Dav9201_hif4_batch_00,,ascend960,32,32,3,3,3,2,2,2,268,160,3680,0,1,0,0,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131072,,FALSE,hif4_batch_00,0,
569+Dav9201,Dav9201_hif4_batch_01,,ascend960,32,32,3,3,3,2,2,2,268,160,3680,0,1,0,0,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131076,,FALSE,hif4_batch_01,0,
570+Dav9201,Dav9201_hif4_batch_10,,ascend960,32,32,3,3,3,2,2,2,268,160,3680,0,1,0,1,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131073,,FALSE,hif4_batch_10,0,
571+Dav9201,Dav9201_hif4_batch_11,,ascend960,32,32,3,3,3,2,2,2,268,160,3680,0,1,0,1,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131077,,FALSE,hif4_batch_11,0,
572+Dav9201,Dav9201_hif4_batch_4d,,ascend960,32,32,4,4,4,2,2,2,268,160,3680,0,1,0,0,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131072,,FALSE,hif4_batch_4d,0,
573+Dav9201,Dav9201_hif4_batch_5d,,ascend960,32,32,5,5,5,2,2,2,268,160,3680,0,1,0,0,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131072,,FALSE,hif4_batch_5d,0,
574+Dav9201,Dav9201_hif4_batch_6d,,ascend960,32,32,6,6,6,2,2,2,268,160,3680,0,1,0,0,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131072,,FALSE,hif4_batch_6d,0,
568Dav9201,Dav9201_large_grid_cube,,ascend960,32,32,2,2,2,0,0,0,8192,4096,8192,0,0,0,0,0,0,INT8,INT8,UINT64,FLOAT,INT32,BF16,ND,ND,TRUE,32,131088,,FALSE,large_grid_cube,0,575Dav9201,Dav9201_large_grid_cube,,ascend960,32,32,2,2,2,0,0,0,8192,4096,8192,0,0,0,0,0,0,INT8,INT8,UINT64,FLOAT,INT32,BF16,ND,ND,TRUE,32,131088,,FALSE,large_grid_cube,0,
569Dav9201,Dav9201_perchannel_bias,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,0,1,0,0,1,INT8,INT8,UINT64,FLOAT,INT32,BF16,ND,ND,TRUE,32,131088,,FALSE,perchannel_bias,0,576Dav9201,Dav9201_perchannel_bias,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,0,1,0,0,1,INT8,INT8,UINT64,FLOAT,INT32,BF16,ND,ND,TRUE,32,131088,,FALSE,perchannel_bias,0,
570Dav9201,Dav9201_pertoken_mix,,ascend960,32,32,2,2,2,0,0,0,256,512,1024,0,1,0,0,0,1,INT8,INT8,FLOAT,FLOAT,INT32,BF16,ND,ND,TRUE,32,131600,,FALSE,pertoken_mix,0,577Dav9201,Dav9201_pertoken_mix,,ascend960,32,32,2,2,2,0,0,0,256,512,1024,0,1,0,0,0,1,INT8,INT8,FLOAT,FLOAT,INT32,BF16,ND,ND,TRUE,32,131600,,FALSE,pertoken_mix,0,
@@ -15,23 +15,18 @@
15#include <cstring>15#include <cstring>
16#include <exception>16#include <exception>
17#include <mutex>17#include <mutex>
18-#include <iostream>
19#include <fstream>18#include <fstream>
20#include <string>19#include <string>
21#include <thread>20#include <thread>
22#include <vector>21#include <vector>
23 22 
24-#include "log/log.h"
25-#include "nlohmann/json.hpp"
26#include "op_common/op_host/util/op_const_def.h"23#include "op_common/op_host/util/op_const_def.h"
27 24 
28#define protected public25#define protected public
29#define private public26#define private public
30-#include "op_host/tiling_templates_registry.h"
31#include "ut_op_util.h"27#include "ut_op_util.h"
32#include "exe_graph/runtime/storage_format.h"28#include "exe_graph/runtime/storage_format.h"
33#include "exe_graph/runtime/storage_shape.h"29#include "exe_graph/runtime/storage_shape.h"
34-#include "exe_graph/runtime/tiling_parse_context.h"
35#include "kernel_run_context_facker.h"30#include "kernel_run_context_facker.h"
36#include "test_cube_util.h"31#include "test_cube_util.h"
37#include "test_quant_batch_matmul_v3_tiling.h"32#include "test_quant_batch_matmul_v3_tiling.h"
@@ -87,6 +82,18 @@ static void SetExpectedTilingFieldIfPresent(std::vector<int32_t>& tilingDataInt,
87 std::memcpy(reinterpret_cast<char*>(tilingDataInt.data()) + fieldOffset, &value, sizeof(T));82 std::memcpy(reinterpret_cast<char*>(tilingDataInt.data()) + fieldOffset, &value, sizeof(T));
88}83}
89 84 
85+static gert::Shape MakeHif4ScaleShape(int64_t batchDimNum, int64_t batchValue, int64_t dim0, int64_t dim1)
86+{
87+ gert::Shape shape;
88+ for (int64_t idx = 0; idx < batchDimNum; ++idx) {
89+ shape.AppendDim(batchValue);
90+ }
91+ shape.AppendDim(dim0);
92+ shape.AppendDim(dim1);
93+ shape.AppendDim(1);
94+ return shape;
95+}
96+ 
90static gert::Shape TransNd2Nz(const gert::Shape& inShape, int64_t c0 = 32)97static gert::Shape TransNd2Nz(const gert::Shape& inShape, int64_t c0 = 32)
91{98{
92 gert::Shape outShape;99 gert::Shape outShape;
@@ -375,6 +382,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c
375 382 
376 if (yDim == 6) {383 if (yDim == 6) {
377 outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, batchC, m, n});384 outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, batchC, m, n});
385+ } else if (yDim == 5) {
386+ outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, m, n});
387+ } else if (yDim == 4) {
388+ outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, m, n});
378 } else if (yDim == 3) {389 } else if (yDim == 3) {
379 outputShape.MutableOriginShape() = gert::Shape({batchC, m, n});390 outputShape.MutableOriginShape() = gert::Shape({batchC, m, n});
380 } else if (yDim == 2) {391 } else if (yDim == 2) {
@@ -386,6 +397,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c
386 if (transA) {397 if (transA) {
387 if (x1Dim == 6) {398 if (x1Dim == 6) {
388 x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, k, m});399 x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, k, m});
400+ } else if (x1Dim == 5) {
401+ x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, k, m});
402+ } else if (x1Dim == 4) {
403+ x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, k, m});
389 } else if (x1Dim == 3) {404 } else if (x1Dim == 3) {
390 x1Shape.MutableOriginShape() = gert::Shape({batchA, k, m});405 x1Shape.MutableOriginShape() = gert::Shape({batchA, k, m});
391 } else if (x1Dim == 2) {406 } else if (x1Dim == 2) {
@@ -396,6 +411,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c
396 } else {411 } else {
397 if (x1Dim == 6) {412 if (x1Dim == 6) {
398 x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, m, k});413 x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, m, k});
414+ } else if (x1Dim == 5) {
415+ x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, m, k});
416+ } else if (x1Dim == 4) {
417+ x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, m, k});
399 } else if (x1Dim == 3) {418 } else if (x1Dim == 3) {
400 x1Shape.MutableOriginShape() = gert::Shape({batchA, m, k});419 x1Shape.MutableOriginShape() = gert::Shape({batchA, m, k});
401 } else if (x1Dim == 2) {420 } else if (x1Dim == 2) {
@@ -408,6 +427,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c
408 if (transB) {427 if (transB) {
409 if (x2Dim == 6) {428 if (x2Dim == 6) {
410 x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, n, k});429 x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, n, k});
430+ } else if (x2Dim == 5) {
431+ x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, n, k});
432+ } else if (x2Dim == 4) {
433+ x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, n, k});
411 } else if (x2Dim == 3) {434 } else if (x2Dim == 3) {
412 x2Shape.MutableOriginShape() = gert::Shape({batchB, n, k});435 x2Shape.MutableOriginShape() = gert::Shape({batchB, n, k});
413 } else if (x2Dim == 2) {436 } else if (x2Dim == 2) {
@@ -418,6 +441,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c
418 } else {441 } else {
419 if (x2Dim == 6) {442 if (x2Dim == 6) {
420 x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, k, n});443 x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, k, n});
444+ } else if (x2Dim == 5) {
445+ x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, k, n});
446+ } else if (x2Dim == 4) {
447+ x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, k, n});
421 } else if (x2Dim == 3) {448 } else if (x2Dim == 3) {
422 x2Shape.MutableOriginShape() = gert::Shape({batchB, k, n});449 x2Shape.MutableOriginShape() = gert::Shape({batchB, k, n});
423 } else if (x2Dim == 2) {450 } else if (x2Dim == 2) {
@@ -491,8 +518,13 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c
491 scaleShape.MutableStorageShape() = gert::Shape({1});518 scaleShape.MutableStorageShape() = gert::Shape({1});
492 } else if (quantMode == 8) { // HIF4 groups: [M, ceil(K/64), 1] and [ceil(K/64), N, 1].519 } else if (quantMode == 8) { // HIF4 groups: [M, ceil(K/64), 1] and [ceil(K/64), N, 1].
493 const int64_t groups = ops::CeilDiv<int64_t>(k, 64);520 const int64_t groups = ops::CeilDiv<int64_t>(k, 64);
494- pertokenShape.MutableStorageShape() = transA ? gert::Shape({groups, m, 1}) : gert::Shape({m, groups, 1});521+ // Batch HIF4 keeps x1Dim - 2 / x2Dim - 2 leading batch dims before the logical scale axes.
495- scaleShape.MutableStorageShape() = transB ? gert::Shape({n, groups, 1}) : gert::Shape({groups, n, 1});522+ const int64_t pertokenBatchDimNum = x1Dim > 2 ? x1Dim - 2 : 0;
523+ const int64_t scaleBatchDimNum = x2Dim > 2 ? x2Dim - 2 : 0;
524+ pertokenShape.MutableStorageShape() = MakeHif4ScaleShape(pertokenBatchDimNum, batchA, transA ? groups : m,
525+ transA ? m : groups);
526+ scaleShape.MutableStorageShape() = MakeHif4ScaleShape(scaleBatchDimNum, batchB, transB ? n : groups,
527+ transB ? groups : n);
496 } else if (quantMode == 7) { // X2 per-group CSV cases use G=128.528 } else if (quantMode == 7) { // X2 per-group CSV cases use G=128.
497 const int64_t groupCnt = ops::CeilDiv<int64_t>(k, 128);529 const int64_t groupCnt = ops::CeilDiv<int64_t>(k, 128);
498 scaleShape.MutableStorageShape() = x2Dim == 3 ? gert::Shape({batchB, groupCnt, n}) : gert::Shape({groupCnt, n});530 scaleShape.MutableStorageShape() = x2Dim == 3 ? gert::Shape({batchB, groupCnt, n}) : gert::Shape({groupCnt, n});
@@ -589,6 +621,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi
589 gert::StorageShape outputShape;621 gert::StorageShape outputShape;
590 if (yDim == 6) {622 if (yDim == 6) {
591 outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, batchC, m, n});623 outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, batchC, m, n});
624+ } else if (yDim == 5) {
625+ outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, m, n});
626+ } else if (yDim == 4) {
627+ outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, m, n});
592 } else if (yDim == 3) {628 } else if (yDim == 3) {
593 outputShape.MutableOriginShape() = gert::Shape({batchC, m, n});629 outputShape.MutableOriginShape() = gert::Shape({batchC, m, n});
594 } else if (yDim == 2) {630 } else if (yDim == 2) {
@@ -600,6 +636,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi
600 if (transA) {636 if (transA) {
601 if (x1Dim == 6) {637 if (x1Dim == 6) {
602 x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, k, m});638 x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, k, m});
639+ } else if (x1Dim == 5) {
640+ x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, k, m});
641+ } else if (x1Dim == 4) {
642+ x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, k, m});
603 } else if (x1Dim == 3) {643 } else if (x1Dim == 3) {
604 x1Shape.MutableOriginShape() = gert::Shape({batchA, k, m});644 x1Shape.MutableOriginShape() = gert::Shape({batchA, k, m});
605 } else if (x1Dim == 2) {645 } else if (x1Dim == 2) {
@@ -610,6 +650,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi
610 } else {650 } else {
611 if (x1Dim == 6) {651 if (x1Dim == 6) {
612 x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, m, k});652 x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, m, k});
653+ } else if (x1Dim == 5) {
654+ x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, m, k});
655+ } else if (x1Dim == 4) {
656+ x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, m, k});
613 } else if (x1Dim == 3) {657 } else if (x1Dim == 3) {
614 x1Shape.MutableOriginShape() = gert::Shape({batchA, m, k});658 x1Shape.MutableOriginShape() = gert::Shape({batchA, m, k});
615 } else if (x1Dim == 2) {659 } else if (x1Dim == 2) {
@@ -622,6 +666,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi
622 if (transB) {666 if (transB) {
623 if (x2Dim == 6) {667 if (x2Dim == 6) {
624 x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, n, k});668 x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, n, k});
669+ } else if (x2Dim == 5) {
670+ x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, n, k});
671+ } else if (x2Dim == 4) {
672+ x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, n, k});
625 } else if (x2Dim == 3) {673 } else if (x2Dim == 3) {
626 x2Shape.MutableOriginShape() = gert::Shape({batchB, n, k});674 x2Shape.MutableOriginShape() = gert::Shape({batchB, n, k});
627 } else if (x2Dim == 2) {675 } else if (x2Dim == 2) {
@@ -632,6 +680,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi
632 } else {680 } else {
633 if (x2Dim == 6) {681 if (x2Dim == 6) {
634 x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, k, n});682 x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, k, n});
683+ } else if (x2Dim == 5) {
684+ x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, k, n});
685+ } else if (x2Dim == 4) {
686+ x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, k, n});
635 } else if (x2Dim == 3) {687 } else if (x2Dim == 3) {
636 x2Shape.MutableOriginShape() = gert::Shape({batchB, k, n});688 x2Shape.MutableOriginShape() = gert::Shape({batchB, k, n});
637 } else if (x2Dim == 2) {689 } else if (x2Dim == 2) {
@@ -716,8 +768,13 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi
716 scaleShape.MutableStorageShape() = gert::Shape({1});768 scaleShape.MutableStorageShape() = gert::Shape({1});
717 } else if (quantMode == 8) { // HIF4 groups: [M, ceil(K/64), 1] and [ceil(K/64), N, 1].769 } else if (quantMode == 8) { // HIF4 groups: [M, ceil(K/64), 1] and [ceil(K/64), N, 1].
718 const int64_t groups = ops::CeilDiv<int64_t>(k, 64);770 const int64_t groups = ops::CeilDiv<int64_t>(k, 64);
719- pertokenShape.MutableStorageShape() = transA ? gert::Shape({groups, m, 1}) : gert::Shape({m, groups, 1});771+ // Batch HIF4 keeps x1Dim - 2 / x2Dim - 2 leading batch dims before the logical scale axes.
720- scaleShape.MutableStorageShape() = transB ? gert::Shape({n, groups, 1}) : gert::Shape({groups, n, 1});772+ const int64_t pertokenBatchDimNum = x1Dim > 2 ? x1Dim - 2 : 0;
773+ const int64_t scaleBatchDimNum = x2Dim > 2 ? x2Dim - 2 : 0;
774+ pertokenShape.MutableStorageShape() = MakeHif4ScaleShape(pertokenBatchDimNum, batchA, transA ? groups : m,
775+ transA ? m : groups);
776+ scaleShape.MutableStorageShape() = MakeHif4ScaleShape(scaleBatchDimNum, batchB, transB ? n : groups,
777+ transB ? groups : n);
721 } else if (quantMode == 7) { // X2 per-group CSV cases use G=128.778 } else if (quantMode == 7) { // X2 per-group CSV cases use G=128.
722 const int64_t groupCnt = ops::CeilDiv<int64_t>(k, 128);779 const int64_t groupCnt = ops::CeilDiv<int64_t>(k, 128);
723 scaleShape.MutableStorageShape() = x2Dim == 3 ? gert::Shape({batchB, groupCnt, n}) : gert::Shape({groupCnt, n});780 scaleShape.MutableStorageShape() = x2Dim == 3 ? gert::Shape({batchB, groupCnt, n}) : gert::Shape({groupCnt, n});