| @@ -303,9 +303,16 @@ __aicore__ inline void QuantMmBatchMX<CMCT_QBMM_MX_KERNEL_FUN_TEM_PARAMS>::AddBa | |||
| 303 | Get<QuantBatchMatmul::IDX_BIAS_OFFSET>(blockOffset_) += batchCOffset_ * params.problemShape.n; | 303 | Get<QuantBatchMatmul::IDX_BIAS_OFFSET>(blockOffset_) += batchCOffset_ * params.problemShape.n; |
| 304 | } | 304 | } |
| 305 | // x1Scale/x2Scale batch offset. e8m0 is 1 byte and never packed, so no sizeShift here. | 305 | // x1Scale/x2Scale batch offset. e8m0 is 1 byte and never packed, so no sizeShift here. |
| 306 | - // Per-batch element count = M(or N) * ceil(K/64) * 2, matching the MX scale stride in GetQuantOffset. | 306 | + // MX per-batch element count = M(or N) * ceil(K/64) * 2, matching the MX scale stride in GetQuantOffset. |
| 307 | - // hif4 scale has no batch dim (3D only), so no batch offset is applied for hif4. | 307 | + // HIF4 per-batch element count = M(or N) * ceil(K/64), since the scale last dim is 1. |
| 308 | - if constexpr (!BlockMmad::IS_HIF4) { | 308 | + if constexpr (BlockMmad::IS_HIF4) { |
| 309 | + Get<QuantBatchMatmul::IDX_X1SCALE_OFFSET>( | ||
| 310 | + blockOffset_) += batchAOffset_ * params.problemShape.m * | ||
| 311 | + Cmct::Gemm::CeilDiv(params.problemShape.k, static_cast<int64_t>(MXFP_DIVISOR_SIZE)); | ||
| 312 | + Get<QuantBatchMatmul::IDX_X2SCALE_OFFSET>( | ||
| 313 | + blockOffset_) += batchBOffset_ * params.problemShape.n * | ||
| 314 | + Cmct::Gemm::CeilDiv(params.problemShape.k, static_cast<int64_t>(MXFP_DIVISOR_SIZE)); | ||
| 315 | + } else { | ||
| 309 | Get<QuantBatchMatmul::IDX_X1SCALE_OFFSET>( | 316 | Get<QuantBatchMatmul::IDX_X1SCALE_OFFSET>( |
| 310 | blockOffset_) += batchAOffset_ * params.problemShape.m * | 317 | blockOffset_) += batchAOffset_ * params.problemShape.m * |
| 311 | Cmct::Gemm::CeilDiv(params.problemShape.k, static_cast<int64_t>(MXFP_DIVISOR_SIZE)) * | 318 | Cmct::Gemm::CeilDiv(params.problemShape.k, static_cast<int64_t>(MXFP_DIVISOR_SIZE)) * |
| @@ -12,9 +12,6 @@ | |||
| 12 | * \file quant_batch_matmul_inplace_add_mx_basic_api_tiling.cpp | 12 | * \file quant_batch_matmul_inplace_add_mx_basic_api_tiling.cpp |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#include <string> | ||
| 16 | -#include <vector> | ||
| 17 | -#include "base/err_msg.h" | ||
| 18 | #include <vector> | 15 | #include <vector> |
| 19 | #include "quant_batch_matmul_inplace_add_mx_basic_api_tiling.h" | 16 | #include "quant_batch_matmul_inplace_add_mx_basic_api_tiling.h" |
| 20 | #include "op_host/tiling_templates_registry.h" | 17 | #include "op_host/tiling_templates_registry.h" |
| @@ -63,19 +60,6 @@ ge::graphStatus QuantBatchMatmulInplaceAddMXBasicAPITiling::DoLibApiTiling() | |||
| 63 | if (ret != ge::GRAPH_SUCCESS) { | 60 | if (ret != ge::GRAPH_SUCCESS) { |
| 64 | return ret; | 61 | return ret; |
| 65 | } | 62 | } |
| 66 | - // TE HIF4新方案合入并且适配后放开,如果未命中QBMMV3中BLAZE tiling | ||
| 67 | - // 基类不会生成 withoutBatch 数据,直接报错避免下发全零 tiling。 | ||
| 68 | - if (inputParams_.isHif4PerGroup && basicTiling_.baseK % (qmmv3_tiling_const::MXFP_DIVISOR_SIZE * 2UL) != 0UL) { | ||
| 69 | - { | ||
| 70 | - const std::string _msg = "When the dtype of x1 and x2 is HIFLOAT4, baseK[" + | ||
| 71 | - std::to_string(basicTiling_.baseK) + "] must be an even multiple of 64."; | ||
| 72 | - OP_LOGE_WITHOUT_REPORT(inputParams_.opName, "%s", _msg.c_str()); | ||
| 73 | - const std::vector<const char*> _msgKey = {"message"}; | ||
| 74 | - const std::vector<const char*> _msgValue = {_msg.c_str()}; | ||
| 75 | - REPORT_PREDEFINED_ERR_MSG("EZ1001", _msgKey, _msgValue); | ||
| 76 | - } | ||
| 77 | - return ge::GRAPH_FAILED; | ||
| 78 | - } | ||
| 79 | UpdateTilingData(); | 63 | UpdateTilingData(); |
| 80 | return ge::GRAPH_SUCCESS; | 64 | return ge::GRAPH_SUCCESS; |
| 81 | } | 65 | } |
| @@ -11,7 +11,6 @@ | |||
| 11 | #include <gtest/gtest.h> | 11 | #include <gtest/gtest.h> |
| 12 | #include <stdlib.h> | 12 | #include <stdlib.h> |
| 13 | 13 | ||
| 14 | -#include <algorithm> | ||
| 15 | #include <cctype> | 14 | #include <cctype> |
| 16 | #include <fstream> | 15 | #include <fstream> |
| 17 | #include <iostream> | 16 | #include <iostream> |
| @@ -19,17 +18,12 @@ | |||
| 19 | #include <string> | 18 | #include <string> |
| 20 | #include <vector> | 19 | #include <vector> |
| 21 | 20 | ||
| 22 | -#include "log/log.h" | ||
| 23 | - | ||
| 24 | #include "exe_graph/runtime/storage_format.h" | 21 | #include "exe_graph/runtime/storage_format.h" |
| 25 | #include "exe_graph/runtime/storage_shape.h" | 22 | #include "exe_graph/runtime/storage_shape.h" |
| 26 | -#include "exe_graph/runtime/tiling_parse_context.h" | ||
| 27 | #include "kernel_run_context_facker.h" | 23 | #include "kernel_run_context_facker.h" |
| 28 | #include "platform/platform_infos_def.h" | 24 | #include "platform/platform_infos_def.h" |
| 29 | #include "register/op_impl_registry.h" | 25 | #include "register/op_impl_registry.h" |
| 30 | -#include "op_host/tiling_templates_registry.h" | ||
| 31 | #include "test_cube_util.h" | 26 | #include "test_cube_util.h" |
| 32 | -#include "ut_op_util.h" | ||
| 33 | #include "../../../op_host/op_tiling/quant_batch_matmul_inplace_add_tiling.h" | 27 | #include "../../../op_host/op_tiling/quant_batch_matmul_inplace_add_tiling.h" |
| 34 | #include "../../../op_kernel/arch35/quant_batch_matmul_inplace_add_tiling_key.h" | 28 | #include "../../../op_kernel/arch35/quant_batch_matmul_inplace_add_tiling_key.h" |
| 35 | #include "../../../../common/op_host/math_util_nn.h" | 29 | #include "../../../../common/op_host/math_util_nn.h" |
| @@ -75,6 +69,18 @@ struct QuantBatchMatmulInplaceAddTilingCsvLoadResult { | |||
| 75 | vector<string> errors; | 69 | vector<string> errors; |
| 76 | }; | 70 | }; |
| 77 | 71 | ||
| 72 | +static ge::DataType ParseInplaceAddDtype(const string& value) | ||
| 73 | +{ | ||
| 74 | + const string dtype = Trim(value); | ||
| 75 | + if (dtype == "HIFLOAT4" || dtype == "DT_HIFLOAT4") { | ||
| 76 | + return Ops::NN::DT_HIFLOAT4; | ||
| 77 | + } | ||
| 78 | + if (dtype == "HIFLOAT4_SCALE" || dtype == "DT_HIFLOAT4_SCALE") { | ||
| 79 | + return Ops::NN::DT_HIFLOAT4_SCALE; | ||
| 80 | + } | ||
| 81 | + return ParseDtype(value); | ||
| 82 | +} | ||
| 83 | + | ||
| 78 | static QuantBatchMatmulInplaceAddTilingCsvLoadResult LoadParams() | 84 | static QuantBatchMatmulInplaceAddTilingCsvLoadResult LoadParams() |
| 79 | { | 85 | { |
| 80 | QuantBatchMatmulInplaceAddTilingCsvLoadResult result; | 86 | QuantBatchMatmulInplaceAddTilingCsvLoadResult result; |
| @@ -125,13 +131,13 @@ static QuantBatchMatmulInplaceAddTilingCsvLoadResult LoadParams() | |||
| 125 | param.groupSize = ParseInt64OrDefault(cols[idx++], 0); | 131 | param.groupSize = ParseInt64OrDefault(cols[idx++], 0); |
| 126 | param.x1Format = ParseFormat(cols[idx++]); | 132 | param.x1Format = ParseFormat(cols[idx++]); |
| 127 | param.x2Format = ParseFormat(cols[idx++]); | 133 | param.x2Format = ParseFormat(cols[idx++]); |
| 128 | - param.x1Dtype = ParseDtype(cols[idx++]); | 134 | + param.x1Dtype = ParseInplaceAddDtype(cols[idx++]); |
| 129 | - param.x2Dtype = ParseDtype(cols[idx++]); | 135 | + param.x2Dtype = ParseInplaceAddDtype(cols[idx++]); |
| 130 | param.hasX1Scale = ParseBool(cols[idx++]); | 136 | param.hasX1Scale = ParseBool(cols[idx++]); |
| 131 | - param.x1ScaleDtype = ParseDtype(cols[idx++]); | 137 | + param.x1ScaleDtype = ParseInplaceAddDtype(cols[idx++]); |
| 132 | param.hasX2Scale = ParseBool(cols[idx++]); | 138 | param.hasX2Scale = ParseBool(cols[idx++]); |
| 133 | - param.x2ScaleDtype = ParseDtype(cols[idx++]); | 139 | + param.x2ScaleDtype = ParseInplaceAddDtype(cols[idx++]); |
| 134 | - param.yDtype = ParseDtype(cols[idx++]); | 140 | + param.yDtype = ParseInplaceAddDtype(cols[idx++]); |
| 135 | param.aicNum = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0)); | 141 | param.aicNum = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0)); |
| 136 | param.aivNum = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0)); | 142 | param.aivNum = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0)); |
| 137 | param.expectBlockDim = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0)); | 143 | param.expectBlockDim = static_cast<uint32_t>(ParseIntOrDefault(cols[idx++], 0)); |
| @@ -241,8 +247,13 @@ static void TestOneParamCase(const QuantBatchMatmulInplaceAddTilingTestParam& pa | |||
| 241 | } | 247 | } |
| 242 | yInputShape.MutableOriginShape() = gert::Shape({param.m, param.n}); | 248 | yInputShape.MutableOriginShape() = gert::Shape({param.m, param.n}); |
| 243 | yInputShape.MutableStorageShape() = gert::Shape({param.m, param.n}); | 249 | yInputShape.MutableStorageShape() = gert::Shape({param.m, param.n}); |
| 244 | - if (param.x1Dtype == ge::DT_HIFLOAT8 && param.x2Dtype == ge::DT_HIFLOAT8 && param.x1ScaleDtype == ge::DT_FLOAT && | 250 | + if (param.x1Dtype == Ops::NN::DT_HIFLOAT4 && param.x2Dtype == Ops::NN::DT_HIFLOAT4) { |
| 245 | - param.x2ScaleDtype == ge::DT_FLOAT) { | 251 | + x1ScaleShape.MutableOriginShape() = gert::Shape({k1, param.m, 1}); |
| 252 | + x1ScaleShape.MutableStorageShape() = gert::Shape({k1, param.m, 1}); | ||
| 253 | + x2ScaleShape.MutableOriginShape() = gert::Shape({k1, param.n, 1}); | ||
| 254 | + x2ScaleShape.MutableStorageShape() = gert::Shape({k1, param.n, 1}); | ||
| 255 | + } else if (param.x1Dtype == ge::DT_HIFLOAT8 && param.x2Dtype == ge::DT_HIFLOAT8 && | ||
| 256 | + param.x1ScaleDtype == ge::DT_FLOAT && param.x2ScaleDtype == ge::DT_FLOAT) { | ||
| 246 | x1ScaleShape.MutableOriginShape() = gert::Shape({1}); | 257 | x1ScaleShape.MutableOriginShape() = gert::Shape({1}); |
| 247 | x1ScaleShape.MutableStorageShape() = gert::Shape({1}); | 258 | x1ScaleShape.MutableStorageShape() = gert::Shape({1}); |
| 248 | x2ScaleShape.MutableOriginShape() = gert::Shape({1}); | 259 | x2ScaleShape.MutableOriginShape() = gert::Shape({1}); |
| @@ -312,7 +323,7 @@ static void TestOneParamCase(const QuantBatchMatmulInplaceAddTilingTestParam& pa | |||
| 312 | {"Short_SoC_version", param.socVersion}, | 323 | {"Short_SoC_version", param.socVersion}, |
| 313 | {"SoC_version", param.socVersion}, | 324 | {"SoC_version", param.socVersion}, |
| 314 | }; | 325 | }; |
| 315 | - const map<string, string> soc2Arch = {{"Dav3510", "3510"}}; | 326 | + const map<string, string> soc2Arch = {{"Dav3510", "3510"}, {"Dav9201", "9201"}}; |
| 316 | auto soc2ArchIter = soc2Arch.find(param.socVersion); | 327 | auto soc2ArchIter = soc2Arch.find(param.socVersion); |
| 317 | if (soc2ArchIter != soc2Arch.end()) { | 328 | if (soc2ArchIter != soc2Arch.end()) { |
| 318 | socVersionInfos.clear(); | 329 | socVersionInfos.clear(); |
| @@ -4,6 +4,7 @@ general,mx_2,Dav3510,1024,512,1024,true,false,32,ND,ND,FP8-E5M2,FP8-E5M2,true,FP | |||
| 4 | general,mx_3,Dav3510,128,512,128,true,false,32,ND,ND,FP8-E5M2,FP8-E5M2,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,16,GRAPH_SUCCESS,1 | 4 | general,mx_3,Dav3510,128,512,128,true,false,32,ND,ND,FP8-E5M2,FP8-E5M2,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,16,GRAPH_SUCCESS,1 |
| 5 | general,mx_4,Dav3510,1024,512,1024,true,false,32,ND,ND,FP8-E4M3,FP8-E4M3,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,1 | 5 | general,mx_4,Dav3510,1024,512,1024,true,false,32,ND,ND,FP8-E4M3,FP8-E4M3,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,1 |
| 6 | general,hif8_tt,Dav3510,128,512,128,true,false,0,ND,ND,HIFLOAT8,HIFLOAT8,true,FP32,true,FP32,FP32,32,64,16,GRAPH_SUCCESS,1 | 6 | general,hif8_tt,Dav3510,128,512,128,true,false,0,ND,ND,HIFLOAT8,HIFLOAT8,true,FP32,true,FP32,FP32,32,64,16,GRAPH_SUCCESS,1 |
| 7 | +general,hif4_m268_k160_n3680,Dav9201,268,160,3680,true,false,4295032896,ND,ND,HIFLOAT4,HIFLOAT4,true,HIFLOAT4_SCALE,true,HIFLOAT4_SCALE,FP32,32,64,32,GRAPH_SUCCESS,17 | ||
| 7 | general,mxfp4_test1,Dav3510,128,512,128,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,4,GRAPH_SUCCESS,1 | 8 | general,mxfp4_test1,Dav3510,128,512,128,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,4,GRAPH_SUCCESS,1 |
| 8 | general,mxfp4_a_full_load,Dav3510,128,512,16384,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,17 | 9 | general,mxfp4_a_full_load,Dav3510,128,512,16384,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,17 |
| 9 | general,mxfp4_test2,Dav3510,1024,512,1024,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,1 | 10 | general,mxfp4_test2,Dav3510,1024,512,1024,true,false,32,ND,ND,FP4-E2M1,FP4-E2M1,true,FP8-E8M0,true,FP8-E8M0,FP32,32,64,32,GRAPH_SUCCESS,1 |
| @@ -44,6 +44,7 @@ static const int64_t MIN_DIM_NUM_ND = 2; | |||
| 44 | static const size_t MX_SCALE_DIM = 3; | 44 | static const size_t MX_SCALE_DIM = 3; |
| 45 | static const size_t MX_SCALE_MIN_DIM = 3; | 45 | static const size_t MX_SCALE_MIN_DIM = 3; |
| 46 | static const size_t HIFP4_SCALE_DIM = 3; | 46 | static const size_t HIFP4_SCALE_DIM = 3; |
| 47 | +static const size_t HIFP4_SCALE_MAX_DIM = 7; | ||
| 47 | static const size_t PENULTIMATE_DIM = 2; | 48 | static const size_t PENULTIMATE_DIM = 2; |
| 48 | static const size_t BATCH_TAILENDER_DIM = 3; | 49 | static const size_t BATCH_TAILENDER_DIM = 3; |
| 49 | static const size_t NZ_K1_INDEX = 3; | 50 | static const size_t NZ_K1_INDEX = 3; |
| @@ -541,21 +542,27 @@ bool QuantMatmulChecker::CheckDimValueMicroScaling() const | |||
| 541 | 542 | ||
| 542 | bool QuantMatmulChecker::CheckDimValueHif4() const | 543 | bool QuantMatmulChecker::CheckDimValueHif4() const |
| 543 | { | 544 | { |
| 544 | - auto x1ScaleMDim = transposeX1_ ? 1 : 0; | 545 | + size_t x1ScaleBatchDimNum = x1Scale_->GetViewShape().GetDimNum() - HIFP4_SCALE_DIM; |
| 545 | - auto x1ScaleKDim = transposeX1_ ? 0 : 1; | 546 | + size_t x2ScaleBatchDimNum = x2Scale_->GetViewShape().GetDimNum() - HIFP4_SCALE_DIM; |
| 546 | - auto x2ScaleNDim = transposeX2_ ? 0 : 1; | 547 | + auto x1ScaleMDimIndex = x1ScaleBatchDimNum + (transposeX1_ ? 1 : 0); |
| 547 | - auto x2ScaleKDim = transposeX2_ ? 1 : 0; | 548 | + auto x1ScaleKDimIndex = x1ScaleBatchDimNum + (transposeX1_ ? 0 : 1); |
| 548 | - | 549 | + auto x2ScaleNDimIndex = x2ScaleBatchDimNum + (transposeX2_ ? 0 : 1); |
| 549 | - bool x1ScaleShapeHasOne = x1Scale_->GetViewShape().GetDim(0) == 1 || x1Scale_->GetViewShape().GetDim(1) == 1; | 550 | + auto x2ScaleKDimIndex = x2ScaleBatchDimNum + (transposeX2_ ? 1 : 0); |
| 550 | - bool x2ScaleShapeHasOne = x2Scale_->GetViewShape().GetDim(0) == 1 || x2Scale_->GetViewShape().GetDim(1) == 1; | 551 | + auto x1ScaleMDim = x1Scale_->GetViewShape().GetDim(x1ScaleMDimIndex); |
| 551 | - if ((!x1ScaleShapeHasOne && x1Scale_->GetViewShape().GetDim(x1ScaleMDim) != x1MDim_) || | 552 | + auto x1ScaleKDim = x1Scale_->GetViewShape().GetDim(x1ScaleKDimIndex); |
| 552 | - (!x2ScaleShapeHasOne && x2Scale_->GetViewShape().GetDim(x2ScaleNDim) != x2NDim_)) { | 553 | + auto x2ScaleNDim = x2Scale_->GetViewShape().GetDim(x2ScaleNDimIndex); |
| 554 | + auto x2ScaleKDim = x2Scale_->GetViewShape().GetDim(x2ScaleKDimIndex); | ||
| 555 | + bool x1ScaleHasOne = (x1ScaleMDim == 1 && x1ScaleKDim == x1MDim_) || (x1ScaleMDim == x1MDim_ && x1ScaleKDim == 1) || | ||
| 556 | + (x1ScaleMDim == 1 && x1ScaleKDim == CeilDiv(x1KDim_, HIFP4_GROUP_SIZE)) || | ||
| 557 | + (x1ScaleMDim == CeilDiv(x1KDim_, HIFP4_GROUP_SIZE) && x1ScaleKDim == 1); | ||
| 558 | + bool x2ScaleHasOne = (x2ScaleNDim == 1 && x2ScaleKDim == x2NDim_) || (x2ScaleNDim == x2NDim_ && x2ScaleKDim == 1) || | ||
| 559 | + (x2ScaleNDim == 1 && x2ScaleKDim == CeilDiv(x2KDim_, HIFP4_GROUP_SIZE)) || | ||
| 560 | + (x2ScaleNDim == CeilDiv(x2KDim_, HIFP4_GROUP_SIZE) && x2ScaleKDim == 1); | ||
| 561 | + if ((!x1ScaleHasOne && x1ScaleMDim != x1MDim_) || (!x2ScaleHasOne && x2ScaleNDim != x2NDim_)) { | ||
| 553 | OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | 562 | OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( |
| 554 | apiName_, | 563 | apiName_, |
| 555 | FormatString("x1 M, %s M, x2 N, %s N", GetX1ScaleName().c_str(), GetX2ScaleName().c_str()).c_str(), | 564 | FormatString("x1 M, %s M, x2 N, %s N", GetX1ScaleName().c_str(), GetX2ScaleName().c_str()).c_str(), |
| 556 | - FormatString("%ld, %ld, %ld, %ld", x1MDim_, x1Scale_->GetViewShape().GetDim(x1ScaleMDim), x2NDim_, | 565 | + FormatString("%ld, %ld, %ld, %ld", x1MDim_, x1ScaleMDim, x2NDim_, x2ScaleNDim).c_str(), |
| 557 | - x2Scale_->GetViewShape().GetDim(x2ScaleNDim)) | ||
| 558 | - .c_str(), | ||
| 559 | FormatString("when the quantization mode is hif4, the M dimension of x1 and %s must be equal, and the N " | 566 | FormatString("when the quantization mode is hif4, the M dimension of x1 and %s must be equal, and the N " |
| 560 | "dimension of x2 and %s must be equal", | 567 | "dimension of x2 and %s must be equal", |
| 561 | GetX1ScaleName().c_str(), GetX2ScaleName().c_str()) | 568 | GetX1ScaleName().c_str(), GetX2ScaleName().c_str()) |
| @@ -563,10 +570,10 @@ bool QuantMatmulChecker::CheckDimValueHif4() const | |||
| 563 | return false; | 570 | return false; |
| 564 | } | 571 | } |
| 565 | 572 | ||
| 566 | - if (!x1ScaleShapeHasOne && CeilDiv(x1KDim_, HIFP4_GROUP_SIZE) != x1Scale_->GetViewShape().GetDim(x1ScaleKDim)) { | 573 | + if (!x1ScaleHasOne && CeilDiv(x1KDim_, HIFP4_GROUP_SIZE) != x1ScaleKDim) { |
| 567 | OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | 574 | OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( |
| 568 | apiName_, FormatString("x1 K, %s K", GetX1ScaleName().c_str()).c_str(), | 575 | apiName_, FormatString("x1 K, %s K", GetX1ScaleName().c_str()).c_str(), |
| 569 | - FormatString("%ld, %ld", x1KDim_, x1Scale_->GetViewShape().GetDim(x1ScaleKDim)).c_str(), | 576 | + FormatString("%ld, %ld", x1KDim_, x1ScaleKDim).c_str(), |
| 570 | FormatString("when the quantization mode is hif4, the K dimension of %s must be equal to the K dimension " | 577 | FormatString("when the quantization mode is hif4, the K dimension of %s must be equal to the K dimension " |
| 571 | "of x1 ceildivided by 64", | 578 | "of x1 ceildivided by 64", |
| 572 | GetX1ScaleName().c_str()) | 579 | GetX1ScaleName().c_str()) |
| @@ -574,10 +581,10 @@ bool QuantMatmulChecker::CheckDimValueHif4() const | |||
| 574 | return false; | 581 | return false; |
| 575 | } | 582 | } |
| 576 | 583 | ||
| 577 | - if (!x2ScaleShapeHasOne && CeilDiv(x2KDim_, HIFP4_GROUP_SIZE) != x2Scale_->GetViewShape().GetDim(x2ScaleKDim)) { | 584 | + if (!x2ScaleHasOne && CeilDiv(x2KDim_, HIFP4_GROUP_SIZE) != x2ScaleKDim) { |
| 578 | OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | 585 | OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( |
| 579 | apiName_, FormatString("x2 K, %s K", GetX2ScaleName().c_str()).c_str(), | 586 | apiName_, FormatString("x2 K, %s K", GetX2ScaleName().c_str()).c_str(), |
| 580 | - FormatString("%ld, %ld", x2KDim_, x2Scale_->GetViewShape().GetDim(x2ScaleKDim)).c_str(), | 587 | + FormatString("%ld, %ld", x2KDim_, x2ScaleKDim).c_str(), |
| 581 | FormatString("when the quantization mode is hif4, the K dimension of %s must be equal to the K dimension " | 588 | FormatString("when the quantization mode is hif4, the K dimension of %s must be equal to the K dimension " |
| 582 | "of x2 ceildivided by 64", | 589 | "of x2 ceildivided by 64", |
| 583 | GetX2ScaleName().c_str()) | 590 | GetX2ScaleName().c_str()) |
| @@ -1211,15 +1218,106 @@ bool QuantMatmulChecker::CheckMxScaleDimRange(size_t x1ScaleDim, size_t x2ScaleD | |||
| 1211 | return true; | 1218 | return true; |
| 1212 | } | 1219 | } |
| 1213 | 1220 | ||
| 1221 | +bool QuantMatmulChecker::CheckHif4ScaleMinDim(size_t x1ScaleDim, size_t x2ScaleDim) const | ||
| 1222 | +{ | ||
| 1223 | + if (x1ScaleDim < HIFP4_SCALE_DIM) { | ||
| 1224 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( | ||
| 1225 | + apiName_, GetX1ScaleName().c_str(), FormatString("%zuD", x1ScaleDim).c_str(), | ||
| 1226 | + FormatString("when the quantization mode is hif4, the shape dim of %s must be at least %zuD", | ||
| 1227 | + GetX1ScaleName().c_str(), HIFP4_SCALE_DIM) | ||
| 1228 | + .c_str()); | ||
| 1229 | + return false; | ||
| 1230 | + } | ||
| 1231 | + if (x2ScaleDim < HIFP4_SCALE_DIM) { | ||
| 1232 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( | ||
| 1233 | + apiName_, GetX2ScaleName().c_str(), FormatString("%zuD", x2ScaleDim).c_str(), | ||
| 1234 | + FormatString("when the quantization mode is hif4, the shape dim of %s must be at least %zuD", | ||
| 1235 | + GetX2ScaleName().c_str(), HIFP4_SCALE_DIM) | ||
| 1236 | + .c_str()); | ||
| 1237 | + return false; | ||
| 1238 | + } | ||
| 1239 | + return true; | ||
| 1240 | +} | ||
| 1241 | + | ||
| 1242 | +bool QuantMatmulChecker::CheckHif4ScaleMaxDim(size_t x1ScaleDim, size_t x2ScaleDim) const | ||
| 1243 | +{ | ||
| 1244 | + if (x1ScaleDim > HIFP4_SCALE_MAX_DIM) { | ||
| 1245 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( | ||
| 1246 | + apiName_, GetX1ScaleName().c_str(), FormatString("%zuD", x1ScaleDim).c_str(), | ||
| 1247 | + FormatString("when the quantization mode is hif4, the shape dim of %s must be at most %zuD", | ||
| 1248 | + GetX1ScaleName().c_str(), HIFP4_SCALE_MAX_DIM) | ||
| 1249 | + .c_str()); | ||
| 1250 | + return false; | ||
| 1251 | + } | ||
| 1252 | + if (x2ScaleDim > HIFP4_SCALE_MAX_DIM) { | ||
| 1253 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( | ||
| 1254 | + apiName_, GetX2ScaleName().c_str(), FormatString("%zuD", x2ScaleDim).c_str(), | ||
| 1255 | + FormatString("when the quantization mode is hif4, the shape dim of %s must be at most %zuD", | ||
| 1256 | + GetX2ScaleName().c_str(), HIFP4_SCALE_MAX_DIM) | ||
| 1257 | + .c_str()); | ||
| 1258 | + return false; | ||
| 1259 | + } | ||
| 1260 | + return true; | ||
| 1261 | +} | ||
| 1262 | + | ||
| 1263 | +bool QuantMatmulChecker::CheckHif4ScaleBatchDimMatch(size_t x1ScaleDim, size_t x2ScaleDim) const | ||
| 1264 | +{ | ||
| 1265 | + // batch dim count alignment: scale removes hif4 fixed 3 dims, x removes last 2 dims | ||
| 1266 | + size_t x1DimNum = x1_->GetViewShape().GetDimNum(); | ||
| 1267 | + size_t x2DimNum = x2_->GetViewShape().GetDimNum(); | ||
| 1268 | + size_t x1BatchDimNum = x1ScaleDim - HIFP4_SCALE_DIM; | ||
| 1269 | + size_t x2BatchDimNum = x2ScaleDim - HIFP4_SCALE_DIM; | ||
| 1270 | + size_t x1InputBatchDimNum = (x1DimNum > MIN_DIM_NUM_ND) ? (x1DimNum - MIN_DIM_NUM_ND) : 0; | ||
| 1271 | + size_t x2InputBatchDimNum = (x2DimNum > MIN_DIM_NUM_ND) ? (x2DimNum - MIN_DIM_NUM_ND) : 0; | ||
| 1272 | + if (x1BatchDimNum != x1InputBatchDimNum) { | ||
| 1273 | + OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | ||
| 1274 | + apiName_, "pertokenBatchDimNum, x1BatchDimNum", | ||
| 1275 | + FormatString("%zu, %zu", x1BatchDimNum, x1InputBatchDimNum).c_str(), | ||
| 1276 | + "when the quantization mode is hif4, the batch dimension num of x1Scale must be equal to that of x1"); | ||
| 1277 | + return false; | ||
| 1278 | + } | ||
| 1279 | + if (x2BatchDimNum != x2InputBatchDimNum) { | ||
| 1280 | + OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | ||
| 1281 | + apiName_, "scaleBatchDimNum, x2BatchDimNum", | ||
| 1282 | + FormatString("%zu, %zu", x2BatchDimNum, x2InputBatchDimNum).c_str(), | ||
| 1283 | + "when the quantization mode is hif4, the batch dimension num of x2Scale must be equal to that of x2"); | ||
| 1284 | + return false; | ||
| 1285 | + } | ||
| 1286 | + // per-batch dim value match | ||
| 1287 | + const auto& x1Shape = x1_->GetViewShape(); | ||
| 1288 | + const auto& x2Shape = x2_->GetViewShape(); | ||
| 1289 | + const auto& x1ScaleShape = x1Scale_->GetViewShape(); | ||
| 1290 | + const auto& x2ScaleShape = x2Scale_->GetViewShape(); | ||
| 1291 | + for (size_t i = 0; i < x1BatchDimNum; ++i) { | ||
| 1292 | + if (x1ScaleShape.GetDim(i) != x1Shape.GetDim(i)) { | ||
| 1293 | + OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | ||
| 1294 | + apiName_, "dimIndex, x1Batch, pertokenBatch", | ||
| 1295 | + FormatString("%zu, %ld, %ld", i, x1Shape.GetDim(i), x1ScaleShape.GetDim(i)).c_str(), | ||
| 1296 | + "when the quantization mode is hif4, the batch dimension of x1Scale must be equal to that of x1"); | ||
| 1297 | + return false; | ||
| 1298 | + } | ||
| 1299 | + } | ||
| 1300 | + for (size_t i = 0; i < x2BatchDimNum; ++i) { | ||
| 1301 | + if (x2ScaleShape.GetDim(i) != x2Shape.GetDim(i)) { | ||
| 1302 | + OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | ||
| 1303 | + apiName_, "dimIndex, x2Batch, scaleBatch", | ||
| 1304 | + FormatString("%zu, %ld, %ld", i, x2Shape.GetDim(i), x2ScaleShape.GetDim(i)).c_str(), | ||
| 1305 | + "when the quantization mode is hif4, the batch dimension of x2Scale must be equal to that of x2"); | ||
| 1306 | + return false; | ||
| 1307 | + } | ||
| 1308 | + } | ||
| 1309 | + return true; | ||
| 1310 | +} | ||
| 1311 | + | ||
| 1214 | bool QuantMatmulChecker::CheckHif4ScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const | 1312 | bool QuantMatmulChecker::CheckHif4ScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const |
hif4 也算 mx 量化吧,为什么不合并到 CheckMxScale 相关的函数中呢 ![]() ![]() | |||
| 1215 | { | 1313 | { |
| 1216 | - if (x1ScaleDim != HIFP4_SCALE_DIM || x2ScaleDim != HIFP4_SCALE_DIM) { | 1314 | + if (!CheckHif4ScaleMinDim(x1ScaleDim, x2ScaleDim)) { |
| 1217 | - OP_LOGE_FOR_INVALID_SHAPEDIMS_WITH_REASON( | 1315 | + return false; |
| 1218 | - apiName_, FormatString("%s, %s", GetX1ScaleName().c_str(), GetX2ScaleName().c_str()).c_str(), | 1316 | + } |
| 1219 | - FormatString("%zuD, %zuD", x1ScaleDim, x2ScaleDim).c_str(), | 1317 | + if (!CheckHif4ScaleMaxDim(x1ScaleDim, x2ScaleDim)) { |
| 1220 | - FormatString("when the quantization mode is hif4, the shape dims of %s and %s must be %zu", | 1318 | + return false; |
| 1221 | - GetX1ScaleName().c_str(), GetX2ScaleName().c_str(), HIFP4_SCALE_DIM) | 1319 | + } |
| 1222 | - .c_str()); | 1320 | + if (!CheckHif4ScaleBatchDimMatch(x1ScaleDim, x2ScaleDim)) { |
| 1223 | return false; | 1321 | return false; |
| 1224 | } | 1322 | } |
| 1225 | return true; | 1323 | return true; |
| @@ -74,6 +74,9 @@ private: | |||
| 74 | bool CheckHif4ScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const; | 74 | bool CheckHif4ScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const; |
| 75 | bool CheckMxScaleMinDim(size_t x1ScaleDim, size_t x2ScaleDim) const; | 75 | bool CheckMxScaleMinDim(size_t x1ScaleDim, size_t x2ScaleDim) const; |
| 76 | bool CheckMxScaleBatchDimMatch(size_t x1ScaleDim, size_t x2ScaleDim) const; | 76 | bool CheckMxScaleBatchDimMatch(size_t x1ScaleDim, size_t x2ScaleDim) const; |
| 77 | + bool CheckHif4ScaleMinDim(size_t x1ScaleDim, size_t x2ScaleDim) const; | ||
| 78 | + bool CheckHif4ScaleMaxDim(size_t x1ScaleDim, size_t x2ScaleDim) const; | ||
| 79 | + bool CheckHif4ScaleBatchDimMatch(size_t x1ScaleDim, size_t x2ScaleDim) const; | ||
| 77 | bool CheckNormalScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const; | 80 | bool CheckNormalScaleDimRange(size_t x1ScaleDim, size_t x2ScaleDim) const; |
| 78 | bool CheckGroupSize() const; | 81 | bool CheckGroupSize() const; |
| 79 | bool CheckOutShape(bool twoDimMatmulCaseFlag, const std::vector<int64_t>& batchRecord) const; | 82 | bool CheckOutShape(bool twoDimMatmulCaseFlag, const std::vector<int64_t>& batchRecord) const; |
| @@ -17,9 +17,7 @@ | |||
| 17 | 17 | ||
| 18 | #include <algorithm> | 18 | #include <algorithm> |
| 19 | 19 | ||
| 20 | -#include "common/op_host/op_tiling/tiling_type_mm.h" | ||
| 21 | #include "log/log.h" | 20 | #include "log/log.h" |
| 22 | -#include "error_util.h" | ||
| 23 | #include "op_host/tiling_templates_registry.h" | 21 | #include "op_host/tiling_templates_registry.h" |
| 24 | #include "base_block_calculator.h" | 22 | #include "base_block_calculator.h" |
| 25 | #include "l1_tiling_data_calculator.h" | 23 | #include "l1_tiling_data_calculator.h" |
| @@ -107,18 +105,7 @@ void AdaptiveSlidingWindowMXBasicAPITiling::ResetTilingData() | |||
| 107 | bool AdaptiveSlidingWindowMXBasicAPITiling::IsWithoutBatchTilingData() const | 105 | bool AdaptiveSlidingWindowMXBasicAPITiling::IsWithoutBatchTilingData() const |
| 108 | { | 106 | { |
| 109 | return IsTensorApiEnabled() && inputParams_.batchC == 1UL && | 107 | return IsTensorApiEnabled() && inputParams_.batchC == 1UL && |
| 110 | - (compileInfo_.npuArch == Ops::Base::DAV_3510 || compileInfo_.npuArch == Ops::Base::DAV_9201) && | 108 | + (compileInfo_.npuArch == Ops::Base::DAV_3510 || compileInfo_.npuArch == NpuArch::DAV_9201); |
| 111 | - IsHif4BlazeCapable(); | ||
| 112 | -} | ||
| 113 | - | ||
| 114 | -bool AdaptiveSlidingWindowMXBasicAPITiling::IsHif4BlazeCapable() const | ||
| 115 | -{ | ||
| 116 | - // 9202 上 hif4 内核只实现 CMCT(basic) 路径,不消费 WithoutBatchTilingData,禁止走 blaze/without-batch | ||
| 117 | - if (inputParams_.isHif4PerGroup && compileInfo_.npuArch != Ops::Base::DAV_9201) { | ||
| 118 | - return false; | ||
| 119 | - } | ||
| 120 | - // tensor-api crd2idx 对 hif4_scale 暂不支持奇数坐标,baseK 需为 64 的偶数倍 | ||
| 121 | - return !(inputParams_.isHif4PerGroup && basicTiling_.baseK % (qmmv3_tiling_const::MXFP_DIVISOR_SIZE * 2UL) != 0UL); | ||
| 122 | } | 109 | } |
| 123 | 110 | ||
| 124 | void AdaptiveSlidingWindowMXBasicAPITiling::AdjustScaleFactorForL0CPingpong(uint32_t& scaleFactor, uint32_t step, | 111 | void AdaptiveSlidingWindowMXBasicAPITiling::AdjustScaleFactorForL0CPingpong(uint32_t& scaleFactor, uint32_t step, |
| @@ -148,10 +135,9 @@ const void* AdaptiveSlidingWindowMXBasicAPITiling::GetTilingData() const | |||
| 148 | 135 | ||
| 149 | uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetApiLevel(NpuArch npuArch) const | 136 | uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetApiLevel(NpuArch npuArch) const |
| 150 | { | 137 | { |
| 151 | - const bool supportsBlaze = npuArch == Ops::Base::DAV_3510 || npuArch == Ops::Base::DAV_9201; | 138 | + const bool supportsBlaze = npuArch == Ops::Base::DAV_3510 || npuArch == NpuArch::DAV_9201; |
| 152 | - return IsTensorApiEnabled() && supportsBlaze && IsHif4BlazeCapable() ? | 139 | + return IsTensorApiEnabled() && supportsBlaze ? static_cast<uint64_t>(QMMApiLevel::BLAZE_LEVEL) : |
| 153 | - static_cast<uint64_t>(QMMApiLevel::BLAZE_LEVEL) : | 140 | + static_cast<uint64_t>(QMMApiLevel::BASIC_LEVEL); |
| 154 | - static_cast<uint64_t>(QMMApiLevel::BASIC_LEVEL); | ||
| 155 | } | 141 | } |
| 156 | 142 | ||
| 157 | uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetBatchMode() const | 143 | uint64_t AdaptiveSlidingWindowMXBasicAPITiling::GetBatchMode() const |
| @@ -72,7 +72,6 @@ private: | |||
| 72 | void SetWithoutBatchTilingData(); | 72 | void SetWithoutBatchTilingData(); |
| 73 | void NormalizeSingleRoundTailSplitBasicBlock(); | 73 | void NormalizeSingleRoundTailSplitBasicBlock(); |
| 74 | void AdjustScaleFactorForL0CPingpong(uint32_t& scaleFactor, uint32_t step, uint32_t baseK) const; | 74 | void AdjustScaleFactorForL0CPingpong(uint32_t& scaleFactor, uint32_t step, uint32_t baseK) const; |
| 75 | - bool IsHif4BlazeCapable() const; | ||
| 76 | 75 | ||
| 77 | protected: | 76 | protected: |
| 78 | void ResetTilingData() override; | 77 | void ResetTilingData() override; |
| @@ -20,7 +20,6 @@ | |||
| 20 | #include "op_host/tiling_templates_registry.h" | 20 | #include "op_host/tiling_templates_registry.h" |
| 21 | #include "error_util.h" | 21 | #include "error_util.h" |
| 22 | #include "log/log.h" | 22 | #include "log/log.h" |
| 23 | -#include "op_cache_tiling.h" | ||
| 24 | #include "quant_batch_matmul_v3_checker_arch_9201.h" | 23 | #include "quant_batch_matmul_v3_checker_arch_9201.h" |
| 25 | #include "quant_batch_matmul_v3_tiling_strategy.h" | 24 | #include "quant_batch_matmul_v3_tiling_strategy.h" |
| 26 | #include "base_block_calculator.h" | 25 | #include "base_block_calculator.h" |
| @@ -68,7 +67,7 @@ QBMMV3StreamKTiling960::QBMMV3StreamKTiling960(gert::TilingContext* context, | |||
| 68 | bool QBMMV3StreamKTiling960::IsCapable() | 67 | bool QBMMV3StreamKTiling960::IsCapable() |
| 69 | { | 68 | { |
| 70 | // 量化场景分流:9201/9202 可走 SK 的量化场景一致(MX per-group + 非MX per-tensor + HIF4 per-group),均走 Blaze。 | 69 | // 量化场景分流:9201/9202 可走 SK 的量化场景一致(MX per-group + 非MX per-tensor + HIF4 per-group),均走 Blaze。 |
| 71 | - bool isQuantCapable = inputParams_.isMxPerGroup || IsPertensorStreamKInput(); | 70 | + bool isQuantCapable = inputParams_.isMxPerGroup || inputParams_.isHif4PerGroup || IsPertensorStreamKInput(); |
Outdated 还是一个疑问,为什么 isHif4PerGroup 和 isMxPerGroup 要分离? ![]() ![]() Outdated [跨仓兼容性检视] 9202 HIF4 StreamK 的 SDK 支持尚未闭合 这里把 本次核验的公开 asc-devkit master
建议明确提供支持 9202 HIF4 的 SDK 版本,并补充 9202 目标编译与数值验证;在该前提未满足前,将 HIF4 StreamK 暂限于 9201,并同步约束 Host 路由、tiling key 与 Kernel 分发。 以上结论限定于所核验的公开 SDK 版本;若目标使用已有 9202 支持的其他 SDK,请提供对应版本/实现和验证结果。相关 Device 分支有存量代码,本意见针对本次开放 Host 可达路径,并不将全部底层限制描述为本次新增回归。 ![]() ![]() | |||
| 72 | if (compileInfo_.aicNum != compileInfo_.aivNum || !isQuantCapable || inputParams_.batchC > 1) { | 71 | if (compileInfo_.aicNum != compileInfo_.aivNum || !isQuantCapable || inputParams_.batchC > 1) { |
| 73 | return false; | 72 | return false; |
| 74 | } | 73 | } |
| @@ -356,6 +355,7 @@ ge::graphStatus QBMMV3StreamKTiling960::PostTiling() | |||
| 356 | return ge::GRAPH_PARAM_INVALID; | 355 | return ge::GRAPH_PARAM_INVALID; |
| 357 | } | 356 | } |
| 358 | context_->SetBlockDim(tilingData_.streamKTiling.usedCoreNum); | 357 | context_->SetBlockDim(tilingData_.streamKTiling.usedCoreNum); |
| 358 | + context_->SetScheduleMode(1); // StreamK kernel uses cross-core synchronization and requires batch mode. | ||
| 359 | context_->GetRawTilingData()->SetDataSize(tilingDataSize_); | 359 | context_->GetRawTilingData()->SetDataSize(tilingDataSize_); |
| 360 | size_t* workspaces = context_->GetWorkspaceSizes(1); // set workspace | 360 | size_t* workspaces = context_->GetWorkspaceSizes(1); // set workspace |
| 361 | OPS_CHECK_NULL_WITH_CONTEXT(context_, workspaces); | 361 | OPS_CHECK_NULL_WITH_CONTEXT(context_, workspaces); |
| @@ -10,9 +10,10 @@ | |||
| 10 | 10 | ||
| 11 | /*! | 11 | /*! |
| 12 | * \file qbmm_streamk_tiling_960.h | 12 | * \file qbmm_streamk_tiling_960.h |
| 13 | - * \brief StreamK tiling implementation for DAV_9201/DAV_9202 QuantBatchMatmulV3, | 13 | + * \brief StreamK tiling implementation for Ascend960DT/PR(DAV_9201/9202) QuantBatchMatmulV3, |
| 14 | - * DAV_9201所有量化走Blaze(复用QbmmMxStreamKBasicApiKernel), | 14 | + * 960DT/960PR(9201/9202)的StreamK量化场景统一走Blaze(复用QbmmMxStreamKBasicApiKernel), |
| 15 | - * DAV_9202走CMCT BasicAPI。tiling data复用QuantBatchMatmulV3StreamKBasicAPITilingData | 15 | + * 非StreamK场景由ASW MX BasicAPI模板分流(9201 Blaze,9202 CMCT)。 |
| 16 | + * tiling data复用QuantBatchMatmulV3StreamKBasicAPITilingData | ||
| 16 | * (与3510 Blaze StreamK共用布局),按SocVersion注册调度。 | 17 | * (与3510 Blaze StreamK共用布局),按SocVersion注册调度。 |
| 17 | */ | 18 | */ |
| 18 | #ifndef QBMM_STREAMK_TILING_960_H | 19 | #ifndef QBMM_STREAMK_TILING_960_H |
| @@ -14,7 +14,6 @@ | |||
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | #include "common/inc/op_host/op_const_def_nn.h" | 16 | #include "common/inc/op_host/op_const_def_nn.h" |
| 17 | -#include "common/op_host/op_tiling/tiling_type_mm.h" | ||
| 18 | #include "log/log.h" | 17 | #include "log/log.h" |
| 19 | #include "matmul/common/op_host/log_format_util.h" | 18 | #include "matmul/common/op_host/log_format_util.h" |
| 20 | #include "quant_batch_matmul_v3_checker.h" | 19 | #include "quant_batch_matmul_v3_checker.h" |
| @@ -1192,7 +1191,7 @@ bool QuantBatchMatmulV3Checker::CheckShape(const std::vector<gert::Shape*>& mand | |||
| 1192 | !CheckInputValidInMxPerGroupMode(scaleShape, pertokenShape, x1Shape, x2Shape, dimValueOfMKN))) { | 1191 | !CheckInputValidInMxPerGroupMode(scaleShape, pertokenShape, x1Shape, x2Shape, dimValueOfMKN))) { |
| 1193 | return false; | 1192 | return false; |
| 1194 | } | 1193 | } |
| 1195 | - if (!CheckExtraShapeValid(scaleShape, pertokenShape, dimValueOfMKN)) { | 1194 | + if (!CheckExtraShapeValid(scaleShape, pertokenShape, x1Shape, x2Shape, dimValueOfMKN)) { |
| 1196 | return false; | 1195 | return false; |
| 1197 | } | 1196 | } |
| 1198 | if (!ExtraInputCheck()) { | 1197 | if (!ExtraInputCheck()) { |
| @@ -1232,8 +1231,11 @@ bool QuantBatchMatmulV3Checker::CheckHif4ScaleDtypeWithPertoken() const { return | |||
| 1232 | 1231 | ||
| 1233 | bool QuantBatchMatmulV3Checker::CheckExtraOutputDtype() const { return true; } | 1232 | bool QuantBatchMatmulV3Checker::CheckExtraOutputDtype() const { return true; } |
| 1234 | 1233 | ||
| 1235 | -bool QuantBatchMatmulV3Checker::CheckExtraShapeValid(const gert::Shape&, const gert::StorageShape*, | 1234 | +bool QuantBatchMatmulV3Checker::CheckExtraShapeValid([[maybe_unused]] const gert::Shape& scaleShape, |
| 1236 | - const std::vector<int64_t>&) const | 1235 | + [[maybe_unused]] const gert::StorageShape* pertokenShape, |
| 1236 | + [[maybe_unused]] const gert::Shape& x1Shape, | ||
| 1237 | + [[maybe_unused]] const gert::Shape& x2Shape, | ||
| 1238 | + [[maybe_unused]] const std::vector<int64_t>& dimValueOfMKN) const | ||
| 1237 | { | 1239 | { |
| 1238 | return true; | 1240 | return true; |
| 1239 | } | 1241 | } |
| @@ -83,6 +83,7 @@ protected: | |||
| 83 | virtual bool CheckHif4ScaleDtypeWithPertoken() const; | 83 | virtual bool CheckHif4ScaleDtypeWithPertoken() const; |
| 84 | virtual bool CheckExtraOutputDtype() const; | 84 | virtual bool CheckExtraOutputDtype() const; |
| 85 | virtual bool CheckExtraShapeValid(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape, | 85 | virtual bool CheckExtraShapeValid(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape, |
| 86 | + const gert::Shape& x1Shape, const gert::Shape& x2Shape, | ||
| 86 | const std::vector<int64_t>& dimValueOfMKN) const; | 87 | const std::vector<int64_t>& dimValueOfMKN) const; |
| 87 | }; | 88 | }; |
| 88 | } // namespace optiling | 89 | } // namespace optiling |
| @@ -14,18 +14,21 @@ | |||
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | #include "common/inc/op_host/op_const_def_nn.h" | 16 | #include "common/inc/op_host/op_const_def_nn.h" |
| 17 | -#include "common/op_host/op_tiling/tiling_type_mm.h" | ||
| 18 | #include "error_util.h" | 17 | #include "error_util.h" |
| 19 | #include "log/log.h" | 18 | #include "log/log.h" |
| 19 | +#include "matmul/common/op_host/log_format_util.h" | ||
| 20 | #include "quant_batch_matmul_v3_checker_arch_9201.h" | 20 | #include "quant_batch_matmul_v3_checker_arch_9201.h" |
| 21 | 21 | ||
| 22 | namespace { | 22 | namespace { |
| 23 | constexpr uint64_t HIF4_GROUP_SIZE = 64UL; | 23 | constexpr uint64_t HIF4_GROUP_SIZE = 64UL; |
| 24 | constexpr size_t SCALE_THREE_DIM = 3; | 24 | constexpr size_t SCALE_THREE_DIM = 3; |
| 25 | +constexpr size_t DIM_NUM_TWO = 2; | ||
| 25 | } // namespace | 26 | } // namespace |
| 26 | 27 | ||
| 27 | namespace optiling { | 28 | namespace optiling { |
| 28 | 29 | ||
| 30 | +using Ops::NN::FormatString; | ||
| 31 | + | ||
| 29 | const std::vector<ge::DataType>& QuantBatchMatmulV3CheckerArch9201::GetLegalInputDtypes() const | 32 | const std::vector<ge::DataType>& QuantBatchMatmulV3CheckerArch9201::GetLegalInputDtypes() const |
| 30 | { | 33 | { |
| 31 | static const std::vector<ge::DataType> legalInputDtypes = {ge::DT_INT8, ge::DT_FLOAT8_E4M3FN, | 34 | static const std::vector<ge::DataType> legalInputDtypes = {ge::DT_INT8, ge::DT_FLOAT8_E4M3FN, |
| @@ -112,22 +115,26 @@ bool QuantBatchMatmulV3CheckerArch9201::CheckExtraOutputDtype() const | |||
| 112 | 115 | ||
| 113 | bool QuantBatchMatmulV3CheckerArch9201::CheckExtraShapeValid(const gert::Shape& scaleShape, | 116 | bool QuantBatchMatmulV3CheckerArch9201::CheckExtraShapeValid(const gert::Shape& scaleShape, |
| 114 | const gert::StorageShape* pertokenShape, | 117 | const gert::StorageShape* pertokenShape, |
| 118 | + const gert::Shape& x1Shape, const gert::Shape& x2Shape, | ||
| 115 | const std::vector<int64_t>& dimValueOfMKN) const | 119 | const std::vector<int64_t>& dimValueOfMKN) const |
| 116 | { | 120 | { |
| 117 | - return CheckInputValidInHif4PerGroupMode(scaleShape, pertokenShape, dimValueOfMKN); | 121 | + return CheckInputValidInHif4PerGroupMode(scaleShape, pertokenShape, x1Shape, x2Shape, dimValueOfMKN); |
| 118 | } | 122 | } |
| 119 | 123 | ||
| 120 | bool QuantBatchMatmulV3CheckerArch9201::Hif4PertokenScaleShapeCheck(const gert::StorageShape* pertokenShape) const | 124 | bool QuantBatchMatmulV3CheckerArch9201::Hif4PertokenScaleShapeCheck(const gert::StorageShape* pertokenShape) const |
| 121 | { | 125 | { |
| 122 | auto& pertoken = pertokenShape->GetStorageShape(); | 126 | auto& pertoken = pertokenShape->GetStorageShape(); |
| 123 | auto pertokenShapeLen = pertoken.GetDimNum(); | 127 | auto pertokenShapeLen = pertoken.GetDimNum(); |
| 124 | - OP_TILING_CHECK(pertokenShapeLen != SCALE_THREE_DIM, | 128 | + OP_TILING_CHECK(pertokenShapeLen < SCALE_THREE_DIM, |
| 125 | - CUBE_INNER_ERR_REPORT(inputParams_.opName, | 129 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( |
| 126 | - "In hif4 quantification, pertokenScale's dimension must be 3, but it is %zu.", | 130 | + inputParams_.opName, "pertokenScale", FormatString("%zuD", pertokenShapeLen).c_str(), |
| 127 | - pertokenShapeLen), | 131 | + FormatString("in hif4 quantification, the shape dim of pertokenScale must be at least %zuD", |
| 132 | + SCALE_THREE_DIM) | ||
| 133 | + .c_str()), | ||
| 128 | return false); | 134 | return false); |
| 129 | - auto mDimIdx = inputParams_.transA ? 1 : 0; | 135 | + size_t batchDimNum = pertokenShapeLen - SCALE_THREE_DIM; |
| 130 | - auto kDimIdx = inputParams_.transA ? 0 : 1; | 136 | + auto mDimIdx = batchDimNum + (inputParams_.transA ? 1 : 0); |
| 137 | + auto kDimIdx = batchDimNum + (inputParams_.transA ? 0 : 1); | ||
| 131 | OP_TILING_CHECK(static_cast<uint64_t>(pertoken.GetDim(pertokenShapeLen - 1)) != 1, | 138 | OP_TILING_CHECK(static_cast<uint64_t>(pertoken.GetDim(pertokenShapeLen - 1)) != 1, |
| 132 | CUBE_INNER_ERR_REPORT(inputParams_.opName, "In hif4 quantification, pertokenScale's last dimension \ | 139 | CUBE_INNER_ERR_REPORT(inputParams_.opName, "In hif4 quantification, pertokenScale's last dimension \ |
| 133 | should be equal to 1, but actual is [%ld].", | 140 | should be equal to 1, but actual is [%ld].", |
| @@ -146,7 +153,7 @@ should be equal to 1, but actual is [%ld].", | |||
| 146 | static_cast<uint64_t>(pertoken.GetDim(kDimIdx)) != ops::CeilDiv(inputParams_.kSize, HIF4_GROUP_SIZE), | 153 | static_cast<uint64_t>(pertoken.GetDim(kDimIdx)) != ops::CeilDiv(inputParams_.kSize, HIF4_GROUP_SIZE), |
| 147 | CUBE_INNER_ERR_REPORT( | 154 | CUBE_INNER_ERR_REPORT( |
| 148 | inputParams_.opName, | 155 | inputParams_.opName, |
| 149 | - "In hif4 quantification, when x1's transposition is %s, pertokenScale's dimension %d shape \ | 156 | + "In hif4 quantification, when x1's transposition is %s, pertokenScale's dimension %zu shape \ |
| 150 | should be equal to x1 k dimension size[%lu] ceil div 64, but actual is [%ld].", | 157 | should be equal to x1 k dimension size[%lu] ceil div 64, but actual is [%ld].", |
| 151 | inputParams_.transA ? "true" : "false", kDimIdx, inputParams_.kSize, pertoken.GetDim(kDimIdx)), | 158 | inputParams_.transA ? "true" : "false", kDimIdx, inputParams_.kSize, pertoken.GetDim(kDimIdx)), |
| 152 | return false); | 159 | return false); |
| @@ -154,7 +161,7 @@ should be equal to x1 k dimension size[%lu] ceil div 64, but actual is [%ld].", | |||
| 154 | static_cast<uint64_t>(pertoken.GetDim(mDimIdx)) != inputParams_.mSize, | 161 | static_cast<uint64_t>(pertoken.GetDim(mDimIdx)) != inputParams_.mSize, |
| 155 | CUBE_INNER_ERR_REPORT( | 162 | CUBE_INNER_ERR_REPORT( |
| 156 | inputParams_.opName, | 163 | inputParams_.opName, |
| 157 | - "In hif4 quantification, when x1's transposition is %s, pertokenScale's dimension %d shape \ | 164 | + "In hif4 quantification, when x1's transposition is %s, pertokenScale's dimension %zu shape \ |
| 158 | should be equal to x1 m dimension size[%lu], but actual is [%ld].", | 165 | should be equal to x1 m dimension size[%lu], but actual is [%ld].", |
| 159 | inputParams_.transA ? "true" : "false", mDimIdx, inputParams_.mSize, pertoken.GetDim(mDimIdx)), | 166 | inputParams_.transA ? "true" : "false", mDimIdx, inputParams_.mSize, pertoken.GetDim(mDimIdx)), |
| 160 | return false); | 167 | return false); |
| @@ -166,12 +173,15 @@ bool QuantBatchMatmulV3CheckerArch9201::Hif4ScaleShapeCheck(const gert::Shape& s | |||
| 166 | { | 173 | { |
| 167 | auto scaleShapeLen = scaleShape.GetDimNum(); | 174 | auto scaleShapeLen = scaleShape.GetDimNum(); |
| 168 | OP_TILING_CHECK( | 175 | OP_TILING_CHECK( |
| 169 | - scaleShapeLen != SCALE_THREE_DIM, | 176 | + scaleShapeLen < SCALE_THREE_DIM, |
| 170 | - CUBE_INNER_ERR_REPORT(inputParams_.opName, | 177 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( |
| 171 | - "In hif4 quantification, scale's dimension must be 3, but it is %zu.", scaleShapeLen), | 178 | + inputParams_.opName, "scale", FormatString("%zuD", scaleShapeLen).c_str(), |
| 179 | + FormatString("in hif4 quantification, the shape dim of scale must be at least %zuD", SCALE_THREE_DIM) | ||
| 180 | + .c_str()), | ||
| 172 | return false); | 181 | return false); |
| 173 | - auto kDimIdx = inputParams_.transB ? 1 : 0; | 182 | + size_t batchDimNum = scaleShapeLen - SCALE_THREE_DIM; |
| 174 | - auto nDimIdx = inputParams_.transB ? 0 : 1; | 183 | + auto kDimIdx = batchDimNum + (inputParams_.transB ? 1 : 0); |
| 184 | + auto nDimIdx = batchDimNum + (inputParams_.transB ? 0 : 1); | ||
| 175 | OP_TILING_CHECK(static_cast<uint64_t>(scaleShape.GetDim(scaleShapeLen - 1)) != 1, | 185 | OP_TILING_CHECK(static_cast<uint64_t>(scaleShape.GetDim(scaleShapeLen - 1)) != 1, |
| 176 | CUBE_INNER_ERR_REPORT(inputParams_.opName, | 186 | CUBE_INNER_ERR_REPORT(inputParams_.opName, |
| 177 | "In hif4 quantification, scale's dimension 2 shape should be equal to 1, but \ | 187 | "In hif4 quantification, scale's dimension 2 shape should be equal to 1, but \ |
| @@ -191,14 +201,14 @@ actual is [%ld].", | |||
| 191 | static_cast<uint64_t>(scaleShape.GetDim(kDimIdx)) != ops::CeilDiv(inputParams_.kSize, HIF4_GROUP_SIZE), | 201 | static_cast<uint64_t>(scaleShape.GetDim(kDimIdx)) != ops::CeilDiv(inputParams_.kSize, HIF4_GROUP_SIZE), |
| 192 | CUBE_INNER_ERR_REPORT( | 202 | CUBE_INNER_ERR_REPORT( |
| 193 | inputParams_.opName, | 203 | inputParams_.opName, |
| 194 | - "In hif4 quantification, when x2's transposition is %s, scale's dimension %d shape should \ | 204 | + "In hif4 quantification, when x2's transposition is %s, scale's dimension %zu shape should \ |
| 195 | be equal to x2 k dimension size[%lu] ceil div 64, but actual is [%ld].", | 205 | be equal to x2 k dimension size[%lu] ceil div 64, but actual is [%ld].", |
| 196 | inputParams_.transB ? "true" : "false", kDimIdx, inputParams_.kSize, scaleShape.GetDim(kDimIdx)), | 206 | inputParams_.transB ? "true" : "false", kDimIdx, inputParams_.kSize, scaleShape.GetDim(kDimIdx)), |
| 197 | return false); | 207 | return false); |
| 198 | OP_TILING_CHECK( | 208 | OP_TILING_CHECK( |
| 199 | static_cast<uint64_t>(scaleShape.GetDim(nDimIdx)) != inputParams_.nSize, | 209 | static_cast<uint64_t>(scaleShape.GetDim(nDimIdx)) != inputParams_.nSize, |
| 200 | CUBE_INNER_ERR_REPORT( | 210 | CUBE_INNER_ERR_REPORT( |
| 201 | - inputParams_.opName, "In hif4 quantification, when x2's transposition is %s, scale's dimension %d \ | 211 | + inputParams_.opName, "In hif4 quantification, when x2's transposition is %s, scale's dimension %zu \ |
| 202 | shape should be equal to x2 n dimension size[%lu], but actual is [%ld].", | 212 | shape should be equal to x2 n dimension size[%lu], but actual is [%ld].", |
| 203 | inputParams_.transB ? "true" : "false", nDimIdx, inputParams_.nSize, scaleShape.GetDim(nDimIdx)), | 213 | inputParams_.transB ? "true" : "false", nDimIdx, inputParams_.nSize, scaleShape.GetDim(nDimIdx)), |
| 204 | return false); | 214 | return false); |
| @@ -206,9 +216,51 @@ shape should be equal to x2 n dimension size[%lu], but actual is [%ld].", | |||
| 206 | return true; | 216 | return true; |
| 207 | } | 217 | } |
| 208 | 218 | ||
| 219 | +bool QuantBatchMatmulV3CheckerArch9201::CheckBatchValidInHif4PerGroupMode(const gert::Shape& scaleShape, | ||
| 220 | + const gert::Shape& pertoken, | ||
| 221 | + const gert::Shape& x1Shape, | ||
| 222 | + const gert::Shape& x2Shape) const | ||
| 223 | +{ | ||
| 224 | + auto x1ShapeLen = x1Shape.GetDimNum(); | ||
| 225 | + auto x2ShapeLen = x2Shape.GetDimNum(); | ||
| 226 | + auto pertokenBatchLen = pertoken.GetDimNum() - SCALE_THREE_DIM; | ||
| 227 | + auto scaleBatchLen = scaleShape.GetDimNum() - SCALE_THREE_DIM; | ||
| 228 | + auto x1BatchLen = (x1ShapeLen > DIM_NUM_TWO) ? (x1ShapeLen - DIM_NUM_TWO) : 0; | ||
| 229 | + auto x2BatchLen = (x2ShapeLen > DIM_NUM_TWO) ? (x2ShapeLen - DIM_NUM_TWO) : 0; | ||
| 230 | + OP_TILING_CHECK(scaleBatchLen != x2BatchLen, | ||
| 231 | + OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | ||
| 232 | + inputParams_.opName, "scaleBatchDimNum, x2BatchDimNum", | ||
| 233 | + FormatString("%zu, %zu", scaleBatchLen, x2BatchLen).c_str(), | ||
| 234 | + "in hif4 quantification, the batch dimension num of scale must be equal to that of x2"), | ||
| 235 | + return false); | ||
| 236 | + OP_TILING_CHECK(pertokenBatchLen != x1BatchLen, | ||
| 237 | + OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | ||
| 238 | + inputParams_.opName, "pertokenBatchDimNum, x1BatchDimNum", | ||
| 239 | + FormatString("%zu, %zu", pertokenBatchLen, x1BatchLen).c_str(), | ||
| 240 | + "in hif4 quantification, the batch dimension num of pertokenScale must be equal to that of x1"), | ||
| 241 | + return false); | ||
| 242 | + for (size_t i = 0; i < x2BatchLen; ++i) { | ||
| 243 | + OP_TILING_CHECK(scaleShape.GetDim(i) != x2Shape.GetDim(i), | ||
| 244 | + OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | ||
| 245 | + inputParams_.opName, "dimIndex, x2Batch, scaleBatch", | ||
| 246 | + FormatString("%zu, %ld, %ld", i, x2Shape.GetDim(i), scaleShape.GetDim(i)).c_str(), | ||
| 247 | + "in hif4 quantification, the batch dimension of scale must be equal to that of x2"), | ||
| 248 | + return false); | ||
| 249 | + } | ||
| 250 | + for (size_t i = 0; i < x1BatchLen; ++i) { | ||
| 251 | + OP_TILING_CHECK(pertoken.GetDim(i) != x1Shape.GetDim(i), | ||
| 252 | + OP_LOGE_FOR_INVALID_VALUES_WITH_REASON( | ||
| 253 | + inputParams_.opName, "dimIndex, x1Batch, pertokenBatch", | ||
| 254 | + FormatString("%zu, %ld, %ld", i, x1Shape.GetDim(i), pertoken.GetDim(i)).c_str(), | ||
| 255 | + "in hif4 quantification, the batch dimension of pertokenScale must be equal to that of x1"), | ||
| 256 | + return false); | ||
| 257 | + } | ||
| 258 | + return true; | ||
| 259 | +} | ||
| 260 | + | ||
| 209 | bool QuantBatchMatmulV3CheckerArch9201::CheckInputValidInHif4PerGroupMode( | 261 | bool QuantBatchMatmulV3CheckerArch9201::CheckInputValidInHif4PerGroupMode( |
| 210 | - const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape, | 262 | + const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape, const gert::Shape& x1Shape, |
| 211 | - const std::vector<int64_t>& dimValueOfMKN) const | 263 | + const gert::Shape& x2Shape, const std::vector<int64_t>& dimValueOfMKN) const |
| 212 | { | 264 | { |
| 213 | if (!inputParams_.isHif4PerGroup) { | 265 | if (!inputParams_.isHif4PerGroup) { |
| 214 | return true; | 266 | return true; |
| @@ -232,6 +284,9 @@ groupSize = groupSizeK | groupSizeN << 16 | groupSizeM << 32.", | |||
| 232 | CUBE_INNER_ERR_REPORT(inputParams_.opName, "The perTokenScale shape check failed."), return false); | 284 | CUBE_INNER_ERR_REPORT(inputParams_.opName, "The perTokenScale shape check failed."), return false); |
| 233 | OP_TILING_CHECK(!Hif4ScaleShapeCheck(scaleShape), | 285 | OP_TILING_CHECK(!Hif4ScaleShapeCheck(scaleShape), |
| 234 | CUBE_INNER_ERR_REPORT(inputParams_.opName, "The scale shape check failed."), return false); | 286 | CUBE_INNER_ERR_REPORT(inputParams_.opName, "The scale shape check failed."), return false); |
| 287 | + if (!CheckBatchValidInHif4PerGroupMode(scaleShape, pertokenShape->GetStorageShape(), x1Shape, x2Shape)) { | ||
| 288 | + return false; | ||
| 289 | + } | ||
| 235 | // hif4 场景下 K 需大于 2(与 aclnn/torch 接口约束一致,K<=2 时无法判断是否转置) | 290 | // hif4 场景下 K 需大于 2(与 aclnn/torch 接口约束一致,K<=2 时无法判断是否转置) |
| 236 | OP_TILING_CHECK( | 291 | OP_TILING_CHECK( |
| 237 | inputParams_.kSize <= 2UL, | 292 | inputParams_.kSize <= 2UL, |
| @@ -36,12 +36,16 @@ protected: | |||
| 36 | bool CheckHif4ScaleDtypeWithPertoken() const override; | 36 | bool CheckHif4ScaleDtypeWithPertoken() const override; |
| 37 | bool CheckExtraOutputDtype() const override; | 37 | bool CheckExtraOutputDtype() const override; |
| 38 | bool CheckExtraShapeValid(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape, | 38 | bool CheckExtraShapeValid(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape, |
| 39 | + const gert::Shape& x1Shape, const gert::Shape& x2Shape, | ||
| 39 | const std::vector<int64_t>& dimValueOfMKN) const override; | 40 | const std::vector<int64_t>& dimValueOfMKN) const override; |
| 40 | 41 | ||
| 41 | private: | 42 | private: |
| 42 | bool Hif4PertokenScaleShapeCheck(const gert::StorageShape* pertokenShape) const; | 43 | bool Hif4PertokenScaleShapeCheck(const gert::StorageShape* pertokenShape) const; |
| 43 | bool Hif4ScaleShapeCheck(const gert::Shape& scaleShape) const; | 44 | bool Hif4ScaleShapeCheck(const gert::Shape& scaleShape) const; |
| 45 | + bool CheckBatchValidInHif4PerGroupMode(const gert::Shape& scaleShape, const gert::Shape& pertoken, | ||
| 46 | + const gert::Shape& x1Shape, const gert::Shape& x2Shape) const; | ||
| 44 | bool CheckInputValidInHif4PerGroupMode(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape, | 47 | bool CheckInputValidInHif4PerGroupMode(const gert::Shape& scaleShape, const gert::StorageShape* pertokenShape, |
| 48 | + const gert::Shape& x1Shape, const gert::Shape& x2Shape, | ||
| 45 | const std::vector<int64_t>& dimValueOfMKN) const; | 49 | const std::vector<int64_t>& dimValueOfMKN) const; |
| 46 | }; | 50 | }; |
| 47 | } // namespace optiling | 51 | } // namespace optiling |
| @@ -800,14 +800,20 @@ bool QuantBatchMatmulV3TilingBase::AnalyzeHif4GroupInfo(const gert::Shape& scale | |||
| 800 | if (inputParams_.groupSizeK == 0UL) { | 800 | if (inputParams_.groupSizeK == 0UL) { |
| 801 | inputParams_.groupSizeK = hif4GroupSize; | 801 | inputParams_.groupSizeK = hif4GroupSize; |
| 802 | } | 802 | } |
| 803 | - if (pertokenShapeLen != hif4ScaleDimNum || scaleShapeLen != hif4ScaleDimNum) { | 803 | + if (pertokenShapeLen < hif4ScaleDimNum || scaleShapeLen < hif4ScaleDimNum) { |
| 804 | // scale 维度非法时由后续 checker 报错 | 804 | // scale 维度非法时由后续 checker 报错 |
| 805 | return true; | 805 | return true; |
| 806 | } | 806 | } |
| 807 | + const size_t pertokenBatchDimNum = pertokenShapeLen - hif4ScaleDimNum; | ||
| 808 | + const size_t scaleBatchDimNum = scaleShapeLen - hif4ScaleDimNum; | ||
| 809 | + const size_t pertokenMDimIdx = pertokenBatchDimNum + (inputParams_.transA ? 1 : 0); | ||
| 810 | + const size_t pertokenKDimIdx = pertokenBatchDimNum + (inputParams_.transA ? 0 : 1); | ||
| 811 | + const size_t scaleKDimIdx = scaleBatchDimNum + (inputParams_.transB ? 1 : 0); | ||
| 812 | + const size_t scaleNDimIdx = scaleBatchDimNum + (inputParams_.transB ? 0 : 1); | ||
| 807 | uint64_t expectedScaleK = ops::CeilDiv(inputParams_.kSize, hif4GroupSize); | 813 | uint64_t expectedScaleK = ops::CeilDiv(inputParams_.kSize, hif4GroupSize); |
| 808 | // 存在 1 维广播时 K 维校验交由 checker 判定,此处只校验非广播场景的 K 维覆盖 ceil(K / 64) | 814 | // 存在 1 维广播时 K 维校验交由 checker 判定,此处只校验非广播场景的 K 维覆盖 ceil(K / 64) |
| 809 | - if (pertoken.GetDim(0) != 1 && pertoken.GetDim(1) != 1) { | 815 | + if (pertoken.GetDim(pertokenMDimIdx) != 1 && pertoken.GetDim(pertokenKDimIdx) != 1) { |
| 810 | - uint64_t pertokenKSize = static_cast<uint64_t>(pertoken.GetDim(inputParams_.transA ? 0 : 1)); | 816 | + uint64_t pertokenKSize = static_cast<uint64_t>(pertoken.GetDim(pertokenKDimIdx)); |
| 811 | OP_TILING_CHECK( | 817 | OP_TILING_CHECK( |
| 812 | pertokenKSize != expectedScaleK, | 818 | pertokenKSize != expectedScaleK, |
| 813 | CUBE_INNER_ERR_REPORT(inputParams_.opName, | 819 | CUBE_INNER_ERR_REPORT(inputParams_.opName, |
| @@ -816,8 +822,8 @@ bool QuantBatchMatmulV3TilingBase::AnalyzeHif4GroupInfo(const gert::Shape& scale | |||
| 816 | pertokenKSize, inputParams_.kSize, expectedScaleK), | 822 | pertokenKSize, inputParams_.kSize, expectedScaleK), |
| 817 | return false); | 823 | return false); |
| 818 | } | 824 | } |
| 819 | - if (scaleShape.GetDim(0) != 1 && scaleShape.GetDim(1) != 1) { | 825 | + if (scaleShape.GetDim(scaleKDimIdx) != 1 && scaleShape.GetDim(scaleNDimIdx) != 1) { |
| 820 | - uint64_t scaleKSize = static_cast<uint64_t>(scaleShape.GetDim(inputParams_.transB ? 1 : 0)); | 826 | + uint64_t scaleKSize = static_cast<uint64_t>(scaleShape.GetDim(scaleKDimIdx)); |
| 821 | OP_TILING_CHECK( | 827 | OP_TILING_CHECK( |
| 822 | scaleKSize != expectedScaleK, | 828 | scaleKSize != expectedScaleK, |
| 823 | CUBE_INNER_ERR_REPORT(inputParams_.opName, | 829 | CUBE_INNER_ERR_REPORT(inputParams_.opName, |
| @@ -448,7 +448,8 @@ UT_STATIC __global__ __aicore__ void quant_batch_matmul_v3(GM_ADDR x1, GM_ADDR x | |||
| 448 | 448 | ||
| 449 | #if QBMMV3_IS_HIF4_DTYPE_TPL && defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202) | 449 | #if QBMMV3_IS_HIF4_DTYPE_TPL && defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202) |
| 450 | if constexpr (DequantBmm::IsHif4Type<DTYPE_X1, DTYPE_SCALE>()) { | 450 | if constexpr (DequantBmm::IsHif4Type<DTYPE_X1, DTYPE_SCALE>()) { |
| 451 | - // hif4: 9202 全部走 CMCT;9201 按 api level 分流 Blaze/CMCT | 451 | + // hif4: 9201 ASW/StreamK 均走 Blaze(BASIC 仅兜底 tensorapi 不可用); |
| 452 | + // 9202 仅 StreamK 走 Blaze,其余 ASW 走 CMCT | ||
| 452 | #if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201) && IS_BLAZE | 453 | #if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201) && IS_BLAZE |
| 453 | if constexpr (TPL_BATCHMODE == TPL_WITHOUT_BATCH && TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI && | 454 | if constexpr (TPL_BATCHMODE == TPL_WITHOUT_BATCH && TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI && |
| 454 | TPL_APILEVEL == TPL_API_LEVEL_BLAZE) { | 455 | TPL_APILEVEL == TPL_API_LEVEL_BLAZE) { |
| @@ -458,28 +459,25 @@ UT_STATIC __global__ __aicore__ void quant_batch_matmul_v3(GM_ADDR x1, GM_ADDR x | |||
| 458 | TPL_APILEVEL == TPL_API_LEVEL_BLAZE) { | 459 | TPL_APILEVEL == TPL_API_LEVEL_BLAZE) { |
| 459 | QUANT_BMMV3_MX_WITHOUT_BATCH_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn, | 460 | QUANT_BMMV3_MX_WITHOUT_BATCH_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn, |
| 460 | Blaze::Gemm::A_FULL_LOAD_MODE); | 461 | Blaze::Gemm::A_FULL_LOAD_MODE); |
| 462 | + } else if constexpr (TPL_BATCHMODE == TPL_WITH_BATCH && TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI && | ||
| 463 | + TPL_APILEVEL == TPL_API_LEVEL_BLAZE) { | ||
| 464 | + QUANT_BMMV3_MX_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn, 0); | ||
🟡 Medium Priority 改动点:kernel 新增 问题:这条路由由三方隐式约定构成——host 侧 建议:在 op_host tiling UT 中新增上述 3 个组合的 tiling key 断言(hif4 9201 batch>1 / batchC==1、hif4 9202),确保 host 下发的 key 与 apt_tiling_key.h 的注册集合、kernel 的 if-constexpr 分支三者一致;若框架允许,可在 kernel 尾部加静态断言/兜底分支替代“静默空 kernel”。 ![]() ![]() Incorrect? | |||
| 461 | } | 465 | } |
| 462 | #endif | 466 | #endif |
| 463 | -#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201) | 467 | + // 9202 ASW / 9201 tensorapi fallback |
| 464 | - if constexpr (TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI && TPL_APILEVEL == TPL_API_LEVEL_BASIC) { | 468 | + if constexpr (TPL_BATCHMODE == TPL_WITH_BATCH && TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI && |
[正确性][必须修复] 🔴 补齐 CMCT 的 HIF4 batch scale 偏移 本 PR 的 checker 已允许 HIF4 的 batch scale,9202 的 batch>1 输入会进入这里,9201 的 TensorAPI fallback 也受影响。例如 transA=transB=false,x1=[2,16,256]、x2=[2,256,32]、pertokenScale=[2,16,4,1]、scale=[2,4,32,1] 时, ![]() ![]() | |||
| 469 | + TPL_APILEVEL == TPL_API_LEVEL_BASIC) { | ||
| 465 | QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign, 0); | 470 | QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign, 0); |
| 466 | - } else if constexpr (TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI && | 471 | + } else if constexpr (TPL_BATCHMODE == TPL_WITH_BATCH && |
| 472 | + TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI && | ||
| 467 | TPL_APILEVEL == TPL_API_LEVEL_BASIC) { | 473 | TPL_APILEVEL == TPL_API_LEVEL_BASIC) { |
| 468 | QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign, | 474 | QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign, |
| 469 | Cmct::Gemm::A_FULL_LOAD_MODE); | 475 | Cmct::Gemm::A_FULL_LOAD_MODE); |
| 470 | } | 476 | } |
| 471 | -#else | 477 | +#if IS_BLAZE |
| 472 | - if constexpr (TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_WITH_MMAPI) { | ||
| 473 | - QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign, 0); | ||
| 474 | - } else if constexpr (TPL_KERNELTYPE == TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI) { | ||
| 475 | - QUANT_BMMV3_MX_CMCT_IMPL_CLASS(CmctALayout, CmctBLayoutNd, Cmct::Gemm::layout::RowMajorAlign, | ||
| 476 | - Cmct::Gemm::A_FULL_LOAD_MODE); | ||
| 477 | - } | ||
| 478 | -#endif | ||
| 479 | -#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202) && IS_BLAZE | ||
| 480 | if constexpr (TPL_BATCHMODE == TPL_WITHOUT_BATCH && TPL_KERNELTYPE == TPL_VEC_EPILOGUE_STREAMK_WITH_MMAPI && | 478 | if constexpr (TPL_BATCHMODE == TPL_WITHOUT_BATCH && TPL_KERNELTYPE == TPL_VEC_EPILOGUE_STREAMK_WITH_MMAPI && |
| 481 | TPL_APILEVEL == TPL_API_LEVEL_BLAZE) { | 479 | TPL_APILEVEL == TPL_API_LEVEL_BLAZE) { |
| 482 | - // hif4 streamk, blaze(9201/9202) | 480 | + // hif4 streamk: 9201/9202 均走 Blaze |
| 483 | QUANT_BMMV3_MX_STREAMK_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn, 0); | 481 | QUANT_BMMV3_MX_STREAMK_BLAZE_IMPL_CLASS(TeALayout, TeBLayoutNd, asc::te::nd_ext_layout_ptn, 0); |
| 484 | } | 482 | } |
| 485 | #endif | 483 | #endif |
| @@ -185,7 +185,7 @@ namespace QuantBatchMatmulV3Arch35TilingKey { | |||
| 185 | 185 | ||
| 186 | // Kernel type 11 is shared by the MX StreamK path, the non-MX per-tensor StreamK path and the HIF4 StreamK path. | 186 | // Kernel type 11 is shared by the MX StreamK path, the non-MX per-tensor StreamK path and the HIF4 StreamK path. |
| 187 | #define SUPPORT_STREAMK_TILING_KEY \ | 187 | #define SUPPORT_STREAMK_TILING_KEY \ |
| 188 | - (SUPPORT_MX_WITHOUT_BATCH_TILING_KEY || SUPPORT_NON_MX_STREAMK_TILING_KEY || QBMMV3_IS_HIF4_DTYPE_TPL) | 188 | + (SUPPORT_MX_WITHOUT_BATCH_TILING_KEY || SUPPORT_NON_MX_STREAMK_TILING_KEY || (IS_BLAZE && QBMMV3_IS_HIF4_DTYPE_TPL)) |
| 189 | 189 | ||
| 190 | // Batch Mode | 190 | // Batch Mode |
| 191 | #define TPL_WITH_BATCH 0 | 191 | #define TPL_WITH_BATCH 0 |
| @@ -361,20 +361,28 @@ ASCENDC_TPL_SEL( | |||
| 361 | #endif | 361 | #endif |
| 362 | #if ((!defined(__CCE_AICORE__)) || \ | 362 | #if ((!defined(__CCE_AICORE__)) || \ |
| 363 | (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202) && QBMMV3_IS_HIF4_DTYPE_TPL)) | 363 | (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202) && QBMMV3_IS_HIF4_DTYPE_TPL)) |
| 364 | - ASCENDC_TPL_ARGS_SEL( // hif4 sliding-window: kernel type {0, 1} * batch mode {0, 1} * api level {1, 2} | 364 | +#if ((!defined(__CCE_AICORE__)) || (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201) && IS_BLAZE)) |
| 365 | + ASCENDC_TPL_ARGS_SEL( // hif4 9201 blaze without batch: kernel type {0, 1} * api level {2} | ||
| 365 | ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1), | 366 | ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1), |
| 366 | ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1), | 367 | ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1), |
| 367 | ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITHOUT_BATCH), | 368 | ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITHOUT_BATCH), |
| 368 | ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI, | 369 | ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI, |
| 369 | TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI), | 370 | TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI), |
| 370 | - ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BASIC, TPL_API_LEVEL_BLAZE)), | 371 | + ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BLAZE)), |
| 371 | - ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), | 372 | + ASCENDC_TPL_ARGS_SEL( // hif4 9201 blaze with batch: kernel type {0} (A full load is disabled for batch > 1) |
| 372 | - ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1), | 373 | + ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1), |
| 373 | - ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1), | 374 | + ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1), |
| 374 | - ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITH_BATCH), | 375 | + ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITH_BATCH), |
| 375 | - ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI, | 376 | + ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI), |
| 376 | - TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI), | 377 | + ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BLAZE)), |
| 377 | - ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BASIC)), | 378 | +#endif |
| 379 | + ASCENDC_TPL_ARGS_SEL( // hif4 CMCT basic (ASW): 9202 and 9201 tensorapi fallback | ||
| 380 | + ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), ASCENDC_TPL_UINT_SEL(ATRANS, ASCENDC_TPL_UI_LIST, 0, 1), | ||
| 381 | + ASCENDC_TPL_UINT_SEL(BTRANS, ASCENDC_TPL_UI_LIST, 0, 1), | ||
| 382 | + ASCENDC_TPL_UINT_SEL(BATCHMODE, ASCENDC_TPL_UI_LIST, TPL_WITH_BATCH), | ||
| 383 | + ASCENDC_TPL_UINT_SEL(KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI, | ||
| 384 | + TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI), | ||
| 385 | + ASCENDC_TPL_UINT_SEL(APILEVEL, ASCENDC_TPL_UI_LIST, TPL_API_LEVEL_BASIC)), | ||
| 378 | #endif | 386 | #endif |
| 379 | 387 | ||
| 380 | #if !defined(__CCE_AICORE__) || \ | 388 | #if !defined(__CCE_AICORE__) || \ |
| @@ -22,6 +22,20 @@ Hif4RejectsGroupN,DAV_9201,16 256,HIFLOAT4,256 32,HIFLOAT4,16 4 1,HIFLOAT4_SCALE | |||
| 22 | Hif4RejectsGroupK,DAV_9201,16 256,HIFLOAT4,256 32,HIFLOAT4,16 4 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,16 32,ACL_BF16,false,false,4295032864,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | 22 | Hif4RejectsGroupK,DAV_9201,16 256,HIFLOAT4,256 32,HIFLOAT4,16 4 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,16 32,ACL_BF16,false,false,4295032864,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false |
| 23 | Hif4RejectsOddInnerAxis,DAV_9201,256 15,HIFLOAT4,256 32,HIFLOAT4,4 15 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,15 32,ACL_BF16,true,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | 23 | Hif4RejectsOddInnerAxis,DAV_9201,256 15,HIFLOAT4,256 32,HIFLOAT4,4 15 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,15 32,ACL_BF16,true,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false |
| 24 | Hif4RejectsKTooSmall,DAV_9201,16 2,HIFLOAT4,2 32,HIFLOAT4,16 1 1,HIFLOAT4_SCALE,1 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | 24 | Hif4RejectsKTooSmall,DAV_9201,16 2,HIFLOAT4,2 32,HIFLOAT4,16 1 1,HIFLOAT4_SCALE,1 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false |
| 25 | +Hif4BatchScale4D,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false | ||
| 26 | +Hif4BatchScale4DTrans,DAV_9201,2 256 16,HIFLOAT4,2 32 256,HIFLOAT4,2 4 16 1,HIFLOAT4_SCALE,2 32 4 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,true,true,4295032896,5,check,ACLNN_SUCCESS,true,0,false | ||
| 27 | +Hif4BatchScale5D,DAV_9201,2 3 16 256,HIFLOAT4,2 3 256 32,HIFLOAT4,2 3 16 4 1,HIFLOAT4_SCALE,2 3 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false | ||
| 28 | +Hif4BatchScale6D,DAV_9201,2 3 4 16 256,HIFLOAT4,2 3 4 256 32,HIFLOAT4,2 3 4 16 4 1,HIFLOAT4_SCALE,2 3 4 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 4 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false | ||
| 29 | +Hif4BatchScale7D,DAV_9201,2 3 4 5 16 256,HIFLOAT4,2 3 4 5 256 32,HIFLOAT4,2 3 4 5 16 4 1,HIFLOAT4_SCALE,2 3 4 5 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 4 5 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false | ||
| 30 | +Hif4BatchMixedScaleRanks,DAV_9201,2 3 16 256,HIFLOAT4,256 32,HIFLOAT4,2 3 16 4 1,HIFLOAT4_SCALE,4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false | ||
| 31 | +Hif4ScaleBatchOnly,DAV_9201,16 256,HIFLOAT4,2 256 32,HIFLOAT4,16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_SUCCESS,true,0,false | ||
| 32 | +Hif4RejectsScaleOver7D,DAV_9201,2 3 4 5 6 16 256,HIFLOAT4,2 3 4 5 6 256 32,HIFLOAT4,2 3 4 5 6 16 4 1,HIFLOAT4_SCALE,2 3 4 5 6 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 3 4 5 6 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | ||
| 33 | +Hif4RejectsPertokenBatchCount,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 3 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | ||
| 34 | +Hif4RejectsPertokenInputBatchCount,DAV_9201,16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | ||
| 35 | +Hif4RejectsScaleBatchCount,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,2 3 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | ||
| 36 | +Hif4RejectsScaleInputBatchCount,DAV_9201,2 16 256,HIFLOAT4,256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | ||
| 37 | +Hif4RejectsPertokenBatchValue,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,3 16 4 1,HIFLOAT4_SCALE,2 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | ||
| 38 | +Hif4RejectsScaleBatchValue,DAV_9201,2 16 256,HIFLOAT4,2 256 32,HIFLOAT4,2 16 4 1,HIFLOAT4_SCALE,3 4 32 1,HIFLOAT4_SCALE,null,ACL_FLOAT,null,ACL_FLOAT,2 16 32,ACL_BF16,false,false,4295032896,5,check,ACLNN_ERR_PARAM_INVALID,true,0,false | ||
| 25 | PergroupValidDAV_3510,DAV_3510,16 1024,ACL_INT4,256 1024,ACL_INT4,16 1,ACL_FLOAT,4 256,ACL_FLOAT,4 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_SUCCESS,true,0,true | 39 | PergroupValidDAV_3510,DAV_3510,16 1024,ACL_INT4,256 1024,ACL_INT4,16 1,ACL_FLOAT,4 256,ACL_FLOAT,4 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_SUCCESS,true,0,true |
| 26 | PergroupValidDAV_9201,DAV_9201,16 1024,ACL_INT4,256 1024,ACL_INT4,16 1,ACL_FLOAT,4 256,ACL_FLOAT,4 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_SUCCESS,true,0,true | 40 | PergroupValidDAV_9201,DAV_9201,16 1024,ACL_INT4,256 1024,ACL_INT4,16 1,ACL_FLOAT,4 256,ACL_FLOAT,4 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_SUCCESS,true,0,true |
| 27 | PergroupRejectsKAlignment,DAV_3510,16 512,ACL_INT4,256 512,ACL_INT4,16 1,ACL_FLOAT,2 256,ACL_FLOAT,2 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_ERR_PARAM_INVALID,true,0,true | 41 | PergroupRejectsKAlignment,DAV_3510,16 512,ACL_INT4,256 512,ACL_INT4,16 1,ACL_FLOAT,2 256,ACL_FLOAT,2 256,ACL_FLOAT16,null,ACL_FLOAT,16 256,ACL_BF16,false,true,256,5,check,ACLNN_ERR_PARAM_INVALID,true,0,true |
| @@ -14,7 +14,6 @@ | |||
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | #include <array> | 16 | #include <array> |
| 17 | -#include <cmath> | ||
| 18 | #include <limits> | 17 | #include <limits> |
| 19 | #include <map> | 18 | #include <map> |
| 20 | #include <memory> | 19 | #include <memory> |
| @@ -25,10 +24,12 @@ | |||
| 25 | #define private public | 24 | #define private public |
| 26 | #define protected public | 25 | #define protected public |
| 27 | #include "../test_quant_batch_matmul_v3_tiling.h" | 26 | #include "../test_quant_batch_matmul_v3_tiling.h" |
| 27 | +#include "../../../../op_kernel/arch35/quant_batch_matmul_v3_apt_tiling_key.h" | ||
| 28 | #include "../../../../op_host/op_tiling/arch35/qbmm_streamk_tiling_960.h" | 28 | #include "../../../../op_host/op_tiling/arch35/qbmm_streamk_tiling_960.h" |
| 29 | #include "../../../../op_host/op_tiling/arch35/quant_batch_matmul_v3_checker_arch_9201.h" | 29 | #include "../../../../op_host/op_tiling/arch35/quant_batch_matmul_v3_checker_arch_9201.h" |
| 30 | #include "../../../../op_host/op_tiling/arch35/tiling_cost_evaluator.h" | 30 | #include "../../../../op_host/op_tiling/arch35/tiling_cost_evaluator.h" |
| 31 | #include "../../../../op_host/op_tiling/arch35/l1_tiling_data_calculator.h" | 31 | #include "../../../../op_host/op_tiling/arch35/l1_tiling_data_calculator.h" |
| 32 | +#include "../../../../op_host/op_tiling/arch35/adaptive_sliding_window_mx_basic_api_tiling.h" | ||
| 32 | #include "../../../../op_host/op_tiling/arch35/adaptive_sliding_window_perblock_basic_api_tiling.h" | 33 | #include "../../../../op_host/op_tiling/arch35/adaptive_sliding_window_perblock_basic_api_tiling.h" |
| 33 | #include "../../../../op_host/op_tiling/arch35/quant_batch_matmul_v3_tiling_util.h" | 34 | #include "../../../../op_host/op_tiling/arch35/quant_batch_matmul_v3_tiling_util.h" |
| 34 | #undef protected | 35 | #undef protected |
| @@ -712,9 +713,11 @@ TEST(QuantBatchMatmulV3Checker960, Hif4GroupValidationRejectsMissingScaleAndWron | |||
| 712 | auto input = MakeHif4Info(); | 713 | auto input = MakeHif4Info(); |
| 713 | const gert::Shape scale({3, 48, 1}); | 714 | const gert::Shape scale({3, 48, 1}); |
| 714 | const gert::StorageShape token({16, 3, 1}, {16, 3, 1}); | 715 | const gert::StorageShape token({16, 3, 1}, {16, 3, 1}); |
| 716 | + const gert::Shape x1Shape({16, 130}); | ||
| 717 | + const gert::Shape x2Shape({130, 48}); | ||
| 715 | QuantBatchMatmulV3CheckerArch9201 valid(nullptr, input); | 718 | QuantBatchMatmulV3CheckerArch9201 valid(nullptr, input); |
| 716 | - EXPECT_TRUE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, {130, 16, 130, 48})); | 719 | + EXPECT_TRUE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {130, 16, 130, 48})); |
| 717 | - EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, nullptr, {130, 16, 130, 48})); | 720 | + EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, nullptr, x1Shape, x2Shape, {130, 16, 130, 48})); |
| 718 | for (int axis = 0; axis < 3; ++axis) { | 721 | for (int axis = 0; axis < 3; ++axis) { |
| 719 | auto invalid = input; | 722 | auto invalid = input; |
| 720 | if (axis == 0) { | 723 | if (axis == 0) { |
| @@ -727,10 +730,150 @@ TEST(QuantBatchMatmulV3Checker960, Hif4GroupValidationRejectsMissingScaleAndWron | |||
| 727 | invalid.groupSizeK = 32; | 730 | invalid.groupSizeK = 32; |
| 728 | } | 731 | } |
| 729 | QuantBatchMatmulV3CheckerArch9201 checker(nullptr, invalid); | 732 | QuantBatchMatmulV3CheckerArch9201 checker(nullptr, invalid); |
| 730 | - EXPECT_FALSE(checker.CheckInputValidInHif4PerGroupMode(scale, &token, {130, 16, 130, 48})); | 733 | + EXPECT_FALSE(checker.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {130, 16, 130, 48})); |
| 734 | + } | ||
| 735 | + EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {129, 16, 130, 48})); | ||
| 736 | + EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {130, 16, 129, 48})); | ||
| 737 | +} | ||
| 738 | + | ||
| 739 | +gert::Shape MakeHif4BatchShape(const std::vector<int64_t>& batch, int64_t dim0, int64_t dim1) | ||
| 740 | +{ | ||
| 741 | + gert::Shape shape; | ||
| 742 | + for (const auto dim : batch) { | ||
| 743 | + shape.AppendDim(dim); | ||
| 744 | + } | ||
| 745 | + shape.AppendDim(dim0); | ||
| 746 | + shape.AppendDim(dim1); | ||
| 747 | + shape.AppendDim(1); | ||
| 748 | + return shape; | ||
| 749 | +} | ||
| 750 | + | ||
| 751 | +gert::Shape MakeHif4InputShape(const std::vector<int64_t>& batch, int64_t dim0, int64_t dim1) | ||
| 752 | +{ | ||
| 753 | + gert::Shape shape; | ||
| 754 | + for (const auto dim : batch) { | ||
| 755 | + shape.AppendDim(dim); | ||
| 756 | + } | ||
| 757 | + shape.AppendDim(dim0); | ||
| 758 | + shape.AppendDim(dim1); | ||
| 759 | + return shape; | ||
| 760 | +} | ||
| 761 | + | ||
| 762 | +TEST(QuantBatchMatmulV3Checker960, Hif4ScaleShapesAcceptBatchDimsUpToSevenD) | ||
| 763 | +{ | ||
| 764 | + auto input = MakeHif4Info(); | ||
| 765 | + // Scale/pertoken ranks 4D~7D carry one to four leading batch dims. | ||
| 766 | + const std::vector<std::vector<int64_t>> batchDims{{2}, {2, 3}, {2, 3, 4}, {2, 3, 4, 5}}; | ||
| 767 | + for (bool transpose : {false, true}) { | ||
| 768 | + SCOPED_TRACE(transpose); | ||
| 769 | + input.transA = input.transB = transpose; | ||
| 770 | + QuantBatchMatmulV3CheckerArch9201 checker(nullptr, input); | ||
| 771 | + for (const auto& batch : batchDims) { | ||
| 772 | + gert::Shape scale = MakeHif4BatchShape(batch, transpose ? 48 : 3, transpose ? 3 : 48); | ||
| 773 | + gert::Shape token = MakeHif4BatchShape(batch, transpose ? 3 : 16, transpose ? 16 : 3); | ||
| 774 | + auto tokenStorage = MakeStorageShape(token); | ||
| 775 | + EXPECT_TRUE(checker.Hif4ScaleShapeCheck(scale)); | ||
| 776 | + EXPECT_TRUE(checker.Hif4PertokenScaleShapeCheck(&tokenStorage)); | ||
| 777 | + gert::Shape x1Shape = MakeHif4InputShape(batch, transpose ? 130 : 16, transpose ? 16 : 130); | ||
| 778 | + gert::Shape x2Shape = MakeHif4InputShape(batch, transpose ? 48 : 130, transpose ? 130 : 48); | ||
| 779 | + EXPECT_TRUE( | ||
| 780 | + checker.CheckInputValidInHif4PerGroupMode(scale, &tokenStorage, x1Shape, x2Shape, {130, 16, 130, 48})); | ||
| 781 | + } | ||
| 782 | + } | ||
| 783 | +} | ||
| 784 | + | ||
| 785 | +TEST(QuantBatchMatmulV3Checker960, Hif4BatchValidationRejectsMismatchedBatchDims) | ||
| 786 | +{ | ||
| 787 | + auto input = MakeHif4Info(); | ||
| 788 | + const gert::Shape scale({2, 3, 48, 1}); | ||
| 789 | + const gert::StorageShape token({2, 16, 3, 1}, {2, 16, 3, 1}); | ||
| 790 | + const gert::Shape x1Shape({2, 16, 130}); | ||
| 791 | + const gert::Shape x2Shape({2, 130, 48}); | ||
| 792 | + QuantBatchMatmulV3CheckerArch9201 checker(nullptr, input); | ||
| 793 | + EXPECT_TRUE(checker.CheckInputValidInHif4PerGroupMode(scale, &token, x1Shape, x2Shape, {130, 16, 130, 48})); | ||
| 794 | + | ||
| 795 | + const gert::StorageShape tokenExtraBatch({2, 2, 16, 3, 1}, {2, 2, 16, 3, 1}); | ||
| 796 | + EXPECT_FALSE( | ||
| 797 | + checker.CheckInputValidInHif4PerGroupMode(scale, &tokenExtraBatch, x1Shape, x2Shape, {130, 16, 130, 48})); | ||
| 798 | + const gert::StorageShape tokenWrongBatch({3, 16, 3, 1}, {3, 16, 3, 1}); | ||
| 799 | + EXPECT_FALSE( | ||
| 800 | + checker.CheckInputValidInHif4PerGroupMode(scale, &tokenWrongBatch, x1Shape, x2Shape, {130, 16, 130, 48})); | ||
| 801 | + | ||
| 802 | + const gert::Shape scaleExtraBatch({2, 2, 3, 48, 1}); | ||
| 803 | + EXPECT_FALSE( | ||
| 804 | + checker.CheckInputValidInHif4PerGroupMode(scaleExtraBatch, &token, x1Shape, x2Shape, {130, 16, 130, 48})); | ||
| 805 | + const gert::Shape scaleWrongBatch({3, 3, 48, 1}); | ||
| 806 | + EXPECT_FALSE( | ||
| 807 | + checker.CheckInputValidInHif4PerGroupMode(scaleWrongBatch, &token, x1Shape, x2Shape, {130, 16, 130, 48})); | ||
| 808 | +} | ||
| 809 | + | ||
| 810 | +class TestableMxBasicApiTiling : public AdaptiveSlidingWindowMXBasicAPITiling { | ||
| 811 | +public: | ||
| 812 | + TestableMxBasicApiTiling() : AdaptiveSlidingWindowMXBasicAPITiling(nullptr) {} | ||
| 813 | + bool IsTensorApiEnabled() const override { return tensorApiEnabled; } | ||
| 814 | + bool tensorApiEnabled = true; | ||
| 815 | +}; | ||
| 816 | + | ||
| 817 | +TEST(QuantBatchMatmulV3Checker960, Hif4BatchUsesBlazeBatchTilingKeyOn9201) | ||
| 818 | +{ | ||
| 819 | + using namespace QuantBatchMatmulV3Arch35TilingKey; | ||
| 820 | + for (bool transpose : {false, true}) { | ||
| 821 | + SCOPED_TRACE(transpose); | ||
| 822 | + const auto trans = static_cast<uint64_t>(transpose); | ||
| 823 | + TestableMxBasicApiTiling tiling; | ||
| 824 | + auto& input = tiling.inputParams_; | ||
| 825 | + input.Reset(); | ||
| 826 | + input.opName = "QuantBatchMatmulV3Hif4BatchTiling"; | ||
| 827 | + input.aDtype = input.bDtype = Ops::NN::DT_HIFLOAT4; | ||
| 828 | + input.scaleDtype = input.perTokenScaleDtype = Ops::NN::DT_HIFLOAT4_SCALE; | ||
| 829 | + input.mSize = 268; | ||
| 830 | + input.nSize = 3680; | ||
| 831 | + input.kSize = 160; | ||
| 832 | + input.batchA = input.batchB = input.batchC = 2; | ||
| 833 | + input.batchA1 = input.batchB1 = input.batchC1 = 2; | ||
| 834 | + input.groupSizeM = input.groupSizeN = 1; | ||
| 835 | + input.groupSizeK = 64; | ||
| 836 | + input.isHif4PerGroup = true; | ||
| 837 | + input.transA = input.transB = transpose; | ||
| 838 | + tiling.compileInfo_.npuArch = NpuArch::DAV_9201; | ||
| 839 | + tiling.isAFullLoad_ = false; | ||
| 840 | + // baseK=64 used to force the HIF4 CMCT fallback; the API level must no longer depend on it. | ||
| 841 | + tiling.basicTiling_.baseK = static_cast<uint32_t>(qmmv3_tiling_const::MXFP_DIVISOR_SIZE); | ||
| 842 | + | ||
| 843 | + EXPECT_FALSE(tiling.IsWithoutBatchTilingData()); | ||
| 844 | + EXPECT_EQ(tiling.GetBatchMode(), static_cast<uint64_t>(TPL_WITH_BATCH)); | ||
| 845 | + EXPECT_EQ(tiling.GetApiLevel(NpuArch::DAV_9201), static_cast<uint64_t>(TPL_API_LEVEL_BLAZE)); | ||
| 846 | + EXPECT_EQ(tiling.GetKernelType(), static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI)); | ||
| 847 | + EXPECT_EQ(tiling.GetTilingKey(), GET_TPL_TILING_KEY(trans, trans, static_cast<uint64_t>(TPL_WITH_BATCH), | ||
| 848 | + static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI), | ||
| 849 | + static_cast<uint64_t>(TPL_API_LEVEL_BLAZE))); | ||
| 850 | + | ||
| 851 | + // 9202 keeps using the CMCT Basic path for the same batched HIF4 shape. | ||
| 852 | + tiling.compileInfo_.npuArch = NpuArch::DAV_9202; | ||
| 853 | + EXPECT_EQ(tiling.GetApiLevel(NpuArch::DAV_9202), static_cast<uint64_t>(TPL_API_LEVEL_BASIC)); | ||
| 854 | + EXPECT_EQ(tiling.GetTilingKey(), GET_TPL_TILING_KEY(trans, trans, static_cast<uint64_t>(TPL_WITH_BATCH), | ||
| 855 | + static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI), | ||
| 856 | + static_cast<uint64_t>(TPL_API_LEVEL_BASIC))); | ||
| 857 | + | ||
| 858 | + // TensorAPI unavailable on 9201 falls back to CMCT Basic with the same batch mode. | ||
| 859 | + tiling.compileInfo_.npuArch = NpuArch::DAV_9201; | ||
| 860 | + tiling.tensorApiEnabled = false; | ||
| 861 | + EXPECT_FALSE(tiling.IsWithoutBatchTilingData()); | ||
| 862 | + EXPECT_EQ(tiling.GetApiLevel(NpuArch::DAV_9201), static_cast<uint64_t>(TPL_API_LEVEL_BASIC)); | ||
| 863 | + EXPECT_EQ(tiling.GetTilingKey(), GET_TPL_TILING_KEY(trans, trans, static_cast<uint64_t>(TPL_WITH_BATCH), | ||
| 864 | + static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI), | ||
| 865 | + static_cast<uint64_t>(TPL_API_LEVEL_BASIC))); | ||
| 866 | + | ||
| 867 | + // batchC == 1 still selects the WithoutBatch Blaze tiling data and key. | ||
| 868 | + tiling.tensorApiEnabled = true; | ||
| 869 | + input.batchA = input.batchB = input.batchC = 1; | ||
| 870 | + input.batchA1 = input.batchB1 = input.batchC1 = 1; | ||
| 871 | + EXPECT_TRUE(tiling.IsWithoutBatchTilingData()); | ||
| 872 | + EXPECT_EQ(tiling.GetBatchMode(), static_cast<uint64_t>(TPL_WITHOUT_BATCH)); | ||
| 873 | + EXPECT_EQ(tiling.GetTilingKey(), GET_TPL_TILING_KEY(trans, trans, static_cast<uint64_t>(TPL_WITHOUT_BATCH), | ||
| 874 | + static_cast<uint64_t>(TPL_NO_VEC_EPILOGUE_WITH_MMAPI), | ||
| 875 | + static_cast<uint64_t>(TPL_API_LEVEL_BLAZE))); | ||
| 731 | } | 876 | } |
| 732 | - EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, {129, 16, 130, 48})); | ||
| 733 | - EXPECT_FALSE(valid.CheckInputValidInHif4PerGroupMode(scale, &token, {130, 16, 129, 48})); | ||
| 734 | } | 877 | } |
| 735 | 878 | ||
| 736 | class QuantBatchMatmulV3StreamK960 : public testing::Test { | 879 | class QuantBatchMatmulV3StreamK960 : public testing::Test { |
| @@ -801,6 +944,26 @@ TEST_F(QuantBatchMatmulV3StreamK960, CapabilityRequiresMatchingCoresAndSingleBat | |||
| 801 | EXPECT_FALSE(tiling->IsCapable()); | 944 | EXPECT_FALSE(tiling->IsCapable()); |
| 802 | } | 945 | } |
| 803 | 946 | ||
| 947 | +TEST_F(QuantBatchMatmulV3StreamK960, CapabilityAcceptsHif4PerGroupWithoutBatch) | ||
| 948 | +{ | ||
| 949 | + auto& input = tiling->inputParams_; | ||
| 950 | + // HIF4 per-group is the only enabled quant mode here, so IsCapable goes through the HIF4 branch. | ||
| 951 | + input.isPerTensor = false; | ||
| 952 | + input.isHif4PerGroup = true; | ||
| 953 | + EXPECT_FALSE(tiling->IsPertensorStreamKInput()); | ||
| 954 | + EXPECT_EQ(tiling->compileInfo_.aicNum, tiling->compileInfo_.aivNum); | ||
| 955 | + EXPECT_EQ(input.batchC, 1U); | ||
| 956 | + EXPECT_EQ(tiling->IsCapable(), IsTensorapiCapable()); | ||
| 957 | + tiling->compileInfo_.aivNum = 64; | ||
| 958 | + EXPECT_FALSE(tiling->IsCapable()); | ||
| 959 | + tiling->compileInfo_.aivNum = 32; | ||
| 960 | + input.batchC = 2; | ||
| 961 | + EXPECT_FALSE(tiling->IsCapable()); | ||
| 962 | + input.batchC = 1; | ||
| 963 | + input.isHif4PerGroup = false; | ||
| 964 | + EXPECT_FALSE(tiling->IsCapable()); | ||
| 965 | +} | ||
| 966 | + | ||
| 804 | TEST_F(QuantBatchMatmulV3StreamK960, FloatingBiasMustMatchScaleAndAccumulationDomain) | 967 | TEST_F(QuantBatchMatmulV3StreamK960, FloatingBiasMustMatchScaleAndAccumulationDomain) |
| 805 | { | 968 | { |
| 806 | auto& input = tiling->inputParams_; | 969 | auto& input = tiling->inputParams_; |
| @@ -857,6 +1020,7 @@ TEST_F(QuantBatchMatmulV3StreamK960, WorkspaceAndSerializedDataMatchKernelContra | |||
| 857 | ASSERT_EQ(tiling->PostTiling(), ge::GRAPH_SUCCESS); | 1020 | ASSERT_EQ(tiling->PostTiling(), ge::GRAPH_SUCCESS); |
| 858 | auto* context = holder.GetContext<gert::TilingContext>(); | 1021 | auto* context = holder.GetContext<gert::TilingContext>(); |
| 859 | EXPECT_EQ(context->GetBlockDim(), 24); | 1022 | EXPECT_EQ(context->GetBlockDim(), 24); |
| 1023 | + EXPECT_EQ(context->GetScheduleMode(), 1U); | ||
| 860 | EXPECT_EQ(context->GetRawTilingData()->GetDataSize(), sizeof(tiling->tilingData_)); | 1024 | EXPECT_EQ(context->GetRawTilingData()->GetDataSize(), sizeof(tiling->tilingData_)); |
| 861 | EXPECT_EQ(context->GetWorkspaceSizes(1)[0], tiling->workspaceSize_); | 1025 | EXPECT_EQ(context->GetWorkspaceSizes(1)[0], tiling->workspaceSize_); |
| 862 | const auto* data = reinterpret_cast<const DequantBmm::QuantBatchMatmulV3StreamKBasicAPITilingData*>( | 1026 | const auto* data = reinterpret_cast<const DequantBmm::QuantBatchMatmulV3StreamKBasicAPITilingData*>( |
| @@ -565,6 +565,13 @@ Dav9201,Dav9201_hif4_group_00,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,0 | |||
| 565 | Dav9201,Dav9201_hif4_group_01,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,0,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131092,,FALSE,hif4_group_01,0, | 565 | Dav9201,Dav9201_hif4_group_01,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,0,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131092,,FALSE,hif4_group_01,0, |
| 566 | Dav9201,Dav9201_hif4_group_10,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,1,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131089,,FALSE,hif4_group_10,0, | 566 | Dav9201,Dav9201_hif4_group_10,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,1,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131089,,FALSE,hif4_group_10,0, |
| 567 | Dav9201,Dav9201_hif4_group_11,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,1,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131093,,FALSE,hif4_group_11,0, | 567 | Dav9201,Dav9201_hif4_group_11,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,1,0,1,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131093,,FALSE,hif4_group_11,0, |
| 568 | +Dav9201,Dav9201_hif4_batch_00,,ascend960,32,32,3,3,3,2,2,2,268,160,3680,0,1,0,0,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131072,,FALSE,hif4_batch_00,0, | ||
| 569 | +Dav9201,Dav9201_hif4_batch_01,,ascend960,32,32,3,3,3,2,2,2,268,160,3680,0,1,0,0,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131076,,FALSE,hif4_batch_01,0, | ||
| 570 | +Dav9201,Dav9201_hif4_batch_10,,ascend960,32,32,3,3,3,2,2,2,268,160,3680,0,1,0,1,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131073,,FALSE,hif4_batch_10,0, | ||
| 571 | +Dav9201,Dav9201_hif4_batch_11,,ascend960,32,32,3,3,3,2,2,2,268,160,3680,0,1,0,1,1,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131077,,FALSE,hif4_batch_11,0, | ||
| 572 | +Dav9201,Dav9201_hif4_batch_4d,,ascend960,32,32,4,4,4,2,2,2,268,160,3680,0,1,0,0,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131072,,FALSE,hif4_batch_4d,0, | ||
| 573 | +Dav9201,Dav9201_hif4_batch_5d,,ascend960,32,32,5,5,5,2,2,2,268,160,3680,0,1,0,0,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131072,,FALSE,hif4_batch_5d,0, | ||
| 574 | +Dav9201,Dav9201_hif4_batch_6d,,ascend960,32,32,6,6,6,2,2,2,268,160,3680,0,1,0,0,0,8,HIFLOAT4,HIFLOAT4,HIFLOAT4_SCALE,HIFLOAT4_SCALE,INT32,BF16,ND,ND,TRUE,32,131072,,FALSE,hif4_batch_6d,0, | ||
| 568 | Dav9201,Dav9201_large_grid_cube,,ascend960,32,32,2,2,2,0,0,0,8192,4096,8192,0,0,0,0,0,0,INT8,INT8,UINT64,FLOAT,INT32,BF16,ND,ND,TRUE,32,131088,,FALSE,large_grid_cube,0, | 575 | Dav9201,Dav9201_large_grid_cube,,ascend960,32,32,2,2,2,0,0,0,8192,4096,8192,0,0,0,0,0,0,INT8,INT8,UINT64,FLOAT,INT32,BF16,ND,ND,TRUE,32,131088,,FALSE,large_grid_cube,0, |
| 569 | Dav9201,Dav9201_perchannel_bias,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,0,1,0,0,1,INT8,INT8,UINT64,FLOAT,INT32,BF16,ND,ND,TRUE,32,131088,,FALSE,perchannel_bias,0, | 576 | Dav9201,Dav9201_perchannel_bias,,ascend960,32,32,2,2,2,0,0,0,1024,512,1024,0,0,1,0,0,1,INT8,INT8,UINT64,FLOAT,INT32,BF16,ND,ND,TRUE,32,131088,,FALSE,perchannel_bias,0, |
| 570 | Dav9201,Dav9201_pertoken_mix,,ascend960,32,32,2,2,2,0,0,0,256,512,1024,0,1,0,0,0,1,INT8,INT8,FLOAT,FLOAT,INT32,BF16,ND,ND,TRUE,32,131600,,FALSE,pertoken_mix,0, | 577 | Dav9201,Dav9201_pertoken_mix,,ascend960,32,32,2,2,2,0,0,0,256,512,1024,0,1,0,0,0,1,INT8,INT8,FLOAT,FLOAT,INT32,BF16,ND,ND,TRUE,32,131600,,FALSE,pertoken_mix,0, |
| @@ -15,23 +15,18 @@ | |||
| 15 | #include <cstring> | 15 | #include <cstring> |
| 16 | #include <exception> | 16 | #include <exception> |
| 17 | #include <mutex> | 17 | #include <mutex> |
| 18 | -#include <iostream> | ||
| 19 | #include <fstream> | 18 | #include <fstream> |
| 20 | #include <string> | 19 | #include <string> |
| 21 | #include <thread> | 20 | #include <thread> |
| 22 | #include <vector> | 21 | #include <vector> |
| 23 | 22 | ||
| 24 | -#include "log/log.h" | ||
| 25 | -#include "nlohmann/json.hpp" | ||
| 26 | #include "op_common/op_host/util/op_const_def.h" | 23 | #include "op_common/op_host/util/op_const_def.h" |
| 27 | 24 | ||
| 28 | #define protected public | 25 | #define protected public |
| 29 | #define private public | 26 | #define private public |
| 30 | -#include "op_host/tiling_templates_registry.h" | ||
| 31 | #include "ut_op_util.h" | 27 | #include "ut_op_util.h" |
| 32 | #include "exe_graph/runtime/storage_format.h" | 28 | #include "exe_graph/runtime/storage_format.h" |
| 33 | #include "exe_graph/runtime/storage_shape.h" | 29 | #include "exe_graph/runtime/storage_shape.h" |
| 34 | -#include "exe_graph/runtime/tiling_parse_context.h" | ||
| 35 | #include "kernel_run_context_facker.h" | 30 | #include "kernel_run_context_facker.h" |
| 36 | #include "test_cube_util.h" | 31 | #include "test_cube_util.h" |
| 37 | #include "test_quant_batch_matmul_v3_tiling.h" | 32 | #include "test_quant_batch_matmul_v3_tiling.h" |
| @@ -87,6 +82,18 @@ static void SetExpectedTilingFieldIfPresent(std::vector<int32_t>& tilingDataInt, | |||
| 87 | std::memcpy(reinterpret_cast<char*>(tilingDataInt.data()) + fieldOffset, &value, sizeof(T)); | 82 | std::memcpy(reinterpret_cast<char*>(tilingDataInt.data()) + fieldOffset, &value, sizeof(T)); |
| 88 | } | 83 | } |
| 89 | 84 | ||
| 85 | +static gert::Shape MakeHif4ScaleShape(int64_t batchDimNum, int64_t batchValue, int64_t dim0, int64_t dim1) | ||
| 86 | +{ | ||
| 87 | + gert::Shape shape; | ||
| 88 | + for (int64_t idx = 0; idx < batchDimNum; ++idx) { | ||
| 89 | + shape.AppendDim(batchValue); | ||
| 90 | + } | ||
| 91 | + shape.AppendDim(dim0); | ||
| 92 | + shape.AppendDim(dim1); | ||
| 93 | + shape.AppendDim(1); | ||
| 94 | + return shape; | ||
| 95 | +} | ||
| 96 | + | ||
| 90 | static gert::Shape TransNd2Nz(const gert::Shape& inShape, int64_t c0 = 32) | 97 | static gert::Shape TransNd2Nz(const gert::Shape& inShape, int64_t c0 = 32) |
| 91 | { | 98 | { |
| 92 | gert::Shape outShape; | 99 | gert::Shape outShape; |
| @@ -375,6 +382,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c | |||
| 375 | 382 | ||
| 376 | if (yDim == 6) { | 383 | if (yDim == 6) { |
| 377 | outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, batchC, m, n}); | 384 | outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, batchC, m, n}); |
| 385 | + } else if (yDim == 5) { | ||
| 386 | + outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, m, n}); | ||
| 387 | + } else if (yDim == 4) { | ||
| 388 | + outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, m, n}); | ||
| 378 | } else if (yDim == 3) { | 389 | } else if (yDim == 3) { |
| 379 | outputShape.MutableOriginShape() = gert::Shape({batchC, m, n}); | 390 | outputShape.MutableOriginShape() = gert::Shape({batchC, m, n}); |
| 380 | } else if (yDim == 2) { | 391 | } else if (yDim == 2) { |
| @@ -386,6 +397,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c | |||
| 386 | if (transA) { | 397 | if (transA) { |
| 387 | if (x1Dim == 6) { | 398 | if (x1Dim == 6) { |
| 388 | x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, k, m}); | 399 | x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, k, m}); |
| 400 | + } else if (x1Dim == 5) { | ||
| 401 | + x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, k, m}); | ||
| 402 | + } else if (x1Dim == 4) { | ||
| 403 | + x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, k, m}); | ||
| 389 | } else if (x1Dim == 3) { | 404 | } else if (x1Dim == 3) { |
| 390 | x1Shape.MutableOriginShape() = gert::Shape({batchA, k, m}); | 405 | x1Shape.MutableOriginShape() = gert::Shape({batchA, k, m}); |
| 391 | } else if (x1Dim == 2) { | 406 | } else if (x1Dim == 2) { |
| @@ -396,6 +411,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c | |||
| 396 | } else { | 411 | } else { |
| 397 | if (x1Dim == 6) { | 412 | if (x1Dim == 6) { |
| 398 | x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, m, k}); | 413 | x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, m, k}); |
| 414 | + } else if (x1Dim == 5) { | ||
| 415 | + x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, m, k}); | ||
| 416 | + } else if (x1Dim == 4) { | ||
| 417 | + x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, m, k}); | ||
| 399 | } else if (x1Dim == 3) { | 418 | } else if (x1Dim == 3) { |
| 400 | x1Shape.MutableOriginShape() = gert::Shape({batchA, m, k}); | 419 | x1Shape.MutableOriginShape() = gert::Shape({batchA, m, k}); |
| 401 | } else if (x1Dim == 2) { | 420 | } else if (x1Dim == 2) { |
| @@ -408,6 +427,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c | |||
| 408 | if (transB) { | 427 | if (transB) { |
| 409 | if (x2Dim == 6) { | 428 | if (x2Dim == 6) { |
| 410 | x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, n, k}); | 429 | x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, n, k}); |
| 430 | + } else if (x2Dim == 5) { | ||
| 431 | + x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, n, k}); | ||
| 432 | + } else if (x2Dim == 4) { | ||
| 433 | + x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, n, k}); | ||
| 411 | } else if (x2Dim == 3) { | 434 | } else if (x2Dim == 3) { |
| 412 | x2Shape.MutableOriginShape() = gert::Shape({batchB, n, k}); | 435 | x2Shape.MutableOriginShape() = gert::Shape({batchB, n, k}); |
| 413 | } else if (x2Dim == 2) { | 436 | } else if (x2Dim == 2) { |
| @@ -418,6 +441,10 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c | |||
| 418 | } else { | 441 | } else { |
| 419 | if (x2Dim == 6) { | 442 | if (x2Dim == 6) { |
| 420 | x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, k, n}); | 443 | x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, k, n}); |
| 444 | + } else if (x2Dim == 5) { | ||
| 445 | + x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, k, n}); | ||
| 446 | + } else if (x2Dim == 4) { | ||
| 447 | + x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, k, n}); | ||
| 421 | } else if (x2Dim == 3) { | 448 | } else if (x2Dim == 3) { |
| 422 | x2Shape.MutableOriginShape() = gert::Shape({batchB, k, n}); | 449 | x2Shape.MutableOriginShape() = gert::Shape({batchB, k, n}); |
| 423 | } else if (x2Dim == 2) { | 450 | } else if (x2Dim == 2) { |
| @@ -491,8 +518,13 @@ void QuantBatchMatmulV3TilingTestParam::Prepare(QuantBatchMatmulV3CompileInfo& c | |||
| 491 | scaleShape.MutableStorageShape() = gert::Shape({1}); | 518 | scaleShape.MutableStorageShape() = gert::Shape({1}); |
| 492 | } else if (quantMode == 8) { // HIF4 groups: [M, ceil(K/64), 1] and [ceil(K/64), N, 1]. | 519 | } else if (quantMode == 8) { // HIF4 groups: [M, ceil(K/64), 1] and [ceil(K/64), N, 1]. |
| 493 | const int64_t groups = ops::CeilDiv<int64_t>(k, 64); | 520 | const int64_t groups = ops::CeilDiv<int64_t>(k, 64); |
| 494 | - pertokenShape.MutableStorageShape() = transA ? gert::Shape({groups, m, 1}) : gert::Shape({m, groups, 1}); | 521 | + // Batch HIF4 keeps x1Dim - 2 / x2Dim - 2 leading batch dims before the logical scale axes. |
| 495 | - scaleShape.MutableStorageShape() = transB ? gert::Shape({n, groups, 1}) : gert::Shape({groups, n, 1}); | 522 | + const int64_t pertokenBatchDimNum = x1Dim > 2 ? x1Dim - 2 : 0; |
| 523 | + const int64_t scaleBatchDimNum = x2Dim > 2 ? x2Dim - 2 : 0; | ||
| 524 | + pertokenShape.MutableStorageShape() = MakeHif4ScaleShape(pertokenBatchDimNum, batchA, transA ? groups : m, | ||
| 525 | + transA ? m : groups); | ||
| 526 | + scaleShape.MutableStorageShape() = MakeHif4ScaleShape(scaleBatchDimNum, batchB, transB ? n : groups, | ||
| 527 | + transB ? groups : n); | ||
| 496 | } else if (quantMode == 7) { // X2 per-group CSV cases use G=128. | 528 | } else if (quantMode == 7) { // X2 per-group CSV cases use G=128. |
| 497 | const int64_t groupCnt = ops::CeilDiv<int64_t>(k, 128); | 529 | const int64_t groupCnt = ops::CeilDiv<int64_t>(k, 128); |
| 498 | scaleShape.MutableStorageShape() = x2Dim == 3 ? gert::Shape({batchB, groupCnt, n}) : gert::Shape({groupCnt, n}); | 530 | scaleShape.MutableStorageShape() = x2Dim == 3 ? gert::Shape({batchB, groupCnt, n}) : gert::Shape({groupCnt, n}); |
| @@ -589,6 +621,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi | |||
| 589 | gert::StorageShape outputShape; | 621 | gert::StorageShape outputShape; |
| 590 | if (yDim == 6) { | 622 | if (yDim == 6) { |
| 591 | outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, batchC, m, n}); | 623 | outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, batchC, m, n}); |
| 624 | + } else if (yDim == 5) { | ||
| 625 | + outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, batchC, m, n}); | ||
| 626 | + } else if (yDim == 4) { | ||
| 627 | + outputShape.MutableOriginShape() = gert::Shape({batchC, batchC, m, n}); | ||
| 592 | } else if (yDim == 3) { | 628 | } else if (yDim == 3) { |
| 593 | outputShape.MutableOriginShape() = gert::Shape({batchC, m, n}); | 629 | outputShape.MutableOriginShape() = gert::Shape({batchC, m, n}); |
| 594 | } else if (yDim == 2) { | 630 | } else if (yDim == 2) { |
| @@ -600,6 +636,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi | |||
| 600 | if (transA) { | 636 | if (transA) { |
| 601 | if (x1Dim == 6) { | 637 | if (x1Dim == 6) { |
| 602 | x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, k, m}); | 638 | x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, k, m}); |
| 639 | + } else if (x1Dim == 5) { | ||
| 640 | + x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, k, m}); | ||
| 641 | + } else if (x1Dim == 4) { | ||
| 642 | + x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, k, m}); | ||
| 603 | } else if (x1Dim == 3) { | 643 | } else if (x1Dim == 3) { |
| 604 | x1Shape.MutableOriginShape() = gert::Shape({batchA, k, m}); | 644 | x1Shape.MutableOriginShape() = gert::Shape({batchA, k, m}); |
| 605 | } else if (x1Dim == 2) { | 645 | } else if (x1Dim == 2) { |
| @@ -610,6 +650,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi | |||
| 610 | } else { | 650 | } else { |
| 611 | if (x1Dim == 6) { | 651 | if (x1Dim == 6) { |
| 612 | x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, m, k}); | 652 | x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, batchA, m, k}); |
| 653 | + } else if (x1Dim == 5) { | ||
| 654 | + x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, batchA, m, k}); | ||
| 655 | + } else if (x1Dim == 4) { | ||
| 656 | + x1Shape.MutableOriginShape() = gert::Shape({batchA, batchA, m, k}); | ||
| 613 | } else if (x1Dim == 3) { | 657 | } else if (x1Dim == 3) { |
| 614 | x1Shape.MutableOriginShape() = gert::Shape({batchA, m, k}); | 658 | x1Shape.MutableOriginShape() = gert::Shape({batchA, m, k}); |
| 615 | } else if (x1Dim == 2) { | 659 | } else if (x1Dim == 2) { |
| @@ -622,6 +666,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi | |||
| 622 | if (transB) { | 666 | if (transB) { |
| 623 | if (x2Dim == 6) { | 667 | if (x2Dim == 6) { |
| 624 | x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, n, k}); | 668 | x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, n, k}); |
| 669 | + } else if (x2Dim == 5) { | ||
| 670 | + x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, n, k}); | ||
| 671 | + } else if (x2Dim == 4) { | ||
| 672 | + x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, n, k}); | ||
| 625 | } else if (x2Dim == 3) { | 673 | } else if (x2Dim == 3) { |
| 626 | x2Shape.MutableOriginShape() = gert::Shape({batchB, n, k}); | 674 | x2Shape.MutableOriginShape() = gert::Shape({batchB, n, k}); |
| 627 | } else if (x2Dim == 2) { | 675 | } else if (x2Dim == 2) { |
| @@ -632,6 +680,10 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi | |||
| 632 | } else { | 680 | } else { |
| 633 | if (x2Dim == 6) { | 681 | if (x2Dim == 6) { |
| 634 | x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, k, n}); | 682 | x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, batchB, k, n}); |
| 683 | + } else if (x2Dim == 5) { | ||
| 684 | + x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, batchB, k, n}); | ||
| 685 | + } else if (x2Dim == 4) { | ||
| 686 | + x2Shape.MutableOriginShape() = gert::Shape({batchB, batchB, k, n}); | ||
| 635 | } else if (x2Dim == 3) { | 687 | } else if (x2Dim == 3) { |
| 636 | x2Shape.MutableOriginShape() = gert::Shape({batchB, k, n}); | 688 | x2Shape.MutableOriginShape() = gert::Shape({batchB, k, n}); |
| 637 | } else if (x2Dim == 2) { | 689 | } else if (x2Dim == 2) { |
| @@ -716,8 +768,13 @@ void QuantBatchMatmulV3TilingTestParam::InvokeTilingFunc(QuantBatchMatmulV3Compi | |||
| 716 | scaleShape.MutableStorageShape() = gert::Shape({1}); | 768 | scaleShape.MutableStorageShape() = gert::Shape({1}); |
| 717 | } else if (quantMode == 8) { // HIF4 groups: [M, ceil(K/64), 1] and [ceil(K/64), N, 1]. | 769 | } else if (quantMode == 8) { // HIF4 groups: [M, ceil(K/64), 1] and [ceil(K/64), N, 1]. |
| 718 | const int64_t groups = ops::CeilDiv<int64_t>(k, 64); | 770 | const int64_t groups = ops::CeilDiv<int64_t>(k, 64); |
| 719 | - pertokenShape.MutableStorageShape() = transA ? gert::Shape({groups, m, 1}) : gert::Shape({m, groups, 1}); | 771 | + // Batch HIF4 keeps x1Dim - 2 / x2Dim - 2 leading batch dims before the logical scale axes. |
| 720 | - scaleShape.MutableStorageShape() = transB ? gert::Shape({n, groups, 1}) : gert::Shape({groups, n, 1}); | 772 | + const int64_t pertokenBatchDimNum = x1Dim > 2 ? x1Dim - 2 : 0; |
| 773 | + const int64_t scaleBatchDimNum = x2Dim > 2 ? x2Dim - 2 : 0; | ||
| 774 | + pertokenShape.MutableStorageShape() = MakeHif4ScaleShape(pertokenBatchDimNum, batchA, transA ? groups : m, | ||
| 775 | + transA ? m : groups); | ||
| 776 | + scaleShape.MutableStorageShape() = MakeHif4ScaleShape(scaleBatchDimNum, batchB, transB ? n : groups, | ||
| 777 | + transB ? groups : n); | ||
| 721 | } else if (quantMode == 7) { // X2 per-group CSV cases use G=128. | 778 | } else if (quantMode == 7) { // X2 per-group CSV cases use G=128. |
| 722 | const int64_t groupCnt = ops::CeilDiv<int64_t>(k, 128); | 779 | const int64_t groupCnt = ops::CeilDiv<int64_t>(k, 128); |
| 723 | scaleShape.MutableStorageShape() = x2Dim == 3 ? gert::Shape({batchB, groupCnt, n}) : gert::Shape({groupCnt, n}); | 780 | scaleShape.MutableStorageShape() = x2Dim == 3 ? gert::Shape({batchB, groupCnt, n}) : gert::Shape({groupCnt, n}); |


没有 ut