| @@ -18,7 +18,7 @@ add_kernel_sources( | |||
| 18 | 18 | ||
| 19 | add_kernel_sources( | 19 | add_kernel_sources( |
| 20 | KERNEL_SRC arch35/weight_quant_batch_matmul_v2.cpp | 20 | KERNEL_SRC arch35/weight_quant_batch_matmul_v2.cpp |
| 21 | - COMPUTE_UNITS ascend950 ascend350 ascend960dt | 21 | + COMPUTE_UNITS ascend950 ascend350 ascend960dt ascend960pr |
| 22 | AUTO_SYNC false | 22 | AUTO_SYNC false |
| 23 | OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true" | 23 | OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true" |
| 24 | ) | 24 | ) |
| @@ -151,7 +151,7 @@ | |||
| 151 | { | 151 | { |
| 152 | "name": "antiquant_offset", | 152 | "name": "antiquant_offset", |
| 153 | "index": 3, | 153 | "index": 3, |
| 154 | - "dtype": "bfloat16", | 154 | + "dtype": "float16", |
| 155 | "format": "ND", | 155 | "format": "ND", |
| 156 | "paramType": "optional", | 156 | "paramType": "optional", |
| 157 | "shape": [ | 157 | "shape": [ |
| @@ -2659,7 +2659,7 @@ | |||
| 2659 | { | 2659 | { |
| 2660 | "name": "antiquant_offset", | 2660 | "name": "antiquant_offset", |
| 2661 | "index": 3, | 2661 | "index": 3, |
| 2662 | - "dtype": "float16", | 2662 | + "dtype": "bfloat16", |
| 2663 | "format": "ND", | 2663 | "format": "ND", |
| 2664 | "paramType": "optional", | 2664 | "paramType": "optional", |
| 2665 | "shape": [ | 2665 | "shape": [ |
| @@ -779,15 +779,7 @@ void WeightQuantBatchMatmulV2TilingASWMix::CalL1Tiling() | |||
| 779 | OP_LOGD(opName_, "coreNum: %u", basicTiling_.usedCoreNum); | 779 | OP_LOGD(opName_, "coreNum: %u", basicTiling_.usedCoreNum); |
| 780 | basicTiling_.baseM = adaptiveWin_.baseM; | 780 | basicTiling_.baseM = adaptiveWin_.baseM; |
| 781 | basicTiling_.baseN = adaptiveWin_.baseN; | 781 | basicTiling_.baseN = adaptiveWin_.baseN; |
| 782 | - | 782 | + basicTiling_.baseK = std::min(adaptiveWin_.baseK, ops::CeilAlign<uint64_t>(matmulInfoPtr_->kSize, B4_C0_SIZE)); |
| 783 | - bool isA16MxFp4 = (matmulInfoPtr_->aDtype == ge::DT_FLOAT16 || matmulInfoPtr_->aDtype == ge::DT_BF16) && | ||
| 784 | - matmulInfoPtr_->bDtype == ge::DT_FLOAT4_E2M1 && matmulInfoPtr_->antiQuantType == QuantType::MX; | ||
| 785 | - if (isA16MxFp4) { | ||
| 786 | - basicTiling_.baseK = std::min(adaptiveWin_.baseK, ops::CeilAlign<uint64_t>(matmulInfoPtr_->kSize, B4_C0_SIZE)); | ||
| 787 | - } else { | ||
| 788 | - basicTiling_.baseK = std::min(adaptiveWin_.baseK, matmulInfoPtr_->kSize); | ||
| 789 | - } | ||
| 790 | - | ||
| 791 | basicTiling_.stepM = 1; | 783 | basicTiling_.stepM = 1; |
| 792 | basicTiling_.stepN = 1; | 784 | basicTiling_.stepN = 1; |
| 793 | basicTiling_.singleCoreM = std::min(matmulInfoPtr_->mSize, static_cast<uint64_t>(basicTiling_.baseM)); | 785 | basicTiling_.singleCoreM = std::min(matmulInfoPtr_->mSize, static_cast<uint64_t>(basicTiling_.baseM)); |
| @@ -423,13 +423,14 @@ bool WeightQuantBatchMatmulV2CheckerFor920R1::CheckInputSize(const gert::Storage | |||
| 423 | return false); | 423 | return false); |
| 424 | 424 | ||
| 425 | isA16MxFp4_ = inputParams_.bDtype == ge::DT_FLOAT4_E2M1 && inputParams_.antiQuantScaleDtype == ge::DT_FLOAT8_E8M0; | 425 | isA16MxFp4_ = inputParams_.bDtype == ge::DT_FLOAT4_E2M1 && inputParams_.antiQuantScaleDtype == ge::DT_FLOAT8_E8M0; |
| 426 | + isA16HiF4_ = inputParams_.bDtype == Ops::NN::DT_HIFLOAT4; | ||
| 426 | inputParams_.nSize = static_cast<uint64_t>(inputParams_.transB ? weightShape->GetOriginShape().GetDim(0) : | 427 | inputParams_.nSize = static_cast<uint64_t>(inputParams_.transB ? weightShape->GetOriginShape().GetDim(0) : |
| 427 | weightLastDim); | 428 | weightLastDim); |
| 428 | auto weightStorageShape = weightShape->GetStorageShape(); | 429 | auto weightStorageShape = weightShape->GetStorageShape(); |
| 429 | if (inputParams_.bFormat == ge::Format::FORMAT_FRACTAL_NZ && inputParams_.transB) { | 430 | if (inputParams_.bFormat == ge::Format::FORMAT_FRACTAL_NZ && inputParams_.transB) { |
| 430 | uint64_t n0 = weightStorageShape.GetDim(2); | 431 | uint64_t n0 = weightStorageShape.GetDim(2); |
| 431 | uint64_t n1 = weightStorageShape.GetDim(1); | 432 | uint64_t n1 = weightStorageShape.GetDim(1); |
| 432 | - OP_TILING_CHECK(!isA16MxFp4_ && n0 * n1 != inputParams_.nSize, | 433 | + OP_TILING_CHECK(!isA16MxFp4_ && !isA16HiF4_ && n0 * n1 != inputParams_.nSize, |
| 433 | OP_LOGE_FOR_INVALID_VALUE_WITH_REASON( | 434 | OP_LOGE_FOR_INVALID_VALUE_WITH_REASON( |
| 434 | inputParams_.opName, "weight", | 435 | inputParams_.opName, "weight", |
| 435 | Ops::NN::FormatString("%llu, %llu", static_cast<unsigned long long>(n0 * n1), | 436 | Ops::NN::FormatString("%llu, %llu", static_cast<unsigned long long>(n0 * n1), |
| @@ -650,8 +651,7 @@ bool WeightQuantBatchMatmulV2CheckerFor920R1::CheckShapeAdvanced() | |||
| 650 | .c_str(), | 651 | .c_str(), |
| 651 | "the K and N dimensions must be divisible by 2"), | 652 | "the K and N dimensions must be divisible by 2"), |
| 652 | return false); | 653 | return false); |
| 653 | - } else if (inputParams_.bDtype == ge::DT_FLOAT4_E2M1 || | 654 | + } else if (inputParams_.bDtype == ge::DT_FLOAT4_E2M1) { |
| 654 | - inputParams_.bDtype == Ops::NN::DT_HIFLOAT4) { // 其余 A16W4 仅支持 n 和 k 对 64 对齐 | ||
| 655 | OP_TILING_CHECK( | 655 | OP_TILING_CHECK( |
| 656 | // 64对齐 | 656 | // 64对齐 |
| 657 | !isA16MxFp4_ && (inputParams_.nSize % 64 != 0 || inputParams_.kSize % 64 != 0), | 657 | !isA16MxFp4_ && (inputParams_.nSize % 64 != 0 || inputParams_.kSize % 64 != 0), |
| @@ -98,7 +98,8 @@ static ge::graphStatus WeightQuantBatchMatmulV2TilingFunc(gert::TilingContext* c | |||
| 98 | } | 98 | } |
| 99 | auto platformInfoPtr = context->GetPlatformInfo(); | 99 | auto platformInfoPtr = context->GetPlatformInfo(); |
| 100 | if (platformInfoPtr != nullptr && | 100 | if (platformInfoPtr != nullptr && |
| 101 | - platform_ascendc::PlatformAscendC(platformInfoPtr).GetCurNpuArch() == NpuArch::DAV_9201) { | 101 | + (platform_ascendc::PlatformAscendC(platformInfoPtr).GetCurNpuArch() == NpuArch::DAV_9201 || |
| 102 | + platform_ascendc::PlatformAscendC(platformInfoPtr).GetCurNpuArch() == NpuArch::DAV_9202)) { | ||
| 102 | WeightQuantBatchMatmulV2CheckerFor920R1 wqbmmv2Checker(context); | 103 | WeightQuantBatchMatmulV2CheckerFor920R1 wqbmmv2Checker(context); |
| 103 | OP_TILING_CHECK( | 104 | OP_TILING_CHECK( |
| 104 | wqbmmv2Checker.Check() != ge::GRAPH_SUCCESS, | 105 | wqbmmv2Checker.Check() != ge::GRAPH_SUCCESS, |
| @@ -615,8 +615,8 @@ public: | |||
| 615 | opbase::ArchConfigHelper::AddConfigByArch(this->AICore(), Ops::Base::DAV_3510, aicore_config_dav3510); | 615 | opbase::ArchConfigHelper::AddConfigByArch(this->AICore(), Ops::Base::DAV_3510, aicore_config_dav3510); |
| 616 | this->AICore().AddConfig("ascend350", aicore_config_dav3510); | 616 | this->AICore().AddConfig("ascend350", aicore_config_dav3510); |
| 617 | 617 | ||
| 618 | - OpAICoreConfig aicore_config_960DT; | 618 | + OpAICoreConfig aicore_config_960; |
| 619 | - aicore_config_960DT.Input("x") | 619 | + aicore_config_960.Input("x") |
| 620 | .ParamType(REQUIRED) | 620 | .ParamType(REQUIRED) |
| 621 | .DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_FLOAT16, | 621 | .DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_FLOAT16, |
| 622 | ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, | 622 | ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, |
| @@ -637,7 +637,7 @@ public: | |||
| 637 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 637 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 638 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 638 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 639 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); | 639 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); |
| 640 | - aicore_config_960DT.Input("weight") | 640 | + aicore_config_960.Input("weight") |
| 641 | .ParamType(REQUIRED) | 641 | .ParamType(REQUIRED) |
| 642 | .DataType({ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, | 642 | .DataType({ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, |
| 643 | ge::DT_INT8, ge::DT_INT4, ge::DT_INT4, ge::DT_FLOAT4_E2M1, | 643 | ge::DT_INT8, ge::DT_INT4, ge::DT_INT4, ge::DT_FLOAT4_E2M1, |
| @@ -677,7 +677,7 @@ public: | |||
| 677 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, | 677 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, |
| 678 | ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, | 678 | ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, |
| 679 | ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ}); | 679 | ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ}); |
| 680 | - aicore_config_960DT.Input("antiquant_scale") | 680 | + aicore_config_960.Input("antiquant_scale") |
| 681 | .ParamType(REQUIRED) | 681 | .ParamType(REQUIRED) |
| 682 | .DataType({ge::DT_UINT64, | 682 | .DataType({ge::DT_UINT64, |
| 683 | ge::DT_UINT64, | 683 | ge::DT_UINT64, |
| @@ -727,7 +727,7 @@ public: | |||
| 727 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 727 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 728 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 728 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 729 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); | 729 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); |
| 730 | - aicore_config_960DT.Input("antiquant_offset") | 730 | + aicore_config_960.Input("antiquant_offset") |
| 731 | .ParamType(OPTIONAL) | 731 | .ParamType(OPTIONAL) |
| 732 | .DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_FLOAT16, | 732 | .DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_FLOAT16, |
| 733 | ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, | 733 | ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, |
| @@ -748,7 +748,7 @@ public: | |||
| 748 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 748 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 749 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 749 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 750 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); | 750 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); |
| 751 | - aicore_config_960DT.Input("quant_scale") | 751 | + aicore_config_960.Input("quant_scale") |
| 752 | .ParamType(OPTIONAL) | 752 | .ParamType(OPTIONAL) |
| 753 | .DataType({ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, | 753 | .DataType({ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, |
| 754 | ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, | 754 | ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, ge::DT_UINT64, |
| @@ -769,7 +769,7 @@ public: | |||
| 769 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 769 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 770 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 770 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 771 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); | 771 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); |
| 772 | - aicore_config_960DT.Input("quant_offset") | 772 | + aicore_config_960.Input("quant_offset") |
| 773 | .ParamType(OPTIONAL) | 773 | .ParamType(OPTIONAL) |
| 774 | .DataType({ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, | 774 | .DataType({ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, |
| 775 | ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, | 775 | ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, |
| @@ -790,7 +790,7 @@ public: | |||
| 790 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 790 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 791 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 791 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 792 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); | 792 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); |
| 793 | - aicore_config_960DT.Input("bias") | 793 | + aicore_config_960.Input("bias") |
| 794 | .ParamType(OPTIONAL) | 794 | .ParamType(OPTIONAL) |
| 795 | .DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_FLOAT16, | 795 | .DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_FLOAT16, |
| 796 | ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_BF16, | 796 | ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_BF16, |
| @@ -811,7 +811,7 @@ public: | |||
| 811 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 811 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 812 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 812 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 813 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); | 813 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); |
| 814 | - aicore_config_960DT.Output("y") | 814 | + aicore_config_960.Output("y") |
| 815 | .ParamType(REQUIRED) | 815 | .ParamType(REQUIRED) |
| 816 | .DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_FLOAT16, | 816 | .DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_FLOAT16, |
| 817 | ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, | 817 | ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_BF16, |
| @@ -832,14 +832,15 @@ public: | |||
| 832 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 832 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 833 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 833 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 834 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); | 834 | ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); |
| 835 | - aicore_config_960DT.DynamicCompileStaticFlag(true) | 835 | + aicore_config_960.DynamicCompileStaticFlag(true) |
| 836 | .DynamicFormatFlag(true) | 836 | .DynamicFormatFlag(true) |
| 837 | .DynamicRankSupportFlag(true) | 837 | .DynamicRankSupportFlag(true) |
| 838 | .DynamicShapeSupportFlag(true) | 838 | .DynamicShapeSupportFlag(true) |
| 839 | .NeedCheckSupportFlag(false) | 839 | .NeedCheckSupportFlag(false) |
| 840 | .PrecisionReduceFlag(true) | 840 | .PrecisionReduceFlag(true) |
| 841 | .ExtendCfgInfo("aclnnSupport.value", "support_aclnn"); | 841 | .ExtendCfgInfo("aclnnSupport.value", "support_aclnn"); |
| 842 | - this->AICore().AddConfig("ascend960dt", aicore_config_960DT); | 842 | + this->AICore().AddConfig("ascend960dt", aicore_config_960); |
| 843 | + this->AICore().AddConfig("ascend960pr", aicore_config_960); | ||
🟡 Medium Priority 变更行 834-843:新增 证据链:本 op 的 kernel 编译单元在 matmul/weight_quant_batch_matmul_v2/CMakeLists.txt 行 21 声明为 失败模式:若构建系统按 COMPUTE_UNITS 逐个平台编译并打包 kernel bin,则 ascend960pr 平台上该算子只有 host 侧的 op 定义、没有对应的 kernel 二进制;图编译/下发阶段会因找不到该 soc 的 kernel 而报 tiling/bin 缺失错误,或在 fallback 到 ascend960dt 的 bin 时与 9202 的实际指令集不匹配。触发条件为在 ascend960pr 平台上运行 weight_quant_batch_matmul_v2。 建议:确认 ascend960pr 的 kernel 产物是否已由构建系统覆盖(必要时在 matmul/weight_quant_batch_matmul_v2/CMakeLists.txt 的 COMPUTE_UNITS 中补充 ascend960pr),避免只注册 host 配置而无对应 kernel bin。 ![]() ![]() Incorrect? | |||
| 843 | } | 844 | } |
| 844 | }; | 845 | }; |
| 845 | 846 | ||
| @@ -355,12 +355,9 @@ __aicore__ inline void WeightQuantBatchMatmulV2ASWMxKernel<LOCAL_TEMPLATE_FUNC_P | |||
| 355 | uint64_t nL1Size = Min<uint64_t>(tiling_->matmulTiling.N - block_.offset_.offsetN, block_.params_.singleCoreN); | 355 | uint64_t nL1Size = Min<uint64_t>(tiling_->matmulTiling.N - block_.offset_.offsetN, block_.params_.singleCoreN); |
| 356 | uint64_t scaleGmOffset = (block_.offset_.offsetN * tiling_->matmulTiling.Kb + kOffset) / MX_GROUP_SIZE; | 356 | uint64_t scaleGmOffset = (block_.offset_.offsetN * tiling_->matmulTiling.Kb + kOffset) / MX_GROUP_SIZE; |
| 357 | 357 | ||
| 358 | - // Mx非对齐场景,weight和weight scale的K轴需要对齐到C0 Size(64) | 358 | + kL1Size = CeilAlign<uint64_t>(kL1Size, C0<wType>::value); |
| 359 | - if constexpr (IsSameType<wType, fp4x2_e2m1_t>::value) { | 359 | + uint64_t kAlignSize = CeilAlign<uint64_t>(tiling_->matmulTiling.Kb, C0<wType>::value); |
| 360 | - kL1Size = CeilAlign<uint64_t>(kL1Size, C0<wType>::value); | 360 | + scaleGmOffset = (block_.offset_.offsetN * kAlignSize + kOffset) / MX_GROUP_SIZE; |
🟡 Medium Priority 变更行(CopyScaleToL1,行 358-360):原代码把「kL1Size 对齐到 C0」「kAlignSize 重算」「scaleGmOffset 按对齐后的 K 重算」三件事都包在 影响范围:同一个 失败模式:对 HIFP4 而言, ![]() ![]() Incorrect? | |||
| 361 | - uint64_t kAlignSize = CeilAlign<uint64_t>(tiling_->matmulTiling.Kb, C0<wType>::value); | ||
| 362 | - scaleGmOffset = (block_.offset_.offsetN * kAlignSize + kOffset) / MX_GROUP_SIZE; | ||
| 363 | - } | ||
| 364 | 361 | ||
| 365 | uint64_t scaleGmSize; | 362 | uint64_t scaleGmSize; |
| 366 | Dn2NzParams dn2NzParams; | 363 | Dn2NzParams dn2NzParams; |
| @@ -434,13 +431,11 @@ __aicore__ inline void WeightQuantBatchMatmulV2ASWMxKernel<LOCAL_TEMPLATE_FUNC_P | |||
| 434 | dataCopyParams.dstStride = CeilAlign<uint64_t>(nL1Size, static_cast<uint64_t>(AscendC::BLOCK_CUBE)) - nL1Size; | 431 | dataCopyParams.dstStride = CeilAlign<uint64_t>(nL1Size, static_cast<uint64_t>(AscendC::BLOCK_CUBE)) - nL1Size; |
| 435 | 432 | ||
| 436 | // weight shape:(k1,n1,(n0=16,k0=64)) | 433 | // weight shape:(k1,n1,(n0=16,k0=64)) |
| 437 | - if constexpr (IsSameType<wType, fp4x2_e2m1_t>::value) { | 434 | + uint64_t nAlignSize = CeilAlign<uint64_t>(tiling_->matmulTiling.N, AscendC::BLOCK_CUBE); |
| 438 | - uint64_t nAlignSize = CeilAlign<uint64_t>(tiling_->matmulTiling.N, AscendC::BLOCK_CUBE); | 435 | + weightGmOffset = (kOffset * nAlignSize + block_.offset_.offsetN * C0<wType>::value) >> 1; |
| 439 | - weightGmOffset = (kOffset * nAlignSize + block_.offset_.offsetN * C0<wType>::value) >> 1; | ||
| 440 | 436 | ||
| 441 | - uint64_t nL1AlignSize = CeilAlign<uint64_t>(nL1Size, AscendC::BLOCK_CUBE); | 437 | + uint64_t nL1AlignSize = CeilAlign<uint64_t>(nL1Size, AscendC::BLOCK_CUBE); |
| 442 | - dataCopyParams.srcStride = nAlignSize - nL1Size; | 438 | + dataCopyParams.srcStride = nAlignSize - nL1Size; |
🟡 Medium Priority 变更行(CopyWeightGm2L1,行 434-438):原来只有 证据链:本文件行 313、367 明确存在 失败模式:若 host tiling 下发的 建议:核实 tiling 侧 N 是否对 HIFP4 保证 BLOCK_CUBE 对齐;未保证时让 srcStride/weightGmOffset 按 wType 分支选择 N 或 CeilAlign(N, BLOCK_CUBE),并补充 N 非 16 倍数的 HIFP4 用例。 ![]() ![]() Incorrect? | |||
| 443 | - } | ||
| 444 | 439 | ||
| 445 | // 最后一个参数enablePreReadIgnoreSync = true表示开启PreAllocation | 440 | // 最后一个参数enablePreReadIgnoreSync = true表示开启PreAllocation |
| 446 | AscendC::DataCopy((l1DbFlag_ == IDX_0) ? bL1PingLocalTensor_.template ReinterpretCast<uint8_t>() : | 441 | AscendC::DataCopy((l1DbFlag_ == IDX_0) ? bL1PingLocalTensor_.template ReinterpretCast<uint8_t>() : |
| @@ -26,14 +26,15 @@ | |||
| 26 | #include "lib/matmul_intf.h" | 26 | #include "lib/matmul_intf.h" |
| 27 | #include "weight_quant_batch_matmul_v2_arch35_tiling_key.h" | 27 | #include "weight_quant_batch_matmul_v2_arch35_tiling_key.h" |
| 28 | #include "weight_quant_batch_matmul_v2_arch35_tiling_data.h" | 28 | #include "weight_quant_batch_matmul_v2_arch35_tiling_data.h" |
| 29 | -#if (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)) | 29 | +#if (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)) |
| 30 | #include "asw/weight_quant_batch_matmul_v2_adaptive_sliding_window.h" | 30 | #include "asw/weight_quant_batch_matmul_v2_adaptive_sliding_window.h" |
| 31 | #include "asw/weight_quant_batch_matmul_v2_adaptive_sliding_window_mix.h" | 31 | #include "asw/weight_quant_batch_matmul_v2_adaptive_sliding_window_mix.h" |
| 32 | #include "asw/weight_quant_batch_matmul_v2_adaptive_sliding_window_mx.h" | 32 | #include "asw/weight_quant_batch_matmul_v2_adaptive_sliding_window_mx.h" |
| 33 | #include "asw/weight_quant_batch_matmul_v2_adaptive_sliding_window_general.h" | 33 | #include "asw/weight_quant_batch_matmul_v2_adaptive_sliding_window_general.h" |
| 34 | #endif | 34 | #endif |
| 35 | 35 | ||
| 36 | -#if !defined(__DAV_310R6__) && !__FIXED_POINT_ONLY_CUBE_TO_L0C__ && !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)) | 36 | +#if !defined(__DAV_310R6__) && !__FIXED_POINT_ONLY_CUBE_TO_L0C__ && \ |
| 37 | + !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)) | ||
| 37 | #if defined(A16) && (defined(WQBMMV2_S4) || defined(FP4) || (defined(WQBMMV2_S8) && defined(WEIGHT_ND))) | 38 | #if defined(A16) && (defined(WQBMMV2_S4) || defined(FP4) || (defined(WQBMMV2_S8) && defined(WEIGHT_ND))) |
| 38 | #include "weight_quant_batch_matmul_v2_reg_base.h" | 39 | #include "weight_quant_batch_matmul_v2_reg_base.h" |
| 39 | #endif | 40 | #endif |
| @@ -77,7 +78,7 @@ constexpr MatmulBatchParams batchParamsNoBatchOut{false, BatchMode::BATCH_LESS_T | |||
| 77 | BatchOutMode::SINGLE_BATCH}; | 78 | BatchOutMode::SINGLE_BATCH}; |
| 78 | constexpr MatmulConfig MM_CFG_MULTI_BATCH_NO_BATCH_OUT = GetMMConfig<configMode>(batchParamsNoBatchOut); | 79 | constexpr MatmulConfig MM_CFG_MULTI_BATCH_NO_BATCH_OUT = GetMMConfig<configMode>(batchParamsNoBatchOut); |
| 79 | 80 | ||
| 80 | -#if !__FIXED_POINT_ONLY_CUBE_TO_L0C__ && !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)) | 81 | +#if !__FIXED_POINT_ONLY_CUBE_TO_L0C__ && !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)) |
| 81 | static constexpr VecAntiQuantConfig VEC_ANTIQUANT_CONFIGS[] = { | 82 | static constexpr VecAntiQuantConfig VEC_ANTIQUANT_CONFIGS[] = { |
| 82 | {2, 512}, // idx 0 | 83 | {2, 512}, // idx 0 |
| 83 | {4, 512}, // idx 1 | 84 | {4, 512}, // idx 1 |
| @@ -148,7 +149,7 @@ static constexpr VecAntiQuantConfig VEC_ANTIQUANT_CONFIG_5 = VEC_ANTIQUANT_CONFI | |||
| 148 | } while (0) | 149 | } while (0) |
| 149 | #endif | 150 | #endif |
| 150 | 151 | ||
| 151 | -#if (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)) | 152 | +#if (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)) |
| 152 | #define INVOKE_WEIGHT_QUANT_BMM_OP_ASW_IMPL_9201(templateClass, ...) \ | 153 | #define INVOKE_WEIGHT_QUANT_BMM_OP_ASW_IMPL_9201(templateClass, ...) \ |
| 153 | do { \ | 154 | do { \ |
| 154 | AscendC::TPipe tPipe; \ | 155 | AscendC::TPipe tPipe; \ |
| @@ -187,14 +188,15 @@ __global__ __aicore__ void weight_quant_batch_matmul_v2(GM_ADDR x, GM_ADDR weigh | |||
| 187 | GM_ADDR quantOffset, GM_ADDR bias, GM_ADDR y, GM_ADDR workspace, | 188 | GM_ADDR quantOffset, GM_ADDR bias, GM_ADDR y, GM_ADDR workspace, |
| 188 | GM_ADDR tiling) | 189 | GM_ADDR tiling) |
| 189 | { | 190 | { |
| 190 | -#if (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)) | 191 | +#if (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)) |
| 191 | GM_ADDR userWS = GetUserWorkspace(workspace); | 192 | GM_ADDR userWS = GetUserWorkspace(workspace); |
| 192 | if (userWS == nullptr) { | 193 | if (userWS == nullptr) { |
| 193 | return; | 194 | return; |
| 194 | } | 195 | } |
| 195 | #endif | 196 | #endif |
| 196 | 197 | ||
| 197 | -#if !defined(__DAV_310R6__) && !__FIXED_POINT_ONLY_CUBE_TO_L0C__ && !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)) | 198 | +#if !defined(__DAV_310R6__) && !__FIXED_POINT_ONLY_CUBE_TO_L0C__ && \ |
| 199 | + !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)) | ||
| 198 | KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_MIX_AIC_1_2); | 200 | KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_MIX_AIC_1_2); |
| 199 | REGISTER_TILING_DEFAULT(wqbmmv2_tiling::WeightQuantBatchMatmulV2ASTilingDataParams); | 201 | REGISTER_TILING_DEFAULT(wqbmmv2_tiling::WeightQuantBatchMatmulV2ASTilingDataParams); |
| 200 | using DtypeBias = AscendC::Std::conditional_t<IS_BIAS_FP32, float, DTYPE_X>; | 202 | using DtypeBias = AscendC::Std::conditional_t<IS_BIAS_FP32, float, DTYPE_X>; |
| @@ -328,7 +330,7 @@ __global__ __aicore__ void weight_quant_batch_matmul_v2(GM_ADDR x, GM_ADDR weigh | |||
| 328 | VEC_ANTIQUANT_CONFIG_5); | 330 | VEC_ANTIQUANT_CONFIG_5); |
| 329 | } | 331 | } |
| 330 | #endif | 332 | #endif |
| 331 | -#elif (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)) | 333 | +#elif (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)) |
| 332 | using DtypeBias = AscendC::Std::conditional_t<IS_BIAS_FP32, float, DTYPE_X>; | 334 | using DtypeBias = AscendC::Std::conditional_t<IS_BIAS_FP32, float, DTYPE_X>; |
| 333 | static constexpr WeightQuantBatchMatmulV2::QuantType | 335 | static constexpr WeightQuantBatchMatmulV2::QuantType |
| 334 | AntiquantType = static_cast<WeightQuantBatchMatmulV2::QuantType>(ANTIQUANT_TYPE); | 336 | AntiquantType = static_cast<WeightQuantBatchMatmulV2::QuantType>(ANTIQUANT_TYPE); |
| @@ -643,7 +643,7 @@ ASCENDC_TPL_SEL( | |||
| 643 | #endif | 643 | #endif |
| 644 | #endif | 644 | #endif |
| 645 | 645 | ||
| 646 | -#if !defined(__CCE_AICORE__) || (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)) | 646 | +#if !defined(__CCE_AICORE__) || (defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201 || __NPU_ARCH__ == 9202)) |
| 647 | #if !defined(__CCE_AICORE__) || (defined(WEIGHT_ND)) | 647 | #if !defined(__CCE_AICORE__) || (defined(WEIGHT_ND)) |
| 648 | #if (defined(A16) && defined(WQBMMV2_HIFP8)) | 648 | #if (defined(A16) && defined(WQBMMV2_HIFP8)) |
| 649 | ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_1), | 649 | ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_1), |
| @@ -694,7 +694,7 @@ vector<CheckerCase> BuildCheckerCases() | |||
| 694 | { | 694 | { |
| 695 | auto c = hiF4(); | 695 | auto c = hiF4(); |
| 696 | c.n = 96; | 696 | c.n = 96; |
| 697 | - add("align_hif4_n_not_64_fail", false, c); | 697 | + add("align_hif4_n_not_64_ok", true, c); |
| 698 | } | 698 | } |
| 699 | { | 699 | { |
| 700 | auto c = w8NzC032PerGroup(); | 700 | auto c = w8NzC032PerGroup(); |


🟡 Medium Priority
建议:仅在 4bit 权重(Is4BitDtype/HiFp4/MxFp4)时用 CeilAlign(kSize, B4_C0_SIZE),其余 dtype 继续用 min(adaptiveWin_.baseK, kSize),保证 baseK 不超过 kSize。