Merged
KirinX90/Kirin9030系列支持QuantBatchMatmulV4/DynmicQuant/RmsNorm/Glue/ScatterNdUpdate/GatherV2/MatMulV3 #13547
chenlong1Add on 18 days ago
KirinX90/Kirin9030系列支持QuantBatchMatmulV4/DynmicQuant/RmsNorm/Glue/ScatterNdUpdate/GatherV2/MatMulV3 #13547
Merged
Merged fromchenlong1/ops-nn_1103masterintocann/ops-nnmaster
Total: 89 files changed+19626-426
| @@ -1,5 +1,5 @@ | |||
| 1 | # ---------------------------------------------------------------------------- | 1 | # ---------------------------------------------------------------------------- |
| 2 | -# Copyright (c) 2025 Huawei Technologies Co., Ltd. | 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 3 | # This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 3 | # This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 4 | # CANN Open Software License Agreement Version 2.0 (the "License"). | 4 | # CANN Open Software License Agreement Version 2.0 (the "License"). |
| 5 | # Please refer to the License for details. You may not use this file except in compliance with the License. | 5 | # Please refer to the License for details. You may not use this file except in compliance with the License. |
| @@ -9,7 +9,15 @@ | |||
| 9 | # ---------------------------------------------------------------------------- | 9 | # ---------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | # 设置算子定义时支持的芯片类型 | 11 | # 设置算子定义时支持的芯片类型 |
| 12 | -set(SUPPORT_COMPUTE_UNIT "ascend950" "mc62" "ascend350" "ascend960dt" "ascend5162") | 12 | +set(SUPPORT_COMPUTE_UNIT "ascend950" "mc62" "ascend350" "ascend960dt" "ascend5162" "kirinx90" "kirin9030") |
| 13 | # 设置每种芯片类型对应的tiling文件目录,即采用op_host目录下哪个文件夹下的tiling文件编译 | 13 | # 设置每种芯片类型对应的tiling文件目录,即采用op_host目录下哪个文件夹下的tiling文件编译 |
| 14 | -set(SUPPORT_TILING_DIR "arch35" "arch35" "arch35" "arch35" "arch35") | 14 | +set(SUPPORT_TILING_DIR "arch35" "arch35" "arch35" "arch35" "arch35" "arch31" "arch31") |
| 15 | add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE gelu ACLNNTYPE aclnn_exclude COMPUTE_UNIT ${SUPPORT_COMPUTE_UNIT} TILING_DIR ${SUPPORT_TILING_DIR} DISABLE_IN_OPP TRUE) | 15 | add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE gelu ACLNNTYPE aclnn_exclude COMPUTE_UNIT ${SUPPORT_COMPUTE_UNIT} TILING_DIR ${SUPPORT_TILING_DIR} DISABLE_IN_OPP TRUE) |
| 16 | + | ||
| 17 | +# arch31: kirinx90/kirin9030 使用 arch31/gelu.cpp 作为 kernel 入口 | ||
| 18 | +add_kernel_sources( | ||
| 19 | + KERNEL_SRC arch31/gelu.cpp | ||
| 20 | + COMPUTE_UNITS kirinx90 kirin9030 | ||
| 21 | + IMPL_MODE high_performance,high_precision | ||
| 22 | + AUTO_SYNC false | ||
| 23 | +) | ||
| @@ -10,6 +10,8 @@ | |||
| 10 | | <term>Atlas 200I/500 A2推理产品</term> | √ | | 10 | | <term>Atlas 200I/500 A2推理产品</term> | √ | |
| 11 | | <term>Atlas推理系列产品</term> | √ | | 11 | | <term>Atlas推理系列产品</term> | √ | |
| 12 | | <term>Atlas训练系列产品</term> | √ | | 12 | | <term>Atlas训练系列产品</term> | √ | |
| 13 | +| <term>KirinX90处理器系列产品</term> | √ | | ||
| 14 | +| <term>Kirin9030处理器系列产品</term> | √ | | ||
| 13 | 15 | ||
| 14 | ## 功能说明 | 16 | ## 功能说明 |
| 15 | 17 | ||
| @@ -69,6 +71,7 @@ | |||
| 69 | </table> | 71 | </table> |
| 70 | 72 | ||
| 71 | - <term>Atlas 200I/500 A2推理产品</term>、<term>Atlas推理系列产品</term>、<term>Atlas训练系列产品</term>:数据类型支持FLOAT、FLOAT16。 | 73 | - <term>Atlas 200I/500 A2推理产品</term>、<term>Atlas推理系列产品</term>、<term>Atlas训练系列产品</term>:数据类型支持FLOAT、FLOAT16。 |
| 74 | + - <term>KirinX90处理器系列产品</term>、<term>Kirin9030处理器系列产品</term>:数据类型支持FLOAT、FLOAT16。 | ||
| 72 | 75 | ||
| 73 | ## 约束说明 | 76 | ## 约束说明 |
| 74 | 77 | ||
| @@ -12,6 +12,8 @@ English | [简体中文](./README.md) | |||
| 12 | | <term>Atlas 200I/500 A2 inference products</term> | √ | | 12 | | <term>Atlas 200I/500 A2 inference products</term> | √ | |
| 13 | | <term>Atlas inference products</term> | √ | | 13 | | <term>Atlas inference products</term> | √ | |
| 14 | | <term>Atlas training products</term> | √ | | 14 | | <term>Atlas training products</term> | √ | |
| 15 | +| <term>KirinX90 processor series products</term> | √ | | ||
| 16 | +| <term>Kirin9030 processor series products</term> | √ | | ||
| 15 | 17 | ||
| 16 | ## Function | 18 | ## Function |
| 17 | 19 | ||
| @@ -71,6 +73,7 @@ English | [简体中文](./README.md) | |||
| 71 | </table> | 73 | </table> |
| 72 | 74 | ||
| 73 | - <term>Atlas 200I/500 A2 inference products</term>, <term>Atlas inference products</term>, and <term>Atlas training products</term>: The supported data types are FLOAT and FLOAT16. | 75 | - <term>Atlas 200I/500 A2 inference products</term>, <term>Atlas inference products</term>, and <term>Atlas training products</term>: The supported data types are FLOAT and FLOAT16. |
| 76 | + - <term>KirinX90 processor series products</term>, <term>Kirin9030 processor series products</term>: The supported data types are FLOAT and FLOAT16. | ||
| 74 | 77 | ||
| 75 | ## Constraints | 78 | ## Constraints |
| 76 | 79 | ||
| @@ -0,0 +1,148 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +#include "gelu_tiling_arch31.h" | ||
| 12 | +#include "tiling/platform/platform_ascendc.h" | ||
| 13 | +#include <graph/utils/type_utils.h> | ||
| 14 | +#include "register/tilingdata_base.h" | ||
| 15 | +#include "register/op_impl_registry.h" | ||
| 16 | +#include "../op_kernel/arch31/gelu_dag_arch31.h" | ||
| 17 | +#include "../op_kernel/arch31/gelu_struct.h" | ||
| 18 | +#include "op_host/tiling_util.h" | ||
| 19 | +#include <iostream> | ||
| 20 | + | ||
| 21 | +namespace optiling { | ||
| 22 | +const size_t ASCEND_WORKSPACE = 0; // 0M | ||
| 23 | +const gert::Shape g_vec_1_shape = {1}; | ||
变量命名风格不统一 ![]() ![]() | |||
| 24 | + | ||
| 25 | +ge::graphStatus GeluTiling::CalcInputDtype() | ||
| 26 | +{ | ||
| 27 | + OP_LOGD(tilingContext->GetNodeName(), "GeluTiling CalcInputDtype enter."); | ||
| 28 | + auto inputDesc = tilingContext->GetInputDesc(0); | ||
| 29 | + OP_CHECK_NULL_WITH_CONTEXT(tilingContext, inputDesc); | ||
| 30 | + this->inputDtype = inputDesc->GetDataType(); | ||
| 31 | + OP_CHECK_IF(this->inputDtype != ge::DT_FLOAT16 && this->inputDtype != ge::DT_FLOAT, | ||
| 32 | + OP_LOGE_FOR_INVALID_DTYPE_WITH_REASON( | ||
| 33 | + tilingContext->GetNodeName(), "x", | ||
| 34 | + ge::TypeUtils::DataTypeToSerialString(static_cast<ge::DataType>(this->inputDtype)), | ||
| 35 | + "The dtype of x must be DT_FLOAT16 or DT_FLOAT"), | ||
| 36 | + return ge::GRAPH_FAILED); | ||
| 37 | + return ge::GRAPH_SUCCESS; | ||
| 38 | +} | ||
| 39 | + | ||
| 40 | +static inline const gert::Shape& EnsureNotScalar(const gert::Shape& in_shape) | ||
| 41 | +{ | ||
| 42 | + if (in_shape.IsScalar()) { | ||
| 43 | + return g_vec_1_shape; | ||
| 44 | + } | ||
| 45 | + return in_shape; | ||
| 46 | +} | ||
| 47 | + | ||
| 48 | +ge::graphStatus GeluTiling::CheckShape() | ||
| 49 | +{ | ||
| 50 | + OP_LOGD(tilingContext->GetNodeName(), "GeluTiling CheckShape enter."); | ||
| 51 | + auto inputStorageShape = tilingContext->GetInputShape(0); | ||
| 52 | + OP_CHECK_NULL_WITH_CONTEXT(tilingContext, inputStorageShape); | ||
| 53 | + const gert::Shape& inputYShape = EnsureNotScalar(inputStorageShape->GetStorageShape()); | ||
| 54 | + | ||
| 55 | + auto outputStorageShape = tilingContext->GetOutputShape(0); | ||
| 56 | + OP_CHECK_NULL_WITH_CONTEXT(tilingContext, outputStorageShape); | ||
| 57 | + const gert::Shape& outputZShape = EnsureNotScalar(outputStorageShape->GetStorageShape()); | ||
| 58 | + | ||
| 59 | + OP_CHECK_IF(inputYShape != outputZShape, | ||
| 60 | + OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON( | ||
| 61 | + tilingContext->GetNodeName(), "x, y", | ||
| 62 | + Ops::Base::ToString(inputYShape) + ", " + Ops::Base::ToString(outputZShape), | ||
| 63 | + "The shapes of x and y must be the same"), | ||
| 64 | + return ge::GRAPH_FAILED); | ||
| 65 | + return ge::GRAPH_SUCCESS; | ||
| 66 | +} | ||
| 67 | + | ||
| 68 | +ge::graphStatus GeluTiling::CalcOutputDtype() | ||
| 69 | +{ | ||
| 70 | + OP_LOGD(tilingContext->GetNodeName(), "GeluTiling CalcOutputDtype enter."); | ||
| 71 | + auto outputDesc = tilingContext->GetOutputDesc(0); | ||
| 72 | + OP_CHECK_NULL_WITH_CONTEXT(tilingContext, outputDesc); | ||
| 73 | + this->outputDtype = outputDesc->GetDataType(); | ||
| 74 | + OP_CHECK_IF(this->outputDtype != this->inputDtype, | ||
| 75 | + OP_LOGE_FOR_INVALID_DTYPES_WITH_REASON( | ||
| 76 | + tilingContext->GetNodeName(), "x, y", | ||
| 77 | + ge::TypeUtils::DataTypeToSerialString(static_cast<ge::DataType>(this->inputDtype)) + ", " + | ||
| 78 | + ge::TypeUtils::DataTypeToSerialString(static_cast<ge::DataType>(this->outputDtype)), | ||
| 79 | + "The dtypes of x and y must be the same"), | ||
| 80 | + return ge::GRAPH_FAILED); | ||
| 81 | + return ge::GRAPH_SUCCESS; | ||
| 82 | +} | ||
| 83 | + | ||
| 84 | +ge::graphStatus GeluTiling::RunTiling() | ||
| 85 | +{ | ||
| 86 | + auto tiling = tilingContext->GetTilingData<Ops::Base::EleBaseTilingData16B>(); | ||
| 87 | + | ||
| 88 | + OP_LOGD(tilingContext->GetNodeName(), "GeluTiling RunTiling enter."); | ||
| 89 | + ElewiseBaseTiling elewiseBaseTiling(tilingContext); | ||
| 90 | + OP_CHECK_IF(CalcInputDtype() == ge::GRAPH_FAILED, OP_LOGE(tilingContext, "get input dtype failed"), | ||
| 91 | + return ge::GRAPH_FAILED); | ||
| 92 | + OP_CHECK_IF(CalcOutputDtype() == ge::GRAPH_FAILED, OP_LOGE(tilingContext, "get output dtype failed"), | ||
| 93 | + return ge::GRAPH_FAILED); | ||
| 94 | + OP_CHECK_IF(CheckShape() == ge::GRAPH_FAILED, OP_LOGE(tilingContext, "check shape failed"), | ||
| 95 | + return ge::GRAPH_FAILED); | ||
| 96 | + | ||
| 97 | + ge::graphStatus baseTilingResult = ge::GRAPH_FAILED; | ||
| 98 | + if (this->outputDtype == ge::DT_FLOAT16) { | ||
| 99 | + dType = TPL_FP16; | ||
| 100 | + baseTilingResult = elewiseBaseTiling.DoTiling<GeluOp::GeluDAG<half>::OpDag>(*tiling); | ||
| 101 | + } else if (this->outputDtype == ge::DT_FLOAT) { | ||
| 102 | + dType = TPL_FP32; | ||
| 103 | + baseTilingResult = elewiseBaseTiling.DoTiling<GeluOp::GeluDAG<float>::OpDag>(*tiling); | ||
| 104 | + } else { | ||
| 105 | + OP_LOGE_FOR_INVALID_DTYPE_WITH_REASON( | ||
| 106 | + tilingContext->GetNodeName(), "y", | ||
| 107 | + ge::TypeUtils::DataTypeToSerialString(static_cast<ge::DataType>(this->outputDtype)), | ||
| 108 | + "The dtype of y must be DT_FLOAT16 or DT_FLOAT"); | ||
| 109 | + return ge::GRAPH_FAILED; | ||
| 110 | + } | ||
| 111 | + OP_CHECK_IF(baseTilingResult == ge::GRAPH_FAILED, OP_LOGE(tilingContext, "elewiseBaseTiling failed"), | ||
| 112 | + return ge::GRAPH_FAILED); | ||
| 113 | + | ||
| 114 | + size_t* currentWorkspace = tilingContext->GetWorkspaceSizes(1); | ||
| 115 | + OP_CHECK_NULL_WITH_CONTEXT(tilingContext, currentWorkspace); | ||
| 116 | + currentWorkspace[0] = ASCEND_WORKSPACE; | ||
| 117 | + | ||
| 118 | + const uint64_t tilingKey = GET_TPL_TILING_KEY(1, dType); | ||
| 119 | + OP_LOGD(tilingContext->GetNodeName(), "[TilingData] : tilingKey=%lu", tilingKey); | ||
| 120 | + tilingContext->SetTilingKey(tilingKey); | ||
| 121 | + tilingContext->SetBlockDim(elewiseBaseTiling.GetBlockDim()); | ||
| 122 | + return ge::GRAPH_SUCCESS; | ||
| 123 | +} | ||
| 124 | + | ||
| 125 | +static ge::graphStatus Tiling4Gelu(gert::TilingContext* tilingContextGen) | ||
| 126 | +{ | ||
| 127 | + OP_LOGD(tilingContextGen->GetNodeName(), "Tiling4Gelu rt2.0 is running."); | ||
| 128 | + auto compileInfo = tilingContextGen->GetCompileInfo<GeluCompileInfo>(); | ||
| 129 | + OP_CHECK_NULL_WITH_CONTEXT(tilingContextGen, compileInfo); | ||
| 130 | + | ||
| 131 | + GeluTiling baseOpTiling(tilingContextGen); | ||
| 132 | + return baseOpTiling.RunTiling(); | ||
| 133 | +} | ||
| 134 | + | ||
| 135 | +ge::graphStatus TilingPrepareForGelu(gert::TilingParseContext* context) | ||
| 136 | +{ | ||
| 137 | + auto compileInfoPtr = context->GetCompiledInfo<GeluCompileInfo>(); | ||
| 138 | + OP_CHECK_NULL_WITH_CONTEXT(context, compileInfoPtr); | ||
| 139 | + fe::PlatFormInfos* platformInfoPtr = context->GetPlatformInfo(); | ||
| 140 | + OP_CHECK_NULL_WITH_CONTEXT(context, platformInfoPtr); | ||
| 141 | + auto ascendcPlatform = platform_ascendc::PlatformAscendC(platformInfoPtr); | ||
| 142 | + compileInfoPtr->coreNum = ascendcPlatform.GetCoreNumAiv(); | ||
| 143 | + ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, compileInfoPtr->ubSize); | ||
| 144 | + return ge::GRAPH_SUCCESS; | ||
| 145 | +} | ||
| 146 | + | ||
| 147 | +IMPL_OP_OPTILING(Gelu).Tiling(Tiling4Gelu).TilingParse<GeluCompileInfo>(TilingPrepareForGelu); | ||
| 148 | +} // namespace optiling | ||
| @@ -0,0 +1,41 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#ifndef OPS_BUILD_IN_OP_TILING_RUNTIME_GELU_REGBASE_OPTILING | ||
| 11 | +#define OPS_BUILD_IN_OP_TILING_RUNTIME_GELU_REGBASE_OPTILING | ||
| 12 | + | ||
| 13 | +#include "register/tilingdata_base.h" | ||
| 14 | +#include "op_common/atvoss/elewise/elewise_tiling.h" | ||
| 15 | + | ||
| 16 | +namespace optiling { | ||
| 17 | +using namespace Ops::Base; | ||
| 18 | + | ||
| 19 | +struct GeluCompileInfo { | ||
| 20 | + uint64_t coreNum; | ||
| 21 | + uint64_t ubSize; | ||
| 22 | +}; | ||
| 23 | + | ||
| 24 | +class GeluTiling { | ||
| 25 | +public: | ||
| 26 | + explicit GeluTiling(gert::TilingContext* context) : tilingContext(context) {}; | ||
| 27 | + ge::graphStatus RunTiling(); | ||
| 28 | + | ||
| 29 | +protected: | ||
| 30 | + ge::graphStatus CalcOutputDtype(); | ||
| 31 | + ge::graphStatus CalcInputDtype(); | ||
| 32 | + ge::graphStatus CheckShape(); | ||
| 33 | + | ||
| 34 | +private: | ||
| 35 | + gert::TilingContext* tilingContext; | ||
| 36 | + ge::DataType outputDtype = ge::DT_UNDEFINED; | ||
| 37 | + ge::DataType inputDtype = ge::DT_UNDEFINED; | ||
| 38 | + uint64_t dType = 0; | ||
| 39 | +}; | ||
| 40 | +} // namespace optiling | ||
| 41 | +#endif // OPS_BUILD_IN_OP_TILING_RUNTIME_GELU_REGBASE_OPTILING | ||
| @@ -1,5 +1,5 @@ | |||
| 1 | /** | 1 | /** |
| 2 | - * Copyright (c) 2025 Huawei Technologies Co., Ltd. | 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 3 | * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 3 | * This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 4 | * CANN Open Software License Agreement Version 2.0 (the "License"). | 4 | * CANN Open Software License Agreement Version 2.0 (the "License"). |
| 5 | * Please refer to the License for details. You may not use this file except in compliance with the License. | 5 | * Please refer to the License for details. You may not use this file except in compliance with the License. |
| @@ -43,6 +43,26 @@ public: | |||
| 43 | this->AICore().AddConfig("ascend350", aicoreConfig); | 43 | this->AICore().AddConfig("ascend350", aicoreConfig); |
| 44 | this->AICore().AddConfig("ascend960dt", aicoreConfig); | 44 | this->AICore().AddConfig("ascend960dt", aicoreConfig); |
| 45 | this->AICore().AddConfig("ascend5162", aicoreConfig); | 45 | this->AICore().AddConfig("ascend5162", aicoreConfig); |
| 46 | + | ||
| 47 | + OpAICoreConfig kirinConfig; | ||
| 48 | + kirinConfig.Input("x") | ||
| 49 | + .ParamType(REQUIRED) | ||
| 50 | + .DataType({ge::DT_FLOAT16, ge::DT_FLOAT}) | ||
| 51 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 52 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}); | ||
| 53 | + kirinConfig.Output("y") | ||
| 54 | + .ParamType(REQUIRED) | ||
| 55 | + .DataType({ge::DT_FLOAT16, ge::DT_FLOAT}) | ||
| 56 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 57 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}); | ||
| 58 | + kirinConfig.DynamicCompileStaticFlag(true) | ||
| 59 | + .DynamicFormatFlag(false) | ||
| 60 | + .DynamicRankSupportFlag(true) | ||
| 61 | + .DynamicShapeSupportFlag(true) | ||
| 62 | + .NeedCheckSupportFlag(false) | ||
| 63 | + .PrecisionReduceFlag(true); | ||
| 64 | + this->AICore().AddConfig("kirinx90", kirinConfig); | ||
| 65 | + this->AICore().AddConfig("kirin9030", kirinConfig); | ||
| 46 | } | 66 | } |
| 47 | }; | 67 | }; |
| 48 | 68 | ||
| @@ -0,0 +1,43 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/* ! | ||
| 12 | + * \file gelu.cpp | ||
| 13 | + * \brief z = gelu(x), KirinX90/Kirin9030 (arch31) kernel entry | ||
| 14 | + */ | ||
| 15 | +#include "kernel_operator.h" | ||
| 16 | +#include "kernel_tiling/kernel_tiling.h" | ||
| 17 | +#include "gelu_dag_arch31.h" | ||
| 18 | +#include "gelu_struct.h" | ||
| 19 | +#include "atvoss/elewise/elewise_sch_with_scalar.h" | ||
| 20 | +#include "atvoss/elewise/elewise_sch_16b.h" | ||
| 21 | +#include "atvoss/util/dfx.h" | ||
| 22 | + | ||
| 23 | +using namespace AscendC; | ||
| 24 | +using namespace Ops::Base; | ||
| 25 | + | ||
| 26 | +template <uint64_t schMode, uint64_t dType> | ||
| 27 | +__global__ __aicore__ void gelu(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) | ||
| 28 | +{ | ||
| 29 | + REGISTER_TILING_DEFAULT(EleBaseTilingData16B); | ||
| 30 | + GET_TILING_DATA_PTR_WITH_STRUCT(EleBaseTilingData16B, tilingData, tiling); | ||
| 31 | + KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_AIV_ONLY); | ||
| 32 | + | ||
| 33 | + if constexpr (dType == TPL_FP16) { | ||
| 34 | + ElementwiseSch16B<schMode, GeluOp::GeluDAG<half>::OpDag> sch(tilingData); | ||
| 35 | + sch.Init(x, y); | ||
| 36 | + sch.Process(); | ||
| 37 | + } else if constexpr (dType == TPL_FP32) { | ||
| 38 | + ElementwiseSch16B<schMode, GeluOp::GeluDAG<float>::OpDag> sch(tilingData); | ||
| 39 | + sch.Init(x, y); | ||
| 40 | + sch.Process(); | ||
| 41 | + } | ||
| 42 | + return; | ||
| 43 | +} | ||
| @@ -0,0 +1,88 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/* ! | ||
| 12 | + * \file gelu_dag_arch31.h | ||
| 13 | + * \brief | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#ifndef CANN_CUSTOM_OPS_GELU_DAG_H | ||
| 17 | +#define CANN_CUSTOM_OPS_GELU_DAG_H | ||
| 18 | + | ||
| 19 | +#include "atvoss/util/dag.h" | ||
| 20 | +#include "atvoss/util/vec.h" | ||
| 21 | +#include "atvoss/util/placeholder.h" | ||
| 22 | + | ||
| 23 | +namespace GeluOp { | ||
| 24 | +using namespace Ops::Base; | ||
| 25 | +const float NEG_SQRT_EIGHT_OVER_PI = -1.595769121 * 0.044715; | ||
| 26 | +const float TANH_APPROX_FACTOR = 1 / 0.044715; | ||
| 27 | +const int CAST_MODE_NONE = 0; | ||
| 28 | +const int CAST_MODE_RINT = 1; | ||
| 29 | +// tbe realization: x / (1 + e^(-1.5957691(x + 0.044715 x * x * x))) | ||
| 30 | +// current realization: x / (1 + e^(-1.5957691*0.044715(x/0.044715 + x^3))) | ||
| 31 | +namespace GeluDag1 { | ||
| 32 | +template <class T> | ||
| 33 | +struct GeluCustom : public Vec::ElemwiseUnaryOP<T, T> { | ||
| 34 | + __aicore__ inline GeluCustom(LocalTensor<T>& dst, LocalTensor<T>& src, uint32_t count) | ||
| 35 | + { | ||
| 36 | +#ifdef __CCE_AICORE__ | ||
| 37 | + uint32_t dtypeSize = sizeof(T); | ||
| 38 | + uint32_t vl = VECTOR_REG_WIDTH / dtypeSize; | ||
| 39 | + uint16_t loopNum = CeilDivision(count, vl); | ||
| 40 | + uint32_t vlSize = vl; | ||
| 41 | + __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); | ||
| 42 | + __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | ||
| 43 | + | ||
| 44 | + MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput; | ||
| 45 | + MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputSqr; | ||
| 46 | + MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputCub; | ||
| 47 | + MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput; | ||
| 48 | + MicroAPI::MaskReg mask; | ||
| 49 | + if constexpr (std::is_same_v<T, float>) { | ||
| 50 | + __VEC_SCOPE__ | ||
| 51 | + { | ||
| 52 | + for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | ||
| 53 | + mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | ||
| 54 | + // OpCopyIn | ||
| 55 | + MicroAPI::DataCopy(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); | ||
| 56 | + MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask); | ||
| 57 | + MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask); | ||
| 58 | + MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask); | ||
| 59 | + MicroAPI::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask); | ||
| 60 | + MicroAPI::Exp(vregInputCub, vregInputCub, mask); | ||
| 61 | + MicroAPI::Adds(vregInputCub, vregInputCub, (float)1.0, mask); | ||
| 62 | + MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask); | ||
| 63 | + | ||
| 64 | + // OpCopyOut | ||
| 65 | + MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | ||
| 66 | + } | ||
| 67 | + } | ||
| 68 | + } | ||
| 69 | +#endif | ||
| 70 | + } | ||
| 71 | +}; | ||
| 72 | +} // namespace GeluDag1 | ||
| 73 | + | ||
| 74 | +template <typename U, typename T = float> | ||
| 75 | +struct GeluDAG { | ||
| 76 | + using OpCopyIn0 = Bind<Vec::CopyIn<U>, Placeholder::In0<U>>; | ||
| 77 | + using OpCopyIn0Cast = Bind<Vec::Cast<T, U, CAST_MODE_NONE>, OpCopyIn0>; | ||
| 78 | + | ||
| 79 | + using OpLogResult = Bind<GeluDag1::GeluCustom<T>, OpCopyIn0Cast>; | ||
| 80 | + using OpResultCast = Bind<Vec::Cast<U, T, CAST_MODE_RINT>, OpLogResult>; | ||
| 81 | + using OpCopyOut = Bind<Vec::CopyOut<U>, Placeholder::Out0<U>, OpResultCast>; | ||
| 82 | + | ||
| 83 | + using Outputs = Elems<OpCopyOut>; | ||
| 84 | + using MemCfg = MemOptCfg<MemLevel::LEVEL_2>; | ||
| 85 | + using OpDag = DAGSch<Outputs, void, MemCfg>; | ||
| 86 | +}; | ||
| 87 | +} // namespace GeluOp | ||
| 88 | +#endif // CANN_CUSTOM_OPS_GELU_DAG_H | ||
| @@ -0,0 +1,34 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/* ! | ||
| 12 | + * \file gelu_struct.h | ||
| 13 | + * \brief | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#include "ascendc/host_api/tiling/template_argument.h" | ||
| 17 | + | ||
| 18 | +#ifndef CANN_CUSTOM_OPS_GELU_STRUCT_H_ | ||
| 19 | +#define CANN_CUSTOM_OPS_GELU_STRUCT_H_ | ||
| 20 | + | ||
| 21 | +#define TPL_FP16 1 | ||
| 22 | +#define TPL_FP32 3 | ||
| 23 | + | ||
| 24 | +#define TPL_SCH_MODE_0 0 | ||
| 25 | +#define TPL_SCH_MODE_1 1 | ||
| 26 | + | ||
| 27 | +ASCENDC_TPL_ARGS_DECL(Gelu, ASCENDC_TPL_UINT_DECL(schMode, 1, ASCENDC_TPL_UI_LIST, TPL_SCH_MODE_0, TPL_SCH_MODE_1), | ||
| 28 | + ASCENDC_TPL_DTYPE_DECL(dType, TPL_FP16, TPL_FP32)); | ||
| 29 | + | ||
| 30 | +ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_UINT_SEL(schMode, ASCENDC_TPL_UI_LIST, TPL_SCH_MODE_0, TPL_SCH_MODE_1), | ||
| 31 | + ASCENDC_TPL_DTYPE_SEL(dType, TPL_FP16)), | ||
| 32 | + ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_UINT_SEL(schMode, ASCENDC_TPL_UI_LIST, TPL_SCH_MODE_0, TPL_SCH_MODE_1), | ||
| 33 | + ASCENDC_TPL_DTYPE_SEL(dType, TPL_FP32))); | ||
| 34 | +#endif // CANN_CUSTOM_OPS_GELU_STRUCT_H_ | ||
| @@ -9,7 +9,7 @@ | |||
| 9 | # ---------------------------------------------------------------------------- | 9 | # ---------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | # 设置算子定义时支持的芯片类型 | 11 | # 设置算子定义时支持的芯片类型 |
| 12 | -set(SUPPORT_COMPUTE_UNIT "ascend950" "mc62" "ascend350" "ascend960dt") | 12 | +set(SUPPORT_COMPUTE_UNIT "ascend950" "mc62" "ascend350" "ascend960dt" "kirinx90" "kirin9030") |
| 13 | # 设置每种芯片类型对应的tiling文件目录,即采用op_host目录下哪个文件夹下的tiling文件编译 | 13 | # 设置每种芯片类型对应的tiling文件目录,即采用op_host目录下哪个文件夹下的tiling文件编译 |
| 14 | -set(SUPPORT_TILING_DIR "arch35" "arch35" "arch35" "arch35") | 14 | +set(SUPPORT_TILING_DIR "arch35" "arch35" "arch35" "arch35" "arch31" "arch31") |
| 15 | add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE gather_v2 ACLNNTYPE aclnn_exclude COMPUTE_UNIT ${SUPPORT_COMPUTE_UNIT} TILING_DIR ${SUPPORT_TILING_DIR} DISABLE_IN_OPP TRUE DEPENDENCIES scatter_update renorm gather_elements gather_v3 linear_index_v2 index_put_with_sort HOSTCPU TRUE) | 15 | add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE gather_v2 ACLNNTYPE aclnn_exclude COMPUTE_UNIT ${SUPPORT_COMPUTE_UNIT} TILING_DIR ${SUPPORT_TILING_DIR} DISABLE_IN_OPP TRUE DEPENDENCIES scatter_update renorm gather_elements gather_v3 linear_index_v2 index_put_with_sort HOSTCPU TRUE) |
| @@ -10,7 +10,8 @@ | |||
| 10 | | <term>Atlas 200I/500 A2推理产品</term> | × | | 10 | | <term>Atlas 200I/500 A2推理产品</term> | × | |
| 11 | | <term>Atlas推理系列产品</term> | × | | 11 | | <term>Atlas推理系列产品</term> | × | |
| 12 | | <term>Atlas训练系列产品</term> | × | | 12 | | <term>Atlas训练系列产品</term> | × | |
| 13 | - | 13 | +| <term>KirinX90处理器系列产品</term> | √ | |
| 14 | +| <term>Kirin9030处理器系列产品</term> | √ | | ||
| 14 | ## 功能说明 | 15 | ## 功能说明 |
| 15 | 16 | ||
| 16 | - 算子功能:从输入Tensor的指定维度dim,按index中的下标序号提取元素,保存到out Tensor中。 | 17 | - 算子功能:从输入Tensor的指定维度dim,按index中的下标序号提取元素,保存到out Tensor中。 |
| @@ -12,6 +12,8 @@ English | [简体中文](./README.md) | |||
| 12 | | <term>Atlas 200I/500 A2 inference products</term> | × | | 12 | | <term>Atlas 200I/500 A2 inference products</term> | × | |
| 13 | | <term>Atlas inference products</term> | × | | 13 | | <term>Atlas inference products</term> | × | |
| 14 | | <term>Atlas training products</term> | × | | 14 | | <term>Atlas training products</term> | × | |
| 15 | +| <term>KirinX90 processor series products</term> | √ | | ||
| 16 | +| <term>Kirin9030 processor series products</term> | √ | | ||
| 15 | 17 | ||
| 16 | ## Function | 18 | ## Function |
| 17 | 19 | ||
| @@ -0,0 +1,79 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file gather_v2_tiling_arch31.h | ||
| 13 | + * \brief | ||
| 14 | + */ | ||
| 15 | +#ifndef OPS_BUILT_IN_OP_TILING_RUNTIME_GATHER_V2_ARCH31_H_ | ||
| 16 | +#define OPS_BUILT_IN_OP_TILING_RUNTIME_GATHER_V2_ARCH31_H_ | ||
| 17 | +#include <cstdint> | ||
| 18 | +#include <nlohmann/json.hpp> | ||
| 19 | +#include <sstream> | ||
| 20 | + | ||
| 21 | +namespace optiling { | ||
| 22 | +struct GatherV2CompileInfo { | ||
| 23 | + int64_t ub_size{1}; | ||
| 24 | + int64_t block_size{1}; | ||
| 25 | + int64_t l1_size{0}; | ||
| 26 | + int64_t core_num{1}; | ||
| 27 | + int64_t params_dsize{1}; | ||
变量命名风格不统一 ![]() ![]() | |||
| 28 | + int64_t indices_dsize{1}; | ||
| 29 | + bool is_tik{false}; | ||
| 30 | + bool is_gather_v2{true}; | ||
| 31 | + int64_t impl_mode{0}; | ||
| 32 | + bool is_preprocessed; | ||
| 33 | + int64_t soc_version{0}; | ||
| 34 | +}; | ||
| 35 | + | ||
| 36 | +struct GatherV2TilingParams { | ||
| 37 | + int64_t tiling_mode = 0; | ||
| 38 | + int64_t params_pre = 1; | ||
| 39 | + int64_t params_axis = 1; | ||
| 40 | + int64_t params_row = 1; | ||
| 41 | + int64_t indices_num = 1; | ||
| 42 | + int64_t cache_params = 0; | ||
| 43 | + int64_t need_core_num = 0; | ||
| 44 | + int64_t tail_process_core = 0; | ||
| 45 | + int64_t indices_num_each_core = 0; | ||
| 46 | + int64_t indices_num_remaining = 0; | ||
| 47 | + int64_t indices_loop_num = 0; | ||
| 48 | + int64_t indices_row_num_once = 0; | ||
| 49 | + int64_t indices_row_num_last = 0; | ||
| 50 | + int64_t row_num_once_ub = 0; | ||
| 51 | + int64_t row_num_once_tail_ub = 0; | ||
| 52 | + int64_t inner_loop_num = 0; | ||
| 53 | + int64_t row_num_last_ub = 0; | ||
| 54 | + int64_t row_num_last_tail_ub = 0; | ||
| 55 | + int64_t inner_loop_num_last = 0; | ||
| 56 | + int64_t params_total = 0; | ||
| 57 | + int64_t one_row_loop = 0; | ||
| 58 | + int64_t one_row_tail = 0; | ||
| 59 | + int64_t params_pre_each_core = 0; | ||
| 60 | + int64_t params_pre_remaining = 0; | ||
| 61 | + int64_t indices_row = 1; | ||
| 62 | + int64_t params_batch_each_core = 1; | ||
| 63 | + int64_t params_batch_remaining = 0; | ||
| 64 | + int64_t params_batch = 1; | ||
| 65 | + int64_t soc_version = 0; | ||
| 66 | + int64_t x_split_num = 1; | ||
| 67 | + int64_t x_buffer_size = 0; | ||
| 68 | + int64_t indices_buffer_size = 0; | ||
| 69 | + int64_t indices_split_num = 0; | ||
| 70 | + int64_t mask_buffer_size = 0; | ||
| 71 | + int64_t total_count_num = 0; | ||
| 72 | + int64_t count_time = 0; | ||
| 73 | + int64_t median_core_id = 0; | ||
| 74 | + int64_t pre_core_compute_size = 0; | ||
| 75 | + int64_t median_core_compute_size = 0; | ||
| 76 | + int64_t post_core_compute_size = 0; | ||
| 77 | +}; | ||
| 78 | +} // namespace optiling | ||
| 79 | +#endif // AIR_CXX_RUNTIME_V2_OP_IMPL_GATHER_V2_RUNTIME2_ARCH31_H_ | ||
| @@ -0,0 +1,231 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file gather_v2_tiling_data_arch31.h | ||
| 13 | + * \brief | ||
| 14 | + */ | ||
| 15 | +#ifndef AIR_CXX_RUNTIME_V2_OP_IMPL_GATHERV2_H | ||
| 16 | +#define AIR_CXX_RUNTIME_V2_OP_IMPL_GATHERV2_H | ||
| 17 | +#include <cmath> | ||
| 18 | +#include "register/tilingdata_base.h" | ||
| 19 | +#include "tiling/tiling_api.h" | ||
| 20 | +#include "op_host/tiling_base.h" | ||
| 21 | +#include "gather_v2_tiling_arch31.h" | ||
| 22 | + | ||
| 23 | +namespace optiling { | ||
| 24 | +BEGIN_TILING_DATA_DEF(GatherV2TilingData) | ||
| 25 | +TILING_DATA_FIELD_DEF(int64_t, needCoreNum); | ||
| 26 | +TILING_DATA_FIELD_DEF(int64_t, threadNum); | ||
| 27 | +TILING_DATA_FIELD_DEF(int64_t, batchSize); | ||
| 28 | +TILING_DATA_FIELD_DEF(int64_t, outerSize); | ||
| 29 | +TILING_DATA_FIELD_DEF(int64_t, gatherDimSize); | ||
| 30 | +TILING_DATA_FIELD_DEF(int64_t, gatherSize); | ||
| 31 | +TILING_DATA_FIELD_DEF(int64_t, innerSize); | ||
| 32 | +TILING_DATA_FIELD_DEF(int64_t, xSize); | ||
| 33 | +TILING_DATA_FIELD_DEF(int64_t, indicesSize); | ||
| 34 | +TILING_DATA_FIELD_DEF(int64_t, ySize); | ||
| 35 | +TILING_DATA_FIELD_DEF(int64_t, perCoreElements); | ||
| 36 | +TILING_DATA_FIELD_DEF(int64_t, lastCoreElements); | ||
| 37 | +TILING_DATA_FIELD_DEF(int64_t, negativeIndexSupport); | ||
| 38 | +TILING_DATA_FIELD_DEF(int64_t, supportOutOfBoundIndex); | ||
| 39 | +TILING_DATA_FIELD_DEF(int64_t, maxElement); | ||
| 40 | +TILING_DATA_FIELD_DEF(int64_t, indiceUbSize); | ||
| 41 | +TILING_DATA_FIELD_DEF(int64_t, dtypeSize); | ||
| 42 | +END_TILING_DATA_DEF; | ||
| 43 | + | ||
| 44 | +REGISTER_TILING_DATA_CLASS(GatherV2, GatherV2TilingData); | ||
| 45 | + | ||
| 46 | +BEGIN_TILING_DATA_DEF(GatherV2LastTilingData) | ||
| 47 | +TILING_DATA_FIELD_DEF(int16_t, needCoreNum); | ||
| 48 | +TILING_DATA_FIELD_DEF(int16_t, indicesNum); | ||
| 49 | +TILING_DATA_FIELD_DEF(int16_t, splitIndices); | ||
| 50 | +TILING_DATA_FIELD_DEF(int16_t, inputNum); | ||
| 51 | +TILING_DATA_FIELD_DEF(int16_t, splitMode); | ||
| 52 | +TILING_DATA_FIELD_DEF(int16_t, coreInCols); | ||
| 53 | +TILING_DATA_FIELD_DEF(int32_t, inputUbSize); | ||
| 54 | +TILING_DATA_FIELD_DEF(int32_t, outUbSize); | ||
| 55 | +TILING_DATA_FIELD_DEF(int32_t, indiceUbSize); | ||
| 56 | +TILING_DATA_FIELD_DEF(int32_t, indiceCastUbSize); | ||
| 57 | +TILING_DATA_FIELD_DEF(int32_t, ubCols); | ||
| 58 | +TILING_DATA_FIELD_DEF(int32_t, ubRows); | ||
| 59 | +TILING_DATA_FIELD_DEF(int32_t, gatherDimSize); | ||
| 60 | +TILING_DATA_FIELD_DEF(int64_t, gatherSize); | ||
| 61 | +TILING_DATA_FIELD_DEF(int64_t, blockFactor); | ||
| 62 | +TILING_DATA_FIELD_DEF(int64_t, tailBlockFactor); | ||
| 63 | +TILING_DATA_FIELD_DEF(int64_t, gFactor); | ||
| 64 | +END_TILING_DATA_DEF; | ||
| 65 | + | ||
| 66 | +REGISTER_TILING_DATA_CLASS(GatherV2_1100000001, GatherV2LastTilingData) | ||
| 67 | +REGISTER_TILING_DATA_CLASS(GatherV2_1100000002, GatherV2LastTilingData) | ||
| 68 | +REGISTER_TILING_DATA_CLASS(GatherV2_1100000004, GatherV2LastTilingData) | ||
| 69 | +REGISTER_TILING_DATA_CLASS(GatherV2_1100000008, GatherV2LastTilingData) | ||
| 70 | +REGISTER_TILING_DATA_CLASS(GatherV2_1100000101, GatherV2LastTilingData) | ||
| 71 | +REGISTER_TILING_DATA_CLASS(GatherV2_1100000102, GatherV2LastTilingData) | ||
| 72 | +REGISTER_TILING_DATA_CLASS(GatherV2_1100000104, GatherV2LastTilingData) | ||
| 73 | +REGISTER_TILING_DATA_CLASS(GatherV2_1100000108, GatherV2LastTilingData) | ||
| 74 | + | ||
| 75 | +BEGIN_TILING_DATA_DEF(GatherV2GaAllLoadTilingData) | ||
| 76 | +TILING_DATA_FIELD_DEF(int64_t, gSize); | ||
| 77 | +TILING_DATA_FIELD_DEF(int64_t, aSize); | ||
| 78 | +TILING_DATA_FIELD_DEF(int64_t, aSizeAligned); | ||
| 79 | +TILING_DATA_FIELD_DEF(int64_t, indicesSize); | ||
| 80 | +TILING_DATA_FIELD_DEF(int64_t, usedCoreNum); | ||
| 81 | +TILING_DATA_FIELD_DEF(int64_t, indicesOuter); | ||
| 82 | +TILING_DATA_FIELD_DEF(int64_t, normalCoreIndicesNum); | ||
| 83 | +TILING_DATA_FIELD_DEF(int64_t, tailCoreIndicesNum); | ||
| 84 | +TILING_DATA_FIELD_DEF(int64_t, pOuter); | ||
| 85 | +TILING_DATA_FIELD_DEF(int64_t, normalCoreGaNum); | ||
| 86 | +TILING_DATA_FIELD_DEF(int64_t, tailCoreGaNum); | ||
| 87 | +TILING_DATA_FIELD_DEF(int64_t, xBufferSize); | ||
| 88 | +TILING_DATA_FIELD_DEF(int64_t, indicesBufferSize); | ||
| 89 | +TILING_DATA_FIELD_DEF(int64_t, yBufferSize); | ||
| 90 | +END_TILING_DATA_DEF; | ||
| 91 | + | ||
| 92 | +REGISTER_TILING_DATA_CLASS(GatherV2_3000, GatherV2GaAllLoadTilingData) | ||
| 93 | +REGISTER_TILING_DATA_CLASS(GatherV2_3100, GatherV2GaAllLoadTilingData) | ||
| 94 | + | ||
| 95 | +BEGIN_TILING_DATA_DEF(GatherV2TilingDataSimdTwoDim) | ||
| 96 | +TILING_DATA_FIELD_DEF(int16_t, needCoreNum); | ||
| 97 | +TILING_DATA_FIELD_DEF(int16_t, negativeIndexSupport); | ||
| 98 | +TILING_DATA_FIELD_DEF(int32_t, indiceFactor); | ||
| 99 | +TILING_DATA_FIELD_DEF(int32_t, dtypeSize); | ||
| 100 | +TILING_DATA_FIELD_DEF(int64_t, gatherDimSize); | ||
| 101 | +TILING_DATA_FIELD_DEF(int64_t, gatherSize); | ||
| 102 | +TILING_DATA_FIELD_DEF(int64_t, innerSize); | ||
| 103 | +TILING_DATA_FIELD_DEF(int64_t, blockFactor); | ||
| 104 | +TILING_DATA_FIELD_DEF(int64_t, tailBlockFactor); | ||
| 105 | +TILING_DATA_FIELD_DEF(int64_t, maxElement); | ||
| 106 | +END_TILING_DATA_DEF; | ||
| 107 | + | ||
| 108 | +REGISTER_TILING_DATA_CLASS(GatherV2_1000000299, GatherV2TilingDataSimdTwoDim) | ||
| 109 | + | ||
| 110 | +BEGIN_TILING_DATA_DEF(GatherV2TilingDataSimtTwoDim) | ||
| 111 | +TILING_DATA_FIELD_DEF(int16_t, needCoreNum); | ||
| 112 | +TILING_DATA_FIELD_DEF(int16_t, negativeIndexSupport); | ||
| 113 | +TILING_DATA_FIELD_DEF(int32_t, threadNum); | ||
| 114 | +TILING_DATA_FIELD_DEF(int64_t, gatherDimSize); | ||
| 115 | +TILING_DATA_FIELD_DEF(int64_t, innerSize); | ||
| 116 | +TILING_DATA_FIELD_DEF(int64_t, perCoreElements); | ||
| 117 | +TILING_DATA_FIELD_DEF(int64_t, lastCoreElements); | ||
| 118 | +END_TILING_DATA_DEF; | ||
| 119 | + | ||
| 120 | +REGISTER_TILING_DATA_CLASS(GatherV2_2000000001, GatherV2TilingDataSimtTwoDim) | ||
| 121 | +REGISTER_TILING_DATA_CLASS(GatherV2_2000000002, GatherV2TilingDataSimtTwoDim) | ||
| 122 | +REGISTER_TILING_DATA_CLASS(GatherV2_2000000004, GatherV2TilingDataSimtTwoDim) | ||
| 123 | +REGISTER_TILING_DATA_CLASS(GatherV2_2000000008, GatherV2TilingDataSimtTwoDim) | ||
| 124 | +REGISTER_TILING_DATA_CLASS(GatherV2_2000000101, GatherV2TilingDataSimtTwoDim) | ||
| 125 | +REGISTER_TILING_DATA_CLASS(GatherV2_2000000102, GatherV2TilingDataSimtTwoDim) | ||
| 126 | +REGISTER_TILING_DATA_CLASS(GatherV2_2000000104, GatherV2TilingDataSimtTwoDim) | ||
| 127 | +REGISTER_TILING_DATA_CLASS(GatherV2_2000000108, GatherV2TilingDataSimtTwoDim) | ||
| 128 | + | ||
| 129 | +BEGIN_TILING_DATA_DEF(GatherV2TilingDataEmptyInput) | ||
| 130 | +TILING_DATA_FIELD_DEF(int64_t, needCoreNum); | ||
| 131 | +TILING_DATA_FIELD_DEF(int64_t, perCoreElements); | ||
| 132 | +TILING_DATA_FIELD_DEF(int64_t, lastCoreElements); | ||
| 133 | +END_TILING_DATA_DEF; | ||
| 134 | +REGISTER_TILING_DATA_CLASS(GatherV2_3000000000, GatherV2TilingDataEmptyInput) | ||
| 135 | + | ||
| 136 | +class Gatherv2TilingBase : public Ops::NN::Optiling::TilingBaseClass { | ||
| 137 | +public: | ||
| 138 | + explicit Gatherv2TilingBase(gert::TilingContext* context) : TilingBaseClass(context) { Reset(); } | ||
| 139 | + ~Gatherv2TilingBase() override = default; | ||
| 140 | + | ||
| 141 | + void Reset(gert::TilingContext* context) override | ||
| 142 | + { | ||
| 143 | + TilingBaseClass::Reset(context); | ||
| 144 | + Reset(); | ||
| 145 | + } | ||
| 146 | + | ||
| 147 | +protected: | ||
| 148 | + bool IsCapable() override { return true; } | ||
| 149 | + // 1、获取平台信息比如CoreNum、UB/L1/L0C资源大小 | ||
| 150 | + ge::graphStatus GetPlatformInfo() override; | ||
| 151 | + // 2、获取INPUT/OUTPUT/ATTR信息 | ||
| 152 | + ge::graphStatus GetShapeAttrsInfo() override; | ||
| 153 | + // 3、计算数据切分TilingData | ||
| 154 | + ge::graphStatus DoOpTiling() override; | ||
| 155 | + // 4、计算高阶API的TilingData | ||
| 156 | + ge::graphStatus DoLibApiTiling() override; | ||
| 157 | + // 5、计算TilingKey | ||
| 158 | + uint64_t GetTilingKey() const override; | ||
| 159 | + // 6、计算Workspace 大小 | ||
| 160 | + ge::graphStatus GetWorkspaceSize() override; | ||
| 161 | + // 7、保存Tiling数据 | ||
| 162 | + ge::graphStatus PostTiling() override; | ||
| 163 | + void DumpTilingInfo() override; | ||
| 164 | + void Reset(); | ||
| 165 | + | ||
| 166 | +private: | ||
| 167 | + inline ge::graphStatus GetXInfoAndCheck(); | ||
| 168 | + inline ge::graphStatus GetIndicesInfoAndCheck(); | ||
| 169 | + inline ge::graphStatus GetAxisInfoAndCheck(); | ||
| 170 | + inline ge::graphStatus GetAttrsInfoAndCheck(); | ||
| 171 | + void ShowBaseTilingData(); | ||
| 172 | + void ShowLastGtaherSimdTilingData(); | ||
| 173 | + ge::graphStatus MargeAxis(); | ||
| 174 | + void CalcCoreElement(); | ||
| 175 | + void CalcSimdTiling(); | ||
| 176 | + ge::graphStatus SimdTwoDimTiling(); | ||
| 177 | + ge::graphStatus SimtTwoDimTiling(); | ||
| 178 | + ge::graphStatus CalFullLoadTiling(); | ||
| 179 | + bool IsAfterGdimFullLoad(); | ||
| 180 | + int64_t XDtypeImprove(); | ||
| 181 | + ge::graphStatus LastGatherTiling(); | ||
| 182 | + ge::graphStatus GaAllLoadTiling(); | ||
| 183 | + ge::graphStatus CalcEmptyCoreElement(); | ||
| 184 | + bool IsLastGatherAndFullLoad(); | ||
| 185 | + bool IsGaAllLoad(); | ||
| 186 | + bool IsSimdTwoDim(); | ||
| 187 | + bool IsSimtTwoDim(); | ||
| 188 | + void LastGatherUbTiling(int32_t& inputUbSize, int32_t& indiceUbSize, int32_t& outUbSize, int32_t& indiceCastUbSize, | ||
| 189 | + int32_t& inputNum, int32_t& indicesNum, int32_t& ubCols, int32_t& ubRows, | ||
| 190 | + int64_t blockFactor, int64_t gFactor); | ||
| 191 | + void CalcMaxUbcolAndIndiceFactor(int32_t& ubCols, int32_t ubRows, int32_t ubSize, int32_t minCols, int32_t inputNum, | ||
| 192 | + int32_t outputNum, int32_t indiceNUm, int32_t castUbRatio); | ||
| 193 | + int64_t aivNum_ = 0; | ||
| 194 | + | ||
| 195 | + const char* opName_ = ""; | ||
| 196 | + GatherV2TilingData gatherV2TilingData_; | ||
| 197 | + GatherV2LastTilingData lastTilingdata_; | ||
| 198 | + GatherV2GaAllLoadTilingData gaAllLoadTilingdata_; | ||
| 199 | + GatherV2TilingDataSimdTwoDim simdTwoDimTilingData_; | ||
| 200 | + GatherV2TilingDataSimtTwoDim simtTwoDimTilingData_; | ||
| 201 | + GatherV2TilingDataEmptyInput emptyTilingData_; | ||
| 202 | +#ifdef DAVID_FPGA | ||
| 203 | + int64_t threadNum_ = 128; | ||
| 204 | +#else | ||
| 205 | + int64_t threadNum_ = 2048; | ||
| 206 | +#endif | ||
| 207 | + gert::Shape xShape_; | ||
| 208 | + gert::Shape indicesShape_; | ||
| 209 | + ge::DataType xDtype_ = ge::DT_FLOAT; | ||
| 210 | + ge::DataType indicesDtype_ = ge::DT_FLOAT; | ||
| 211 | + int64_t axis_ = 0; | ||
| 212 | + int64_t batchDims_ = 0; | ||
| 213 | + int64_t ySize_ = 1; | ||
| 214 | + int32_t improveDtypeSize_ = 0; | ||
| 215 | + int32_t indicesDtypeSize_ = 0; | ||
| 216 | + int64_t gatherDimSize_ = 0; | ||
| 217 | + int64_t batchSize_ = 1; | ||
| 218 | + int64_t outerSize_ = 1; | ||
| 219 | + int64_t gatherSize_ = 0; | ||
| 220 | + int64_t innerSize_ = 1; | ||
| 221 | + int64_t ubSize_ = 0; | ||
| 222 | + int64_t needCoreNum_ = 0; | ||
| 223 | + int64_t inputBatchDims_ = 0; | ||
| 224 | + bool negativeIndexSupport_ = false; | ||
| 225 | + bool supportOutOfBoundIndex_ = false; | ||
| 226 | + int32_t tilingMode_ = 0; | ||
| 227 | + int32_t ubBlockSize_ = 32; | ||
| 228 | + int32_t vRegSize_ = 256; | ||
| 229 | +}; | ||
| 230 | +} // namespace optiling | ||
| 231 | +#endif // AIR_CXX_RUNTIME_V2_OP_IMPL_GATHERV2_H | ||
| @@ -15,6 +15,40 @@ | |||
| 15 | #include "register/op_def_registry.h" | 15 | #include "register/op_def_registry.h" |
| 16 | 16 | ||
| 17 | namespace ops { | 17 | namespace ops { |
| 18 | + | ||
| 19 | +// kirinx90 / kirin9030 支持的数据类型组合(保守精简:去掉 bf16/float8/complex,保留基本类型) | ||
| 20 | +// 组合数:5 种 x (fp16/fp32/int8/int32/int64) × 2 种 indices (int32/int64) × 2 种 axis (int32/int64) = 20 组 | ||
| 21 | +static const std::vector<ge::DataType> kirinXDataType = { | ||
| 22 | + // fp16 × 4 | ||
| 23 | + ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, | ||
| 24 | + // fp32 × 4 | ||
| 25 | + ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, | ||
| 26 | + // int8 × 4 | ||
| 27 | + ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, | ||
| 28 | + // int32 × 4 | ||
| 29 | + ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, | ||
| 30 | + // int64 × 4 | ||
| 31 | + ge::DT_INT64, ge::DT_INT64, ge::DT_INT64, ge::DT_INT64}; | ||
| 32 | +static const std::vector<ge::DataType> kirinIndicesDataType = { | ||
| 33 | + // 每种 x 重复 4 组:(int32,int32,int64,int64) | ||
| 34 | + ge::DT_INT32, ge::DT_INT32, ge::DT_INT64, ge::DT_INT64, ge::DT_INT32, ge::DT_INT32, ge::DT_INT64, | ||
| 35 | + ge::DT_INT64, ge::DT_INT32, ge::DT_INT32, ge::DT_INT64, ge::DT_INT64, ge::DT_INT32, ge::DT_INT32, | ||
| 36 | + ge::DT_INT64, ge::DT_INT64, ge::DT_INT32, ge::DT_INT32, ge::DT_INT64, ge::DT_INT64}; | ||
| 37 | +static const std::vector<ge::DataType> kirinAxisDataType = { | ||
| 38 | + // 每组 axis 在 (int32, int64) 间交替 | ||
| 39 | + ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, | ||
| 40 | + ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, | ||
| 41 | + ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64}; | ||
| 42 | +static const std::vector<ge::DataType> kirinYDataType = { | ||
| 43 | + // y 类型 = x 类型 | ||
| 44 | + ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, | ||
| 45 | + ge::DT_FLOAT, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT32, ge::DT_INT32, | ||
| 46 | + ge::DT_INT32, ge::DT_INT32, ge::DT_INT64, ge::DT_INT64, ge::DT_INT64, ge::DT_INT64}; | ||
| 47 | +static const std::vector<ge::Format> gatherV2FormatKirin = { | ||
| 48 | + ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | ||
| 49 | + ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | ||
| 50 | + ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}; | ||
| 51 | + | ||
| 18 | class GatherV2 : public OpDef { | 52 | class GatherV2 : public OpDef { |
| 19 | public: | 53 | public: |
| 20 | explicit GatherV2(const char* name) : OpDef(name) | 54 | explicit GatherV2(const char* name) : OpDef(name) |
| @@ -210,6 +244,37 @@ public: | |||
| 210 | this->AICore().AddConfig("ascend950", aicoreConfig); | 244 | this->AICore().AddConfig("ascend950", aicoreConfig); |
| 211 | this->AICore().AddConfig("ascend350", aicoreConfig); | 245 | this->AICore().AddConfig("ascend350", aicoreConfig); |
| 212 | this->AICore().AddConfig("ascend960dt", aicoreConfig); | 246 | this->AICore().AddConfig("ascend960dt", aicoreConfig); |
| 247 | + | ||
| 248 | + OpAICoreConfig config_kirin; | ||
| 249 | + config_kirin.DynamicCompileStaticFlag(true) | ||
| 250 | + .DynamicFormatFlag(true) | ||
| 251 | + .DynamicRankSupportFlag(true) | ||
| 252 | + .DynamicShapeSupportFlag(true) | ||
| 253 | + .NeedCheckSupportFlag(false) | ||
| 254 | + .PrecisionReduceFlag(true); | ||
| 255 | + config_kirin.Input("x") | ||
| 256 | + .ParamType(REQUIRED) | ||
| 257 | + .DataType(kirinXDataType) | ||
| 258 | + .Format(gatherV2FormatKirin) | ||
| 259 | + .UnknownShapeFormat(gatherV2FormatKirin); | ||
| 260 | + config_kirin.Input("indices") | ||
| 261 | + .ParamType(REQUIRED) | ||
| 262 | + .DataType(kirinIndicesDataType) | ||
| 263 | + .Format(gatherV2FormatKirin) | ||
| 264 | + .UnknownShapeFormat(gatherV2FormatKirin); | ||
| 265 | + config_kirin.Input("axis") | ||
| 266 | + .ParamType(REQUIRED) | ||
| 267 | + .DataType(kirinAxisDataType) | ||
| 268 | + .Format(gatherV2FormatKirin) | ||
| 269 | + .UnknownShapeFormat(gatherV2FormatKirin) | ||
| 270 | + .ValueDepend(OPTIONAL); | ||
| 271 | + config_kirin.Output("y") | ||
| 272 | + .ParamType(REQUIRED) | ||
| 273 | + .DataType(kirinYDataType) | ||
| 274 | + .Format(gatherV2FormatKirin) | ||
| 275 | + .UnknownShapeFormat(gatherV2FormatKirin); | ||
| 276 | + this->AICore().AddConfig("kirinx90", config_kirin); | ||
| 277 | + this->AICore().AddConfig("kirin9030", config_kirin); | ||
| 213 | } | 278 | } |
| 214 | }; | 279 | }; |
| 215 | 280 | ||
| @@ -22,3 +22,10 @@ add_kernel_sources( | |||
| 22 | IMPL_MODE high_performance,high_precision,support_out_of_bound_index | 22 | IMPL_MODE high_performance,high_precision,support_out_of_bound_index |
| 23 | OPTIONS "--cce-no-dcache-flush" | 23 | OPTIONS "--cce-no-dcache-flush" |
| 24 | ) | 24 | ) |
| 25 | +# arch31(kirinx90/kirin9030) 使用 arch31/gather_v2.cpp 作为 kernel 入口 | ||
| 26 | +add_kernel_sources( | ||
| 27 | + KERNEL_SRC arch31/gather_v2.cpp | ||
| 28 | + COMPUTE_UNITS kirinx90 kirin9030 | ||
| 29 | + IMPL_MODE high_performance,high_precision,support_out_of_bound_index | ||
| 30 | + AUTO_SYNC false | ||
| 31 | +) | ||
| @@ -0,0 +1,208 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +// KirinX90 适配:去除所有 simt 依赖模板,改用纯 SIMD 新模板 | ||
| 12 | +// 新增: gather_v2_simd_1d.h(Gatherv2Simd1D) / gather_v2_simd_2d.h(Gatherv2Simd2D) / | ||
| 13 | +// gather_v2_simd_full_load.h(Gatherv2SimdFullLoad) 删除原 simt 头: gather_v2.h / gather_v2_simt_two_dim.h / | ||
| 14 | +// gather_v2_after_gather_full_load.h | ||
| 15 | +#include "gather_v2_simd_1d.h" // 替代 gather_v2.h (Gatherv2) | ||
| 16 | +#include "gather_v2_simd_arch31.h" | ||
| 17 | +#include "gather_v2_simd_two_dim_arch31.h" | ||
| 18 | +#include "gather_v2_simd_full_load.h" // 替代 gather_v2_after_gather_full_load.h (Gatherv2FullLoad) | ||
| 19 | +#include "gather_v2_simd_2d.h" // 替代 gather_v2_simt_two_dim.h (Gatherv2SimtTwoDim) | ||
| 20 | +#include "gather_v2_empty_arch31.h" | ||
| 21 | + | ||
| 22 | +using namespace AscendC; | ||
| 23 | +using namespace gatherv2; | ||
| 24 | + | ||
| 25 | +#define TILING_KEY_X_B64_INDEX_SIZE_64 10000000000000000031UL | ||
| 26 | +#define TILING_KEY_X_B32_INDEX_SIZE_64 10000000000000000021UL | ||
| 27 | +#define TILING_KEY_X_B16_INDEX_SIZE_64 10000000000000000011UL | ||
| 28 | +#define TILING_KEY_X_B8_INDEX_SIZE_64 10000000000000000001UL | ||
| 29 | +#define TILING_KEY_X_B64_INDEX_SIZE_32 10000000000000000030UL | ||
| 30 | +#define TILING_KEY_X_B32_INDEX_SIZE_32 10000000000000000020UL | ||
| 31 | +#define TILING_KEY_X_B16_INDEX_SIZE_32 10000000000000000010UL | ||
| 32 | +#define TILING_KEY_X_B8_INDEX_SIZE_32 10000000000000000000UL | ||
| 33 | +#define SIMD_TILING_KEY 1000000099 | ||
| 34 | +#define SIMD_TILING_KEY_TWO_DIM 1000000299 | ||
| 35 | +#define SIMD_LAST_GATHER_B8_TILING_KEY 1100000001UL | ||
| 36 | +#define SIMD_LAST_GATHER_B16_TILING_KEY 1100000002UL | ||
| 37 | +#define SIMD_LAST_GATHER_B32_TILING_KEY 1100000004UL | ||
| 38 | +#define SIMD_LAST_GATHER_B64_TILING_KEY 1100000008UL | ||
| 39 | +#define SIMD_LAST_GATHER_B8_SUPPORT_NEG_INDICE_TILING_KEY 1100000101UL | ||
| 40 | +#define SIMD_LAST_GATHER_B16_SUPPORT_NEG_INDICE_TILING_KEY 1100000102UL | ||
| 41 | +#define SIMD_LAST_GATHER_B32_SUPPORT_NEG_INDICE_TILING_KEY 1100000104UL | ||
| 42 | +#define SIMD_LAST_GATHER_B64_SUPPORT_NEG_INDICE_TILING_KEY 1100000108UL | ||
| 43 | +#define SIMD_GA_ALL_LOAD_TILING_KEY 3000UL | ||
| 44 | +#define SIMD_GA_ALL_LOAD_SUPPORT_NEG_INDICE_TILING_KEY 3100UL | ||
| 45 | + | ||
| 46 | +#define TILING_KEY_X_B64_INDEX_SIZE_2D 10000000000000000032UL | ||
| 47 | +#define TILING_KEY_X_B32_INDEX_SIZE_2D 10000000000000000022UL | ||
| 48 | +#define TILING_KEY_X_B16_INDEX_SIZE_2D 10000000000000000012UL | ||
| 49 | +#define TILING_KEY_X_B8_INDEX_SIZE_2D 10000000000000000002UL | ||
| 50 | + | ||
| 51 | +#define TILING_KEY_EMPTY 3000000000UL | ||
| 52 | + | ||
| 53 | +#define SIMT_TWO_DIM_B8_INDEX_SIZE_32_TILING_KEY 2000000001UL | ||
| 54 | +#define SIMT_TWO_DIM_B16_INDEX_SIZE_32_TILING_KEY 2000000002UL | ||
| 55 | +#define SIMT_TWO_DIM_B32_INDEX_SIZE_32_TILING_KEY 2000000004UL | ||
| 56 | +#define SIMT_TWO_DIM_B64_INDEX_SIZE_32_TILING_KEY 2000000008UL | ||
| 57 | +#define SIMT_TWO_DIM_B8_INDEX_SIZE_64_TILING_KEY 2000000101UL | ||
| 58 | +#define SIMT_TWO_DIM_B16_INDEX_SIZE_64_TILING_KEY 2000000102UL | ||
| 59 | +#define SIMT_TWO_DIM_B32_INDEX_SIZE_64_TILING_KEY 2000000104UL | ||
| 60 | +#define SIMT_TWO_DIM_B64_INDEX_SIZE_64_TILING_KEY 2000000108UL | ||
| 61 | + | ||
| 62 | +extern "C" __global__ __aicore__ void gather_v2(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, GM_ADDR workspace, | ||
| 63 | + GM_ADDR tiling) | ||
| 64 | +{ | ||
| 65 | + TPipe pipe; | ||
| 66 | + KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_AIV_ONLY); | ||
| 67 | + | ||
| 68 | + TILING_KEY_IS(SIMD_TILING_KEY); | ||
| 69 | + TILING_KEY_IS(SIMD_TILING_KEY_TWO_DIM); | ||
| 70 | + TILING_KEY_IS(SIMT_TWO_DIM_B8_INDEX_SIZE_32_TILING_KEY); | ||
| 71 | + TILING_KEY_IS(SIMT_TWO_DIM_B16_INDEX_SIZE_32_TILING_KEY); | ||
| 72 | + TILING_KEY_IS(SIMT_TWO_DIM_B32_INDEX_SIZE_32_TILING_KEY); | ||
| 73 | + TILING_KEY_IS(SIMT_TWO_DIM_B64_INDEX_SIZE_32_TILING_KEY); | ||
| 74 | + TILING_KEY_IS(SIMT_TWO_DIM_B8_INDEX_SIZE_64_TILING_KEY); | ||
| 75 | + TILING_KEY_IS(SIMT_TWO_DIM_B16_INDEX_SIZE_64_TILING_KEY); | ||
| 76 | + TILING_KEY_IS(SIMT_TWO_DIM_B32_INDEX_SIZE_64_TILING_KEY); | ||
| 77 | + TILING_KEY_IS(SIMT_TWO_DIM_B64_INDEX_SIZE_64_TILING_KEY); | ||
| 78 | + TILING_KEY_IS(TILING_KEY_X_B64_INDEX_SIZE_64); | ||
| 79 | + TILING_KEY_IS(TILING_KEY_X_B32_INDEX_SIZE_64); | ||
| 80 | + TILING_KEY_IS(TILING_KEY_X_B16_INDEX_SIZE_64); | ||
| 81 | + TILING_KEY_IS(TILING_KEY_X_B8_INDEX_SIZE_64); | ||
| 82 | + TILING_KEY_IS(TILING_KEY_X_B64_INDEX_SIZE_32); | ||
| 83 | + TILING_KEY_IS(TILING_KEY_X_B32_INDEX_SIZE_32); | ||
| 84 | + TILING_KEY_IS(TILING_KEY_X_B16_INDEX_SIZE_32); | ||
| 85 | + TILING_KEY_IS(TILING_KEY_X_B8_INDEX_SIZE_32); | ||
| 86 | + TILING_KEY_IS(TILING_KEY_X_B64_INDEX_SIZE_2D); | ||
| 87 | + TILING_KEY_IS(TILING_KEY_X_B32_INDEX_SIZE_2D); | ||
| 88 | + TILING_KEY_IS(TILING_KEY_X_B16_INDEX_SIZE_2D); | ||
| 89 | + TILING_KEY_IS(TILING_KEY_X_B8_INDEX_SIZE_2D); | ||
| 90 | + TILING_KEY_IS(TILING_KEY_EMPTY); | ||
| 91 | + | ||
| 92 | +#if TILING_KEY_VAR == SIMD_TILING_KEY | ||
| 93 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 94 | + Gatherv2Simd<DTYPE_INDICES> gatherv2Op(&pipe); | ||
| 95 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 96 | + gatherv2Op.Process(); | ||
| 97 | +#elif TILING_KEY_VAR == SIMD_TILING_KEY_TWO_DIM | ||
| 98 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimdTwoDim, tilingDataIn, tiling); | ||
| 99 | + Gatherv2SimdTwoDim<DTYPE_INDICES> gatherv2Op(&pipe); | ||
| 100 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 101 | + gatherv2Op.Process(); | ||
| 102 | +#elif TILING_KEY_VAR == SIMT_TWO_DIM_B8_INDEX_SIZE_64_TILING_KEY | ||
| 103 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling); | ||
| 104 | + Gatherv2Simd2D<int8_t, DTYPE_INDICES, uint64_t> gatherv2Op; | ||
| 105 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 106 | + gatherv2Op.Process(); | ||
| 107 | +#elif TILING_KEY_VAR == SIMT_TWO_DIM_B16_INDEX_SIZE_64_TILING_KEY | ||
| 108 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling); | ||
| 109 | + Gatherv2Simd2D<int16_t, DTYPE_INDICES, uint64_t> gatherv2Op; | ||
| 110 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 111 | + gatherv2Op.Process(); | ||
| 112 | +#elif TILING_KEY_VAR == SIMT_TWO_DIM_B32_INDEX_SIZE_64_TILING_KEY | ||
| 113 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling); | ||
| 114 | + Gatherv2Simd2D<int32_t, DTYPE_INDICES, uint64_t> gatherv2Op; | ||
| 115 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 116 | + gatherv2Op.Process(); | ||
| 117 | +#elif TILING_KEY_VAR == SIMT_TWO_DIM_B64_INDEX_SIZE_64_TILING_KEY | ||
| 118 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling); | ||
| 119 | + Gatherv2Simd2D<int64_t, DTYPE_INDICES, uint64_t> gatherv2Op; | ||
| 120 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 121 | + gatherv2Op.Process(); | ||
| 122 | +#elif TILING_KEY_VAR == SIMT_TWO_DIM_B8_INDEX_SIZE_32_TILING_KEY | ||
| 123 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling); | ||
| 124 | + Gatherv2Simd2D<int8_t, DTYPE_INDICES, uint32_t> gatherv2Op; | ||
| 125 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 126 | + gatherv2Op.Process(); | ||
| 127 | +#elif TILING_KEY_VAR == SIMT_TWO_DIM_B16_INDEX_SIZE_32_TILING_KEY | ||
| 128 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling); | ||
| 129 | + Gatherv2Simd2D<int16_t, DTYPE_INDICES, uint32_t> gatherv2Op; | ||
| 130 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 131 | + gatherv2Op.Process(); | ||
| 132 | +#elif TILING_KEY_VAR == SIMT_TWO_DIM_B32_INDEX_SIZE_32_TILING_KEY | ||
| 133 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling); | ||
| 134 | + Gatherv2Simd2D<int32_t, DTYPE_INDICES, uint32_t> gatherv2Op; | ||
| 135 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 136 | + gatherv2Op.Process(); | ||
| 137 | +#elif TILING_KEY_VAR == SIMT_TWO_DIM_B64_INDEX_SIZE_32_TILING_KEY | ||
| 138 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling); | ||
| 139 | + Gatherv2Simd2D<int64_t, DTYPE_INDICES, uint32_t> gatherv2Op; | ||
| 140 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 141 | + gatherv2Op.Process(); | ||
| 142 | +#elif TILING_KEY_VAR == TILING_KEY_X_B8_INDEX_SIZE_64 | ||
| 143 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 144 | + Gatherv2Simd1D<int8_t, DTYPE_INDICES, uint64_t> gatherv2Op; | ||
| 145 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 146 | + gatherv2Op.Process(); | ||
| 147 | +#elif TILING_KEY_VAR == TILING_KEY_X_B16_INDEX_SIZE_64 | ||
| 148 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 149 | + Gatherv2Simd1D<int16_t, DTYPE_INDICES, uint64_t> gatherv2Op; | ||
| 150 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 151 | + gatherv2Op.Process(); | ||
| 152 | +#elif TILING_KEY_VAR == TILING_KEY_X_B32_INDEX_SIZE_64 | ||
| 153 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 154 | + Gatherv2Simd1D<int32_t, DTYPE_INDICES, uint64_t> gatherv2Op; | ||
| 155 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 156 | + gatherv2Op.Process(); | ||
| 157 | +#elif TILING_KEY_VAR == TILING_KEY_X_B64_INDEX_SIZE_64 | ||
| 158 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 159 | + Gatherv2Simd1D<int64_t, DTYPE_INDICES, uint64_t> gatherv2Op; | ||
| 160 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 161 | + gatherv2Op.Process(); | ||
| 162 | +#elif TILING_KEY_VAR == TILING_KEY_X_B8_INDEX_SIZE_32 | ||
| 163 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 164 | + Gatherv2Simd1D<int8_t, DTYPE_INDICES, uint32_t> gatherv2Op; | ||
| 165 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 166 | + gatherv2Op.Process(); | ||
| 167 | +#elif TILING_KEY_VAR == TILING_KEY_X_B16_INDEX_SIZE_32 | ||
| 168 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 169 | + Gatherv2Simd1D<int16_t, DTYPE_INDICES, uint32_t> gatherv2Op; | ||
| 170 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 171 | + gatherv2Op.Process(); | ||
| 172 | +#elif TILING_KEY_VAR == TILING_KEY_X_B32_INDEX_SIZE_32 | ||
| 173 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 174 | + Gatherv2Simd1D<int32_t, DTYPE_INDICES, uint32_t> gatherv2Op; | ||
| 175 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 176 | + gatherv2Op.Process(); | ||
| 177 | +#elif TILING_KEY_VAR == TILING_KEY_X_B64_INDEX_SIZE_32 | ||
| 178 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 179 | + Gatherv2Simd1D<int64_t, DTYPE_INDICES, uint32_t> gatherv2Op; | ||
| 180 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 181 | + gatherv2Op.Process(); | ||
| 182 | +#elif TILING_KEY_VAR == TILING_KEY_X_B8_INDEX_SIZE_2D | ||
| 183 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 184 | + Gatherv2SimdFullLoad<int8_t, DTYPE_INDICES, uint32_t> gatherv2Op(&pipe); | ||
| 185 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 186 | + gatherv2Op.Process(); | ||
| 187 | +#elif TILING_KEY_VAR == TILING_KEY_X_B16_INDEX_SIZE_2D | ||
| 188 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 189 | + Gatherv2SimdFullLoad<int16_t, DTYPE_INDICES, uint32_t> gatherv2Op(&pipe); | ||
| 190 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 191 | + gatherv2Op.Process(); | ||
| 192 | +#elif TILING_KEY_VAR == TILING_KEY_X_B32_INDEX_SIZE_2D | ||
| 193 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 194 | + Gatherv2SimdFullLoad<int32_t, DTYPE_INDICES, uint32_t> gatherv2Op(&pipe); | ||
| 195 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 196 | + gatherv2Op.Process(); | ||
| 197 | +#elif TILING_KEY_VAR == TILING_KEY_X_B64_INDEX_SIZE_2D | ||
| 198 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling); | ||
| 199 | + Gatherv2SimdFullLoad<int64_t, DTYPE_INDICES, uint32_t> gatherv2Op(&pipe); | ||
| 200 | + gatherv2Op.Init(x, indices, axis, y, &tilingDataIn); | ||
| 201 | + gatherv2Op.Process(); | ||
| 202 | +#elif TILING_KEY_VAR == TILING_KEY_EMPTY | ||
| 203 | + GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataEmptyInput, tilingDataIn, tiling); | ||
| 204 | + Gatherv2Empty<int8_t> gatherv2Op; | ||
| 205 | + gatherv2Op.Init(y, &tilingDataIn); | ||
| 206 | + gatherv2Op.Process(); | ||
| 207 | +#endif | ||
| 208 | +} | ||
| @@ -0,0 +1,70 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file gather_v2_empty_arch31.h | ||
| 13 | + * \brief | ||
| 14 | + */ | ||
| 15 | +#ifndef GATHER_V2_EMPTY | ||
| 16 | +#define GATHER_V2_EMPTY | ||
| 17 | + | ||
| 18 | +#if ASC_DEVKIT_MAJOR >= 9 | ||
| 19 | +#include "basic_api/kernel_vec_intf.h" | ||
| 20 | +#else | ||
| 21 | +#include "kernel_operator.h" | ||
| 22 | +#endif | ||
| 23 | +#include "op_kernel/platform_util.h" | ||
| 24 | + | ||
| 25 | +namespace gatherv2 { | ||
| 26 | +using namespace AscendC; | ||
| 27 | + | ||
| 28 | +template <typename T> | ||
| 29 | +class Gatherv2Empty { | ||
| 30 | +public: | ||
| 31 | + __aicore__ inline Gatherv2Empty(){}; | ||
| 32 | + __aicore__ inline void Init(GM_ADDR y, const GatherV2TilingDataEmptyInput* tilingData); | ||
| 33 | + __aicore__ inline void Process(); | ||
| 34 | + | ||
| 35 | +private: | ||
| 36 | + GlobalTensor<T> yGm_; | ||
| 37 | + | ||
| 38 | + int64_t needCoreNum_; | ||
| 39 | + int64_t perCoreElements_; | ||
| 40 | + int64_t lastCoreElements_; | ||
| 41 | + | ||
| 42 | + int64_t extentElements_; | ||
| 43 | + uint32_t blockIdx_; | ||
| 44 | + const GatherV2TilingDataEmptyInput* tilingData_; | ||
| 45 | +}; | ||
| 46 | + | ||
| 47 | +template <typename T> | ||
| 48 | +__aicore__ inline void Gatherv2Empty<T>::Init(GM_ADDR y, const GatherV2TilingDataEmptyInput* tilingData) | ||
| 49 | +{ | ||
| 50 | + tilingData_ = tilingData; | ||
| 51 | + yGm_.SetGlobalBuffer((__gm__ T*)y); | ||
| 52 | +} | ||
| 53 | + | ||
| 54 | +template <typename T> | ||
| 55 | +__aicore__ inline void Gatherv2Empty<T>::Process() | ||
| 56 | +{ | ||
| 57 | + needCoreNum_ = tilingData_->needCoreNum; | ||
| 58 | + perCoreElements_ = tilingData_->perCoreElements; | ||
| 59 | + lastCoreElements_ = tilingData_->lastCoreElements; | ||
| 60 | + extentElements_ = perCoreElements_; | ||
| 61 | + | ||
| 62 | + blockIdx_ = GetBlockIdx(); | ||
| 63 | + if (blockIdx_ + 1 == (uint32_t)needCoreNum_) { | ||
| 64 | + extentElements_ = lastCoreElements_; | ||
| 65 | + } | ||
| 66 | + | ||
| 67 | + InitOutput<T>(yGm_, extentElements_, 0); | ||
| 68 | +} | ||
| 69 | +} // namespace gatherv2 | ||
| 70 | +#endif // GATHER_V2_EMPTY | ||
| @@ -0,0 +1,634 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file gather_v2_ga_all_load_arch31.h | ||
| 13 | + * \brief | ||
| 14 | + */ | ||
| 15 | +#ifndef GATHER_V2_GA_ALL_LOAD_H | ||
| 16 | +#define GATHER_V2_GA_ALL_LOAD_H | ||
| 17 | + | ||
| 18 | +#ifndef K_MAX_SHAPE_DIM | ||
| 19 | +#define K_MAX_SHAPE_DIM 0 | ||
| 20 | +#endif | ||
| 21 | +#if ASC_DEVKIT_MAJOR >= 9 | ||
| 22 | +#include "basic_api/kernel_vec_intf.h" | ||
| 23 | +#else | ||
| 24 | +#include "kernel_operator.h" | ||
| 25 | +#endif | ||
| 26 | +#include "op_kernel/platform_util.h" | ||
| 27 | + | ||
| 28 | +namespace gatherv2 { | ||
| 29 | +using namespace AscendC; | ||
| 30 | + | ||
| 31 | +#if 0 // KirinX90 支持 MicroAPI(DataCopyGather/Gather/Adds/Copy/UpdateMask/CompareScalar | ||
| 32 | + // 等),不过这里可以用兜底路径实现,可以先封上 参照 cast 算子 arch31 适配经验,本文件大量使用 | ||
| 33 | + // MicroAPI,整体注释避免编译错误 如后续 KirinX90 支持相关指令或需用 SIMD 重写,解开本 #if 0 即可恢复参考实现 | ||
| 34 | +constexpr int32_t DOUBLE_BUFFER = 2; | ||
| 35 | +constexpr int32_t HELP_BUFFER_SIZE = 256; | ||
| 36 | +constexpr int32_t GATHER_ENABLE_SIZE = 512; | ||
| 37 | + | ||
| 38 | +template <typename INDICES_T, const bool NIS> | ||
| 39 | +class Gatherv2GaAllLoad { | ||
| 40 | +public: | ||
| 41 | + __aicore__ inline Gatherv2GaAllLoad(TPipe* pipe) : pipe_(pipe){}; | ||
| 42 | + __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, | ||
| 43 | + const GatherV2GaAllLoadTilingData* tilingData); | ||
| 44 | + __aicore__ inline void GenIndexBuf(); | ||
| 45 | + __aicore__ inline void Process(); | ||
| 46 | + __aicore__ inline void IndicesProcess(int32_t indicesNumPro, int32_t indicesNumOffset); | ||
| 47 | + __aicore__ inline void xProcess(int32_t gaCoreOffset, int32_t gaNumPro); | ||
| 48 | + __aicore__ inline void yProcess(int32_t indicesNumPro, int32_t indicesNumOffset, int32_t gaNumPro, | ||
| 49 | + int32_t gaCoreOffset); | ||
| 50 | + __aicore__ inline void GatherProcess(int32_t indicesUbOffset, int32_t indicesNumCurPro, | ||
| 51 | + __local_mem__ int32_t* indicesAddr, __local_mem__ int8_t* xAddr, | ||
| 52 | + __local_mem__ int8_t* yAddr); | ||
| 53 | + __aicore__ inline void FixIndices(__local_mem__ INDICES_T* indicesAddr, int32_t indicesNumPro); | ||
| 54 | + __aicore__ inline void CopyInIndices(LocalTensor<INDICES_T>& indicesLocal, int32_t burstLen, int32_t coreOffset); | ||
| 55 | + __aicore__ inline void CopyInX(LocalTensor<int8_t>& xLocal, int32_t gaCoreOffset, int32_t gaNumPro); | ||
| 56 | + __aicore__ inline void CopyOutY(int32_t nBurst, int32_t indicesCoreOffset, int32_t pCoreOffset, int32_t gaNumPro); | ||
| 57 | + __aicore__ inline void GatherProcessVfWithGA(int32_t indicesNumCurPro, uint16_t gaNumPro, | ||
| 58 | + __local_mem__ int32_t* curIndicesAddr, __local_mem__ int8_t* xAddr, | ||
| 59 | + __local_mem__ int8_t* yAddr); | ||
| 60 | + __aicore__ inline void GatherProcessWithGA2(int32_t indicesNumCurPro, uint16_t gaNumPro, | ||
| 61 | + __local_mem__ int32_t* curIndicesAddr, __local_mem__ int8_t* xAddr, | ||
| 62 | + __local_mem__ int8_t* yAddr); | ||
| 63 | + __aicore__ inline void InitializationX(int32_t xBufGaNum); | ||
| 64 | + | ||
| 65 | +private: | ||
| 66 | + GlobalTensor<int8_t> xGm_; | ||
| 67 | + GlobalTensor<INDICES_T> indicesGm_; | ||
| 68 | + GlobalTensor<int8_t> yGm_; | ||
| 69 | + TPipe* pipe_; | ||
| 70 | + TBuf<QuePosition::VECCALC> xBuf_; | ||
| 71 | + TBuf<QuePosition::VECCALC> indicesBuf_; | ||
| 72 | + TBuf<QuePosition::VECCALC> tmpIndexBuf_; | ||
| 73 | + TQue<QuePosition::VECOUT, DOUBLE_BUFFER> yQueue_; | ||
| 74 | + const GatherV2GaAllLoadTilingData* tilingData_; | ||
| 75 | + | ||
| 76 | + int32_t blockIdx_; | ||
| 77 | + bool enableGather_; | ||
| 78 | + | ||
| 79 | + int64_t curCoreIndicesNum_; | ||
| 80 | + int64_t curCoreGaNum_; | ||
| 81 | + | ||
| 82 | + int64_t xGmOffset_; | ||
| 83 | + int64_t indicesGmOffset_; | ||
| 84 | +}; | ||
| 85 | + | ||
| 86 | +template <typename INDICES_T, const bool NIS> | ||
| 87 | +__aicore__ inline void Gatherv2GaAllLoad<INDICES_T, NIS>::Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, | ||
| 88 | + const GatherV2GaAllLoadTilingData* tilingData) | ||
| 89 | +{ | ||
| 90 | + tilingData_ = tilingData; | ||
| 91 | + blockIdx_ = GetBlockIdx(); | ||
| 92 | + | ||
| 93 | + // 先分核在分UB | ||
| 94 | + int64_t pIndex = blockIdx_ / tilingData_->indicesOuter; | ||
| 95 | + int64_t indicesIndex = blockIdx_ % tilingData_->indicesOuter; | ||
| 96 | + | ||
| 97 | + indicesGmOffset_ = indicesIndex * tilingData_->normalCoreIndicesNum; | ||
| 98 | + xGmOffset_ = pIndex * tilingData_->normalCoreGaNum; | ||
| 99 | + | ||
| 100 | + curCoreIndicesNum_ = (indicesIndex + 1 == tilingData_->indicesOuter) ? tilingData_->tailCoreIndicesNum : | ||
| @@ -0,0 +1,107 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file gather_v2_simd_1d.h | ||
| 13 | + * \brief KirinX90 纯 SIMD 1D 小 shape 模板(替代原 Gatherv2 simt 实现,去除 simt_api 依赖) | ||
| 14 | + */ | ||
| 15 | +#ifndef GATHER_V2_SIMD_1D_H | ||
| 16 | +#define GATHER_V2_SIMD_1D_H | ||
| 17 | + | ||
| 18 | +#include "kernel_operator.h" | ||
| 19 | +// 已删除: simt_api/asc_simt.h、basic_api/kernel_vec_intf.h(KirinX90 硬件不支持 simt) | ||
| 20 | + | ||
| 21 | +namespace gatherv2 { | ||
| 22 | +using namespace AscendC; | ||
| 23 | + | ||
| 24 | +template <typename X_T, typename INDICES_T, typename INDEX_SIZE_T> | ||
| 25 | +class Gatherv2Simd1D { | ||
| 26 | +public: | ||
| 27 | + __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, | ||
| 28 | + const GatherV2TilingData* tilingData) | ||
| 29 | + { | ||
| 30 | + tilingData_ = tilingData; | ||
| 31 | + xGm_.SetGlobalBuffer((__gm__ X_T*)x); | ||
| 32 | + indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices); | ||
| 33 | + yGm_.SetGlobalBuffer((__gm__ X_T*)y); | ||
| 34 | + } | ||
| 35 | + | ||
| 36 | + __aicore__ inline void Process() | ||
| 37 | + { | ||
| 38 | + int32_t blockIdx = static_cast<int32_t>(GetBlockIdx()); | ||
| 39 | + int32_t needCoreNum = static_cast<int32_t>(tilingData_->needCoreNum); | ||
| 40 | + if (blockIdx >= needCoreNum) { | ||
| 41 | + return; | ||
| 42 | + } | ||
| 43 | + | ||
| 44 | + INDEX_SIZE_T outerSize = static_cast<INDEX_SIZE_T>(tilingData_->outerSize); | ||
| 45 | + INDEX_SIZE_T gatherDimSize = static_cast<INDEX_SIZE_T>(tilingData_->gatherDimSize); | ||
| 46 | + INDEX_SIZE_T gatherSize = static_cast<INDEX_SIZE_T>(tilingData_->gatherSize); | ||
| 47 | + INDEX_SIZE_T innerSize = static_cast<INDEX_SIZE_T>(tilingData_->innerSize); | ||
| 48 | + bool nis = static_cast<bool>(tilingData_->negativeIndexSupport); | ||
| 49 | + | ||
| 50 | + INDEX_SIZE_T currentCoreElements = static_cast<INDEX_SIZE_T>(tilingData_->perCoreElements); | ||
| 51 | + if (blockIdx == tilingData_->needCoreNum - 1) { | ||
| 52 | + currentCoreElements = static_cast<INDEX_SIZE_T>(tilingData_->lastCoreElements); | ||
| 53 | + } | ||
| 54 | + INDEX_SIZE_T yIndexBase = static_cast<INDEX_SIZE_T>(blockIdx) * tilingData_->perCoreElements; | ||
| 55 | + | ||
| 56 | + __gm__ X_T* xPtr = (__gm__ X_T*)xGm_.GetPhyAddr(); | ||
| 57 | + __gm__ INDICES_T* indicesPtr = (__gm__ INDICES_T*)indicesGm_.GetPhyAddr(); | ||
| 58 | + __gm__ volatile X_T* yPtr = (__gm__ volatile X_T*)yGm_.GetPhyAddr(); | ||
| 59 | + | ||
| 60 | + if (nis) { | ||
| 61 | + GatherCore<true>(xPtr, indicesPtr, yPtr, yIndexBase, currentCoreElements, gatherSize, outerSize, innerSize, | ||
| 62 | + gatherDimSize); | ||
| 63 | + } else { | ||
| 64 | + GatherCore<false>(xPtr, indicesPtr, yPtr, yIndexBase, currentCoreElements, gatherSize, outerSize, innerSize, | ||
| 65 | + gatherDimSize); | ||
| 66 | + } | ||
| 67 | + } | ||
| 68 | + | ||
| 69 | +private: | ||
| 70 | + template <const bool NIS> | ||
| 71 | + __aicore__ inline void GatherCore(__gm__ X_T* x, __gm__ INDICES_T* indices, __gm__ volatile X_T* y, | ||
| 72 | + INDEX_SIZE_T yIndexBase, INDEX_SIZE_T currentCoreElements, | ||
| 73 | + INDEX_SIZE_T gatherSize, INDEX_SIZE_T outerSize, INDEX_SIZE_T innerSize, | ||
| 74 | + INDEX_SIZE_T gatherDimSize) | ||
| 75 | + { | ||
| 76 | + // 原 GatherSimt 计算逻辑原样保留;线程级(threadIdx/asc_vf_call)→核内串行循环; | ||
| 77 | + // UintDiv 快速除法→普通除法,去除 simt 依赖 | ||
| 78 | + for (INDEX_SIZE_T index = 0; index < currentCoreElements; index++) { | ||
| 79 | + INDEX_SIZE_T yIndex = yIndexBase + index; | ||
| 80 | + INDEX_SIZE_T tmp = yIndex / innerSize; // 原 UintDiv(yIndex, m0, shift0) | ||
| 81 | + INDEX_SIZE_T tmp1 = tmp / gatherSize; // 原 UintDiv(tmp, m1, shift1) | ||
| 82 | + INDEX_SIZE_T gatherI = tmp - tmp1 * gatherSize; | ||
| 83 | + INDEX_SIZE_T batchI = tmp1 / outerSize; // 原 UintDiv(tmp1, m2, shift2) | ||
| 84 | + INDEX_SIZE_T innerI = yIndex - tmp * innerSize; | ||
| 85 | + INDEX_SIZE_T indicesIndex = batchI * gatherSize + gatherI; | ||
| 86 | + | ||
| 87 | + INDICES_T indicesValue = indices[indicesIndex]; | ||
| 88 | + if constexpr (NIS) { | ||
| 89 | + if (unlikely(indicesValue < 0)) { | ||
| 90 | + indicesValue += static_cast<INDICES_T>(gatherDimSize); | ||
| 91 | + } | ||
| 92 | + } | ||
| 93 | + INDEX_SIZE_T indicesValueI = static_cast<INDEX_SIZE_T>(indicesValue); | ||
| 94 | + INDEX_SIZE_T xIndex = (tmp1 * gatherDimSize + indicesValueI) * innerSize + innerI; | ||
| 95 | + bool oob = indicesValue < 0 || indicesValueI >= gatherDimSize; | ||
| 96 | + y[yIndex] = oob ? static_cast<X_T>(0) : x[xIndex]; | ||
| 97 | + } | ||
| 98 | + } | ||
| 99 | + | ||
| 100 | +private: | ||
| 101 | + GlobalTensor<X_T> xGm_; | ||
| 102 | + GlobalTensor<INDICES_T> indicesGm_; | ||
| 103 | + GlobalTensor<X_T> yGm_; | ||
| 104 | + const GatherV2TilingData* tilingData_ = nullptr; | ||
| 105 | +}; | ||
| 106 | +} // namespace gatherv2 | ||
| 107 | +#endif // GATHER_V2_SIMD_1D_H | ||
| @@ -0,0 +1,97 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file gather_v2_simd_2d.h | ||
| 13 | + * \brief KirinX90 纯 SIMD 2D 小 shape 模板(替代原 Gatherv2SimtTwoDim simt 实现,去除 simt_api 依赖) | ||
| 14 | + */ | ||
| 15 | +#ifndef GATHER_V2_SIMD_2D_H | ||
| 16 | +#define GATHER_V2_SIMD_2D_H | ||
| 17 | + | ||
| 18 | +#include "kernel_operator.h" | ||
| 19 | +// 已删除: simt_api/asc_simt.h、basic_api/kernel_vec_intf.h(KirinX90 硬件不支持 simt) | ||
| 20 | + | ||
| 21 | +namespace gatherv2 { | ||
| 22 | +using namespace AscendC; | ||
| 23 | + | ||
| 24 | +template <typename X_T, typename INDICES_T, typename INDEX_SIZE_T> | ||
| 25 | +class Gatherv2Simd2D { | ||
| 26 | +public: | ||
| 27 | + __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, | ||
| 28 | + const GatherV2TilingDataSimtTwoDim* tilingData) | ||
| 29 | + { | ||
| 30 | + tilingData_ = tilingData; | ||
| 31 | + xGm_.SetGlobalBuffer((__gm__ X_T*)x); | ||
| 32 | + indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices); | ||
| 33 | + yGm_.SetGlobalBuffer((__gm__ X_T*)y); | ||
| 34 | + } | ||
| 35 | + | ||
| 36 | + __aicore__ inline void Process() | ||
| 37 | + { | ||
| 38 | + int32_t blockIdx = static_cast<int32_t>(GetBlockIdx()); | ||
| 39 | + int32_t needCoreNum = static_cast<int32_t>(tilingData_->needCoreNum); | ||
| 40 | + if (blockIdx >= needCoreNum) { | ||
| 41 | + return; | ||
| 42 | + } | ||
| 43 | + | ||
| 44 | + INDEX_SIZE_T gatherDimSize = static_cast<INDEX_SIZE_T>(tilingData_->gatherDimSize); | ||
| 45 | + INDEX_SIZE_T innerSize = static_cast<INDEX_SIZE_T>(tilingData_->innerSize); | ||
| 46 | + bool nis = static_cast<bool>(tilingData_->negativeIndexSupport); | ||
| 47 | + | ||
| 48 | + INDEX_SIZE_T currentCoreElements = static_cast<INDEX_SIZE_T>(tilingData_->perCoreElements); | ||
| 49 | + if (blockIdx == tilingData_->needCoreNum - 1) { | ||
| 50 | + currentCoreElements = static_cast<INDEX_SIZE_T>(tilingData_->lastCoreElements); | ||
| 51 | + } | ||
| 52 | + INDEX_SIZE_T yIndexBase = static_cast<INDEX_SIZE_T>(blockIdx) * tilingData_->perCoreElements; | ||
| 53 | + | ||
| 54 | + __gm__ X_T* xPtr = (__gm__ X_T*)xGm_.GetPhyAddr(); | ||
| 55 | + __gm__ INDICES_T* indicesPtr = (__gm__ INDICES_T*)indicesGm_.GetPhyAddr(); | ||
| 56 | + __gm__ volatile X_T* yPtr = (__gm__ volatile X_T*)yGm_.GetPhyAddr(); | ||
| 57 | + | ||
| 58 | + if (nis) { | ||
| 59 | + GatherCore<true>(xPtr, indicesPtr, yPtr, yIndexBase, currentCoreElements, innerSize, gatherDimSize); | ||
| 60 | + } else { | ||
| 61 | + GatherCore<false>(xPtr, indicesPtr, yPtr, yIndexBase, currentCoreElements, innerSize, gatherDimSize); | ||
| 62 | + } | ||
| 63 | + } | ||
| 64 | + | ||
| 65 | +private: | ||
| 66 | + template <const bool NIS> | ||
| 67 | + __aicore__ inline void GatherCore(__gm__ X_T* x, __gm__ INDICES_T* indices, __gm__ volatile X_T* y, | ||
| 68 | + INDEX_SIZE_T yIndexBase, INDEX_SIZE_T currentCoreElements, INDEX_SIZE_T innerSize, | ||
| 69 | + INDEX_SIZE_T gatherDimSize) | ||
| 70 | + { | ||
| 71 | + // 2D: batch==1 && outer==1,yIndex 只分解成 gatherI/innerI 两维 | ||
| 72 | + for (INDEX_SIZE_T index = 0; index < currentCoreElements; index++) { | ||
| 73 | + INDEX_SIZE_T yIndex = yIndexBase + index; | ||
| 74 | + INDEX_SIZE_T gatherI = yIndex / innerSize; // 原 UintDiv(yIndex, m0, shift0) | ||
| 75 | + INDEX_SIZE_T innerI = yIndex - gatherI * innerSize; | ||
| 76 | + | ||
| 77 | + INDICES_T indicesValue = indices[gatherI]; | ||
| 78 | + if constexpr (NIS) { | ||
| 79 | + if (unlikely(indicesValue < 0)) { | ||
| 80 | + indicesValue += static_cast<INDICES_T>(gatherDimSize); | ||
| 81 | + } | ||
| 82 | + } | ||
| 83 | + INDEX_SIZE_T indicesValueI = static_cast<INDEX_SIZE_T>(indicesValue); | ||
| 84 | + INDEX_SIZE_T xIndex = indicesValueI * innerSize + innerI; | ||
| 85 | + bool oob = indicesValue < 0 || indicesValueI >= gatherDimSize; | ||
| 86 | + y[yIndex] = oob ? static_cast<X_T>(0) : x[xIndex]; | ||
| 87 | + } | ||
| 88 | + } | ||
| 89 | + | ||
| 90 | +private: | ||
| 91 | + GlobalTensor<X_T> xGm_; | ||
| 92 | + GlobalTensor<INDICES_T> indicesGm_; | ||
| 93 | + GlobalTensor<X_T> yGm_; | ||
| 94 | + const GatherV2TilingDataSimtTwoDim* tilingData_ = nullptr; | ||
| 95 | +}; | ||
| 96 | +} // namespace gatherv2 | ||
| 97 | +#endif // GATHER_V2_SIMD_2D_H | ||
| @@ -0,0 +1,305 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file gather_v2_simd_arch31.h | ||
| 13 | + * \brief | ||
| 14 | + */ | ||
| 15 | +#ifndef GATHER_V2_SIMD_H | ||
| 16 | +#define GATHER_V2_SIMD_H | ||
| 17 | + | ||
| 18 | +#ifndef K_MAX_SHAPE_DIM | ||
| 19 | +#define K_MAX_SHAPE_DIM 0 | ||
| 20 | +#endif | ||
| 21 | + | ||
| 22 | +#if ASC_DEVKIT_MAJOR >= 9 | ||
| 23 | +#include "basic_api/kernel_vec_intf.h" | ||
| 24 | +#else | ||
| 25 | +#include "kernel_operator.h" | ||
| 26 | +#endif | ||
| 27 | +#include "op_kernel/platform_util.h" | ||
| 28 | + | ||
| 29 | +namespace gatherv2 { | ||
| 30 | +using namespace AscendC; | ||
| 31 | + | ||
| 32 | +constexpr int32_t BUFFER_NUM_SIMD = 2; | ||
| 33 | + | ||
| 34 | +template <typename INDICES_T> | ||
| 35 | +class Gatherv2Simd { | ||
| 36 | +public: | ||
| 37 | + __aicore__ inline Gatherv2Simd(TPipe* pipe) : pipe_(pipe){}; | ||
| 38 | + __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, | ||
| 39 | + const GatherV2TilingData* tilingData); | ||
| 40 | + __aicore__ inline void Process(); | ||
| 41 | + template <typename T> | ||
| 42 | + __aicore__ inline void CopyIn(LocalTensor<T> xLocal, GlobalTensor<T> xGm, int64_t offset, uint32_t nBurst, | ||
| 43 | + uint32_t copyLen); | ||
| 44 | + __aicore__ inline INDICES_T GetIndex(int64_t idx, int64_t endIdx); | ||
| 45 | + __aicore__ inline void CopyOut(int64_t offset, uint32_t nBurst, uint32_t copyLen); | ||
| 46 | + __aicore__ inline void NoSplitColProcess(int64_t colsAlign); | ||
| 47 | + __aicore__ inline void SplitColProcess(int64_t colsAlign); | ||
| 48 | + | ||
| 49 | +private: | ||
| 50 | + GlobalTensor<int8_t> xGm_; | ||
| 51 | + GlobalTensor<INDICES_T> indicesGm_; | ||
| 52 | + GlobalTensor<int8_t> yGm_; | ||
| 53 | + TPipe* pipe_; | ||
| 54 | + TQueBind<QuePosition::VECIN, QuePosition::VECOUT, BUFFER_NUM_SIMD> inQueue_; | ||
| 55 | + TBuf<QuePosition::VECCALC> indexBuf_; | ||
| 56 | + const GatherV2TilingData* tilingData_; | ||
| 57 | + | ||
| 58 | + int32_t needCoreNum_; | ||
| 59 | + int32_t threadNum_; | ||
| 60 | + int64_t batchSize_; | ||
| 61 | + int64_t outerSize_; | ||
| 62 | + int64_t gatherDimSize_; | ||
| 63 | + int64_t gatherSize_; | ||
| 64 | + int64_t innerSize_; | ||
| 65 | + int64_t xSize_; | ||
| 66 | + int64_t indicesSize_; | ||
| 67 | + int64_t ySize_; | ||
| 68 | + int64_t yOffsetBase_ = 0; | ||
| 69 | + int64_t indicesOffsetBase_ = 0; | ||
| 70 | + int64_t maxIndex_ = 0; | ||
| 71 | + int64_t curIndexSize_ = 0; | ||
| 72 | + bool negativeIndexSupport_; | ||
| 73 | + | ||
| 74 | + int64_t currentCoreElements_; | ||
| 75 | + int32_t blockIdx_; | ||
| 76 | +}; | ||
| 77 | + | ||
| 78 | +template <typename INDICES_T> | ||
| 79 | +__aicore__ inline void Gatherv2Simd<INDICES_T>::Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, | ||
| 80 | + const GatherV2TilingData* tilingData) | ||
| 81 | +{ | ||
| 82 | + tilingData_ = tilingData; | ||
| 83 | + blockIdx_ = static_cast<int32_t>(GetBlockIdx()); | ||
| 84 | + needCoreNum_ = static_cast<int32_t>(tilingData_->needCoreNum); | ||
| 85 | + threadNum_ = static_cast<int32_t>(tilingData_->threadNum); | ||
| 86 | + batchSize_ = static_cast<int64_t>(tilingData_->batchSize); | ||
| 87 | + outerSize_ = static_cast<int64_t>(tilingData_->outerSize); | ||
| 88 | + gatherDimSize_ = static_cast<int64_t>(tilingData_->gatherDimSize); | ||
| 89 | + gatherSize_ = static_cast<int64_t>(tilingData_->gatherSize); | ||
| 90 | + innerSize_ = static_cast<int64_t>(tilingData_->innerSize); | ||
| 91 | + | ||
| 92 | + xSize_ = static_cast<int64_t>(tilingData_->xSize); | ||
| 93 | + indicesSize_ = static_cast<int64_t>(tilingData_->indicesSize); | ||
| 94 | + ySize_ = static_cast<int64_t>(tilingData_->ySize); | ||
| 95 | + negativeIndexSupport_ = static_cast<bool>(tilingData_->negativeIndexSupport); | ||
| 96 | + | ||
| 97 | + xGm_.SetGlobalBuffer((__gm__ int8_t*)x); | ||
| 98 | + | ||
| 99 | + if (GetBlockIdx() < tilingData_->lastCoreElements) { | ||
| 100 | + yOffsetBase_ = (tilingData_->perCoreElements + 1) * GetBlockIdx(); | ||
| 101 | + currentCoreElements_ = tilingData_->perCoreElements + 1; | ||
| 102 | + } else { | ||
| 103 | + yOffsetBase_ = tilingData_->perCoreElements * GetBlockIdx() + tilingData_->lastCoreElements; | ||
| 104 | + currentCoreElements_ = tilingData_->perCoreElements; | ||
| 105 | + } | ||
| 106 | + | ||
| 107 | + indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices); | ||
| 108 | + yGm_.SetGlobalBuffer((__gm__ int8_t*)y); | ||
| 109 | + pipe_->InitBuffer(inQueue_, BUFFER_NUM_SIMD, tilingData_->maxElement * tilingData_->dtypeSize); | ||
| 110 | + pipe_->InitBuffer(indexBuf_, tilingData_->indiceUbSize); | ||
| 111 | + maxIndex_ = tilingData_->indiceUbSize / sizeof(INDICES_T); | ||
| 112 | +} | ||
| 113 | + | ||
| 114 | +template <typename INDICES_T> | ||
| 115 | +template <typename T> | ||
| 116 | +__aicore__ inline void Gatherv2Simd<INDICES_T>::CopyIn(LocalTensor<T> xLocal, GlobalTensor<T> xGm, int64_t offset, | ||
| 117 | + uint32_t nBurst, uint32_t copyLen) | ||
| 118 | +{ | ||
| 119 | + DataCopyPadExtParams<T> dataCopyPadExtParams; | ||
| 120 | + dataCopyPadExtParams.isPad = false; | ||
| 121 | + dataCopyPadExtParams.leftPadding = 0; | ||
| 122 | + dataCopyPadExtParams.rightPadding = 0; | ||
| 123 | + dataCopyPadExtParams.paddingValue = 0; | ||
| 124 | + | ||
| 125 | + DataCopyExtParams dataCoptExtParams; | ||
| 126 | + dataCoptExtParams.blockCount = nBurst; | ||
| 127 | + dataCoptExtParams.blockLen = copyLen * sizeof(T); | ||
| 128 | + dataCoptExtParams.srcStride = 0; | ||
| 129 | + dataCoptExtParams.dstStride = 0; | ||
| 130 | + DataCopyPad(xLocal, xGm[offset], dataCoptExtParams, dataCopyPadExtParams); | ||
| 131 | +} | ||
| 132 | + | ||
| 133 | +template <typename INDICES_T> | ||
| 134 | +__aicore__ inline void Gatherv2Simd<INDICES_T>::CopyOut(int64_t offset, uint32_t nBurst, uint32_t copyLen) | ||
| 135 | +{ | ||
| 136 | + DataCopyExtParams dataCoptExtParams; | ||
| 137 | + dataCoptExtParams.blockCount = nBurst; | ||
| 138 | + dataCoptExtParams.blockLen = copyLen; | ||
| 139 | + dataCoptExtParams.srcStride = 0; | ||
| 140 | + dataCoptExtParams.dstStride = 0; | ||
| 141 | + LocalTensor<int8_t> yLocal = inQueue_.DeQue<int8_t>(); | ||
| 142 | + event_t eventIdVtoMTE3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | ||
| 143 | + SetFlag<HardEvent::V_MTE3>(eventIdVtoMTE3); | ||
| 144 | + WaitFlag<HardEvent::V_MTE3>(eventIdVtoMTE3); | ||
| 145 | + DataCopyPad(yGm_[offset], yLocal, dataCoptExtParams); | ||
| 146 | + inQueue_.FreeTensor(yLocal); | ||
| 147 | +} | ||
| 148 | + | ||
| 149 | +template <typename INDICES_T> | ||
| 150 | +__aicore__ inline INDICES_T Gatherv2Simd<INDICES_T>::GetIndex(int64_t idx, int64_t endIdx) | ||
| 151 | +{ | ||
| 152 | + if (idx >= indicesOffsetBase_ + curIndexSize_ || idx < indicesOffsetBase_) { | ||
| 153 | + int64_t startBIdx = idx / gatherSize_; | ||
| 154 | + int64_t startGIdx = idx % gatherSize_; | ||
| 155 | + int64_t endBIdx = endIdx / gatherSize_; | ||
| 156 | + int64_t endGIdx = endIdx % gatherSize_; | ||
| 157 | + int64_t copyLen = 0; | ||
| 158 | + if (maxIndex_ >= (endBIdx - startBIdx + 1) * gatherSize_) { | ||
| 159 | + copyLen = (endBIdx - startBIdx + 1) * gatherSize_; | ||
| 160 | + indicesOffsetBase_ = startBIdx * gatherSize_; | ||
| 161 | + } else if (maxIndex_ >= gatherSize_) { | ||
| 162 | + int64_t maxBSize = maxIndex_ / gatherSize_; | ||
| 163 | + copyLen = startBIdx + maxBSize > endBIdx ? (endBIdx - startBIdx + 1) * gatherSize_ : maxBSize * gatherSize_; | ||
| 164 | + indicesOffsetBase_ = startBIdx * gatherSize_; | ||
| 165 | + } else { | ||
| 166 | + copyLen = startGIdx + maxIndex_ >= (endBIdx + 1) * gatherSize_ ? (endBIdx + 1) * gatherSize_ - idx : | ||
| 167 | + maxIndex_; | ||
| 168 | + indicesOffsetBase_ = idx; | ||
| 169 | + } | ||
| 170 | + curIndexSize_ = copyLen; | ||
| 171 | + LocalTensor<INDICES_T> tmpLocal = indexBuf_.Get<INDICES_T>(); | ||
| 172 | + CopyIn(tmpLocal, indicesGm_, indicesOffsetBase_, 1, copyLen); | ||
| 173 | + | ||
| 174 | + event_t eventIdMTE2toS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_S)); | ||
| 175 | + SetFlag<HardEvent::MTE2_S>(eventIdMTE2toS); | ||
| 176 | + WaitFlag<HardEvent::MTE2_S>(eventIdMTE2toS); | ||
| 177 | + } | ||
| 178 | + LocalTensor<INDICES_T> indexLocal = indexBuf_.Get<INDICES_T>(); | ||
| 179 | + return indexLocal.GetValue(idx - indicesOffsetBase_); | ||
| 180 | +} | ||
| 181 | +template <typename INDICES_T> | ||
| 182 | +__aicore__ inline void Gatherv2Simd<INDICES_T>::NoSplitColProcess(int64_t colsAlign) | ||
| 183 | +{ | ||
| 184 | + int64_t onceMaxRows = tilingData_->maxElement * tilingData_->dtypeSize / colsAlign; | ||
| 185 | + int64_t loopNum = (currentCoreElements_ + onceMaxRows - 1) / onceMaxRows; | ||
| 186 | + | ||
| 187 | + // indice -> b, g | ||
| 188 | + // output -> b, o, g, i | ||
| 189 | + int64_t yStart = yOffsetBase_; | ||
| 190 | + int64_t yEnd = yOffsetBase_ + currentCoreElements_ - 1; | ||
| 191 | + int64_t endBatch = yEnd / (outerSize_ * gatherSize_); | ||
| 192 | + int64_t endGIdx = yEnd % gatherSize_; | ||
| 193 | + int64_t indiceEndIdx = endBatch * gatherSize_ + endGIdx; | ||
| 194 | + for (int64_t i = 0; i < loopNum; i++) { | ||
| 195 | + int64_t rows = (i == loopNum - 1) ? (currentCoreElements_ - i * onceMaxRows) : onceMaxRows; | ||
| 196 | + int64_t cols = tilingData_->innerSize; | ||
| 197 | + LocalTensor<int8_t> xLocal = inQueue_.AllocTensor<int8_t>(); | ||
| 198 | + event_t eventIdMTE3toV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); | ||
| 199 | + SetFlag<HardEvent::MTE3_V>(eventIdMTE3toV); | ||
| 200 | + WaitFlag<HardEvent::MTE3_V>(eventIdMTE3toV); | ||
| 201 | + int64_t preIdx0 = -1; | ||
| 202 | + int64_t preIdx1 = -1; | ||
| 203 | + int64_t preIdx2 = -1; | ||
| 204 | + int32_t backOffset0 = 0; | ||
| 205 | + int32_t backOffset1 = 0; | ||
| 206 | + int32_t backOffset2 = 0; | ||
| 207 | + for (int64_t j = 0; j < rows; j++) { | ||
| 208 | + int64_t yIdx = yStart + i * onceMaxRows + j; | ||
| 209 | + int64_t batchI = yIdx / (outerSize_ * gatherSize_); | ||
| 210 | + int64_t outerI = yIdx % (outerSize_ * gatherSize_) / gatherSize_; | ||
| 211 | + int64_t gatherI = yIdx % gatherSize_; | ||
| 212 | + int64_t indiceOffset = batchI * gatherSize_ + gatherI; | ||
| 213 | + INDICES_T index = GetIndex(indiceOffset, indiceEndIdx); | ||
| 214 | + if (negativeIndexSupport_) { | ||
| 215 | + index = index < 0 ? index + gatherDimSize_ : index; | ||
| 216 | + } | ||
| 217 | + int64_t xIndex = ((batchI * outerSize_ + outerI) * gatherDimSize_ + index) * innerSize_; | ||
| 218 | + int64_t offset = xIndex * tilingData_->dtypeSize; | ||
| 219 | + if (likely(index >= 0 && index < gatherDimSize_)) { | ||
| 220 | + if (unlikely(xIndex == preIdx0 || xIndex == preIdx1 || xIndex == preIdx2)) { | ||
| 221 | + event_t eventIdMTE2toV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); | ||
| 222 | + SetFlag<HardEvent::MTE2_V>(eventIdMTE2toV); | ||
| 223 | + WaitFlag<HardEvent::MTE2_V>(eventIdMTE2toV); | ||
| 224 | + int32_t backStep = (xIndex == preIdx0) ? backOffset0 : | ||
| 225 | + (xIndex == preIdx1) ? backOffset1 : | ||
| 226 | + backOffset2; | ||
| 227 | + DataCopy(xLocal[j * colsAlign], xLocal[backStep], tilingData_->innerSize * tilingData_->dtypeSize); | ||
| 228 | + } else { | ||
| 229 | + CopyIn(xLocal[j * colsAlign], xGm_, offset, 1, tilingData_->innerSize * tilingData_->dtypeSize); | ||
| 230 | + preIdx2 = preIdx1; | ||
| 231 | + preIdx1 = preIdx0; | ||
| 232 | + preIdx0 = xIndex; | ||
| 233 | + backOffset2 = backOffset1; | ||
| 234 | + backOffset1 = backOffset0; | ||
| 235 | + backOffset0 = j * colsAlign; | ||
| 236 | + } | ||
| 237 | + } else { | ||
| 238 | + Duplicate<int8_t>(xLocal[j * colsAlign], 0, tilingData_->innerSize * tilingData_->dtypeSize); | ||
| 239 | + } | ||
| 240 | + } | ||
| 241 | + inQueue_.EnQue<int8_t>(xLocal); | ||
| 242 | + int64_t yOffset = (yStart + i * onceMaxRows) * tilingData_->innerSize * tilingData_->dtypeSize; | ||
| 243 | + CopyOut(yOffset, rows, cols * tilingData_->dtypeSize); | ||
| 244 | + } | ||
| 245 | +} | ||
| 246 | + | ||
| 247 | +template <typename INDICES_T> | ||
| 248 | +__aicore__ inline void Gatherv2Simd<INDICES_T>::SplitColProcess(int64_t colsAlign) | ||
| 249 | +{ | ||
| 250 | + // indice -> b, g | ||
| 251 | + // output -> b, o, g, i | ||
| 252 | + int64_t yStart = yOffsetBase_; | ||
| 253 | + int64_t yEnd = yOffsetBase_ + currentCoreElements_ - 1; | ||
| 254 | + int64_t endBatch = yEnd / (outerSize_ * gatherSize_); | ||
| 255 | + int64_t endGIdx = yEnd % gatherSize_; | ||
| 256 | + int64_t indiceEndIdx = endBatch * gatherSize_ + endGIdx; | ||
| 257 | + int64_t loopSize = (tilingData_->innerSize + tilingData_->maxElement - 1) / tilingData_->maxElement; | ||
| 258 | + for (int64_t i = 0; i < currentCoreElements_; i++) { | ||
| 259 | + int64_t yIdx = yStart + i; | ||
| 260 | + int64_t batchI = yIdx / (outerSize_ * gatherSize_); | ||
| 261 | + int64_t outerI = yIdx % (outerSize_ * gatherSize_) / gatherSize_; | ||
| 262 | + int64_t gatherI = yIdx % gatherSize_; | ||
| 263 | + int64_t indiceOffset = batchI * gatherSize_ + gatherI; | ||
| 264 | + INDICES_T index = GetIndex(indiceOffset, indiceEndIdx); | ||
| 265 | + if (negativeIndexSupport_) { | ||
| 266 | + index = index < 0 ? index + gatherDimSize_ : index; | ||
| 267 | + } | ||
| 268 | + int64_t xIndex = ((batchI * outerSize_ + outerI) * gatherDimSize_ + index) * innerSize_; | ||
| 269 | + int64_t yIndex = (yStart + i) * tilingData_->innerSize; | ||
| 270 | + for (int64_t j = 0; j < loopSize; j++) { | ||
| 271 | + int64_t cols = (j == loopSize - 1) ? (tilingData_->innerSize - j * tilingData_->maxElement) : | ||
| 272 | + tilingData_->maxElement; | ||
| 273 | + LocalTensor<int8_t> xLocal = inQueue_.AllocTensor<int8_t>(); | ||
| 274 | + int64_t offset = (xIndex + j * tilingData_->maxElement) * tilingData_->dtypeSize; | ||
| 275 | + if (likely(index >= 0 && index < gatherDimSize_)) { | ||
| 276 | + CopyIn(xLocal[0], xGm_, offset, 1, cols * tilingData_->dtypeSize); | ||
| 277 | + } else { | ||
| 278 | + event_t eventIdMTE3toV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); | ||
| 279 | + SetFlag<HardEvent::MTE3_V>(eventIdMTE3toV); | ||
| 280 | + WaitFlag<HardEvent::MTE3_V>(eventIdMTE3toV); | ||
| 281 | + Duplicate<int8_t>(xLocal[0], 0, cols * tilingData_->dtypeSize); | ||
| 282 | + } | ||
| 283 | + inQueue_.EnQue<int8_t>(xLocal); | ||
| 284 | + int64_t yOffset = (yIndex + j * tilingData_->maxElement) * tilingData_->dtypeSize; | ||
| 285 | + CopyOut(yOffset, 1, cols * tilingData_->dtypeSize); | ||
| 286 | + } | ||
| 287 | + } | ||
| 288 | +} | ||
| 289 | + | ||
| 290 | +template <typename INDICES_T> | ||
| 291 | +__aicore__ inline void Gatherv2Simd<INDICES_T>::Process() | ||
| 292 | +{ | ||
| 293 | + if (blockIdx_ >= tilingData_->needCoreNum) { | ||
| 294 | + return; | ||
| 295 | + } | ||
| 296 | + int64_t ubBlockSize = static_cast<int64_t>(Ops::Base::GetUbBlockSize()); | ||
| 297 | + int64_t colsAlign = (tilingData_->innerSize * tilingData_->dtypeSize + ubBlockSize - 1) / ubBlockSize * ubBlockSize; | ||
| 298 | + if (colsAlign <= tilingData_->maxElement * tilingData_->dtypeSize) { | ||
| 299 | + NoSplitColProcess(colsAlign); | ||
| 300 | + } else { | ||
| 301 | + SplitColProcess(colsAlign); | ||
| 302 | + } | ||
| 303 | +} | ||
| 304 | +} // namespace gatherv2 | ||
| 305 | +#endif // GATHER_V2_SIMD_H | ||
| @@ -0,0 +1,276 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file gather_v2_simd_full_load.h | ||
| 13 | + * \brief KirinX90 纯 SIMD full-load 模板(替代原 Gatherv2FullLoad simt 实现,去除 simt_api 依赖) | ||
| 14 | + * 保留 UB 缓冲(CopyIn/DataCopyPad/LoopCol/NoLoop)逻辑,仅把 GatherSimt3D/2D 换成 GatherCore3D/2D | ||
| 15 | + */ | ||
| 16 | +#ifndef GATHER_V2_SIMD_FULL_LOAD_H | ||
| 17 | +#define GATHER_V2_SIMD_FULL_LOAD_H | ||
| 18 | + | ||
| 19 | +#include "kernel_operator.h" | ||
| 20 | +#include "op_kernel/platform_util.h" | ||
| 21 | +// 已删除: simt_api/asc_simt.h、basic_api/kernel_vec_intf.h(KirinX90 硬件不支持 simt) | ||
| 22 | + | ||
| 23 | +namespace gatherv2 { | ||
| 24 | +using namespace AscendC; | ||
| 25 | + | ||
| 26 | +template <typename T, typename INDICES_T, typename U> | ||
| 27 | +class Gatherv2SimdFullLoad { | ||
| 28 | +public: | ||
| 29 | + __aicore__ inline Gatherv2SimdFullLoad(TPipe* pipe) : pipe_(pipe){}; | ||
| 30 | + __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, | ||
| 31 | + const GatherV2TilingData* tilingData); | ||
| 32 | + __aicore__ inline void CopyIn(LocalTensor<T> xLocal, GlobalTensor<T> xGm, int64_t offset, uint32_t nBurst, | ||
| 33 | + uint64_t copyLen); | ||
| 34 | + __aicore__ inline void Process(); | ||
| 35 | + __aicore__ inline void LoopColProcess(); | ||
| 36 | + __aicore__ inline void NoLoopProcess(); | ||
| 37 | + | ||
| 38 | +private: | ||
| 39 | + template <const bool NIS> | ||
| 40 | + __aicore__ inline void GatherCore3D(const U yIndexBase, U currentCoreElements, int64_t pOffset, U gatherSize, | ||
| 41 | + U innerSize, U gatherDimSize, __ubuf__ T* srcTensor, __gm__ INDICES_T* indices, | ||
| 42 | + __gm__ volatile T* y); | ||
| 43 | + template <const bool NIS> | ||
| 44 | + __aicore__ inline void GatherCore2D(const U yIndexBase, U currentCoreElements, U innerSize, U gatherDimSize, | ||
| 45 | + __ubuf__ T* srcTensor, __gm__ INDICES_T* indices, __gm__ volatile T* y); | ||
| 46 | + | ||
| 47 | +private: | ||
| 48 | + GlobalTensor<T> xGm_; | ||
| 49 | + GlobalTensor<INDICES_T> indicesGm_; | ||
| 50 | + GlobalTensor<T> yGm_; | ||
| 51 | + | ||
| 52 | + TPipe* pipe_; | ||
| 53 | + TQue<QuePosition::VECIN, 1> inQueue_; | ||
| 54 | + const GatherV2TilingData* tilingData_; | ||
| 55 | + | ||
| 56 | + int32_t blockIdx_; | ||
| 57 | + int32_t needCoreNum_; | ||
| 58 | + bool negativeIndexSupport_; | ||
| 59 | + | ||
| 60 | + U outerSize_; | ||
| 61 | + U gatherDimSize_; | ||
| 62 | + U gatherSize_; | ||
| 63 | + U innerSize_; | ||
| 64 | + U batchSize_; | ||
| 65 | + U ubAviable_; | ||
| 66 | + U currentCoreElements_; | ||
| 67 | + U loopNum_; | ||
| 68 | + U perLoopCoreElements_; | ||
| 69 | +}; | ||
| 70 | + | ||
| 71 | +// GatherCore3D:替代 GatherSimt3D(去 simt,普通除法,核内串行) | ||
| 72 | +template <typename T, typename INDICES_T, typename U> | ||
| 73 | +template <const bool NIS> | ||
| 74 | +__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::GatherCore3D(const U yIndexBase, U currentCoreElements, | ||
| 75 | + int64_t pOffset, U gatherSize, U innerSize, | ||
| 76 | + U gatherDimSize, __ubuf__ T* srcTensor, | ||
| 77 | + __gm__ INDICES_T* indices, | ||
| 78 | + __gm__ volatile T* y) | ||
| 79 | +{ | ||
| 80 | + for (U index = 0; index < currentCoreElements; index++) { | ||
| 81 | + U yIndex = yIndexBase + index; | ||
| 82 | + U tmp = yIndex / innerSize; // 原 UintDiv(yIndex, m0, shift0) | ||
| 83 | + U tmp1 = tmp / gatherSize; // 原 UintDiv(tmp, m1, shift1) | ||
| 84 | + U gatherI = tmp - tmp1 * gatherSize; | ||
| 85 | + U innerI = yIndex - tmp * innerSize; | ||
| 86 | + | ||
| 87 | + INDICES_T indicesValue = indices[gatherI]; | ||
| 88 | + if constexpr (NIS) { | ||
| 89 | + if (unlikely(indicesValue < 0)) { | ||
| 90 | + indicesValue += static_cast<INDICES_T>(gatherDimSize); | ||
| 91 | + } | ||
| 92 | + } | ||
| 93 | + U indicesValueI = static_cast<U>(indicesValue); | ||
| 94 | + U xIndex = ((tmp1 - pOffset) * gatherDimSize + indicesValueI) * innerSize + innerI; | ||
| 95 | + bool oob = indicesValue < 0 || indicesValueI >= gatherDimSize; | ||
| 96 | + y[yIndex] = oob ? static_cast<T>(0) : srcTensor[xIndex]; // 从 UB(srcTensor) 读,写 GM(y) | ||
| 97 | + } | ||
| 98 | +} | ||
| 99 | + | ||
| 100 | +// GatherCore2D:替代 GatherSimt2D | ||
| 101 | +template <typename T, typename INDICES_T, typename U> | ||
| 102 | +template <const bool NIS> | ||
| 103 | +__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::GatherCore2D(const U yIndexBase, U currentCoreElements, | ||
| 104 | + U innerSize, U gatherDimSize, | ||
| 105 | + __ubuf__ T* srcTensor, | ||
| 106 | + __gm__ INDICES_T* indices, | ||
| 107 | + __gm__ volatile T* y) | ||
| 108 | +{ | ||
| 109 | + for (U index = 0; index < currentCoreElements; index++) { | ||
| 110 | + U yIndex = yIndexBase + index; | ||
| 111 | + U gatherI = yIndex / innerSize; | ||
| 112 | + U innerI = yIndex - gatherI * innerSize; | ||
| 113 | + | ||
| 114 | + INDICES_T indicesValue = indices[gatherI]; | ||
| 115 | + if constexpr (NIS) { | ||
| 116 | + if (unlikely(indicesValue < 0)) { | ||
| 117 | + indicesValue += static_cast<INDICES_T>(gatherDimSize); | ||
| 118 | + } | ||
| 119 | + } | ||
| 120 | + U indicesValueI = static_cast<U>(indicesValue); | ||
| 121 | + U xIndex = indicesValueI * innerSize + innerI; | ||
| 122 | + bool oob = indicesValue < 0 || indicesValueI >= gatherDimSize; | ||
| 123 | + y[yIndex] = oob ? static_cast<T>(0) : srcTensor[xIndex]; | ||
| 124 | + } | ||
| 125 | +} | ||
| 126 | + | ||
| 127 | +// CopyIn 保留原样(DataCopyPad 搬 x: GM→UB,非 simt) | ||
| 128 | +template <typename T, typename INDICES_T, typename U> | ||
| 129 | +__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::CopyIn(LocalTensor<T> xLocal, GlobalTensor<T> xGm, | ||
| 130 | + int64_t offset, uint32_t nBurst, uint64_t copyLen) | ||
| 131 | +{ | ||
| 132 | + DataCopyPadExtParams<T> dataCopyPadExtParams; | ||
| 133 | + dataCopyPadExtParams.isPad = false; | ||
| 134 | + dataCopyPadExtParams.leftPadding = 0; | ||
| 135 | + dataCopyPadExtParams.rightPadding = 0; | ||
| 136 | + dataCopyPadExtParams.paddingValue = 0; | ||
| 137 | + | ||
| 138 | + DataCopyExtParams dataCoptExtParams; | ||
| 139 | + dataCoptExtParams.blockCount = nBurst; | ||
| 140 | + dataCoptExtParams.blockLen = copyLen; | ||
| 141 | + dataCoptExtParams.srcStride = 0; | ||
| 142 | + dataCoptExtParams.dstStride = 0; | ||
| 143 | + DataCopyPad(xLocal, xGm[offset], dataCoptExtParams, dataCopyPadExtParams); | ||
| 144 | +} | ||
| 145 | + | ||
| 146 | +// LoopColProcess:保留 UB 循环结构,GatherSimt3D→GatherCore3D,去 asc_vf_call/dim3/UintDiv | ||
| 147 | +template <typename T, typename INDICES_T, typename U> | ||
| 148 | +__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::LoopColProcess() | ||
| 149 | +{ | ||
| 150 | + int64_t gDimInerSize = gatherDimSize_ * innerSize_; | ||
| 151 | + int64_t gInerSize = gatherSize_ * innerSize_; | ||
| 152 | + | ||
| 153 | + int64_t maxGdimA = ubAviable_ / gDimInerSize; | ||
| 154 | + int64_t start = blockIdx_ * tilingData_->perCoreElements; | ||
| 155 | + int64_t curCoreData = (blockIdx_ == needCoreNum_ - 1) ? tilingData_->lastCoreElements : | ||
| 156 | + tilingData_->perCoreElements; | ||
| 157 | + int64_t end = start + curCoreData - 1; | ||
| 158 | + | ||
| 159 | + int64_t startP = start / gatherSize_; | ||
| 160 | + int64_t startG = start % gatherSize_; | ||
| 161 | + int64_t endP = end / gatherSize_; | ||
| 162 | + int64_t startGOffset = startG * innerSize_; | ||
| 163 | + | ||
| 164 | + loopNum_ = (endP - startP + 1 + maxGdimA - 1) / maxGdimA; | ||
| 165 | + U yIndexBase = blockIdx_ * tilingData_->perCoreElements * innerSize_; | ||
| 166 | + int64_t pOffset = startP; | ||
| 167 | + | ||
| 168 | + for (int64_t loop = 0; loop < loopNum_; ++loop) { | ||
| 169 | + int64_t copySize; | ||
| 170 | + int64_t startGroup = startP + loop * maxGdimA; | ||
| 171 | + int64_t endGroup = startGroup + maxGdimA; | ||
| 172 | + int64_t groupCount = endGroup - startGroup; | ||
| 173 | + int64_t startOffset = pOffset * gDimInerSize; | ||
| 174 | + | ||
| 175 | + if (loopNum_ > 1) { | ||
| 176 | + copySize = groupCount * gDimInerSize; | ||
| 177 | + if (loop == 0) { | ||
| 178 | + currentCoreElements_ = groupCount * gInerSize - startGOffset; | ||
| 179 | + } else if (loop == (loopNum_ - 1)) { | ||
| 180 | + copySize = ((endP - startP + 1) % maxGdimA) * gDimInerSize; | ||
| 181 | + currentCoreElements_ = perLoopCoreElements_ - (loop * maxGdimA) * gInerSize + startGOffset; | ||
| 182 | + } else { | ||
| 183 | + currentCoreElements_ = groupCount * gInerSize; | ||
| 184 | + } | ||
| 185 | + } else { | ||
| 186 | + copySize = (endP - startP + 1) * gDimInerSize; | ||
| 187 | + } | ||
| 188 | + | ||
| 189 | + // 从 GM 搬运数据到 UB(保留) | ||
| 190 | + LocalTensor<T> gaBuffer = inQueue_.AllocTensor<T>(); | ||
| 191 | + CopyIn(gaBuffer, xGm_, startOffset, 1, copySize * tilingData_->dtypeSize); | ||
| 192 | + | ||
| 193 | + inQueue_.EnQue(gaBuffer); | ||
| 194 | + inQueue_.DeQue<T>(); | ||
| 195 | + // gather 计算:GatherSimt3D→GatherCore3D(去 asc_vf_call/dim3,去 m0/shift0/m1/shift1) | ||
| 196 | + if (unlikely(negativeIndexSupport_)) { | ||
| 197 | + GatherCore3D<true>(yIndexBase, currentCoreElements_, pOffset, gatherSize_, innerSize_, gatherDimSize_, | ||
| 198 | + (__ubuf__ T*)(gaBuffer.GetPhyAddr()), (__gm__ INDICES_T*)(indicesGm_.GetPhyAddr()), | ||
| 199 | + (__gm__ volatile T*)(yGm_.GetPhyAddr())); | ||
| 200 | + } else { | ||
| 201 | + GatherCore3D<false>(yIndexBase, currentCoreElements_, pOffset, gatherSize_, innerSize_, gatherDimSize_, | ||
| 202 | + (__ubuf__ T*)(gaBuffer.GetPhyAddr()), (__gm__ INDICES_T*)(indicesGm_.GetPhyAddr()), | ||
| 203 | + (__gm__ volatile T*)(yGm_.GetPhyAddr())); | ||
| 204 | + } | ||
| 205 | + // 计算每次循环 yIndexBase(保留) | ||
| 206 | + if (loopNum_ > 1) { | ||
| 207 | + yIndexBase += currentCoreElements_; | ||
| 208 | + pOffset = yIndexBase / gInerSize; | ||
| 209 | + } | ||
| 210 | + inQueue_.FreeTensor(gaBuffer); | ||
| 211 | + } | ||
| 212 | +} | ||
| 213 | + | ||
| 214 | +// NoLoopProcess:保留 UB,GatherSimt2D→GatherCore2D | ||
| 215 | +template <typename T, typename INDICES_T, typename U> | ||
| 216 | +__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::NoLoopProcess() | ||
| 217 | +{ | ||
| 218 | + int64_t copy_size = gatherDimSize_ * innerSize_; | ||
| 219 | + LocalTensor<T> gaBuffer = inQueue_.AllocTensor<T>(); | ||
| 220 | + CopyIn(gaBuffer, xGm_, 0, 1, copy_size * tilingData_->dtypeSize); | ||
| 221 | + | ||
| 222 | + inQueue_.EnQue(gaBuffer); | ||
| 223 | + inQueue_.DeQue<T>(); | ||
| 224 | + U yIndexBase = blockIdx_ * tilingData_->perCoreElements * innerSize_; | ||
| 225 | + if (unlikely(negativeIndexSupport_)) { | ||
| 226 | + GatherCore2D<true>(yIndexBase, currentCoreElements_, innerSize_, gatherDimSize_, | ||
| 227 | + (__ubuf__ T*)(gaBuffer.GetPhyAddr()), (__gm__ INDICES_T*)(indicesGm_.GetPhyAddr()), | ||
| 228 | + (__gm__ volatile T*)(yGm_.GetPhyAddr())); | ||
| 229 | + } else { | ||
| 230 | + GatherCore2D<false>(yIndexBase, currentCoreElements_, innerSize_, gatherDimSize_, | ||
| 231 | + (__ubuf__ T*)(gaBuffer.GetPhyAddr()), (__gm__ INDICES_T*)(indicesGm_.GetPhyAddr()), | ||
| 232 | + (__gm__ volatile T*)(yGm_.GetPhyAddr())); | ||
| 233 | + } | ||
| 234 | + inQueue_.FreeTensor(gaBuffer); | ||
| 235 | +} | ||
| 236 | + | ||
| 237 | +template <typename T, typename INDICES_T, typename U> | ||
| 238 | +__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, | ||
| 239 | + const GatherV2TilingData* tilingData) | ||
| 240 | +{ | ||
| 241 | + tilingData_ = tilingData; | ||
| 242 | + xGm_.SetGlobalBuffer((__gm__ T*)x); | ||
| 243 | + indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices); | ||
| 244 | + yGm_.SetGlobalBuffer((__gm__ T*)y); | ||
| 245 | + | ||
| 246 | + pipe_->InitBuffer(inQueue_, 1, (tilingData_->maxElement * tilingData_->dtypeSize)); | ||
| 247 | +} | ||
| 248 | + | ||
| 249 | +template <typename T, typename INDICES_T, typename U> | ||
| 250 | +__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::Process() | ||
| 251 | +{ | ||
| 252 | + blockIdx_ = static_cast<int32_t>(GetBlockIdx()); | ||
| 253 | + needCoreNum_ = static_cast<int32_t>(tilingData_->needCoreNum); | ||
| 254 | + | ||
| 255 | + batchSize_ = static_cast<U>(tilingData_->batchSize); | ||
| 256 | + outerSize_ = static_cast<U>(tilingData_->outerSize); | ||
| 257 | + gatherDimSize_ = static_cast<U>(tilingData_->gatherDimSize); | ||
| 258 | + gatherSize_ = static_cast<U>(tilingData_->gatherSize); | ||
| 259 | + innerSize_ = static_cast<U>(tilingData_->innerSize); | ||
| 260 | + ubAviable_ = static_cast<U>(tilingData_->maxElement); | ||
| 261 | + | ||
| 262 | + negativeIndexSupport_ = static_cast<bool>(tilingData_->negativeIndexSupport); | ||
| 263 | + currentCoreElements_ = static_cast<U>(tilingData_->perCoreElements * innerSize_); | ||
| 264 | + if (blockIdx_ == tilingData_->needCoreNum - 1) { | ||
| 265 | + currentCoreElements_ = static_cast<U>(tilingData_->lastCoreElements * innerSize_); | ||
| 266 | + } | ||
| 267 | + perLoopCoreElements_ = currentCoreElements_; | ||
| 268 | + | ||
| 269 | + if (batchSize_ == 1 && outerSize_ > 1) { | ||
| 270 | + LoopColProcess(); | ||
| 271 | + } else { | ||
| 272 | + NoLoopProcess(); | ||
| 273 | + } | ||
| 274 | +} | ||
| 275 | +} // namespace gatherv2 | ||
| 276 | +#endif // GATHER_V2_SIMD_FULL_LOAD_H | ||
| @@ -0,0 +1,407 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file gather_v2_simd_last_gather_arch31.h | ||
| 13 | + * \brief | ||
| 14 | + */ | ||
| 15 | +#ifndef GATHER_V2_SIMD_LAST_GATHER | ||
| 16 | +#define GATHER_V2_SIMD_LAST_GATHER | ||
| 17 | + | ||
| 18 | +#ifndef K_MAX_SHAPE_DIM | ||
| 19 | +#define K_MAX_SHAPE_DIM 0 | ||
| 20 | +#endif | ||
| 21 | +#if ASC_DEVKIT_MAJOR >= 9 | ||
| 22 | +#include "basic_api/kernel_vec_intf.h" | ||
| 23 | +#else | ||
| 24 | +#include "kernel_operator.h" | ||
| 25 | +#endif | ||
| 26 | +#include "op_kernel/platform_util.h" | ||
| 27 | + | ||
| 28 | +namespace gatherv2 { | ||
| 29 | +using namespace AscendC; | ||
| 30 | + | ||
| 31 | +#if 0 // KirinX90 不支持 | ||
| 32 | + // MicroAPI(Interleave/DeInterleave/MaskInterleave/MaskDeInterleave/Pack/CompareScalar/MaskAnd/DataCopyGather | ||
| 33 | + // 等) 参照 cast 算子 arch31 适配经验,本文件全部方法实现依赖 MicroAPI,整体注释避免编译错误 如后续 KirinX90 | ||
| 34 | + // 支持相关指令或需用 SIMD 重写,解开本 #if 0 即可恢复参考实现 | ||
| 35 | +template <typename T> | ||
| 36 | +struct IndexTypeGet { | ||
| 37 | + using type = typename std::conditional<sizeof(T) == sizeof(int8_t) || sizeof(T) == sizeof(int16_t), uint16_t, | ||
| 38 | + uint32_t>::type; | ||
| 39 | +}; | ||
| 40 | + | ||
| 41 | +template <typename TARGET_T, typename ORG_T> | ||
| 42 | +__simd_callee__ inline void LoadIndices(MicroAPI::RegTensor<TARGET_T>& vregIndcie, MicroAPI::MaskReg& gatherMask, | ||
| 43 | + __local_mem__ ORG_T* indiceAddr, MicroAPI::MaskReg preg, ORG_T dimsize) | ||
| 44 | +{ | ||
| 45 | + if constexpr (sizeof(ORG_T) == sizeof(int64_t)) { | ||
| 46 | + if constexpr (sizeof(TARGET_T) == sizeof(int32_t)) { | ||
| 47 | + MicroAPI::RegTensor<int64_t, MicroAPI::RegTraitNumTwo> tmpReg; | ||
| 48 | + MicroAPI::DataCopy(tmpReg, indiceAddr); | ||
| 49 | + MicroAPI::MaskReg gtPreg; | ||
| 50 | + MicroAPI::MaskReg ltPreg; | ||
| 51 | + MicroAPI::CompareScalar<int64_t, CMPMODE::GE>(gtPreg, tmpReg, 0, preg); | ||
| 52 | + MicroAPI::CompareScalar<int64_t, CMPMODE::LT>(ltPreg, tmpReg, dimsize, preg); | ||
| 53 | + MicroAPI::MaskAnd(gatherMask, gtPreg, ltPreg, preg); | ||
| 54 | + MicroAPI::Pack((MicroAPI::RegTensor<uint32_t>&)vregIndcie, tmpReg); | ||
| 55 | + } else { | ||
| 56 | + constexpr int16_t vfLen = AscendC::VECTOR_REG_WIDTH / sizeof(int32_t); | ||
| 57 | + MicroAPI::RegTensor<int64_t, MicroAPI::RegTraitNumTwo> tmpReg0, tmpReg1; | ||
| 58 | + MicroAPI::RegTensor<int32_t> tmpB32Reg0, tmpB32Reg1; | ||
| 59 | + MicroAPI::MaskReg lowPreg, highPreg; | ||
| 60 | + MicroAPI::MaskInterleave<int16_t>(lowPreg, highPreg, preg, preg); | ||
| 61 | + MicroAPI::DataCopy(tmpReg0, indiceAddr); | ||
| 62 | + MicroAPI::DataCopy(tmpReg1, indiceAddr + vfLen); | ||
| 63 | + MicroAPI::Pack((MicroAPI::RegTensor<uint32_t>&)tmpB32Reg0, tmpReg0); | ||
| 64 | + MicroAPI::Pack((MicroAPI::RegTensor<uint32_t>&)tmpB32Reg1, tmpReg1); | ||
| 65 | + MicroAPI::DeInterleave<int16_t>( | ||
| 66 | + (MicroAPI::RegTensor<int16_t>&)vregIndcie, (MicroAPI::RegTensor<int16_t>&)tmpB32Reg0, | ||
| 67 | + (MicroAPI::RegTensor<int16_t>&)tmpB32Reg0, (MicroAPI::RegTensor<int16_t>&)tmpB32Reg1); | ||
| 68 | + MicroAPI::MaskReg gtPreg, ltPreg; | ||
| 69 | + MicroAPI::CompareScalar<int64_t, CMPMODE::GE>(gtPreg, tmpReg0, 0, lowPreg); | ||
| 70 | + MicroAPI::CompareScalar<int64_t, CMPMODE::LT>(ltPreg, tmpReg0, dimsize, lowPreg); | ||
| 71 | + MicroAPI::MaskAnd(lowPreg, gtPreg, ltPreg, lowPreg); | ||
| 72 | + MicroAPI::CompareScalar<int64_t, CMPMODE::GE>(gtPreg, tmpReg1, 0, highPreg); | ||
| 73 | + MicroAPI::CompareScalar<int64_t, CMPMODE::LT>(ltPreg, tmpReg1, dimsize, highPreg); | ||
| 74 | + MicroAPI::MaskAnd(highPreg, gtPreg, ltPreg, highPreg); | ||
| 75 | + MicroAPI::MaskDeInterleave<int16_t>(gatherMask, ltPreg, lowPreg, highPreg); | ||
| 76 | + } | ||
| 77 | + } else { | ||
| 78 | + if constexpr (sizeof(TARGET_T) == sizeof(int32_t)) { | ||
| 79 | + MicroAPI::DataCopy((MicroAPI::RegTensor<int32_t>&)vregIndcie, indiceAddr); | ||
| 80 | + MicroAPI::MaskReg gtPreg; | ||
| 81 | + MicroAPI::MaskReg ltPreg; | ||
| 82 | + MicroAPI::CompareScalar<int32_t, CMPMODE::GE>(gtPreg, (MicroAPI::RegTensor<int32_t>&)vregIndcie, 0, preg); | ||
| 83 | + MicroAPI::CompareScalar<int32_t, CMPMODE::LT>(ltPreg, (MicroAPI::RegTensor<int32_t>&)vregIndcie, dimsize, | ||
| 84 | + preg); | ||
| 85 | + MicroAPI::MaskAnd(gatherMask, gtPreg, ltPreg, preg); | ||
| 86 | + } else { | ||
| 87 | + constexpr int16_t vfLen = AscendC::VECTOR_REG_WIDTH / sizeof(int32_t); | ||
| 88 | + MicroAPI::RegTensor<int32_t> tmpReg0, tmpReg1; | ||
| 89 | + MicroAPI::RegTensor<int32_t> tmpB32; | ||
| 90 | + MicroAPI::MaskReg lowPreg, highPreg; | ||
| 91 | + MicroAPI::MaskInterleave<int16_t>(lowPreg, highPreg, preg, preg); | ||
| 92 | + MicroAPI::DataCopy(tmpReg0, indiceAddr); | ||
| 93 | + MicroAPI::DataCopy(tmpReg1, indiceAddr + vfLen); | ||
| 94 | + | ||
| 95 | + MicroAPI::DeInterleave<int16_t>( | ||
| 96 | + (MicroAPI::RegTensor<int16_t>&)vregIndcie, (MicroAPI::RegTensor<int16_t>&)tmpB32, | ||
| 97 | + (MicroAPI::RegTensor<int16_t>&)tmpReg0, (MicroAPI::RegTensor<int16_t>&)tmpReg1); | ||
| 98 | + MicroAPI::MaskReg gtPreg, ltPreg; | ||
| 99 | + MicroAPI::CompareScalar<int32_t, CMPMODE::GE>(gtPreg, tmpReg0, 0, lowPreg); | ||
| 100 | + MicroAPI::CompareScalar<int32_t, CMPMODE::LT>(ltPreg, tmpReg0, dimsize, lowPreg); | ||
| 101 | + MicroAPI::MaskAnd(lowPreg, gtPreg, ltPreg, lowPreg); | ||
| 102 | + MicroAPI::CompareScalar<int32_t, CMPMODE::GE>(gtPreg, tmpReg1, 0, highPreg); | ||
| 103 | + MicroAPI::CompareScalar<int32_t, CMPMODE::LT>(ltPreg, tmpReg1, dimsize, highPreg); | ||
| 104 | + MicroAPI::MaskAnd(highPreg, gtPreg, ltPreg, highPreg); | ||
| 105 | + MicroAPI::MaskDeInterleave<int16_t>(gatherMask, ltPreg, lowPreg, highPreg); | ||
| 106 | + } | ||
| 107 | + } | ||
| 108 | +} | ||
| 109 | + | ||
| 110 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 111 | +class Gatherv2SimdLastGather { | ||
| 112 | +public: | ||
| 113 | + __aicore__ inline Gatherv2SimdLastGather(TPipe* pipe) : pipe_(pipe){}; | ||
| 114 | + __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, | ||
| 115 | + const GatherV2LastTilingData* tilingData); | ||
| 116 | + __aicore__ inline void Process(); | ||
| 117 | + __aicore__ inline void CopyInIndices(int64_t offset, uint32_t nBurst, uint32_t copyLen); | ||
| 118 | + __aicore__ inline void CopyInX(int64_t offset, uint32_t nBurst, uint32_t copyLen); | ||
| 119 | + __aicore__ inline void CopyOut(int64_t offset, uint32_t nBurst, uint32_t copyLen); | ||
| 120 | + __aicore__ inline void ConvertNegIndices(const LocalTensor<int32_t>& indicesLocal, int32_t num); | ||
| 121 | + __aicore__ inline void GatherComputeMultiRows(const LocalTensor<X_T>& xLocal, | ||
| 122 | + const LocalTensor<int32_t>& indicesLocal, int32_t rows, int32_t cols, | ||
| 123 | + int64_t yOffset); | ||
| 124 | + __aicore__ inline void SplitIndicesProcess(); | ||
| 125 | + __aicore__ inline void NoSplitIndicesProcess(); | ||
| 126 | + | ||
| 127 | +private: | ||
| 128 | + GlobalTensor<X_T> xGm_; | ||
| 129 | + GlobalTensor<INDICES_T> indicesGm_; | ||
| 130 | + GlobalTensor<X_T> yGm_; | ||
| 131 | + TPipe* pipe_; | ||
| 132 | + TQue<QuePosition::VECIN, 1> inQueue_; | ||
| 133 | + TQue<QuePosition::VECIN, 1> indexQue_; | ||
| 134 | + TQue<QuePosition::VECOUT, 1> outQueue_; | ||
| 135 | + TBuf<QuePosition::VECCALC> indexBuf_; | ||
| 136 | + const GatherV2LastTilingData* tilingData_; | ||
| 137 | + int32_t blockIdx_{0}; | ||
| 138 | + int32_t pBlock_{0}; | ||
| 139 | + int32_t gBlock_{0}; | ||
| 140 | + int32_t curGatherFactor_{0}; | ||
| 141 | +}; | ||
| 142 | + | ||
| 143 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 144 | +__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::Init( | ||
| 145 | + GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, const GatherV2LastTilingData* tilingData) | ||
| 146 | +{ | ||
| 147 | + tilingData_ = tilingData; | ||
| 148 | + blockIdx_ = static_cast<int32_t>(GetBlockIdx()); | ||
| 149 | + pBlock_ = blockIdx_ / tilingData->coreInCols; | ||
| 150 | + gBlock_ = blockIdx_ % tilingData->coreInCols; | ||
| 151 | + xGm_.SetGlobalBuffer((__gm__ X_T*)x + pBlock_ * tilingData->blockFactor * tilingData->gatherDimSize); | ||
| 152 | + | ||
| 153 | + indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices + gBlock_ * tilingData->gFactor); | ||
| 154 | + yGm_.SetGlobalBuffer((__gm__ X_T*)y + pBlock_ * tilingData->blockFactor * tilingData->gatherSize + | ||
| 155 | + gBlock_ * tilingData->gFactor); | ||
| 156 | + pipe_->InitBuffer(inQueue_, tilingData_->inputNum, tilingData_->inputUbSize); | ||
| 157 | + pipe_->InitBuffer(outQueue_, 2, tilingData_->outUbSize); | ||
| 158 | + pipe_->InitBuffer(indexQue_, tilingData_->indicesNum, tilingData_->indiceUbSize); | ||
| 159 | + pipe_->InitBuffer(indexBuf_, tilingData_->indiceCastUbSize); | ||
| 160 | + curGatherFactor_ = (gBlock_ == tilingData->coreInCols - 1) ? | ||
| 161 | + tilingData->gatherSize - (tilingData->coreInCols - 1) * tilingData->gFactor : | ||
| 162 | + tilingData->gFactor; | ||
| 163 | +} | ||
| 164 | + | ||
| 165 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 166 | +__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::CopyInIndices(int64_t offset, | ||
| 167 | + uint32_t nBurst, | ||
| 168 | + uint32_t copyLen) | ||
| 169 | +{ | ||
| 170 | + DataCopyPadExtParams<INDICES_T> dataCopyPadExtParams; | ||
| 171 | + dataCopyPadExtParams.isPad = false; | ||
| 172 | + dataCopyPadExtParams.leftPadding = 0; | ||
| 173 | + dataCopyPadExtParams.rightPadding = 0; | ||
| 174 | + dataCopyPadExtParams.paddingValue = 0; | ||
| 175 | + LocalTensor<INDICES_T> indicesLocal = indexQue_.AllocTensor<INDICES_T>(); | ||
| 176 | + DataCopyExtParams dataCoptExtParams; | ||
| 177 | + dataCoptExtParams.blockCount = nBurst; | ||
| 178 | + dataCoptExtParams.blockLen = copyLen * sizeof(INDICES_T); | ||
| 179 | + dataCoptExtParams.srcStride = 0; | ||
| 180 | + dataCoptExtParams.dstStride = 0; | ||
| 181 | + DataCopyPad(indicesLocal, indicesGm_[offset], dataCoptExtParams, dataCopyPadExtParams); | ||
| 182 | + indexQue_.EnQue<INDICES_T>(indicesLocal); | ||
| 183 | +} | ||
| 184 | + | ||
| 185 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 186 | +__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::CopyInX(int64_t offset, | ||
| 187 | + uint32_t nBurst, | ||
| 188 | + uint32_t copyLen) | ||
| 189 | +{ | ||
| 190 | + DataCopyPadExtParams<X_T> dataCopyPadExtParams; | ||
| 191 | + dataCopyPadExtParams.isPad = false; | ||
| 192 | + dataCopyPadExtParams.leftPadding = 0; | ||
| 193 | + dataCopyPadExtParams.rightPadding = 0; | ||
| 194 | + dataCopyPadExtParams.paddingValue = 0; | ||
| 195 | + LocalTensor<X_T> xLocal = inQueue_.AllocTensor<X_T>(); | ||
| 196 | + DataCopyExtParams dataCoptExtParams; | ||
| 197 | + dataCoptExtParams.blockCount = nBurst; | ||
| 198 | + dataCoptExtParams.blockLen = copyLen * sizeof(X_T); | ||
| 199 | + dataCoptExtParams.srcStride = 0; | ||
| 200 | + dataCoptExtParams.dstStride = 0; | ||
| 201 | + DataCopyPad(xLocal, xGm_[offset], dataCoptExtParams, dataCopyPadExtParams); | ||
| 202 | + inQueue_.EnQue<X_T>(xLocal); | ||
| 203 | +} | ||
| 204 | + | ||
| 205 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 206 | +__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::CopyOut(int64_t offset, | ||
| 207 | + uint32_t nBurst, | ||
| 208 | + uint32_t copyLen) | ||
| 209 | +{ | ||
| 210 | + int32_t ubBlockSize = static_cast<int64_t>(Ops::Base::GetUbBlockSize()); | ||
| 211 | + int32_t srcStride = (tilingData_->ubCols - copyLen) * sizeof(X_T) / ubBlockSize; | ||
| 212 | + int32_t dstStride = (tilingData_->gatherSize - copyLen) * sizeof(X_T); | ||
| 213 | + DataCopyExtParams dataCoptExtParams; | ||
| 214 | + dataCoptExtParams.blockCount = nBurst; | ||
| 215 | + dataCoptExtParams.blockLen = copyLen * sizeof(X_T); | ||
| 216 | + dataCoptExtParams.srcStride = srcStride; | ||
| 217 | + dataCoptExtParams.dstStride = dstStride; | ||
| 218 | + LocalTensor<X_T> yLocal = outQueue_.DeQue<X_T>(); | ||
| 219 | + DataCopyPad(yGm_[offset], yLocal, dataCoptExtParams); | ||
| 220 | + outQueue_.FreeTensor(yLocal); | ||
| 221 | +} | ||
| 222 | + | ||
| 223 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 224 | +__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::SplitIndicesProcess() | ||
| 225 | +{ | ||
| 226 | + int32_t inputFactor = tilingData_->ubRows; | ||
| 227 | + int64_t curBlockFactor = (GetBlockIdx() == tilingData_->needCoreNum - 1) ? tilingData_->tailBlockFactor : | ||
| 228 | + tilingData_->blockFactor; | ||
| 229 | + int64_t inputLoop = (curBlockFactor + inputFactor - 1) / inputFactor; | ||
| 230 | + int32_t tailInputFactor = curBlockFactor - (inputLoop - 1) * inputFactor; | ||
| 231 | + int32_t maxOutCols = tilingData_->ubCols; | ||
| 232 | + int64_t outLoopNum = (curGatherFactor_ + maxOutCols - 1) / maxOutCols; | ||
| 233 | + int32_t tailOutCols = curGatherFactor_ - (outLoopNum - 1) * maxOutCols; | ||
| 234 | + for (int64_t i = 0; i < inputLoop; i++) { | ||
| 235 | + int32_t curInputFactor = (i == inputLoop - 1) ? tailInputFactor : inputFactor; | ||
| 236 | + CopyInX(i * inputFactor * tilingData_->gatherDimSize, 1, curInputFactor * tilingData_->gatherDimSize); | ||
| 237 | + LocalTensor<X_T> xLocal = inQueue_.DeQue<X_T>(); | ||
| 238 | + for (int64_t j = 0; j < outLoopNum; j++) { | ||
| 239 | + int32_t curOutCols = (j == outLoopNum - 1) ? tailOutCols : maxOutCols; | ||
| 240 | + CopyInIndices(j * maxOutCols, 1, curOutCols); | ||
| 241 | + LocalTensor<INDICES_T> indicesLocal = indexQue_.DeQue<INDICES_T>(); | ||
| 242 | + if constexpr (sizeof(INDICES_T) == sizeof(int64_t)) { | ||
| 243 | + LocalTensor<int32_t> tmpLocal = indexBuf_.Get<int32_t>(); | ||
| 244 | + Cast(tmpLocal, indicesLocal, AscendC::RoundMode::CAST_NONE, curOutCols); | ||
| 245 | + indexQue_.FreeTensor(indicesLocal); | ||
| 246 | + ConvertNegIndices(tmpLocal, curOutCols); | ||
| 247 | + GatherComputeMultiRows(xLocal, tmpLocal, curInputFactor, curOutCols, | ||
| 248 | + i * inputFactor * tilingData_->gatherSize + j * maxOutCols); | ||
| 249 | + } else { | ||
| 250 | + ConvertNegIndices(indicesLocal, curOutCols); | ||
| 251 | + GatherComputeMultiRows(xLocal, indicesLocal, curInputFactor, curOutCols, | ||
| 252 | + i * inputFactor * tilingData_->gatherSize + j * maxOutCols); | ||
| 253 | + indexQue_.FreeTensor(indicesLocal); | ||
| 254 | + } | ||
| 255 | + } | ||
| 256 | + inQueue_.FreeTensor(xLocal); | ||
| 257 | + } | ||
| 258 | +} | ||
| 259 | + | ||
| 260 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 261 | +__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::NoSplitIndicesProcess() | ||
| 262 | +{ | ||
| 263 | + int32_t maxIndcieNum = tilingData_->outUbSize / sizeof(X_T); | ||
| 264 | + int64_t indiceLoopNum = (curGatherFactor_ + maxIndcieNum - 1) / maxIndcieNum; | ||
| 265 | + int32_t tailIndicesNum = curGatherFactor_ - (indiceLoopNum - 1) * maxIndcieNum; | ||
| 266 | + int32_t inputFactor = tilingData_->ubRows; | ||
| 267 | + int64_t curBlockFactor = (GetBlockIdx() == tilingData_->needCoreNum - 1) ? tilingData_->tailBlockFactor : | ||
| 268 | + tilingData_->blockFactor; | ||
| 269 | + int64_t inputLoop = (curBlockFactor + inputFactor - 1) / inputFactor; | ||
| 270 | + int32_t tailInputFactor = curBlockFactor - (inputLoop - 1) * inputFactor; | ||
| 271 | + CopyInIndices(0, 1, curGatherFactor_); | ||
| 272 | + LocalTensor<INDICES_T> indicesLocal = indexQue_.DeQue<INDICES_T>(); | ||
| 273 | + LocalTensor<int32_t> tmpLocal = indexBuf_.Get<int32_t>(); | ||
| 274 | + if constexpr (sizeof(INDICES_T) == sizeof(int64_t)) { | ||
| 275 | + Cast(tmpLocal, indicesLocal, AscendC::RoundMode::CAST_NONE, curGatherFactor_); | ||
| 276 | + ConvertNegIndices(tmpLocal, curGatherFactor_); | ||
| 277 | + } else { | ||
| 278 | + ConvertNegIndices(indicesLocal, curGatherFactor_); | ||
| 279 | + } | ||
| 280 | + | ||
| 281 | + for (int64_t i = 0; i < inputLoop; i++) { | ||
| 282 | + int32_t curInputFactor = (i == inputLoop - 1) ? tailInputFactor : inputFactor; | ||
| 283 | + CopyInX(i * inputFactor * tilingData_->gatherDimSize, 1, curInputFactor * tilingData_->gatherDimSize); | ||
| 284 | + LocalTensor<X_T> xLocal = inQueue_.DeQue<X_T>(); | ||
| 285 | + if constexpr (sizeof(INDICES_T) == sizeof(int64_t)) { | ||
| 286 | + GatherComputeMultiRows(xLocal, tmpLocal, curInputFactor, curGatherFactor_, | ||
| 287 | + i * inputFactor * tilingData_->gatherSize); | ||
| 288 | + } else { | ||
| 289 | + GatherComputeMultiRows(xLocal, indicesLocal, curInputFactor, curGatherFactor_, | ||
| 290 | + i * inputFactor * tilingData_->gatherSize); | ||
| 291 | + } | ||
| 292 | + inQueue_.FreeTensor(xLocal); | ||
| 293 | + } | ||
| 294 | + indexQue_.FreeTensor(indicesLocal); | ||
| 295 | +} | ||
| 296 | + | ||
| 297 | +template <typename X_T, typename indiceType> | ||
| 298 | +__simd_vf__ inline void GatherComputeMultiRowsVF(__ubuf__ int32_t* indiceAddr, __ubuf__ X_T* xAddr, __ubuf__ X_T* yAddr, | ||
| 299 | + int32_t curCols, int32_t indicesOffset, uint16_t vfLoopNum, | ||
| 300 | + int32_t dimSize, int16_t vfLen, uint16_t vfRowsLoop, int32_t colInUb) | ||
| 301 | +{ | ||
| 302 | + using RegDstT = typename std::conditional<sizeof(X_T) == sizeof(int64_t), | ||
| 303 | + MicroAPI::RegTensor<X_T, MicroAPI::RegTraitNumTwo>, | ||
| 304 | + MicroAPI::RegTensor<X_T>>::type; | ||
| 305 | + RegDstT vd0; | ||
| 306 | + MicroAPI::RegTensor<indiceType> vregIndcie; | ||
| 307 | + MicroAPI::MaskReg gatherMask; | ||
| 308 | + uint32_t size = curCols; | ||
| 309 | + __local_mem__ int32_t* curIndiceAddr = indiceAddr + indicesOffset; | ||
| 310 | + for (uint16_t i = 0; i < vfLoopNum; i++) { | ||
| 311 | + MicroAPI::MaskReg preg0 = AscendC::MicroAPI::UpdateMask<indiceType>(size); | ||
| 312 | + LoadIndices<indiceType, int32_t>(vregIndcie, gatherMask, curIndiceAddr, preg0, dimSize); | ||
| 313 | + curIndiceAddr += vfLen; | ||
| 314 | + __local_mem__ X_T* curyAddr = yAddr + i * vfLen; | ||
| 315 | + __local_mem__ X_T* curXaddr = xAddr; | ||
| 316 | + for (uint16_t j = 0; j < vfRowsLoop; j++) { | ||
| 317 | + if constexpr (sizeof(X_T) == 1) { | ||
| 318 | + MicroAPI::DataCopyGather((MicroAPI::RegTensor<int16_t>&)vd0, curXaddr, vregIndcie, gatherMask); | ||
| 319 | + MicroAPI::DataCopy<X_T, AscendC::MicroAPI::StoreDist::DIST_PACK_B16>(curyAddr, vd0, preg0); | ||
| 320 | + } else { | ||
| 321 | + MicroAPI::DataCopyGather(vd0, curXaddr, vregIndcie, gatherMask); | ||
| 322 | + MicroAPI::DataCopy(curyAddr, vd0, preg0); | ||
| 323 | + } | ||
| 324 | + curXaddr += dimSize; | ||
| 325 | + curyAddr += colInUb; | ||
| 326 | + } | ||
| 327 | + } | ||
| 328 | +} | ||
| 329 | + | ||
| 330 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 331 | +__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::GatherComputeMultiRows( | ||
| 332 | + const LocalTensor<X_T>& xLocal, const LocalTensor<int32_t>& indicesLocal, int32_t rows, int32_t cols, | ||
| 333 | + int64_t yOffset) | ||
| 334 | +{ | ||
| 335 | + __local_mem__ X_T* xAddr = (__local_mem__ X_T*)xLocal.GetPhyAddr(); | ||
| 336 | + __local_mem__ int32_t* indiceAddr = (__local_mem__ int32_t*)indicesLocal.GetPhyAddr(); | ||
| 337 | + using indiceType = typename IndexTypeGet<X_T>::type; | ||
| 338 | + int32_t colInUb = tilingData_->ubCols; | ||
| 339 | + int32_t loopInCols = (cols + colInUb - 1) / colInUb; | ||
| 340 | + int32_t tailCols = cols - (loopInCols - 1) * colInUb; | ||
| 341 | + | ||
| 342 | + for (int32_t colIdx = 0; colIdx < loopInCols; colIdx++) { | ||
| 343 | + LocalTensor<X_T> yLocal = outQueue_.AllocTensor<X_T>(); | ||
| 344 | + __local_mem__ X_T* yAddr = (__local_mem__ X_T*)yLocal.GetPhyAddr(); | ||
| 345 | + int32_t curCols = (colIdx == loopInCols - 1) ? tailCols : colInUb; | ||
| 346 | + constexpr int16_t vfLen = AscendC::VECTOR_REG_WIDTH / sizeof(indiceType); | ||
| 347 | + uint16_t vfLoopNum = (curCols + vfLen - 1) / vfLen; | ||
| 348 | + int32_t dimSize = tilingData_->gatherDimSize; | ||
| 349 | + uint16_t vfRowsLoop = rows; | ||
| 350 | + int32_t indicesOffset = colIdx * colInUb; | ||
| 351 | + GatherComputeMultiRowsVF<X_T, indiceType>(indiceAddr, xAddr, yAddr, curCols, indicesOffset, vfLoopNum, dimSize, | ||
| 352 | + vfLen, vfRowsLoop, colInUb); | ||
| 353 | + | ||
| 354 | + outQueue_.EnQue(yLocal); | ||
| 355 | + CopyOut(yOffset + colIdx * colInUb, rows, curCols); | ||
| 356 | + } | ||
| 357 | +} | ||
| 358 | + | ||
| 359 | +__simd_vf__ inline void ConvertNegIndicesVF(__ubuf__ int32_t* indiceAddr, int16_t vfLen, uint16_t vfLoopNum, | ||
| 360 | + int32_t dimSize, int32_t num) | ||
| 361 | +{ | ||
| 362 | + MicroAPI::RegTensor<int32_t> indice; | ||
| 363 | + MicroAPI::RegTensor<int32_t> dst; | ||
| 364 | + MicroAPI::MaskReg ltPreg; | ||
| 365 | + uint32_t size = num; | ||
| 366 | + __local_mem__ int32_t* curIndiceAddr = indiceAddr; | ||
| 367 | + for (uint16_t i = 0; i < vfLoopNum; i++) { | ||
| 368 | + MicroAPI::MaskReg preg = AscendC::MicroAPI::UpdateMask<int32_t>(size); | ||
| 369 | + MicroAPI::DataCopy(indice, curIndiceAddr); | ||
| 370 | + MicroAPI::CompareScalar<int32_t, CMPMODE::LT>(ltPreg, indice, 0, preg); | ||
| 371 | + MicroAPI::Adds(dst, indice, dimSize, ltPreg); | ||
| 372 | + MicroAPI::Copy<int32_t, MicroAPI::MaskMergeMode::MERGING>(indice, dst, ltPreg); | ||
| 373 | + MicroAPI::DataCopy(curIndiceAddr, indice, preg); | ||
| 374 | + curIndiceAddr += vfLen; | ||
| 375 | + } | ||
| 376 | +} | ||
| 377 | + | ||
| 378 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 379 | +__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::ConvertNegIndices( | ||
| 380 | + const LocalTensor<int32_t>& indicesLocal, int32_t num) | ||
| 381 | +{ | ||
| 382 | + if constexpr (NEG_INDICE_SUPPORT) { | ||
| 383 | + __local_mem__ int32_t* indiceAddr = (__local_mem__ int32_t*)indicesLocal.GetPhyAddr(); | ||
| 384 | + constexpr int16_t vfLen = AscendC::VECTOR_REG_WIDTH / sizeof(int32_t); | ||
| 385 | + uint16_t vfLoopNum = (num + vfLen - 1) / vfLen; | ||
| 386 | + int32_t dimSize = tilingData_->gatherDimSize; | ||
| 387 | + ConvertNegIndicesVF(indiceAddr, vfLen, vfLoopNum, dimSize, num); | ||
| 388 | + } | ||
| 389 | +} | ||
| 390 | + | ||
| 391 | +template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT> | ||
| 392 | +__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::Process() | ||
| 393 | +{ | ||
| 394 | + if (blockIdx_ >= tilingData_->needCoreNum) { | ||
| 395 | + return; | ||
| 396 | + } | ||
| 397 | + | ||
| 398 | + if (tilingData_->splitMode == 0) { | ||
| 399 | + NoSplitIndicesProcess(); | ||
| 400 | + } else if (tilingData_->splitMode == 1) { | ||
| 401 | + SplitIndicesProcess(); | ||
| 402 | + } | ||
| 403 | +} | ||
| 404 | + | ||
| 405 | +#endif // KirinX90 不支持 MicroAPI | ||
| 406 | +} // namespace gatherv2 | ||
| 407 | +#endif // GATHER_V2_SIMD_LAST_GATHER | ||


修改Copyright为2026