Merged
KirinX90/Kirin9030系列支持QuantBatchMatmulV4/DynmicQuant/RmsNorm/Glue/ScatterNdUpdate/GatherV2/MatMulV3 #13547
chenlong1Add on 18 days ago
KirinX90/Kirin9030系列支持QuantBatchMatmulV4/DynmicQuant/RmsNorm/Glue/ScatterNdUpdate/GatherV2/MatMulV3 #13547
Merged
chenlong1Add on 18 days ago
Total: 89 files changed+19626-426
@@ -1,5 +1,5 @@
1# ----------------------------------------------------------------------------1# ----------------------------------------------------------------------------
2-# Copyright (c) 2025 Huawei Technologies Co., Ltd.2+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
3# This program is free software, you can redistribute it and/or modify it under the terms and conditions of3# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4# CANN Open Software License Agreement Version 2.0 (the "License").4# CANN Open Software License Agreement Version 2.0 (the "License").
5# Please refer to the License for details. You may not use this file except in compliance with the License.5# Please refer to the License for details. You may not use this file except in compliance with the License.
@@ -9,7 +9,15 @@
9# ----------------------------------------------------------------------------9# ----------------------------------------------------------------------------
10 10 
11# 设置算子定义时支持的芯片类型11# 设置算子定义时支持的芯片类型
12-set(SUPPORT_COMPUTE_UNIT "ascend950" "mc62" "ascend350" "ascend960dt" "ascend5162")12+set(SUPPORT_COMPUTE_UNIT "ascend950" "mc62" "ascend350" "ascend960dt" "ascend5162" "kirinx90" "kirin9030")
13# 设置每种芯片类型对应的tiling文件目录,即采用op_host目录下哪个文件夹下的tiling文件编译13# 设置每种芯片类型对应的tiling文件目录,即采用op_host目录下哪个文件夹下的tiling文件编译
14-set(SUPPORT_TILING_DIR "arch35" "arch35" "arch35" "arch35" "arch35")14+set(SUPPORT_TILING_DIR "arch35" "arch35" "arch35" "arch35" "arch35" "arch31" "arch31")
15add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE gelu ACLNNTYPE aclnn_exclude COMPUTE_UNIT ${SUPPORT_COMPUTE_UNIT} TILING_DIR ${SUPPORT_TILING_DIR} DISABLE_IN_OPP TRUE)15add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE gelu ACLNNTYPE aclnn_exclude COMPUTE_UNIT ${SUPPORT_COMPUTE_UNIT} TILING_DIR ${SUPPORT_TILING_DIR} DISABLE_IN_OPP TRUE)
16+ 
17+# arch31: kirinx90/kirin9030 使用 arch31/gelu.cpp 作为 kernel 入口
18+add_kernel_sources(
19+ KERNEL_SRC arch31/gelu.cpp
20+ COMPUTE_UNITS kirinx90 kirin9030
21+ IMPL_MODE high_performance,high_precision
22+ AUTO_SYNC false
23+)
@@ -10,6 +10,8 @@
10| <term>Atlas 200I/500 A2推理产品</term> | √ |10| <term>Atlas 200I/500 A2推理产品</term> | √ |
11| <term>Atlas推理系列产品</term> | √ |11| <term>Atlas推理系列产品</term> | √ |
12| <term>Atlas训练系列产品</term> | √ |12| <term>Atlas训练系列产品</term> | √ |
13+| <term>KirinX90处理器系列产品</term> | √ |
14+| <term>Kirin9030处理器系列产品</term> | √ |
13 15 
14## 功能说明16## 功能说明
15 17 
@@ -69,6 +71,7 @@
69 </table>71 </table>
70 72 
71 - <term>Atlas 200I/500 A2推理产品</term>、<term>Atlas推理系列产品</term>、<term>Atlas训练系列产品</term>:数据类型支持FLOAT、FLOAT16。73 - <term>Atlas 200I/500 A2推理产品</term>、<term>Atlas推理系列产品</term>、<term>Atlas训练系列产品</term>:数据类型支持FLOAT、FLOAT16。
74+ - <term>KirinX90处理器系列产品</term>、<term>Kirin9030处理器系列产品</term>:数据类型支持FLOAT、FLOAT16。
72 75 
73## 约束说明76## 约束说明
74 77 
@@ -12,6 +12,8 @@ English | [简体中文](./README.md)
12| <term>Atlas 200I/500 A2 inference products</term> | √ |12| <term>Atlas 200I/500 A2 inference products</term> | √ |
13| <term>Atlas inference products</term> | √ |13| <term>Atlas inference products</term> | √ |
14| <term>Atlas training products</term> | √ |14| <term>Atlas training products</term> | √ |
15+| <term>KirinX90 processor series products</term> | √ |
16+| <term>Kirin9030 processor series products</term> | √ |
15 17 
16## Function18## Function
17 19 
@@ -71,6 +73,7 @@ English | [简体中文](./README.md)
71 </table>73 </table>
72 74 
73 - <term>Atlas 200I/500 A2 inference products</term>, <term>Atlas inference products</term>, and <term>Atlas training products</term>: The supported data types are FLOAT and FLOAT16.75 - <term>Atlas 200I/500 A2 inference products</term>, <term>Atlas inference products</term>, and <term>Atlas training products</term>: The supported data types are FLOAT and FLOAT16.
76+ - <term>KirinX90 processor series products</term>, <term>Kirin9030 processor series products</term>: The supported data types are FLOAT and FLOAT16.
74 77 
75## Constraints78## Constraints
76 79 
@@ -0,0 +1,148 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
zengjuan
zengjuanzengjuan18 days ago

修改Copyright为2026

likedislike
xiongtingxiang
xiongtingxiang
17 days ago Comments:
likedislike
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include "gelu_tiling_arch31.h"
12+#include "tiling/platform/platform_ascendc.h"
13+#include <graph/utils/type_utils.h>
14+#include "register/tilingdata_base.h"
15+#include "register/op_impl_registry.h"
16+#include "../op_kernel/arch31/gelu_dag_arch31.h"
17+#include "../op_kernel/arch31/gelu_struct.h"
18+#include "op_host/tiling_util.h"
19+#include <iostream>
20+ 
21+namespace optiling {
22+const size_t ASCEND_WORKSPACE = 0; // 0M
23+const gert::Shape g_vec_1_shape = {1};
zengjuan
zengjuanzengjuan18 days ago

变量命名风格不统一

likedislike
xiongtingxiang
xiongtingxiang
18 days ago Comments:
likedislike
24+ 
25+ge::graphStatus GeluTiling::CalcInputDtype()
26+{
27+ OP_LOGD(tilingContext->GetNodeName(), "GeluTiling CalcInputDtype enter.");
28+ auto inputDesc = tilingContext->GetInputDesc(0);
29+ OP_CHECK_NULL_WITH_CONTEXT(tilingContext, inputDesc);
30+ this->inputDtype = inputDesc->GetDataType();
31+ OP_CHECK_IF(this->inputDtype != ge::DT_FLOAT16 && this->inputDtype != ge::DT_FLOAT,
32+ OP_LOGE_FOR_INVALID_DTYPE_WITH_REASON(
33+ tilingContext->GetNodeName(), "x",
34+ ge::TypeUtils::DataTypeToSerialString(static_cast<ge::DataType>(this->inputDtype)),
35+ "The dtype of x must be DT_FLOAT16 or DT_FLOAT"),
36+ return ge::GRAPH_FAILED);
37+ return ge::GRAPH_SUCCESS;
38+}
39+ 
40+static inline const gert::Shape& EnsureNotScalar(const gert::Shape& in_shape)
41+{
42+ if (in_shape.IsScalar()) {
43+ return g_vec_1_shape;
44+ }
45+ return in_shape;
46+}
47+ 
48+ge::graphStatus GeluTiling::CheckShape()
49+{
50+ OP_LOGD(tilingContext->GetNodeName(), "GeluTiling CheckShape enter.");
51+ auto inputStorageShape = tilingContext->GetInputShape(0);
52+ OP_CHECK_NULL_WITH_CONTEXT(tilingContext, inputStorageShape);
53+ const gert::Shape& inputYShape = EnsureNotScalar(inputStorageShape->GetStorageShape());
54+ 
55+ auto outputStorageShape = tilingContext->GetOutputShape(0);
56+ OP_CHECK_NULL_WITH_CONTEXT(tilingContext, outputStorageShape);
57+ const gert::Shape& outputZShape = EnsureNotScalar(outputStorageShape->GetStorageShape());
58+ 
59+ OP_CHECK_IF(inputYShape != outputZShape,
60+ OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON(
61+ tilingContext->GetNodeName(), "x, y",
62+ Ops::Base::ToString(inputYShape) + ", " + Ops::Base::ToString(outputZShape),
63+ "The shapes of x and y must be the same"),
64+ return ge::GRAPH_FAILED);
65+ return ge::GRAPH_SUCCESS;
66+}
67+ 
68+ge::graphStatus GeluTiling::CalcOutputDtype()
69+{
70+ OP_LOGD(tilingContext->GetNodeName(), "GeluTiling CalcOutputDtype enter.");
71+ auto outputDesc = tilingContext->GetOutputDesc(0);
72+ OP_CHECK_NULL_WITH_CONTEXT(tilingContext, outputDesc);
73+ this->outputDtype = outputDesc->GetDataType();
74+ OP_CHECK_IF(this->outputDtype != this->inputDtype,
75+ OP_LOGE_FOR_INVALID_DTYPES_WITH_REASON(
76+ tilingContext->GetNodeName(), "x, y",
77+ ge::TypeUtils::DataTypeToSerialString(static_cast<ge::DataType>(this->inputDtype)) + ", " +
78+ ge::TypeUtils::DataTypeToSerialString(static_cast<ge::DataType>(this->outputDtype)),
79+ "The dtypes of x and y must be the same"),
80+ return ge::GRAPH_FAILED);
81+ return ge::GRAPH_SUCCESS;
82+}
83+ 
84+ge::graphStatus GeluTiling::RunTiling()
85+{
86+ auto tiling = tilingContext->GetTilingData<Ops::Base::EleBaseTilingData16B>();
87+ 
88+ OP_LOGD(tilingContext->GetNodeName(), "GeluTiling RunTiling enter.");
89+ ElewiseBaseTiling elewiseBaseTiling(tilingContext);
90+ OP_CHECK_IF(CalcInputDtype() == ge::GRAPH_FAILED, OP_LOGE(tilingContext, "get input dtype failed"),
91+ return ge::GRAPH_FAILED);
92+ OP_CHECK_IF(CalcOutputDtype() == ge::GRAPH_FAILED, OP_LOGE(tilingContext, "get output dtype failed"),
93+ return ge::GRAPH_FAILED);
94+ OP_CHECK_IF(CheckShape() == ge::GRAPH_FAILED, OP_LOGE(tilingContext, "check shape failed"),
95+ return ge::GRAPH_FAILED);
96+ 
97+ ge::graphStatus baseTilingResult = ge::GRAPH_FAILED;
98+ if (this->outputDtype == ge::DT_FLOAT16) {
99+ dType = TPL_FP16;
100+ baseTilingResult = elewiseBaseTiling.DoTiling<GeluOp::GeluDAG<half>::OpDag>(*tiling);
101+ } else if (this->outputDtype == ge::DT_FLOAT) {
102+ dType = TPL_FP32;
103+ baseTilingResult = elewiseBaseTiling.DoTiling<GeluOp::GeluDAG<float>::OpDag>(*tiling);
104+ } else {
105+ OP_LOGE_FOR_INVALID_DTYPE_WITH_REASON(
106+ tilingContext->GetNodeName(), "y",
107+ ge::TypeUtils::DataTypeToSerialString(static_cast<ge::DataType>(this->outputDtype)),
108+ "The dtype of y must be DT_FLOAT16 or DT_FLOAT");
109+ return ge::GRAPH_FAILED;
110+ }
111+ OP_CHECK_IF(baseTilingResult == ge::GRAPH_FAILED, OP_LOGE(tilingContext, "elewiseBaseTiling failed"),
112+ return ge::GRAPH_FAILED);
113+ 
114+ size_t* currentWorkspace = tilingContext->GetWorkspaceSizes(1);
115+ OP_CHECK_NULL_WITH_CONTEXT(tilingContext, currentWorkspace);
116+ currentWorkspace[0] = ASCEND_WORKSPACE;
117+ 
118+ const uint64_t tilingKey = GET_TPL_TILING_KEY(1, dType);
119+ OP_LOGD(tilingContext->GetNodeName(), "[TilingData] : tilingKey=%lu", tilingKey);
120+ tilingContext->SetTilingKey(tilingKey);
121+ tilingContext->SetBlockDim(elewiseBaseTiling.GetBlockDim());
122+ return ge::GRAPH_SUCCESS;
123+}
124+ 
125+static ge::graphStatus Tiling4Gelu(gert::TilingContext* tilingContextGen)
126+{
127+ OP_LOGD(tilingContextGen->GetNodeName(), "Tiling4Gelu rt2.0 is running.");
128+ auto compileInfo = tilingContextGen->GetCompileInfo<GeluCompileInfo>();
129+ OP_CHECK_NULL_WITH_CONTEXT(tilingContextGen, compileInfo);
130+ 
131+ GeluTiling baseOpTiling(tilingContextGen);
132+ return baseOpTiling.RunTiling();
133+}
134+ 
135+ge::graphStatus TilingPrepareForGelu(gert::TilingParseContext* context)
136+{
137+ auto compileInfoPtr = context->GetCompiledInfo<GeluCompileInfo>();
138+ OP_CHECK_NULL_WITH_CONTEXT(context, compileInfoPtr);
139+ fe::PlatFormInfos* platformInfoPtr = context->GetPlatformInfo();
140+ OP_CHECK_NULL_WITH_CONTEXT(context, platformInfoPtr);
141+ auto ascendcPlatform = platform_ascendc::PlatformAscendC(platformInfoPtr);
142+ compileInfoPtr->coreNum = ascendcPlatform.GetCoreNumAiv();
143+ ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, compileInfoPtr->ubSize);
144+ return ge::GRAPH_SUCCESS;
145+}
146+ 
147+IMPL_OP_OPTILING(Gelu).Tiling(Tiling4Gelu).TilingParse<GeluCompileInfo>(TilingPrepareForGelu);
148+} // namespace optiling
@@ -0,0 +1,41 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+#ifndef OPS_BUILD_IN_OP_TILING_RUNTIME_GELU_REGBASE_OPTILING
11+#define OPS_BUILD_IN_OP_TILING_RUNTIME_GELU_REGBASE_OPTILING
12+ 
13+#include "register/tilingdata_base.h"
14+#include "op_common/atvoss/elewise/elewise_tiling.h"
15+ 
16+namespace optiling {
17+using namespace Ops::Base;
18+ 
19+struct GeluCompileInfo {
20+ uint64_t coreNum;
21+ uint64_t ubSize;
22+};
23+ 
24+class GeluTiling {
25+public:
26+ explicit GeluTiling(gert::TilingContext* context) : tilingContext(context) {};
27+ ge::graphStatus RunTiling();
28+ 
29+protected:
30+ ge::graphStatus CalcOutputDtype();
31+ ge::graphStatus CalcInputDtype();
32+ ge::graphStatus CheckShape();
33+ 
34+private:
35+ gert::TilingContext* tilingContext;
36+ ge::DataType outputDtype = ge::DT_UNDEFINED;
37+ ge::DataType inputDtype = ge::DT_UNDEFINED;
38+ uint64_t dType = 0;
39+};
40+} // namespace optiling
41+#endif // OPS_BUILD_IN_OP_TILING_RUNTIME_GELU_REGBASE_OPTILING
@@ -1,5 +1,5 @@
1/**1/**
2- * Copyright (c) 2025 Huawei Technologies Co., Ltd.2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3 * This program is free software, you can redistribute it and/or modify it under the terms and conditions of3 * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4 * CANN Open Software License Agreement Version 2.0 (the "License").4 * CANN Open Software License Agreement Version 2.0 (the "License").
5 * Please refer to the License for details. You may not use this file except in compliance with the License.5 * Please refer to the License for details. You may not use this file except in compliance with the License.
@@ -43,6 +43,26 @@ public:
43 this->AICore().AddConfig("ascend350", aicoreConfig);43 this->AICore().AddConfig("ascend350", aicoreConfig);
44 this->AICore().AddConfig("ascend960dt", aicoreConfig);44 this->AICore().AddConfig("ascend960dt", aicoreConfig);
45 this->AICore().AddConfig("ascend5162", aicoreConfig);45 this->AICore().AddConfig("ascend5162", aicoreConfig);
46+ 
47+ OpAICoreConfig kirinConfig;
48+ kirinConfig.Input("x")
49+ .ParamType(REQUIRED)
50+ .DataType({ge::DT_FLOAT16, ge::DT_FLOAT})
51+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
52+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND});
53+ kirinConfig.Output("y")
54+ .ParamType(REQUIRED)
55+ .DataType({ge::DT_FLOAT16, ge::DT_FLOAT})
56+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
57+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND});
58+ kirinConfig.DynamicCompileStaticFlag(true)
59+ .DynamicFormatFlag(false)
60+ .DynamicRankSupportFlag(true)
61+ .DynamicShapeSupportFlag(true)
62+ .NeedCheckSupportFlag(false)
63+ .PrecisionReduceFlag(true);
64+ this->AICore().AddConfig("kirinx90", kirinConfig);
65+ this->AICore().AddConfig("kirin9030", kirinConfig);
46 }66 }
47};67};
48 68 
@@ -0,0 +1,43 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/* !
12+ * \file gelu.cpp
13+ * \brief z = gelu(x), KirinX90/Kirin9030 (arch31) kernel entry
14+ */
15+#include "kernel_operator.h"
16+#include "kernel_tiling/kernel_tiling.h"
17+#include "gelu_dag_arch31.h"
18+#include "gelu_struct.h"
19+#include "atvoss/elewise/elewise_sch_with_scalar.h"
20+#include "atvoss/elewise/elewise_sch_16b.h"
21+#include "atvoss/util/dfx.h"
22+ 
23+using namespace AscendC;
24+using namespace Ops::Base;
25+ 
26+template <uint64_t schMode, uint64_t dType>
27+__global__ __aicore__ void gelu(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling)
28+{
29+ REGISTER_TILING_DEFAULT(EleBaseTilingData16B);
30+ GET_TILING_DATA_PTR_WITH_STRUCT(EleBaseTilingData16B, tilingData, tiling);
31+ KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_AIV_ONLY);
32+ 
33+ if constexpr (dType == TPL_FP16) {
34+ ElementwiseSch16B<schMode, GeluOp::GeluDAG<half>::OpDag> sch(tilingData);
35+ sch.Init(x, y);
36+ sch.Process();
37+ } else if constexpr (dType == TPL_FP32) {
38+ ElementwiseSch16B<schMode, GeluOp::GeluDAG<float>::OpDag> sch(tilingData);
39+ sch.Init(x, y);
40+ sch.Process();
41+ }
42+ return;
43+}
@@ -0,0 +1,88 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/* !
12+ * \file gelu_dag_arch31.h
13+ * \brief
14+ */
15+ 
16+#ifndef CANN_CUSTOM_OPS_GELU_DAG_H
17+#define CANN_CUSTOM_OPS_GELU_DAG_H
18+ 
19+#include "atvoss/util/dag.h"
20+#include "atvoss/util/vec.h"
21+#include "atvoss/util/placeholder.h"
22+ 
23+namespace GeluOp {
24+using namespace Ops::Base;
25+const float NEG_SQRT_EIGHT_OVER_PI = -1.595769121 * 0.044715;
26+const float TANH_APPROX_FACTOR = 1 / 0.044715;
27+const int CAST_MODE_NONE = 0;
28+const int CAST_MODE_RINT = 1;
29+// tbe realization: x / (1 + e^(-1.5957691(x + 0.044715 x * x * x)))
30+// current realization: x / (1 + e^(-1.5957691*0.044715(x/0.044715 + x^3)))
31+namespace GeluDag1 {
32+template <class T>
33+struct GeluCustom : public Vec::ElemwiseUnaryOP<T, T> {
34+ __aicore__ inline GeluCustom(LocalTensor<T>& dst, LocalTensor<T>& src, uint32_t count)
35+ {
36+#ifdef __CCE_AICORE__
37+ uint32_t dtypeSize = sizeof(T);
38+ uint32_t vl = VECTOR_REG_WIDTH / dtypeSize;
39+ uint16_t loopNum = CeilDivision(count, vl);
40+ uint32_t vlSize = vl;
41+ __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr();
42+ __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr();
43+ 
44+ MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput;
45+ MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputSqr;
46+ MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputCub;
47+ MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput;
48+ MicroAPI::MaskReg mask;
49+ if constexpr (std::is_same_v<T, float>) {
50+ __VEC_SCOPE__
51+ {
52+ for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
53+ mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count);
54+ // OpCopyIn
55+ MicroAPI::DataCopy(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize));
56+ MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask);
57+ MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask);
58+ MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask);
59+ MicroAPI::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask);
60+ MicroAPI::Exp(vregInputCub, vregInputCub, mask);
61+ MicroAPI::Adds(vregInputCub, vregInputCub, (float)1.0, mask);
62+ MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask);
63+ 
64+ // OpCopyOut
65+ MicroAPI::DataCopy((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask);
66+ }
67+ }
68+ }
69+#endif
70+ }
71+};
72+} // namespace GeluDag1
73+ 
74+template <typename U, typename T = float>
75+struct GeluDAG {
76+ using OpCopyIn0 = Bind<Vec::CopyIn<U>, Placeholder::In0<U>>;
77+ using OpCopyIn0Cast = Bind<Vec::Cast<T, U, CAST_MODE_NONE>, OpCopyIn0>;
78+ 
79+ using OpLogResult = Bind<GeluDag1::GeluCustom<T>, OpCopyIn0Cast>;
80+ using OpResultCast = Bind<Vec::Cast<U, T, CAST_MODE_RINT>, OpLogResult>;
81+ using OpCopyOut = Bind<Vec::CopyOut<U>, Placeholder::Out0<U>, OpResultCast>;
82+ 
83+ using Outputs = Elems<OpCopyOut>;
84+ using MemCfg = MemOptCfg<MemLevel::LEVEL_2>;
85+ using OpDag = DAGSch<Outputs, void, MemCfg>;
86+};
87+} // namespace GeluOp
88+#endif // CANN_CUSTOM_OPS_GELU_DAG_H
@@ -0,0 +1,34 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/* !
12+ * \file gelu_struct.h
13+ * \brief
14+ */
15+ 
16+#include "ascendc/host_api/tiling/template_argument.h"
17+ 
18+#ifndef CANN_CUSTOM_OPS_GELU_STRUCT_H_
19+#define CANN_CUSTOM_OPS_GELU_STRUCT_H_
20+ 
21+#define TPL_FP16 1
22+#define TPL_FP32 3
23+ 
24+#define TPL_SCH_MODE_0 0
25+#define TPL_SCH_MODE_1 1
26+ 
27+ASCENDC_TPL_ARGS_DECL(Gelu, ASCENDC_TPL_UINT_DECL(schMode, 1, ASCENDC_TPL_UI_LIST, TPL_SCH_MODE_0, TPL_SCH_MODE_1),
28+ ASCENDC_TPL_DTYPE_DECL(dType, TPL_FP16, TPL_FP32));
29+ 
30+ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_UINT_SEL(schMode, ASCENDC_TPL_UI_LIST, TPL_SCH_MODE_0, TPL_SCH_MODE_1),
31+ ASCENDC_TPL_DTYPE_SEL(dType, TPL_FP16)),
32+ ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_UINT_SEL(schMode, ASCENDC_TPL_UI_LIST, TPL_SCH_MODE_0, TPL_SCH_MODE_1),
33+ ASCENDC_TPL_DTYPE_SEL(dType, TPL_FP32)));
34+#endif // CANN_CUSTOM_OPS_GELU_STRUCT_H_
@@ -9,7 +9,7 @@
9# ----------------------------------------------------------------------------9# ----------------------------------------------------------------------------
10 10 
11# 设置算子定义时支持的芯片类型11# 设置算子定义时支持的芯片类型
12-set(SUPPORT_COMPUTE_UNIT "ascend950" "mc62" "ascend350" "ascend960dt")12+set(SUPPORT_COMPUTE_UNIT "ascend950" "mc62" "ascend350" "ascend960dt" "kirinx90" "kirin9030")
13# 设置每种芯片类型对应的tiling文件目录,即采用op_host目录下哪个文件夹下的tiling文件编译13# 设置每种芯片类型对应的tiling文件目录,即采用op_host目录下哪个文件夹下的tiling文件编译
14-set(SUPPORT_TILING_DIR "arch35" "arch35" "arch35" "arch35")14+set(SUPPORT_TILING_DIR "arch35" "arch35" "arch35" "arch35" "arch31" "arch31")
15add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE gather_v2 ACLNNTYPE aclnn_exclude COMPUTE_UNIT ${SUPPORT_COMPUTE_UNIT} TILING_DIR ${SUPPORT_TILING_DIR} DISABLE_IN_OPP TRUE DEPENDENCIES scatter_update renorm gather_elements gather_v3 linear_index_v2 index_put_with_sort HOSTCPU TRUE)15add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE gather_v2 ACLNNTYPE aclnn_exclude COMPUTE_UNIT ${SUPPORT_COMPUTE_UNIT} TILING_DIR ${SUPPORT_TILING_DIR} DISABLE_IN_OPP TRUE DEPENDENCIES scatter_update renorm gather_elements gather_v3 linear_index_v2 index_put_with_sort HOSTCPU TRUE)
@@ -10,7 +10,8 @@
10| <term>Atlas 200I/500 A2推理产品</term> | × |10| <term>Atlas 200I/500 A2推理产品</term> | × |
11| <term>Atlas推理系列产品</term> | × |11| <term>Atlas推理系列产品</term> | × |
12| <term>Atlas训练系列产品</term> | × |12| <term>Atlas训练系列产品</term> | × |
13- 13+| <term>KirinX90处理器系列产品</term> | √ |
14+| <term>Kirin9030处理器系列产品</term> | √ |
14## 功能说明15## 功能说明
15 16 
16- 算子功能:从输入Tensor的指定维度dim,按index中的下标序号提取元素,保存到out Tensor中。17- 算子功能:从输入Tensor的指定维度dim,按index中的下标序号提取元素,保存到out Tensor中。
@@ -12,6 +12,8 @@ English | [简体中文](./README.md)
12| <term>Atlas 200I/500 A2 inference products</term> | × |12| <term>Atlas 200I/500 A2 inference products</term> | × |
13| <term>Atlas inference products</term> | × |13| <term>Atlas inference products</term> | × |
14| <term>Atlas training products</term> | × |14| <term>Atlas training products</term> | × |
15+| <term>KirinX90 processor series products</term> | √ |
16+| <term>Kirin9030 processor series products</term> | √ |
15 17 
16## Function18## Function
17 19 
@@ -0,0 +1,79 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file gather_v2_tiling_arch31.h
13+ * \brief
14+ */
15+#ifndef OPS_BUILT_IN_OP_TILING_RUNTIME_GATHER_V2_ARCH31_H_
16+#define OPS_BUILT_IN_OP_TILING_RUNTIME_GATHER_V2_ARCH31_H_
17+#include <cstdint>
18+#include <nlohmann/json.hpp>
19+#include <sstream>
20+ 
21+namespace optiling {
22+struct GatherV2CompileInfo {
23+ int64_t ub_size{1};
24+ int64_t block_size{1};
25+ int64_t l1_size{0};
26+ int64_t core_num{1};
27+ int64_t params_dsize{1};
zengjuan
zengjuanzengjuan18 days ago

变量命名风格不统一

likedislike
LiuYuzhou
LiuYuzhou
17 days ago Comments:
likedislike
28+ int64_t indices_dsize{1};
29+ bool is_tik{false};
30+ bool is_gather_v2{true};
31+ int64_t impl_mode{0};
32+ bool is_preprocessed;
33+ int64_t soc_version{0};
34+};
35+ 
36+struct GatherV2TilingParams {
37+ int64_t tiling_mode = 0;
38+ int64_t params_pre = 1;
39+ int64_t params_axis = 1;
40+ int64_t params_row = 1;
41+ int64_t indices_num = 1;
42+ int64_t cache_params = 0;
43+ int64_t need_core_num = 0;
44+ int64_t tail_process_core = 0;
45+ int64_t indices_num_each_core = 0;
46+ int64_t indices_num_remaining = 0;
47+ int64_t indices_loop_num = 0;
48+ int64_t indices_row_num_once = 0;
49+ int64_t indices_row_num_last = 0;
50+ int64_t row_num_once_ub = 0;
51+ int64_t row_num_once_tail_ub = 0;
52+ int64_t inner_loop_num = 0;
53+ int64_t row_num_last_ub = 0;
54+ int64_t row_num_last_tail_ub = 0;
55+ int64_t inner_loop_num_last = 0;
56+ int64_t params_total = 0;
57+ int64_t one_row_loop = 0;
58+ int64_t one_row_tail = 0;
59+ int64_t params_pre_each_core = 0;
60+ int64_t params_pre_remaining = 0;
61+ int64_t indices_row = 1;
62+ int64_t params_batch_each_core = 1;
63+ int64_t params_batch_remaining = 0;
64+ int64_t params_batch = 1;
65+ int64_t soc_version = 0;
66+ int64_t x_split_num = 1;
67+ int64_t x_buffer_size = 0;
68+ int64_t indices_buffer_size = 0;
69+ int64_t indices_split_num = 0;
70+ int64_t mask_buffer_size = 0;
71+ int64_t total_count_num = 0;
72+ int64_t count_time = 0;
73+ int64_t median_core_id = 0;
74+ int64_t pre_core_compute_size = 0;
75+ int64_t median_core_compute_size = 0;
76+ int64_t post_core_compute_size = 0;
77+};
78+} // namespace optiling
79+#endif // AIR_CXX_RUNTIME_V2_OP_IMPL_GATHER_V2_RUNTIME2_ARCH31_H_
@@ -0,0 +1,231 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file gather_v2_tiling_data_arch31.h
13+ * \brief
14+ */
15+#ifndef AIR_CXX_RUNTIME_V2_OP_IMPL_GATHERV2_H
16+#define AIR_CXX_RUNTIME_V2_OP_IMPL_GATHERV2_H
17+#include <cmath>
18+#include "register/tilingdata_base.h"
19+#include "tiling/tiling_api.h"
20+#include "op_host/tiling_base.h"
21+#include "gather_v2_tiling_arch31.h"
22+ 
23+namespace optiling {
24+BEGIN_TILING_DATA_DEF(GatherV2TilingData)
25+TILING_DATA_FIELD_DEF(int64_t, needCoreNum);
26+TILING_DATA_FIELD_DEF(int64_t, threadNum);
27+TILING_DATA_FIELD_DEF(int64_t, batchSize);
28+TILING_DATA_FIELD_DEF(int64_t, outerSize);
29+TILING_DATA_FIELD_DEF(int64_t, gatherDimSize);
30+TILING_DATA_FIELD_DEF(int64_t, gatherSize);
31+TILING_DATA_FIELD_DEF(int64_t, innerSize);
32+TILING_DATA_FIELD_DEF(int64_t, xSize);
33+TILING_DATA_FIELD_DEF(int64_t, indicesSize);
34+TILING_DATA_FIELD_DEF(int64_t, ySize);
35+TILING_DATA_FIELD_DEF(int64_t, perCoreElements);
36+TILING_DATA_FIELD_DEF(int64_t, lastCoreElements);
37+TILING_DATA_FIELD_DEF(int64_t, negativeIndexSupport);
38+TILING_DATA_FIELD_DEF(int64_t, supportOutOfBoundIndex);
39+TILING_DATA_FIELD_DEF(int64_t, maxElement);
40+TILING_DATA_FIELD_DEF(int64_t, indiceUbSize);
41+TILING_DATA_FIELD_DEF(int64_t, dtypeSize);
42+END_TILING_DATA_DEF;
43+ 
44+REGISTER_TILING_DATA_CLASS(GatherV2, GatherV2TilingData);
45+ 
46+BEGIN_TILING_DATA_DEF(GatherV2LastTilingData)
47+TILING_DATA_FIELD_DEF(int16_t, needCoreNum);
48+TILING_DATA_FIELD_DEF(int16_t, indicesNum);
49+TILING_DATA_FIELD_DEF(int16_t, splitIndices);
50+TILING_DATA_FIELD_DEF(int16_t, inputNum);
51+TILING_DATA_FIELD_DEF(int16_t, splitMode);
52+TILING_DATA_FIELD_DEF(int16_t, coreInCols);
53+TILING_DATA_FIELD_DEF(int32_t, inputUbSize);
54+TILING_DATA_FIELD_DEF(int32_t, outUbSize);
55+TILING_DATA_FIELD_DEF(int32_t, indiceUbSize);
56+TILING_DATA_FIELD_DEF(int32_t, indiceCastUbSize);
57+TILING_DATA_FIELD_DEF(int32_t, ubCols);
58+TILING_DATA_FIELD_DEF(int32_t, ubRows);
59+TILING_DATA_FIELD_DEF(int32_t, gatherDimSize);
60+TILING_DATA_FIELD_DEF(int64_t, gatherSize);
61+TILING_DATA_FIELD_DEF(int64_t, blockFactor);
62+TILING_DATA_FIELD_DEF(int64_t, tailBlockFactor);
63+TILING_DATA_FIELD_DEF(int64_t, gFactor);
64+END_TILING_DATA_DEF;
65+ 
66+REGISTER_TILING_DATA_CLASS(GatherV2_1100000001, GatherV2LastTilingData)
67+REGISTER_TILING_DATA_CLASS(GatherV2_1100000002, GatherV2LastTilingData)
68+REGISTER_TILING_DATA_CLASS(GatherV2_1100000004, GatherV2LastTilingData)
69+REGISTER_TILING_DATA_CLASS(GatherV2_1100000008, GatherV2LastTilingData)
70+REGISTER_TILING_DATA_CLASS(GatherV2_1100000101, GatherV2LastTilingData)
71+REGISTER_TILING_DATA_CLASS(GatherV2_1100000102, GatherV2LastTilingData)
72+REGISTER_TILING_DATA_CLASS(GatherV2_1100000104, GatherV2LastTilingData)
73+REGISTER_TILING_DATA_CLASS(GatherV2_1100000108, GatherV2LastTilingData)
74+ 
75+BEGIN_TILING_DATA_DEF(GatherV2GaAllLoadTilingData)
76+TILING_DATA_FIELD_DEF(int64_t, gSize);
77+TILING_DATA_FIELD_DEF(int64_t, aSize);
78+TILING_DATA_FIELD_DEF(int64_t, aSizeAligned);
79+TILING_DATA_FIELD_DEF(int64_t, indicesSize);
80+TILING_DATA_FIELD_DEF(int64_t, usedCoreNum);
81+TILING_DATA_FIELD_DEF(int64_t, indicesOuter);
82+TILING_DATA_FIELD_DEF(int64_t, normalCoreIndicesNum);
83+TILING_DATA_FIELD_DEF(int64_t, tailCoreIndicesNum);
84+TILING_DATA_FIELD_DEF(int64_t, pOuter);
85+TILING_DATA_FIELD_DEF(int64_t, normalCoreGaNum);
86+TILING_DATA_FIELD_DEF(int64_t, tailCoreGaNum);
87+TILING_DATA_FIELD_DEF(int64_t, xBufferSize);
88+TILING_DATA_FIELD_DEF(int64_t, indicesBufferSize);
89+TILING_DATA_FIELD_DEF(int64_t, yBufferSize);
90+END_TILING_DATA_DEF;
91+ 
92+REGISTER_TILING_DATA_CLASS(GatherV2_3000, GatherV2GaAllLoadTilingData)
93+REGISTER_TILING_DATA_CLASS(GatherV2_3100, GatherV2GaAllLoadTilingData)
94+ 
95+BEGIN_TILING_DATA_DEF(GatherV2TilingDataSimdTwoDim)
96+TILING_DATA_FIELD_DEF(int16_t, needCoreNum);
97+TILING_DATA_FIELD_DEF(int16_t, negativeIndexSupport);
98+TILING_DATA_FIELD_DEF(int32_t, indiceFactor);
99+TILING_DATA_FIELD_DEF(int32_t, dtypeSize);
100+TILING_DATA_FIELD_DEF(int64_t, gatherDimSize);
101+TILING_DATA_FIELD_DEF(int64_t, gatherSize);
102+TILING_DATA_FIELD_DEF(int64_t, innerSize);
103+TILING_DATA_FIELD_DEF(int64_t, blockFactor);
104+TILING_DATA_FIELD_DEF(int64_t, tailBlockFactor);
105+TILING_DATA_FIELD_DEF(int64_t, maxElement);
106+END_TILING_DATA_DEF;
107+ 
108+REGISTER_TILING_DATA_CLASS(GatherV2_1000000299, GatherV2TilingDataSimdTwoDim)
109+ 
110+BEGIN_TILING_DATA_DEF(GatherV2TilingDataSimtTwoDim)
111+TILING_DATA_FIELD_DEF(int16_t, needCoreNum);
112+TILING_DATA_FIELD_DEF(int16_t, negativeIndexSupport);
113+TILING_DATA_FIELD_DEF(int32_t, threadNum);
114+TILING_DATA_FIELD_DEF(int64_t, gatherDimSize);
115+TILING_DATA_FIELD_DEF(int64_t, innerSize);
116+TILING_DATA_FIELD_DEF(int64_t, perCoreElements);
117+TILING_DATA_FIELD_DEF(int64_t, lastCoreElements);
118+END_TILING_DATA_DEF;
119+ 
120+REGISTER_TILING_DATA_CLASS(GatherV2_2000000001, GatherV2TilingDataSimtTwoDim)
121+REGISTER_TILING_DATA_CLASS(GatherV2_2000000002, GatherV2TilingDataSimtTwoDim)
122+REGISTER_TILING_DATA_CLASS(GatherV2_2000000004, GatherV2TilingDataSimtTwoDim)
123+REGISTER_TILING_DATA_CLASS(GatherV2_2000000008, GatherV2TilingDataSimtTwoDim)
124+REGISTER_TILING_DATA_CLASS(GatherV2_2000000101, GatherV2TilingDataSimtTwoDim)
125+REGISTER_TILING_DATA_CLASS(GatherV2_2000000102, GatherV2TilingDataSimtTwoDim)
126+REGISTER_TILING_DATA_CLASS(GatherV2_2000000104, GatherV2TilingDataSimtTwoDim)
127+REGISTER_TILING_DATA_CLASS(GatherV2_2000000108, GatherV2TilingDataSimtTwoDim)
128+ 
129+BEGIN_TILING_DATA_DEF(GatherV2TilingDataEmptyInput)
130+TILING_DATA_FIELD_DEF(int64_t, needCoreNum);
131+TILING_DATA_FIELD_DEF(int64_t, perCoreElements);
132+TILING_DATA_FIELD_DEF(int64_t, lastCoreElements);
133+END_TILING_DATA_DEF;
134+REGISTER_TILING_DATA_CLASS(GatherV2_3000000000, GatherV2TilingDataEmptyInput)
135+ 
136+class Gatherv2TilingBase : public Ops::NN::Optiling::TilingBaseClass {
137+public:
138+ explicit Gatherv2TilingBase(gert::TilingContext* context) : TilingBaseClass(context) { Reset(); }
139+ ~Gatherv2TilingBase() override = default;
140+ 
141+ void Reset(gert::TilingContext* context) override
142+ {
143+ TilingBaseClass::Reset(context);
144+ Reset();
145+ }
146+ 
147+protected:
148+ bool IsCapable() override { return true; }
149+ // 1、获取平台信息比如CoreNum、UB/L1/L0C资源大小
150+ ge::graphStatus GetPlatformInfo() override;
151+ // 2、获取INPUT/OUTPUT/ATTR信息
152+ ge::graphStatus GetShapeAttrsInfo() override;
153+ // 3、计算数据切分TilingData
154+ ge::graphStatus DoOpTiling() override;
155+ // 4、计算高阶API的TilingData
156+ ge::graphStatus DoLibApiTiling() override;
157+ // 5、计算TilingKey
158+ uint64_t GetTilingKey() const override;
159+ // 6、计算Workspace 大小
160+ ge::graphStatus GetWorkspaceSize() override;
161+ // 7、保存Tiling数据
162+ ge::graphStatus PostTiling() override;
163+ void DumpTilingInfo() override;
164+ void Reset();
165+ 
166+private:
167+ inline ge::graphStatus GetXInfoAndCheck();
168+ inline ge::graphStatus GetIndicesInfoAndCheck();
169+ inline ge::graphStatus GetAxisInfoAndCheck();
170+ inline ge::graphStatus GetAttrsInfoAndCheck();
171+ void ShowBaseTilingData();
172+ void ShowLastGtaherSimdTilingData();
173+ ge::graphStatus MargeAxis();
174+ void CalcCoreElement();
175+ void CalcSimdTiling();
176+ ge::graphStatus SimdTwoDimTiling();
177+ ge::graphStatus SimtTwoDimTiling();
178+ ge::graphStatus CalFullLoadTiling();
179+ bool IsAfterGdimFullLoad();
180+ int64_t XDtypeImprove();
181+ ge::graphStatus LastGatherTiling();
182+ ge::graphStatus GaAllLoadTiling();
183+ ge::graphStatus CalcEmptyCoreElement();
184+ bool IsLastGatherAndFullLoad();
185+ bool IsGaAllLoad();
186+ bool IsSimdTwoDim();
187+ bool IsSimtTwoDim();
188+ void LastGatherUbTiling(int32_t& inputUbSize, int32_t& indiceUbSize, int32_t& outUbSize, int32_t& indiceCastUbSize,
189+ int32_t& inputNum, int32_t& indicesNum, int32_t& ubCols, int32_t& ubRows,
190+ int64_t blockFactor, int64_t gFactor);
191+ void CalcMaxUbcolAndIndiceFactor(int32_t& ubCols, int32_t ubRows, int32_t ubSize, int32_t minCols, int32_t inputNum,
192+ int32_t outputNum, int32_t indiceNUm, int32_t castUbRatio);
193+ int64_t aivNum_ = 0;
194+ 
195+ const char* opName_ = "";
196+ GatherV2TilingData gatherV2TilingData_;
197+ GatherV2LastTilingData lastTilingdata_;
198+ GatherV2GaAllLoadTilingData gaAllLoadTilingdata_;
199+ GatherV2TilingDataSimdTwoDim simdTwoDimTilingData_;
200+ GatherV2TilingDataSimtTwoDim simtTwoDimTilingData_;
201+ GatherV2TilingDataEmptyInput emptyTilingData_;
202+#ifdef DAVID_FPGA
203+ int64_t threadNum_ = 128;
204+#else
205+ int64_t threadNum_ = 2048;
206+#endif
207+ gert::Shape xShape_;
208+ gert::Shape indicesShape_;
209+ ge::DataType xDtype_ = ge::DT_FLOAT;
210+ ge::DataType indicesDtype_ = ge::DT_FLOAT;
211+ int64_t axis_ = 0;
212+ int64_t batchDims_ = 0;
213+ int64_t ySize_ = 1;
214+ int32_t improveDtypeSize_ = 0;
215+ int32_t indicesDtypeSize_ = 0;
216+ int64_t gatherDimSize_ = 0;
217+ int64_t batchSize_ = 1;
218+ int64_t outerSize_ = 1;
219+ int64_t gatherSize_ = 0;
220+ int64_t innerSize_ = 1;
221+ int64_t ubSize_ = 0;
222+ int64_t needCoreNum_ = 0;
223+ int64_t inputBatchDims_ = 0;
224+ bool negativeIndexSupport_ = false;
225+ bool supportOutOfBoundIndex_ = false;
226+ int32_t tilingMode_ = 0;
227+ int32_t ubBlockSize_ = 32;
228+ int32_t vRegSize_ = 256;
229+};
230+} // namespace optiling
231+#endif // AIR_CXX_RUNTIME_V2_OP_IMPL_GATHERV2_H
@@ -15,6 +15,40 @@
15#include "register/op_def_registry.h"15#include "register/op_def_registry.h"
16 16 
17namespace ops {17namespace ops {
18+ 
19+// kirinx90 / kirin9030 支持的数据类型组合(保守精简:去掉 bf16/float8/complex,保留基本类型)
20+// 组合数:5 种 x (fp16/fp32/int8/int32/int64) × 2 种 indices (int32/int64) × 2 种 axis (int32/int64) = 20 组
21+static const std::vector<ge::DataType> kirinXDataType = {
22+ // fp16 × 4
23+ ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16,
24+ // fp32 × 4
25+ ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT,
26+ // int8 × 4
27+ ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8,
28+ // int32 × 4
29+ ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32,
30+ // int64 × 4
31+ ge::DT_INT64, ge::DT_INT64, ge::DT_INT64, ge::DT_INT64};
32+static const std::vector<ge::DataType> kirinIndicesDataType = {
33+ // 每种 x 重复 4 组:(int32,int32,int64,int64)
34+ ge::DT_INT32, ge::DT_INT32, ge::DT_INT64, ge::DT_INT64, ge::DT_INT32, ge::DT_INT32, ge::DT_INT64,
35+ ge::DT_INT64, ge::DT_INT32, ge::DT_INT32, ge::DT_INT64, ge::DT_INT64, ge::DT_INT32, ge::DT_INT32,
36+ ge::DT_INT64, ge::DT_INT64, ge::DT_INT32, ge::DT_INT32, ge::DT_INT64, ge::DT_INT64};
37+static const std::vector<ge::DataType> kirinAxisDataType = {
38+ // 每组 axis 在 (int32, int64) 间交替
39+ ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32,
40+ ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64,
41+ ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64, ge::DT_INT32, ge::DT_INT64};
42+static const std::vector<ge::DataType> kirinYDataType = {
43+ // y 类型 = x 类型
44+ ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT,
45+ ge::DT_FLOAT, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT32, ge::DT_INT32,
46+ ge::DT_INT32, ge::DT_INT32, ge::DT_INT64, ge::DT_INT64, ge::DT_INT64, ge::DT_INT64};
47+static const std::vector<ge::Format> gatherV2FormatKirin = {
48+ ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND,
49+ ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND,
50+ ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND};
51+ 
18class GatherV2 : public OpDef {52class GatherV2 : public OpDef {
19public:53public:
20 explicit GatherV2(const char* name) : OpDef(name)54 explicit GatherV2(const char* name) : OpDef(name)
@@ -210,6 +244,37 @@ public:
210 this->AICore().AddConfig("ascend950", aicoreConfig);244 this->AICore().AddConfig("ascend950", aicoreConfig);
211 this->AICore().AddConfig("ascend350", aicoreConfig);245 this->AICore().AddConfig("ascend350", aicoreConfig);
212 this->AICore().AddConfig("ascend960dt", aicoreConfig);246 this->AICore().AddConfig("ascend960dt", aicoreConfig);
247+ 
248+ OpAICoreConfig config_kirin;
249+ config_kirin.DynamicCompileStaticFlag(true)
250+ .DynamicFormatFlag(true)
251+ .DynamicRankSupportFlag(true)
252+ .DynamicShapeSupportFlag(true)
253+ .NeedCheckSupportFlag(false)
254+ .PrecisionReduceFlag(true);
255+ config_kirin.Input("x")
256+ .ParamType(REQUIRED)
257+ .DataType(kirinXDataType)
258+ .Format(gatherV2FormatKirin)
259+ .UnknownShapeFormat(gatherV2FormatKirin);
260+ config_kirin.Input("indices")
261+ .ParamType(REQUIRED)
262+ .DataType(kirinIndicesDataType)
263+ .Format(gatherV2FormatKirin)
264+ .UnknownShapeFormat(gatherV2FormatKirin);
265+ config_kirin.Input("axis")
266+ .ParamType(REQUIRED)
267+ .DataType(kirinAxisDataType)
268+ .Format(gatherV2FormatKirin)
269+ .UnknownShapeFormat(gatherV2FormatKirin)
270+ .ValueDepend(OPTIONAL);
271+ config_kirin.Output("y")
272+ .ParamType(REQUIRED)
273+ .DataType(kirinYDataType)
274+ .Format(gatherV2FormatKirin)
275+ .UnknownShapeFormat(gatherV2FormatKirin);
276+ this->AICore().AddConfig("kirinx90", config_kirin);
277+ this->AICore().AddConfig("kirin9030", config_kirin);
213 }278 }
214};279};
215 280 
@@ -22,3 +22,10 @@ add_kernel_sources(
22 IMPL_MODE high_performance,high_precision,support_out_of_bound_index22 IMPL_MODE high_performance,high_precision,support_out_of_bound_index
23 OPTIONS "--cce-no-dcache-flush"23 OPTIONS "--cce-no-dcache-flush"
24)24)
25+# arch31(kirinx90/kirin9030) 使用 arch31/gather_v2.cpp 作为 kernel 入口
26+add_kernel_sources(
27+ KERNEL_SRC arch31/gather_v2.cpp
28+ COMPUTE_UNITS kirinx90 kirin9030
29+ IMPL_MODE high_performance,high_precision,support_out_of_bound_index
30+ AUTO_SYNC false
31+)
@@ -0,0 +1,208 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+// KirinX90 适配:去除所有 simt 依赖模板,改用纯 SIMD 新模板
12+// 新增: gather_v2_simd_1d.h(Gatherv2Simd1D) / gather_v2_simd_2d.h(Gatherv2Simd2D) /
13+// gather_v2_simd_full_load.h(Gatherv2SimdFullLoad) 删除原 simt 头: gather_v2.h / gather_v2_simt_two_dim.h /
14+// gather_v2_after_gather_full_load.h
15+#include "gather_v2_simd_1d.h" // 替代 gather_v2.h (Gatherv2)
16+#include "gather_v2_simd_arch31.h"
17+#include "gather_v2_simd_two_dim_arch31.h"
18+#include "gather_v2_simd_full_load.h" // 替代 gather_v2_after_gather_full_load.h (Gatherv2FullLoad)
19+#include "gather_v2_simd_2d.h" // 替代 gather_v2_simt_two_dim.h (Gatherv2SimtTwoDim)
20+#include "gather_v2_empty_arch31.h"
21+ 
22+using namespace AscendC;
23+using namespace gatherv2;
24+ 
25+#define TILING_KEY_X_B64_INDEX_SIZE_64 10000000000000000031UL
26+#define TILING_KEY_X_B32_INDEX_SIZE_64 10000000000000000021UL
27+#define TILING_KEY_X_B16_INDEX_SIZE_64 10000000000000000011UL
28+#define TILING_KEY_X_B8_INDEX_SIZE_64 10000000000000000001UL
29+#define TILING_KEY_X_B64_INDEX_SIZE_32 10000000000000000030UL
30+#define TILING_KEY_X_B32_INDEX_SIZE_32 10000000000000000020UL
31+#define TILING_KEY_X_B16_INDEX_SIZE_32 10000000000000000010UL
32+#define TILING_KEY_X_B8_INDEX_SIZE_32 10000000000000000000UL
33+#define SIMD_TILING_KEY 1000000099
34+#define SIMD_TILING_KEY_TWO_DIM 1000000299
35+#define SIMD_LAST_GATHER_B8_TILING_KEY 1100000001UL
36+#define SIMD_LAST_GATHER_B16_TILING_KEY 1100000002UL
37+#define SIMD_LAST_GATHER_B32_TILING_KEY 1100000004UL
38+#define SIMD_LAST_GATHER_B64_TILING_KEY 1100000008UL
39+#define SIMD_LAST_GATHER_B8_SUPPORT_NEG_INDICE_TILING_KEY 1100000101UL
40+#define SIMD_LAST_GATHER_B16_SUPPORT_NEG_INDICE_TILING_KEY 1100000102UL
41+#define SIMD_LAST_GATHER_B32_SUPPORT_NEG_INDICE_TILING_KEY 1100000104UL
42+#define SIMD_LAST_GATHER_B64_SUPPORT_NEG_INDICE_TILING_KEY 1100000108UL
43+#define SIMD_GA_ALL_LOAD_TILING_KEY 3000UL
44+#define SIMD_GA_ALL_LOAD_SUPPORT_NEG_INDICE_TILING_KEY 3100UL
45+ 
46+#define TILING_KEY_X_B64_INDEX_SIZE_2D 10000000000000000032UL
47+#define TILING_KEY_X_B32_INDEX_SIZE_2D 10000000000000000022UL
48+#define TILING_KEY_X_B16_INDEX_SIZE_2D 10000000000000000012UL
49+#define TILING_KEY_X_B8_INDEX_SIZE_2D 10000000000000000002UL
50+ 
51+#define TILING_KEY_EMPTY 3000000000UL
52+ 
53+#define SIMT_TWO_DIM_B8_INDEX_SIZE_32_TILING_KEY 2000000001UL
54+#define SIMT_TWO_DIM_B16_INDEX_SIZE_32_TILING_KEY 2000000002UL
55+#define SIMT_TWO_DIM_B32_INDEX_SIZE_32_TILING_KEY 2000000004UL
56+#define SIMT_TWO_DIM_B64_INDEX_SIZE_32_TILING_KEY 2000000008UL
57+#define SIMT_TWO_DIM_B8_INDEX_SIZE_64_TILING_KEY 2000000101UL
58+#define SIMT_TWO_DIM_B16_INDEX_SIZE_64_TILING_KEY 2000000102UL
59+#define SIMT_TWO_DIM_B32_INDEX_SIZE_64_TILING_KEY 2000000104UL
60+#define SIMT_TWO_DIM_B64_INDEX_SIZE_64_TILING_KEY 2000000108UL
61+ 
62+extern "C" __global__ __aicore__ void gather_v2(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, GM_ADDR workspace,
63+ GM_ADDR tiling)
64+{
65+ TPipe pipe;
66+ KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_AIV_ONLY);
67+ 
68+ TILING_KEY_IS(SIMD_TILING_KEY);
69+ TILING_KEY_IS(SIMD_TILING_KEY_TWO_DIM);
70+ TILING_KEY_IS(SIMT_TWO_DIM_B8_INDEX_SIZE_32_TILING_KEY);
71+ TILING_KEY_IS(SIMT_TWO_DIM_B16_INDEX_SIZE_32_TILING_KEY);
72+ TILING_KEY_IS(SIMT_TWO_DIM_B32_INDEX_SIZE_32_TILING_KEY);
73+ TILING_KEY_IS(SIMT_TWO_DIM_B64_INDEX_SIZE_32_TILING_KEY);
74+ TILING_KEY_IS(SIMT_TWO_DIM_B8_INDEX_SIZE_64_TILING_KEY);
75+ TILING_KEY_IS(SIMT_TWO_DIM_B16_INDEX_SIZE_64_TILING_KEY);
76+ TILING_KEY_IS(SIMT_TWO_DIM_B32_INDEX_SIZE_64_TILING_KEY);
77+ TILING_KEY_IS(SIMT_TWO_DIM_B64_INDEX_SIZE_64_TILING_KEY);
78+ TILING_KEY_IS(TILING_KEY_X_B64_INDEX_SIZE_64);
79+ TILING_KEY_IS(TILING_KEY_X_B32_INDEX_SIZE_64);
80+ TILING_KEY_IS(TILING_KEY_X_B16_INDEX_SIZE_64);
81+ TILING_KEY_IS(TILING_KEY_X_B8_INDEX_SIZE_64);
82+ TILING_KEY_IS(TILING_KEY_X_B64_INDEX_SIZE_32);
83+ TILING_KEY_IS(TILING_KEY_X_B32_INDEX_SIZE_32);
84+ TILING_KEY_IS(TILING_KEY_X_B16_INDEX_SIZE_32);
85+ TILING_KEY_IS(TILING_KEY_X_B8_INDEX_SIZE_32);
86+ TILING_KEY_IS(TILING_KEY_X_B64_INDEX_SIZE_2D);
87+ TILING_KEY_IS(TILING_KEY_X_B32_INDEX_SIZE_2D);
88+ TILING_KEY_IS(TILING_KEY_X_B16_INDEX_SIZE_2D);
89+ TILING_KEY_IS(TILING_KEY_X_B8_INDEX_SIZE_2D);
90+ TILING_KEY_IS(TILING_KEY_EMPTY);
91+ 
92+#if TILING_KEY_VAR == SIMD_TILING_KEY
93+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
94+ Gatherv2Simd<DTYPE_INDICES> gatherv2Op(&pipe);
95+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
96+ gatherv2Op.Process();
97+#elif TILING_KEY_VAR == SIMD_TILING_KEY_TWO_DIM
98+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimdTwoDim, tilingDataIn, tiling);
99+ Gatherv2SimdTwoDim<DTYPE_INDICES> gatherv2Op(&pipe);
100+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
101+ gatherv2Op.Process();
102+#elif TILING_KEY_VAR == SIMT_TWO_DIM_B8_INDEX_SIZE_64_TILING_KEY
103+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling);
104+ Gatherv2Simd2D<int8_t, DTYPE_INDICES, uint64_t> gatherv2Op;
105+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
106+ gatherv2Op.Process();
107+#elif TILING_KEY_VAR == SIMT_TWO_DIM_B16_INDEX_SIZE_64_TILING_KEY
108+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling);
109+ Gatherv2Simd2D<int16_t, DTYPE_INDICES, uint64_t> gatherv2Op;
110+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
111+ gatherv2Op.Process();
112+#elif TILING_KEY_VAR == SIMT_TWO_DIM_B32_INDEX_SIZE_64_TILING_KEY
113+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling);
114+ Gatherv2Simd2D<int32_t, DTYPE_INDICES, uint64_t> gatherv2Op;
115+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
116+ gatherv2Op.Process();
117+#elif TILING_KEY_VAR == SIMT_TWO_DIM_B64_INDEX_SIZE_64_TILING_KEY
118+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling);
119+ Gatherv2Simd2D<int64_t, DTYPE_INDICES, uint64_t> gatherv2Op;
120+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
121+ gatherv2Op.Process();
122+#elif TILING_KEY_VAR == SIMT_TWO_DIM_B8_INDEX_SIZE_32_TILING_KEY
123+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling);
124+ Gatherv2Simd2D<int8_t, DTYPE_INDICES, uint32_t> gatherv2Op;
125+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
126+ gatherv2Op.Process();
127+#elif TILING_KEY_VAR == SIMT_TWO_DIM_B16_INDEX_SIZE_32_TILING_KEY
128+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling);
129+ Gatherv2Simd2D<int16_t, DTYPE_INDICES, uint32_t> gatherv2Op;
130+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
131+ gatherv2Op.Process();
132+#elif TILING_KEY_VAR == SIMT_TWO_DIM_B32_INDEX_SIZE_32_TILING_KEY
133+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling);
134+ Gatherv2Simd2D<int32_t, DTYPE_INDICES, uint32_t> gatherv2Op;
135+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
136+ gatherv2Op.Process();
137+#elif TILING_KEY_VAR == SIMT_TWO_DIM_B64_INDEX_SIZE_32_TILING_KEY
138+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataSimtTwoDim, tilingDataIn, tiling);
139+ Gatherv2Simd2D<int64_t, DTYPE_INDICES, uint32_t> gatherv2Op;
140+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
141+ gatherv2Op.Process();
142+#elif TILING_KEY_VAR == TILING_KEY_X_B8_INDEX_SIZE_64
143+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
144+ Gatherv2Simd1D<int8_t, DTYPE_INDICES, uint64_t> gatherv2Op;
145+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
146+ gatherv2Op.Process();
147+#elif TILING_KEY_VAR == TILING_KEY_X_B16_INDEX_SIZE_64
148+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
149+ Gatherv2Simd1D<int16_t, DTYPE_INDICES, uint64_t> gatherv2Op;
150+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
151+ gatherv2Op.Process();
152+#elif TILING_KEY_VAR == TILING_KEY_X_B32_INDEX_SIZE_64
153+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
154+ Gatherv2Simd1D<int32_t, DTYPE_INDICES, uint64_t> gatherv2Op;
155+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
156+ gatherv2Op.Process();
157+#elif TILING_KEY_VAR == TILING_KEY_X_B64_INDEX_SIZE_64
158+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
159+ Gatherv2Simd1D<int64_t, DTYPE_INDICES, uint64_t> gatherv2Op;
160+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
161+ gatherv2Op.Process();
162+#elif TILING_KEY_VAR == TILING_KEY_X_B8_INDEX_SIZE_32
163+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
164+ Gatherv2Simd1D<int8_t, DTYPE_INDICES, uint32_t> gatherv2Op;
165+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
166+ gatherv2Op.Process();
167+#elif TILING_KEY_VAR == TILING_KEY_X_B16_INDEX_SIZE_32
168+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
169+ Gatherv2Simd1D<int16_t, DTYPE_INDICES, uint32_t> gatherv2Op;
170+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
171+ gatherv2Op.Process();
172+#elif TILING_KEY_VAR == TILING_KEY_X_B32_INDEX_SIZE_32
173+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
174+ Gatherv2Simd1D<int32_t, DTYPE_INDICES, uint32_t> gatherv2Op;
175+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
176+ gatherv2Op.Process();
177+#elif TILING_KEY_VAR == TILING_KEY_X_B64_INDEX_SIZE_32
178+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
179+ Gatherv2Simd1D<int64_t, DTYPE_INDICES, uint32_t> gatherv2Op;
180+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
181+ gatherv2Op.Process();
182+#elif TILING_KEY_VAR == TILING_KEY_X_B8_INDEX_SIZE_2D
183+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
184+ Gatherv2SimdFullLoad<int8_t, DTYPE_INDICES, uint32_t> gatherv2Op(&pipe);
185+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
186+ gatherv2Op.Process();
187+#elif TILING_KEY_VAR == TILING_KEY_X_B16_INDEX_SIZE_2D
188+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
189+ Gatherv2SimdFullLoad<int16_t, DTYPE_INDICES, uint32_t> gatherv2Op(&pipe);
190+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
191+ gatherv2Op.Process();
192+#elif TILING_KEY_VAR == TILING_KEY_X_B32_INDEX_SIZE_2D
193+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
194+ Gatherv2SimdFullLoad<int32_t, DTYPE_INDICES, uint32_t> gatherv2Op(&pipe);
195+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
196+ gatherv2Op.Process();
197+#elif TILING_KEY_VAR == TILING_KEY_X_B64_INDEX_SIZE_2D
198+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingData, tilingDataIn, tiling);
199+ Gatherv2SimdFullLoad<int64_t, DTYPE_INDICES, uint32_t> gatherv2Op(&pipe);
200+ gatherv2Op.Init(x, indices, axis, y, &tilingDataIn);
201+ gatherv2Op.Process();
202+#elif TILING_KEY_VAR == TILING_KEY_EMPTY
203+ GET_TILING_DATA_WITH_STRUCT(GatherV2TilingDataEmptyInput, tilingDataIn, tiling);
204+ Gatherv2Empty<int8_t> gatherv2Op;
205+ gatherv2Op.Init(y, &tilingDataIn);
206+ gatherv2Op.Process();
207+#endif
208+}
@@ -0,0 +1,70 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file gather_v2_empty_arch31.h
13+ * \brief
14+ */
15+#ifndef GATHER_V2_EMPTY
16+#define GATHER_V2_EMPTY
17+ 
18+#if ASC_DEVKIT_MAJOR >= 9
19+#include "basic_api/kernel_vec_intf.h"
20+#else
21+#include "kernel_operator.h"
22+#endif
23+#include "op_kernel/platform_util.h"
24+ 
25+namespace gatherv2 {
26+using namespace AscendC;
27+ 
28+template <typename T>
29+class Gatherv2Empty {
30+public:
31+ __aicore__ inline Gatherv2Empty(){};
32+ __aicore__ inline void Init(GM_ADDR y, const GatherV2TilingDataEmptyInput* tilingData);
33+ __aicore__ inline void Process();
34+ 
35+private:
36+ GlobalTensor<T> yGm_;
37+ 
38+ int64_t needCoreNum_;
39+ int64_t perCoreElements_;
40+ int64_t lastCoreElements_;
41+ 
42+ int64_t extentElements_;
43+ uint32_t blockIdx_;
44+ const GatherV2TilingDataEmptyInput* tilingData_;
45+};
46+ 
47+template <typename T>
48+__aicore__ inline void Gatherv2Empty<T>::Init(GM_ADDR y, const GatherV2TilingDataEmptyInput* tilingData)
49+{
50+ tilingData_ = tilingData;
51+ yGm_.SetGlobalBuffer((__gm__ T*)y);
52+}
53+ 
54+template <typename T>
55+__aicore__ inline void Gatherv2Empty<T>::Process()
56+{
57+ needCoreNum_ = tilingData_->needCoreNum;
58+ perCoreElements_ = tilingData_->perCoreElements;
59+ lastCoreElements_ = tilingData_->lastCoreElements;
60+ extentElements_ = perCoreElements_;
61+ 
62+ blockIdx_ = GetBlockIdx();
63+ if (blockIdx_ + 1 == (uint32_t)needCoreNum_) {
64+ extentElements_ = lastCoreElements_;
65+ }
66+ 
67+ InitOutput<T>(yGm_, extentElements_, 0);
68+}
69+} // namespace gatherv2
70+#endif // GATHER_V2_EMPTY
@@ -0,0 +1,634 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file gather_v2_ga_all_load_arch31.h
13+ * \brief
14+ */
15+#ifndef GATHER_V2_GA_ALL_LOAD_H
16+#define GATHER_V2_GA_ALL_LOAD_H
17+ 
18+#ifndef K_MAX_SHAPE_DIM
19+#define K_MAX_SHAPE_DIM 0
20+#endif
21+#if ASC_DEVKIT_MAJOR >= 9
22+#include "basic_api/kernel_vec_intf.h"
23+#else
24+#include "kernel_operator.h"
25+#endif
26+#include "op_kernel/platform_util.h"
27+ 
28+namespace gatherv2 {
29+using namespace AscendC;
30+ 
31+#if 0 // KirinX90 支持 MicroAPI(DataCopyGather/Gather/Adds/Copy/UpdateMask/CompareScalar
32+ // 等),不过这里可以用兜底路径实现,可以先封上 参照 cast 算子 arch31 适配经验,本文件大量使用
33+ // MicroAPI,整体注释避免编译错误 如后续 KirinX90 支持相关指令或需用 SIMD 重写,解开本 #if 0 即可恢复参考实现
34+constexpr int32_t DOUBLE_BUFFER = 2;
35+constexpr int32_t HELP_BUFFER_SIZE = 256;
36+constexpr int32_t GATHER_ENABLE_SIZE = 512;
37+ 
38+template <typename INDICES_T, const bool NIS>
39+class Gatherv2GaAllLoad {
40+public:
41+ __aicore__ inline Gatherv2GaAllLoad(TPipe* pipe) : pipe_(pipe){};
42+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y,
43+ const GatherV2GaAllLoadTilingData* tilingData);
44+ __aicore__ inline void GenIndexBuf();
45+ __aicore__ inline void Process();
46+ __aicore__ inline void IndicesProcess(int32_t indicesNumPro, int32_t indicesNumOffset);
47+ __aicore__ inline void xProcess(int32_t gaCoreOffset, int32_t gaNumPro);
48+ __aicore__ inline void yProcess(int32_t indicesNumPro, int32_t indicesNumOffset, int32_t gaNumPro,
49+ int32_t gaCoreOffset);
50+ __aicore__ inline void GatherProcess(int32_t indicesUbOffset, int32_t indicesNumCurPro,
51+ __local_mem__ int32_t* indicesAddr, __local_mem__ int8_t* xAddr,
52+ __local_mem__ int8_t* yAddr);
53+ __aicore__ inline void FixIndices(__local_mem__ INDICES_T* indicesAddr, int32_t indicesNumPro);
54+ __aicore__ inline void CopyInIndices(LocalTensor<INDICES_T>& indicesLocal, int32_t burstLen, int32_t coreOffset);
55+ __aicore__ inline void CopyInX(LocalTensor<int8_t>& xLocal, int32_t gaCoreOffset, int32_t gaNumPro);
56+ __aicore__ inline void CopyOutY(int32_t nBurst, int32_t indicesCoreOffset, int32_t pCoreOffset, int32_t gaNumPro);
57+ __aicore__ inline void GatherProcessVfWithGA(int32_t indicesNumCurPro, uint16_t gaNumPro,
58+ __local_mem__ int32_t* curIndicesAddr, __local_mem__ int8_t* xAddr,
59+ __local_mem__ int8_t* yAddr);
60+ __aicore__ inline void GatherProcessWithGA2(int32_t indicesNumCurPro, uint16_t gaNumPro,
61+ __local_mem__ int32_t* curIndicesAddr, __local_mem__ int8_t* xAddr,
62+ __local_mem__ int8_t* yAddr);
63+ __aicore__ inline void InitializationX(int32_t xBufGaNum);
64+ 
65+private:
66+ GlobalTensor<int8_t> xGm_;
67+ GlobalTensor<INDICES_T> indicesGm_;
68+ GlobalTensor<int8_t> yGm_;
69+ TPipe* pipe_;
70+ TBuf<QuePosition::VECCALC> xBuf_;
71+ TBuf<QuePosition::VECCALC> indicesBuf_;
72+ TBuf<QuePosition::VECCALC> tmpIndexBuf_;
73+ TQue<QuePosition::VECOUT, DOUBLE_BUFFER> yQueue_;
74+ const GatherV2GaAllLoadTilingData* tilingData_;
75+ 
76+ int32_t blockIdx_;
77+ bool enableGather_;
78+ 
79+ int64_t curCoreIndicesNum_;
80+ int64_t curCoreGaNum_;
81+ 
82+ int64_t xGmOffset_;
83+ int64_t indicesGmOffset_;
84+};
85+ 
86+template <typename INDICES_T, const bool NIS>
87+__aicore__ inline void Gatherv2GaAllLoad<INDICES_T, NIS>::Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y,
88+ const GatherV2GaAllLoadTilingData* tilingData)
89+{
90+ tilingData_ = tilingData;
91+ blockIdx_ = GetBlockIdx();
92+ 
93+ // 先分核在分UB
94+ int64_t pIndex = blockIdx_ / tilingData_->indicesOuter;
95+ int64_t indicesIndex = blockIdx_ % tilingData_->indicesOuter;
96+ 
97+ indicesGmOffset_ = indicesIndex * tilingData_->normalCoreIndicesNum;
98+ xGmOffset_ = pIndex * tilingData_->normalCoreGaNum;
99+ 
100+ curCoreIndicesNum_ = (indicesIndex + 1 == tilingData_->indicesOuter) ? tilingData_->tailCoreIndicesNum :
@@ -0,0 +1,107 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file gather_v2_simd_1d.h
13+ * \brief KirinX90 纯 SIMD 1D 小 shape 模板(替代原 Gatherv2 simt 实现,去除 simt_api 依赖)
14+ */
15+#ifndef GATHER_V2_SIMD_1D_H
16+#define GATHER_V2_SIMD_1D_H
17+ 
18+#include "kernel_operator.h"
19+// 已删除: simt_api/asc_simt.h、basic_api/kernel_vec_intf.h(KirinX90 硬件不支持 simt)
20+ 
21+namespace gatherv2 {
22+using namespace AscendC;
23+ 
24+template <typename X_T, typename INDICES_T, typename INDEX_SIZE_T>
25+class Gatherv2Simd1D {
26+public:
27+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y,
28+ const GatherV2TilingData* tilingData)
29+ {
30+ tilingData_ = tilingData;
31+ xGm_.SetGlobalBuffer((__gm__ X_T*)x);
32+ indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices);
33+ yGm_.SetGlobalBuffer((__gm__ X_T*)y);
34+ }
35+ 
36+ __aicore__ inline void Process()
37+ {
38+ int32_t blockIdx = static_cast<int32_t>(GetBlockIdx());
39+ int32_t needCoreNum = static_cast<int32_t>(tilingData_->needCoreNum);
40+ if (blockIdx >= needCoreNum) {
41+ return;
42+ }
43+ 
44+ INDEX_SIZE_T outerSize = static_cast<INDEX_SIZE_T>(tilingData_->outerSize);
45+ INDEX_SIZE_T gatherDimSize = static_cast<INDEX_SIZE_T>(tilingData_->gatherDimSize);
46+ INDEX_SIZE_T gatherSize = static_cast<INDEX_SIZE_T>(tilingData_->gatherSize);
47+ INDEX_SIZE_T innerSize = static_cast<INDEX_SIZE_T>(tilingData_->innerSize);
48+ bool nis = static_cast<bool>(tilingData_->negativeIndexSupport);
49+ 
50+ INDEX_SIZE_T currentCoreElements = static_cast<INDEX_SIZE_T>(tilingData_->perCoreElements);
51+ if (blockIdx == tilingData_->needCoreNum - 1) {
52+ currentCoreElements = static_cast<INDEX_SIZE_T>(tilingData_->lastCoreElements);
53+ }
54+ INDEX_SIZE_T yIndexBase = static_cast<INDEX_SIZE_T>(blockIdx) * tilingData_->perCoreElements;
55+ 
56+ __gm__ X_T* xPtr = (__gm__ X_T*)xGm_.GetPhyAddr();
57+ __gm__ INDICES_T* indicesPtr = (__gm__ INDICES_T*)indicesGm_.GetPhyAddr();
58+ __gm__ volatile X_T* yPtr = (__gm__ volatile X_T*)yGm_.GetPhyAddr();
59+ 
60+ if (nis) {
61+ GatherCore<true>(xPtr, indicesPtr, yPtr, yIndexBase, currentCoreElements, gatherSize, outerSize, innerSize,
62+ gatherDimSize);
63+ } else {
64+ GatherCore<false>(xPtr, indicesPtr, yPtr, yIndexBase, currentCoreElements, gatherSize, outerSize, innerSize,
65+ gatherDimSize);
66+ }
67+ }
68+ 
69+private:
70+ template <const bool NIS>
71+ __aicore__ inline void GatherCore(__gm__ X_T* x, __gm__ INDICES_T* indices, __gm__ volatile X_T* y,
72+ INDEX_SIZE_T yIndexBase, INDEX_SIZE_T currentCoreElements,
73+ INDEX_SIZE_T gatherSize, INDEX_SIZE_T outerSize, INDEX_SIZE_T innerSize,
74+ INDEX_SIZE_T gatherDimSize)
75+ {
76+ // 原 GatherSimt 计算逻辑原样保留;线程级(threadIdx/asc_vf_call)→核内串行循环;
77+ // UintDiv 快速除法→普通除法,去除 simt 依赖
78+ for (INDEX_SIZE_T index = 0; index < currentCoreElements; index++) {
79+ INDEX_SIZE_T yIndex = yIndexBase + index;
80+ INDEX_SIZE_T tmp = yIndex / innerSize; // 原 UintDiv(yIndex, m0, shift0)
81+ INDEX_SIZE_T tmp1 = tmp / gatherSize; // 原 UintDiv(tmp, m1, shift1)
82+ INDEX_SIZE_T gatherI = tmp - tmp1 * gatherSize;
83+ INDEX_SIZE_T batchI = tmp1 / outerSize; // 原 UintDiv(tmp1, m2, shift2)
84+ INDEX_SIZE_T innerI = yIndex - tmp * innerSize;
85+ INDEX_SIZE_T indicesIndex = batchI * gatherSize + gatherI;
86+ 
87+ INDICES_T indicesValue = indices[indicesIndex];
88+ if constexpr (NIS) {
89+ if (unlikely(indicesValue < 0)) {
90+ indicesValue += static_cast<INDICES_T>(gatherDimSize);
91+ }
92+ }
93+ INDEX_SIZE_T indicesValueI = static_cast<INDEX_SIZE_T>(indicesValue);
94+ INDEX_SIZE_T xIndex = (tmp1 * gatherDimSize + indicesValueI) * innerSize + innerI;
95+ bool oob = indicesValue < 0 || indicesValueI >= gatherDimSize;
96+ y[yIndex] = oob ? static_cast<X_T>(0) : x[xIndex];
97+ }
98+ }
99+ 
100+private:
101+ GlobalTensor<X_T> xGm_;
102+ GlobalTensor<INDICES_T> indicesGm_;
103+ GlobalTensor<X_T> yGm_;
104+ const GatherV2TilingData* tilingData_ = nullptr;
105+};
106+} // namespace gatherv2
107+#endif // GATHER_V2_SIMD_1D_H
@@ -0,0 +1,97 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file gather_v2_simd_2d.h
13+ * \brief KirinX90 纯 SIMD 2D 小 shape 模板(替代原 Gatherv2SimtTwoDim simt 实现,去除 simt_api 依赖)
14+ */
15+#ifndef GATHER_V2_SIMD_2D_H
16+#define GATHER_V2_SIMD_2D_H
17+ 
18+#include "kernel_operator.h"
19+// 已删除: simt_api/asc_simt.h、basic_api/kernel_vec_intf.h(KirinX90 硬件不支持 simt)
20+ 
21+namespace gatherv2 {
22+using namespace AscendC;
23+ 
24+template <typename X_T, typename INDICES_T, typename INDEX_SIZE_T>
25+class Gatherv2Simd2D {
26+public:
27+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y,
28+ const GatherV2TilingDataSimtTwoDim* tilingData)
29+ {
30+ tilingData_ = tilingData;
31+ xGm_.SetGlobalBuffer((__gm__ X_T*)x);
32+ indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices);
33+ yGm_.SetGlobalBuffer((__gm__ X_T*)y);
34+ }
35+ 
36+ __aicore__ inline void Process()
37+ {
38+ int32_t blockIdx = static_cast<int32_t>(GetBlockIdx());
39+ int32_t needCoreNum = static_cast<int32_t>(tilingData_->needCoreNum);
40+ if (blockIdx >= needCoreNum) {
41+ return;
42+ }
43+ 
44+ INDEX_SIZE_T gatherDimSize = static_cast<INDEX_SIZE_T>(tilingData_->gatherDimSize);
45+ INDEX_SIZE_T innerSize = static_cast<INDEX_SIZE_T>(tilingData_->innerSize);
46+ bool nis = static_cast<bool>(tilingData_->negativeIndexSupport);
47+ 
48+ INDEX_SIZE_T currentCoreElements = static_cast<INDEX_SIZE_T>(tilingData_->perCoreElements);
49+ if (blockIdx == tilingData_->needCoreNum - 1) {
50+ currentCoreElements = static_cast<INDEX_SIZE_T>(tilingData_->lastCoreElements);
51+ }
52+ INDEX_SIZE_T yIndexBase = static_cast<INDEX_SIZE_T>(blockIdx) * tilingData_->perCoreElements;
53+ 
54+ __gm__ X_T* xPtr = (__gm__ X_T*)xGm_.GetPhyAddr();
55+ __gm__ INDICES_T* indicesPtr = (__gm__ INDICES_T*)indicesGm_.GetPhyAddr();
56+ __gm__ volatile X_T* yPtr = (__gm__ volatile X_T*)yGm_.GetPhyAddr();
57+ 
58+ if (nis) {
59+ GatherCore<true>(xPtr, indicesPtr, yPtr, yIndexBase, currentCoreElements, innerSize, gatherDimSize);
60+ } else {
61+ GatherCore<false>(xPtr, indicesPtr, yPtr, yIndexBase, currentCoreElements, innerSize, gatherDimSize);
62+ }
63+ }
64+ 
65+private:
66+ template <const bool NIS>
67+ __aicore__ inline void GatherCore(__gm__ X_T* x, __gm__ INDICES_T* indices, __gm__ volatile X_T* y,
68+ INDEX_SIZE_T yIndexBase, INDEX_SIZE_T currentCoreElements, INDEX_SIZE_T innerSize,
69+ INDEX_SIZE_T gatherDimSize)
70+ {
71+ // 2D: batch==1 && outer==1,yIndex 只分解成 gatherI/innerI 两维
72+ for (INDEX_SIZE_T index = 0; index < currentCoreElements; index++) {
73+ INDEX_SIZE_T yIndex = yIndexBase + index;
74+ INDEX_SIZE_T gatherI = yIndex / innerSize; // 原 UintDiv(yIndex, m0, shift0)
75+ INDEX_SIZE_T innerI = yIndex - gatherI * innerSize;
76+ 
77+ INDICES_T indicesValue = indices[gatherI];
78+ if constexpr (NIS) {
79+ if (unlikely(indicesValue < 0)) {
80+ indicesValue += static_cast<INDICES_T>(gatherDimSize);
81+ }
82+ }
83+ INDEX_SIZE_T indicesValueI = static_cast<INDEX_SIZE_T>(indicesValue);
84+ INDEX_SIZE_T xIndex = indicesValueI * innerSize + innerI;
85+ bool oob = indicesValue < 0 || indicesValueI >= gatherDimSize;
86+ y[yIndex] = oob ? static_cast<X_T>(0) : x[xIndex];
87+ }
88+ }
89+ 
90+private:
91+ GlobalTensor<X_T> xGm_;
92+ GlobalTensor<INDICES_T> indicesGm_;
93+ GlobalTensor<X_T> yGm_;
94+ const GatherV2TilingDataSimtTwoDim* tilingData_ = nullptr;
95+};
96+} // namespace gatherv2
97+#endif // GATHER_V2_SIMD_2D_H
@@ -0,0 +1,305 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file gather_v2_simd_arch31.h
13+ * \brief
14+ */
15+#ifndef GATHER_V2_SIMD_H
16+#define GATHER_V2_SIMD_H
17+ 
18+#ifndef K_MAX_SHAPE_DIM
19+#define K_MAX_SHAPE_DIM 0
20+#endif
21+ 
22+#if ASC_DEVKIT_MAJOR >= 9
23+#include "basic_api/kernel_vec_intf.h"
24+#else
25+#include "kernel_operator.h"
26+#endif
27+#include "op_kernel/platform_util.h"
28+ 
29+namespace gatherv2 {
30+using namespace AscendC;
31+ 
32+constexpr int32_t BUFFER_NUM_SIMD = 2;
33+ 
34+template <typename INDICES_T>
35+class Gatherv2Simd {
36+public:
37+ __aicore__ inline Gatherv2Simd(TPipe* pipe) : pipe_(pipe){};
38+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y,
39+ const GatherV2TilingData* tilingData);
40+ __aicore__ inline void Process();
41+ template <typename T>
42+ __aicore__ inline void CopyIn(LocalTensor<T> xLocal, GlobalTensor<T> xGm, int64_t offset, uint32_t nBurst,
43+ uint32_t copyLen);
44+ __aicore__ inline INDICES_T GetIndex(int64_t idx, int64_t endIdx);
45+ __aicore__ inline void CopyOut(int64_t offset, uint32_t nBurst, uint32_t copyLen);
46+ __aicore__ inline void NoSplitColProcess(int64_t colsAlign);
47+ __aicore__ inline void SplitColProcess(int64_t colsAlign);
48+ 
49+private:
50+ GlobalTensor<int8_t> xGm_;
51+ GlobalTensor<INDICES_T> indicesGm_;
52+ GlobalTensor<int8_t> yGm_;
53+ TPipe* pipe_;
54+ TQueBind<QuePosition::VECIN, QuePosition::VECOUT, BUFFER_NUM_SIMD> inQueue_;
55+ TBuf<QuePosition::VECCALC> indexBuf_;
56+ const GatherV2TilingData* tilingData_;
57+ 
58+ int32_t needCoreNum_;
59+ int32_t threadNum_;
60+ int64_t batchSize_;
61+ int64_t outerSize_;
62+ int64_t gatherDimSize_;
63+ int64_t gatherSize_;
64+ int64_t innerSize_;
65+ int64_t xSize_;
66+ int64_t indicesSize_;
67+ int64_t ySize_;
68+ int64_t yOffsetBase_ = 0;
69+ int64_t indicesOffsetBase_ = 0;
70+ int64_t maxIndex_ = 0;
71+ int64_t curIndexSize_ = 0;
72+ bool negativeIndexSupport_;
73+ 
74+ int64_t currentCoreElements_;
75+ int32_t blockIdx_;
76+};
77+ 
78+template <typename INDICES_T>
79+__aicore__ inline void Gatherv2Simd<INDICES_T>::Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y,
80+ const GatherV2TilingData* tilingData)
81+{
82+ tilingData_ = tilingData;
83+ blockIdx_ = static_cast<int32_t>(GetBlockIdx());
84+ needCoreNum_ = static_cast<int32_t>(tilingData_->needCoreNum);
85+ threadNum_ = static_cast<int32_t>(tilingData_->threadNum);
86+ batchSize_ = static_cast<int64_t>(tilingData_->batchSize);
87+ outerSize_ = static_cast<int64_t>(tilingData_->outerSize);
88+ gatherDimSize_ = static_cast<int64_t>(tilingData_->gatherDimSize);
89+ gatherSize_ = static_cast<int64_t>(tilingData_->gatherSize);
90+ innerSize_ = static_cast<int64_t>(tilingData_->innerSize);
91+ 
92+ xSize_ = static_cast<int64_t>(tilingData_->xSize);
93+ indicesSize_ = static_cast<int64_t>(tilingData_->indicesSize);
94+ ySize_ = static_cast<int64_t>(tilingData_->ySize);
95+ negativeIndexSupport_ = static_cast<bool>(tilingData_->negativeIndexSupport);
96+ 
97+ xGm_.SetGlobalBuffer((__gm__ int8_t*)x);
98+ 
99+ if (GetBlockIdx() < tilingData_->lastCoreElements) {
100+ yOffsetBase_ = (tilingData_->perCoreElements + 1) * GetBlockIdx();
101+ currentCoreElements_ = tilingData_->perCoreElements + 1;
102+ } else {
103+ yOffsetBase_ = tilingData_->perCoreElements * GetBlockIdx() + tilingData_->lastCoreElements;
104+ currentCoreElements_ = tilingData_->perCoreElements;
105+ }
106+ 
107+ indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices);
108+ yGm_.SetGlobalBuffer((__gm__ int8_t*)y);
109+ pipe_->InitBuffer(inQueue_, BUFFER_NUM_SIMD, tilingData_->maxElement * tilingData_->dtypeSize);
110+ pipe_->InitBuffer(indexBuf_, tilingData_->indiceUbSize);
111+ maxIndex_ = tilingData_->indiceUbSize / sizeof(INDICES_T);
112+}
113+ 
114+template <typename INDICES_T>
115+template <typename T>
116+__aicore__ inline void Gatherv2Simd<INDICES_T>::CopyIn(LocalTensor<T> xLocal, GlobalTensor<T> xGm, int64_t offset,
117+ uint32_t nBurst, uint32_t copyLen)
118+{
119+ DataCopyPadExtParams<T> dataCopyPadExtParams;
120+ dataCopyPadExtParams.isPad = false;
121+ dataCopyPadExtParams.leftPadding = 0;
122+ dataCopyPadExtParams.rightPadding = 0;
123+ dataCopyPadExtParams.paddingValue = 0;
124+ 
125+ DataCopyExtParams dataCoptExtParams;
126+ dataCoptExtParams.blockCount = nBurst;
127+ dataCoptExtParams.blockLen = copyLen * sizeof(T);
128+ dataCoptExtParams.srcStride = 0;
129+ dataCoptExtParams.dstStride = 0;
130+ DataCopyPad(xLocal, xGm[offset], dataCoptExtParams, dataCopyPadExtParams);
131+}
132+ 
133+template <typename INDICES_T>
134+__aicore__ inline void Gatherv2Simd<INDICES_T>::CopyOut(int64_t offset, uint32_t nBurst, uint32_t copyLen)
135+{
136+ DataCopyExtParams dataCoptExtParams;
137+ dataCoptExtParams.blockCount = nBurst;
138+ dataCoptExtParams.blockLen = copyLen;
139+ dataCoptExtParams.srcStride = 0;
140+ dataCoptExtParams.dstStride = 0;
141+ LocalTensor<int8_t> yLocal = inQueue_.DeQue<int8_t>();
142+ event_t eventIdVtoMTE3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));
143+ SetFlag<HardEvent::V_MTE3>(eventIdVtoMTE3);
144+ WaitFlag<HardEvent::V_MTE3>(eventIdVtoMTE3);
145+ DataCopyPad(yGm_[offset], yLocal, dataCoptExtParams);
146+ inQueue_.FreeTensor(yLocal);
147+}
148+ 
149+template <typename INDICES_T>
150+__aicore__ inline INDICES_T Gatherv2Simd<INDICES_T>::GetIndex(int64_t idx, int64_t endIdx)
151+{
152+ if (idx >= indicesOffsetBase_ + curIndexSize_ || idx < indicesOffsetBase_) {
153+ int64_t startBIdx = idx / gatherSize_;
154+ int64_t startGIdx = idx % gatherSize_;
155+ int64_t endBIdx = endIdx / gatherSize_;
156+ int64_t endGIdx = endIdx % gatherSize_;
157+ int64_t copyLen = 0;
158+ if (maxIndex_ >= (endBIdx - startBIdx + 1) * gatherSize_) {
159+ copyLen = (endBIdx - startBIdx + 1) * gatherSize_;
160+ indicesOffsetBase_ = startBIdx * gatherSize_;
161+ } else if (maxIndex_ >= gatherSize_) {
162+ int64_t maxBSize = maxIndex_ / gatherSize_;
163+ copyLen = startBIdx + maxBSize > endBIdx ? (endBIdx - startBIdx + 1) * gatherSize_ : maxBSize * gatherSize_;
164+ indicesOffsetBase_ = startBIdx * gatherSize_;
165+ } else {
166+ copyLen = startGIdx + maxIndex_ >= (endBIdx + 1) * gatherSize_ ? (endBIdx + 1) * gatherSize_ - idx :
167+ maxIndex_;
168+ indicesOffsetBase_ = idx;
169+ }
170+ curIndexSize_ = copyLen;
171+ LocalTensor<INDICES_T> tmpLocal = indexBuf_.Get<INDICES_T>();
172+ CopyIn(tmpLocal, indicesGm_, indicesOffsetBase_, 1, copyLen);
173+ 
174+ event_t eventIdMTE2toS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_S));
175+ SetFlag<HardEvent::MTE2_S>(eventIdMTE2toS);
176+ WaitFlag<HardEvent::MTE2_S>(eventIdMTE2toS);
177+ }
178+ LocalTensor<INDICES_T> indexLocal = indexBuf_.Get<INDICES_T>();
179+ return indexLocal.GetValue(idx - indicesOffsetBase_);
180+}
181+template <typename INDICES_T>
182+__aicore__ inline void Gatherv2Simd<INDICES_T>::NoSplitColProcess(int64_t colsAlign)
183+{
184+ int64_t onceMaxRows = tilingData_->maxElement * tilingData_->dtypeSize / colsAlign;
185+ int64_t loopNum = (currentCoreElements_ + onceMaxRows - 1) / onceMaxRows;
186+ 
187+ // indice -> b, g
188+ // output -> b, o, g, i
189+ int64_t yStart = yOffsetBase_;
190+ int64_t yEnd = yOffsetBase_ + currentCoreElements_ - 1;
191+ int64_t endBatch = yEnd / (outerSize_ * gatherSize_);
192+ int64_t endGIdx = yEnd % gatherSize_;
193+ int64_t indiceEndIdx = endBatch * gatherSize_ + endGIdx;
194+ for (int64_t i = 0; i < loopNum; i++) {
195+ int64_t rows = (i == loopNum - 1) ? (currentCoreElements_ - i * onceMaxRows) : onceMaxRows;
196+ int64_t cols = tilingData_->innerSize;
197+ LocalTensor<int8_t> xLocal = inQueue_.AllocTensor<int8_t>();
198+ event_t eventIdMTE3toV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V));
199+ SetFlag<HardEvent::MTE3_V>(eventIdMTE3toV);
200+ WaitFlag<HardEvent::MTE3_V>(eventIdMTE3toV);
201+ int64_t preIdx0 = -1;
202+ int64_t preIdx1 = -1;
203+ int64_t preIdx2 = -1;
204+ int32_t backOffset0 = 0;
205+ int32_t backOffset1 = 0;
206+ int32_t backOffset2 = 0;
207+ for (int64_t j = 0; j < rows; j++) {
208+ int64_t yIdx = yStart + i * onceMaxRows + j;
209+ int64_t batchI = yIdx / (outerSize_ * gatherSize_);
210+ int64_t outerI = yIdx % (outerSize_ * gatherSize_) / gatherSize_;
211+ int64_t gatherI = yIdx % gatherSize_;
212+ int64_t indiceOffset = batchI * gatherSize_ + gatherI;
213+ INDICES_T index = GetIndex(indiceOffset, indiceEndIdx);
214+ if (negativeIndexSupport_) {
215+ index = index < 0 ? index + gatherDimSize_ : index;
216+ }
217+ int64_t xIndex = ((batchI * outerSize_ + outerI) * gatherDimSize_ + index) * innerSize_;
218+ int64_t offset = xIndex * tilingData_->dtypeSize;
219+ if (likely(index >= 0 && index < gatherDimSize_)) {
220+ if (unlikely(xIndex == preIdx0 || xIndex == preIdx1 || xIndex == preIdx2)) {
221+ event_t eventIdMTE2toV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V));
222+ SetFlag<HardEvent::MTE2_V>(eventIdMTE2toV);
223+ WaitFlag<HardEvent::MTE2_V>(eventIdMTE2toV);
224+ int32_t backStep = (xIndex == preIdx0) ? backOffset0 :
225+ (xIndex == preIdx1) ? backOffset1 :
226+ backOffset2;
227+ DataCopy(xLocal[j * colsAlign], xLocal[backStep], tilingData_->innerSize * tilingData_->dtypeSize);
228+ } else {
229+ CopyIn(xLocal[j * colsAlign], xGm_, offset, 1, tilingData_->innerSize * tilingData_->dtypeSize);
230+ preIdx2 = preIdx1;
231+ preIdx1 = preIdx0;
232+ preIdx0 = xIndex;
233+ backOffset2 = backOffset1;
234+ backOffset1 = backOffset0;
235+ backOffset0 = j * colsAlign;
236+ }
237+ } else {
238+ Duplicate<int8_t>(xLocal[j * colsAlign], 0, tilingData_->innerSize * tilingData_->dtypeSize);
239+ }
240+ }
241+ inQueue_.EnQue<int8_t>(xLocal);
242+ int64_t yOffset = (yStart + i * onceMaxRows) * tilingData_->innerSize * tilingData_->dtypeSize;
243+ CopyOut(yOffset, rows, cols * tilingData_->dtypeSize);
244+ }
245+}
246+ 
247+template <typename INDICES_T>
248+__aicore__ inline void Gatherv2Simd<INDICES_T>::SplitColProcess(int64_t colsAlign)
249+{
250+ // indice -> b, g
251+ // output -> b, o, g, i
252+ int64_t yStart = yOffsetBase_;
253+ int64_t yEnd = yOffsetBase_ + currentCoreElements_ - 1;
254+ int64_t endBatch = yEnd / (outerSize_ * gatherSize_);
255+ int64_t endGIdx = yEnd % gatherSize_;
256+ int64_t indiceEndIdx = endBatch * gatherSize_ + endGIdx;
257+ int64_t loopSize = (tilingData_->innerSize + tilingData_->maxElement - 1) / tilingData_->maxElement;
258+ for (int64_t i = 0; i < currentCoreElements_; i++) {
259+ int64_t yIdx = yStart + i;
260+ int64_t batchI = yIdx / (outerSize_ * gatherSize_);
261+ int64_t outerI = yIdx % (outerSize_ * gatherSize_) / gatherSize_;
262+ int64_t gatherI = yIdx % gatherSize_;
263+ int64_t indiceOffset = batchI * gatherSize_ + gatherI;
264+ INDICES_T index = GetIndex(indiceOffset, indiceEndIdx);
265+ if (negativeIndexSupport_) {
266+ index = index < 0 ? index + gatherDimSize_ : index;
267+ }
268+ int64_t xIndex = ((batchI * outerSize_ + outerI) * gatherDimSize_ + index) * innerSize_;
269+ int64_t yIndex = (yStart + i) * tilingData_->innerSize;
270+ for (int64_t j = 0; j < loopSize; j++) {
271+ int64_t cols = (j == loopSize - 1) ? (tilingData_->innerSize - j * tilingData_->maxElement) :
272+ tilingData_->maxElement;
273+ LocalTensor<int8_t> xLocal = inQueue_.AllocTensor<int8_t>();
274+ int64_t offset = (xIndex + j * tilingData_->maxElement) * tilingData_->dtypeSize;
275+ if (likely(index >= 0 && index < gatherDimSize_)) {
276+ CopyIn(xLocal[0], xGm_, offset, 1, cols * tilingData_->dtypeSize);
277+ } else {
278+ event_t eventIdMTE3toV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V));
279+ SetFlag<HardEvent::MTE3_V>(eventIdMTE3toV);
280+ WaitFlag<HardEvent::MTE3_V>(eventIdMTE3toV);
281+ Duplicate<int8_t>(xLocal[0], 0, cols * tilingData_->dtypeSize);
282+ }
283+ inQueue_.EnQue<int8_t>(xLocal);
284+ int64_t yOffset = (yIndex + j * tilingData_->maxElement) * tilingData_->dtypeSize;
285+ CopyOut(yOffset, 1, cols * tilingData_->dtypeSize);
286+ }
287+ }
288+}
289+ 
290+template <typename INDICES_T>
291+__aicore__ inline void Gatherv2Simd<INDICES_T>::Process()
292+{
293+ if (blockIdx_ >= tilingData_->needCoreNum) {
294+ return;
295+ }
296+ int64_t ubBlockSize = static_cast<int64_t>(Ops::Base::GetUbBlockSize());
297+ int64_t colsAlign = (tilingData_->innerSize * tilingData_->dtypeSize + ubBlockSize - 1) / ubBlockSize * ubBlockSize;
298+ if (colsAlign <= tilingData_->maxElement * tilingData_->dtypeSize) {
299+ NoSplitColProcess(colsAlign);
300+ } else {
301+ SplitColProcess(colsAlign);
302+ }
303+}
304+} // namespace gatherv2
305+#endif // GATHER_V2_SIMD_H
@@ -0,0 +1,276 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file gather_v2_simd_full_load.h
13+ * \brief KirinX90 纯 SIMD full-load 模板(替代原 Gatherv2FullLoad simt 实现,去除 simt_api 依赖)
14+ * 保留 UB 缓冲(CopyIn/DataCopyPad/LoopCol/NoLoop)逻辑,仅把 GatherSimt3D/2D 换成 GatherCore3D/2D
15+ */
16+#ifndef GATHER_V2_SIMD_FULL_LOAD_H
17+#define GATHER_V2_SIMD_FULL_LOAD_H
18+ 
19+#include "kernel_operator.h"
20+#include "op_kernel/platform_util.h"
21+// 已删除: simt_api/asc_simt.h、basic_api/kernel_vec_intf.h(KirinX90 硬件不支持 simt)
22+ 
23+namespace gatherv2 {
24+using namespace AscendC;
25+ 
26+template <typename T, typename INDICES_T, typename U>
27+class Gatherv2SimdFullLoad {
28+public:
29+ __aicore__ inline Gatherv2SimdFullLoad(TPipe* pipe) : pipe_(pipe){};
30+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y,
31+ const GatherV2TilingData* tilingData);
32+ __aicore__ inline void CopyIn(LocalTensor<T> xLocal, GlobalTensor<T> xGm, int64_t offset, uint32_t nBurst,
33+ uint64_t copyLen);
34+ __aicore__ inline void Process();
35+ __aicore__ inline void LoopColProcess();
36+ __aicore__ inline void NoLoopProcess();
37+ 
38+private:
39+ template <const bool NIS>
40+ __aicore__ inline void GatherCore3D(const U yIndexBase, U currentCoreElements, int64_t pOffset, U gatherSize,
41+ U innerSize, U gatherDimSize, __ubuf__ T* srcTensor, __gm__ INDICES_T* indices,
42+ __gm__ volatile T* y);
43+ template <const bool NIS>
44+ __aicore__ inline void GatherCore2D(const U yIndexBase, U currentCoreElements, U innerSize, U gatherDimSize,
45+ __ubuf__ T* srcTensor, __gm__ INDICES_T* indices, __gm__ volatile T* y);
46+ 
47+private:
48+ GlobalTensor<T> xGm_;
49+ GlobalTensor<INDICES_T> indicesGm_;
50+ GlobalTensor<T> yGm_;
51+ 
52+ TPipe* pipe_;
53+ TQue<QuePosition::VECIN, 1> inQueue_;
54+ const GatherV2TilingData* tilingData_;
55+ 
56+ int32_t blockIdx_;
57+ int32_t needCoreNum_;
58+ bool negativeIndexSupport_;
59+ 
60+ U outerSize_;
61+ U gatherDimSize_;
62+ U gatherSize_;
63+ U innerSize_;
64+ U batchSize_;
65+ U ubAviable_;
66+ U currentCoreElements_;
67+ U loopNum_;
68+ U perLoopCoreElements_;
69+};
70+ 
71+// GatherCore3D:替代 GatherSimt3D(去 simt,普通除法,核内串行)
72+template <typename T, typename INDICES_T, typename U>
73+template <const bool NIS>
74+__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::GatherCore3D(const U yIndexBase, U currentCoreElements,
75+ int64_t pOffset, U gatherSize, U innerSize,
76+ U gatherDimSize, __ubuf__ T* srcTensor,
77+ __gm__ INDICES_T* indices,
78+ __gm__ volatile T* y)
79+{
80+ for (U index = 0; index < currentCoreElements; index++) {
81+ U yIndex = yIndexBase + index;
82+ U tmp = yIndex / innerSize; // 原 UintDiv(yIndex, m0, shift0)
83+ U tmp1 = tmp / gatherSize; // 原 UintDiv(tmp, m1, shift1)
84+ U gatherI = tmp - tmp1 * gatherSize;
85+ U innerI = yIndex - tmp * innerSize;
86+ 
87+ INDICES_T indicesValue = indices[gatherI];
88+ if constexpr (NIS) {
89+ if (unlikely(indicesValue < 0)) {
90+ indicesValue += static_cast<INDICES_T>(gatherDimSize);
91+ }
92+ }
93+ U indicesValueI = static_cast<U>(indicesValue);
94+ U xIndex = ((tmp1 - pOffset) * gatherDimSize + indicesValueI) * innerSize + innerI;
95+ bool oob = indicesValue < 0 || indicesValueI >= gatherDimSize;
96+ y[yIndex] = oob ? static_cast<T>(0) : srcTensor[xIndex]; // 从 UB(srcTensor) 读,写 GM(y)
97+ }
98+}
99+ 
100+// GatherCore2D:替代 GatherSimt2D
101+template <typename T, typename INDICES_T, typename U>
102+template <const bool NIS>
103+__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::GatherCore2D(const U yIndexBase, U currentCoreElements,
104+ U innerSize, U gatherDimSize,
105+ __ubuf__ T* srcTensor,
106+ __gm__ INDICES_T* indices,
107+ __gm__ volatile T* y)
108+{
109+ for (U index = 0; index < currentCoreElements; index++) {
110+ U yIndex = yIndexBase + index;
111+ U gatherI = yIndex / innerSize;
112+ U innerI = yIndex - gatherI * innerSize;
113+ 
114+ INDICES_T indicesValue = indices[gatherI];
115+ if constexpr (NIS) {
116+ if (unlikely(indicesValue < 0)) {
117+ indicesValue += static_cast<INDICES_T>(gatherDimSize);
118+ }
119+ }
120+ U indicesValueI = static_cast<U>(indicesValue);
121+ U xIndex = indicesValueI * innerSize + innerI;
122+ bool oob = indicesValue < 0 || indicesValueI >= gatherDimSize;
123+ y[yIndex] = oob ? static_cast<T>(0) : srcTensor[xIndex];
124+ }
125+}
126+ 
127+// CopyIn 保留原样(DataCopyPad 搬 x: GM→UB,非 simt)
128+template <typename T, typename INDICES_T, typename U>
129+__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::CopyIn(LocalTensor<T> xLocal, GlobalTensor<T> xGm,
130+ int64_t offset, uint32_t nBurst, uint64_t copyLen)
131+{
132+ DataCopyPadExtParams<T> dataCopyPadExtParams;
133+ dataCopyPadExtParams.isPad = false;
134+ dataCopyPadExtParams.leftPadding = 0;
135+ dataCopyPadExtParams.rightPadding = 0;
136+ dataCopyPadExtParams.paddingValue = 0;
137+ 
138+ DataCopyExtParams dataCoptExtParams;
139+ dataCoptExtParams.blockCount = nBurst;
140+ dataCoptExtParams.blockLen = copyLen;
141+ dataCoptExtParams.srcStride = 0;
142+ dataCoptExtParams.dstStride = 0;
143+ DataCopyPad(xLocal, xGm[offset], dataCoptExtParams, dataCopyPadExtParams);
144+}
145+ 
146+// LoopColProcess:保留 UB 循环结构,GatherSimt3D→GatherCore3D,去 asc_vf_call/dim3/UintDiv
147+template <typename T, typename INDICES_T, typename U>
148+__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::LoopColProcess()
149+{
150+ int64_t gDimInerSize = gatherDimSize_ * innerSize_;
151+ int64_t gInerSize = gatherSize_ * innerSize_;
152+ 
153+ int64_t maxGdimA = ubAviable_ / gDimInerSize;
154+ int64_t start = blockIdx_ * tilingData_->perCoreElements;
155+ int64_t curCoreData = (blockIdx_ == needCoreNum_ - 1) ? tilingData_->lastCoreElements :
156+ tilingData_->perCoreElements;
157+ int64_t end = start + curCoreData - 1;
158+ 
159+ int64_t startP = start / gatherSize_;
160+ int64_t startG = start % gatherSize_;
161+ int64_t endP = end / gatherSize_;
162+ int64_t startGOffset = startG * innerSize_;
163+ 
164+ loopNum_ = (endP - startP + 1 + maxGdimA - 1) / maxGdimA;
165+ U yIndexBase = blockIdx_ * tilingData_->perCoreElements * innerSize_;
166+ int64_t pOffset = startP;
167+ 
168+ for (int64_t loop = 0; loop < loopNum_; ++loop) {
169+ int64_t copySize;
170+ int64_t startGroup = startP + loop * maxGdimA;
171+ int64_t endGroup = startGroup + maxGdimA;
172+ int64_t groupCount = endGroup - startGroup;
173+ int64_t startOffset = pOffset * gDimInerSize;
174+ 
175+ if (loopNum_ > 1) {
176+ copySize = groupCount * gDimInerSize;
177+ if (loop == 0) {
178+ currentCoreElements_ = groupCount * gInerSize - startGOffset;
179+ } else if (loop == (loopNum_ - 1)) {
180+ copySize = ((endP - startP + 1) % maxGdimA) * gDimInerSize;
181+ currentCoreElements_ = perLoopCoreElements_ - (loop * maxGdimA) * gInerSize + startGOffset;
182+ } else {
183+ currentCoreElements_ = groupCount * gInerSize;
184+ }
185+ } else {
186+ copySize = (endP - startP + 1) * gDimInerSize;
187+ }
188+ 
189+ // 从 GM 搬运数据到 UB(保留)
190+ LocalTensor<T> gaBuffer = inQueue_.AllocTensor<T>();
191+ CopyIn(gaBuffer, xGm_, startOffset, 1, copySize * tilingData_->dtypeSize);
192+ 
193+ inQueue_.EnQue(gaBuffer);
194+ inQueue_.DeQue<T>();
195+ // gather 计算:GatherSimt3D→GatherCore3D(去 asc_vf_call/dim3,去 m0/shift0/m1/shift1)
196+ if (unlikely(negativeIndexSupport_)) {
197+ GatherCore3D<true>(yIndexBase, currentCoreElements_, pOffset, gatherSize_, innerSize_, gatherDimSize_,
198+ (__ubuf__ T*)(gaBuffer.GetPhyAddr()), (__gm__ INDICES_T*)(indicesGm_.GetPhyAddr()),
199+ (__gm__ volatile T*)(yGm_.GetPhyAddr()));
200+ } else {
201+ GatherCore3D<false>(yIndexBase, currentCoreElements_, pOffset, gatherSize_, innerSize_, gatherDimSize_,
202+ (__ubuf__ T*)(gaBuffer.GetPhyAddr()), (__gm__ INDICES_T*)(indicesGm_.GetPhyAddr()),
203+ (__gm__ volatile T*)(yGm_.GetPhyAddr()));
204+ }
205+ // 计算每次循环 yIndexBase(保留)
206+ if (loopNum_ > 1) {
207+ yIndexBase += currentCoreElements_;
208+ pOffset = yIndexBase / gInerSize;
209+ }
210+ inQueue_.FreeTensor(gaBuffer);
211+ }
212+}
213+ 
214+// NoLoopProcess:保留 UB,GatherSimt2D→GatherCore2D
215+template <typename T, typename INDICES_T, typename U>
216+__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::NoLoopProcess()
217+{
218+ int64_t copy_size = gatherDimSize_ * innerSize_;
219+ LocalTensor<T> gaBuffer = inQueue_.AllocTensor<T>();
220+ CopyIn(gaBuffer, xGm_, 0, 1, copy_size * tilingData_->dtypeSize);
221+ 
222+ inQueue_.EnQue(gaBuffer);
223+ inQueue_.DeQue<T>();
224+ U yIndexBase = blockIdx_ * tilingData_->perCoreElements * innerSize_;
225+ if (unlikely(negativeIndexSupport_)) {
226+ GatherCore2D<true>(yIndexBase, currentCoreElements_, innerSize_, gatherDimSize_,
227+ (__ubuf__ T*)(gaBuffer.GetPhyAddr()), (__gm__ INDICES_T*)(indicesGm_.GetPhyAddr()),
228+ (__gm__ volatile T*)(yGm_.GetPhyAddr()));
229+ } else {
230+ GatherCore2D<false>(yIndexBase, currentCoreElements_, innerSize_, gatherDimSize_,
231+ (__ubuf__ T*)(gaBuffer.GetPhyAddr()), (__gm__ INDICES_T*)(indicesGm_.GetPhyAddr()),
232+ (__gm__ volatile T*)(yGm_.GetPhyAddr()));
233+ }
234+ inQueue_.FreeTensor(gaBuffer);
235+}
236+ 
237+template <typename T, typename INDICES_T, typename U>
238+__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y,
239+ const GatherV2TilingData* tilingData)
240+{
241+ tilingData_ = tilingData;
242+ xGm_.SetGlobalBuffer((__gm__ T*)x);
243+ indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices);
244+ yGm_.SetGlobalBuffer((__gm__ T*)y);
245+ 
246+ pipe_->InitBuffer(inQueue_, 1, (tilingData_->maxElement * tilingData_->dtypeSize));
247+}
248+ 
249+template <typename T, typename INDICES_T, typename U>
250+__aicore__ inline void Gatherv2SimdFullLoad<T, INDICES_T, U>::Process()
251+{
252+ blockIdx_ = static_cast<int32_t>(GetBlockIdx());
253+ needCoreNum_ = static_cast<int32_t>(tilingData_->needCoreNum);
254+ 
255+ batchSize_ = static_cast<U>(tilingData_->batchSize);
256+ outerSize_ = static_cast<U>(tilingData_->outerSize);
257+ gatherDimSize_ = static_cast<U>(tilingData_->gatherDimSize);
258+ gatherSize_ = static_cast<U>(tilingData_->gatherSize);
259+ innerSize_ = static_cast<U>(tilingData_->innerSize);
260+ ubAviable_ = static_cast<U>(tilingData_->maxElement);
261+ 
262+ negativeIndexSupport_ = static_cast<bool>(tilingData_->negativeIndexSupport);
263+ currentCoreElements_ = static_cast<U>(tilingData_->perCoreElements * innerSize_);
264+ if (blockIdx_ == tilingData_->needCoreNum - 1) {
265+ currentCoreElements_ = static_cast<U>(tilingData_->lastCoreElements * innerSize_);
266+ }
267+ perLoopCoreElements_ = currentCoreElements_;
268+ 
269+ if (batchSize_ == 1 && outerSize_ > 1) {
270+ LoopColProcess();
271+ } else {
272+ NoLoopProcess();
273+ }
274+}
275+} // namespace gatherv2
276+#endif // GATHER_V2_SIMD_FULL_LOAD_H
@@ -0,0 +1,407 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file gather_v2_simd_last_gather_arch31.h
13+ * \brief
14+ */
15+#ifndef GATHER_V2_SIMD_LAST_GATHER
16+#define GATHER_V2_SIMD_LAST_GATHER
17+ 
18+#ifndef K_MAX_SHAPE_DIM
19+#define K_MAX_SHAPE_DIM 0
20+#endif
21+#if ASC_DEVKIT_MAJOR >= 9
22+#include "basic_api/kernel_vec_intf.h"
23+#else
24+#include "kernel_operator.h"
25+#endif
26+#include "op_kernel/platform_util.h"
27+ 
28+namespace gatherv2 {
29+using namespace AscendC;
30+ 
31+#if 0 // KirinX90 不支持
32+ // MicroAPI(Interleave/DeInterleave/MaskInterleave/MaskDeInterleave/Pack/CompareScalar/MaskAnd/DataCopyGather
33+ // 等) 参照 cast 算子 arch31 适配经验,本文件全部方法实现依赖 MicroAPI,整体注释避免编译错误 如后续 KirinX90
34+ // 支持相关指令或需用 SIMD 重写,解开本 #if 0 即可恢复参考实现
35+template <typename T>
36+struct IndexTypeGet {
37+ using type = typename std::conditional<sizeof(T) == sizeof(int8_t) || sizeof(T) == sizeof(int16_t), uint16_t,
38+ uint32_t>::type;
39+};
40+ 
41+template <typename TARGET_T, typename ORG_T>
42+__simd_callee__ inline void LoadIndices(MicroAPI::RegTensor<TARGET_T>& vregIndcie, MicroAPI::MaskReg& gatherMask,
43+ __local_mem__ ORG_T* indiceAddr, MicroAPI::MaskReg preg, ORG_T dimsize)
44+{
45+ if constexpr (sizeof(ORG_T) == sizeof(int64_t)) {
46+ if constexpr (sizeof(TARGET_T) == sizeof(int32_t)) {
47+ MicroAPI::RegTensor<int64_t, MicroAPI::RegTraitNumTwo> tmpReg;
48+ MicroAPI::DataCopy(tmpReg, indiceAddr);
49+ MicroAPI::MaskReg gtPreg;
50+ MicroAPI::MaskReg ltPreg;
51+ MicroAPI::CompareScalar<int64_t, CMPMODE::GE>(gtPreg, tmpReg, 0, preg);
52+ MicroAPI::CompareScalar<int64_t, CMPMODE::LT>(ltPreg, tmpReg, dimsize, preg);
53+ MicroAPI::MaskAnd(gatherMask, gtPreg, ltPreg, preg);
54+ MicroAPI::Pack((MicroAPI::RegTensor<uint32_t>&)vregIndcie, tmpReg);
55+ } else {
56+ constexpr int16_t vfLen = AscendC::VECTOR_REG_WIDTH / sizeof(int32_t);
57+ MicroAPI::RegTensor<int64_t, MicroAPI::RegTraitNumTwo> tmpReg0, tmpReg1;
58+ MicroAPI::RegTensor<int32_t> tmpB32Reg0, tmpB32Reg1;
59+ MicroAPI::MaskReg lowPreg, highPreg;
60+ MicroAPI::MaskInterleave<int16_t>(lowPreg, highPreg, preg, preg);
61+ MicroAPI::DataCopy(tmpReg0, indiceAddr);
62+ MicroAPI::DataCopy(tmpReg1, indiceAddr + vfLen);
63+ MicroAPI::Pack((MicroAPI::RegTensor<uint32_t>&)tmpB32Reg0, tmpReg0);
64+ MicroAPI::Pack((MicroAPI::RegTensor<uint32_t>&)tmpB32Reg1, tmpReg1);
65+ MicroAPI::DeInterleave<int16_t>(
66+ (MicroAPI::RegTensor<int16_t>&)vregIndcie, (MicroAPI::RegTensor<int16_t>&)tmpB32Reg0,
67+ (MicroAPI::RegTensor<int16_t>&)tmpB32Reg0, (MicroAPI::RegTensor<int16_t>&)tmpB32Reg1);
68+ MicroAPI::MaskReg gtPreg, ltPreg;
69+ MicroAPI::CompareScalar<int64_t, CMPMODE::GE>(gtPreg, tmpReg0, 0, lowPreg);
70+ MicroAPI::CompareScalar<int64_t, CMPMODE::LT>(ltPreg, tmpReg0, dimsize, lowPreg);
71+ MicroAPI::MaskAnd(lowPreg, gtPreg, ltPreg, lowPreg);
72+ MicroAPI::CompareScalar<int64_t, CMPMODE::GE>(gtPreg, tmpReg1, 0, highPreg);
73+ MicroAPI::CompareScalar<int64_t, CMPMODE::LT>(ltPreg, tmpReg1, dimsize, highPreg);
74+ MicroAPI::MaskAnd(highPreg, gtPreg, ltPreg, highPreg);
75+ MicroAPI::MaskDeInterleave<int16_t>(gatherMask, ltPreg, lowPreg, highPreg);
76+ }
77+ } else {
78+ if constexpr (sizeof(TARGET_T) == sizeof(int32_t)) {
79+ MicroAPI::DataCopy((MicroAPI::RegTensor<int32_t>&)vregIndcie, indiceAddr);
80+ MicroAPI::MaskReg gtPreg;
81+ MicroAPI::MaskReg ltPreg;
82+ MicroAPI::CompareScalar<int32_t, CMPMODE::GE>(gtPreg, (MicroAPI::RegTensor<int32_t>&)vregIndcie, 0, preg);
83+ MicroAPI::CompareScalar<int32_t, CMPMODE::LT>(ltPreg, (MicroAPI::RegTensor<int32_t>&)vregIndcie, dimsize,
84+ preg);
85+ MicroAPI::MaskAnd(gatherMask, gtPreg, ltPreg, preg);
86+ } else {
87+ constexpr int16_t vfLen = AscendC::VECTOR_REG_WIDTH / sizeof(int32_t);
88+ MicroAPI::RegTensor<int32_t> tmpReg0, tmpReg1;
89+ MicroAPI::RegTensor<int32_t> tmpB32;
90+ MicroAPI::MaskReg lowPreg, highPreg;
91+ MicroAPI::MaskInterleave<int16_t>(lowPreg, highPreg, preg, preg);
92+ MicroAPI::DataCopy(tmpReg0, indiceAddr);
93+ MicroAPI::DataCopy(tmpReg1, indiceAddr + vfLen);
94+ 
95+ MicroAPI::DeInterleave<int16_t>(
96+ (MicroAPI::RegTensor<int16_t>&)vregIndcie, (MicroAPI::RegTensor<int16_t>&)tmpB32,
97+ (MicroAPI::RegTensor<int16_t>&)tmpReg0, (MicroAPI::RegTensor<int16_t>&)tmpReg1);
98+ MicroAPI::MaskReg gtPreg, ltPreg;
99+ MicroAPI::CompareScalar<int32_t, CMPMODE::GE>(gtPreg, tmpReg0, 0, lowPreg);
100+ MicroAPI::CompareScalar<int32_t, CMPMODE::LT>(ltPreg, tmpReg0, dimsize, lowPreg);
101+ MicroAPI::MaskAnd(lowPreg, gtPreg, ltPreg, lowPreg);
102+ MicroAPI::CompareScalar<int32_t, CMPMODE::GE>(gtPreg, tmpReg1, 0, highPreg);
103+ MicroAPI::CompareScalar<int32_t, CMPMODE::LT>(ltPreg, tmpReg1, dimsize, highPreg);
104+ MicroAPI::MaskAnd(highPreg, gtPreg, ltPreg, highPreg);
105+ MicroAPI::MaskDeInterleave<int16_t>(gatherMask, ltPreg, lowPreg, highPreg);
106+ }
107+ }
108+}
109+ 
110+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
111+class Gatherv2SimdLastGather {
112+public:
113+ __aicore__ inline Gatherv2SimdLastGather(TPipe* pipe) : pipe_(pipe){};
114+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y,
115+ const GatherV2LastTilingData* tilingData);
116+ __aicore__ inline void Process();
117+ __aicore__ inline void CopyInIndices(int64_t offset, uint32_t nBurst, uint32_t copyLen);
118+ __aicore__ inline void CopyInX(int64_t offset, uint32_t nBurst, uint32_t copyLen);
119+ __aicore__ inline void CopyOut(int64_t offset, uint32_t nBurst, uint32_t copyLen);
120+ __aicore__ inline void ConvertNegIndices(const LocalTensor<int32_t>& indicesLocal, int32_t num);
121+ __aicore__ inline void GatherComputeMultiRows(const LocalTensor<X_T>& xLocal,
122+ const LocalTensor<int32_t>& indicesLocal, int32_t rows, int32_t cols,
123+ int64_t yOffset);
124+ __aicore__ inline void SplitIndicesProcess();
125+ __aicore__ inline void NoSplitIndicesProcess();
126+ 
127+private:
128+ GlobalTensor<X_T> xGm_;
129+ GlobalTensor<INDICES_T> indicesGm_;
130+ GlobalTensor<X_T> yGm_;
131+ TPipe* pipe_;
132+ TQue<QuePosition::VECIN, 1> inQueue_;
133+ TQue<QuePosition::VECIN, 1> indexQue_;
134+ TQue<QuePosition::VECOUT, 1> outQueue_;
135+ TBuf<QuePosition::VECCALC> indexBuf_;
136+ const GatherV2LastTilingData* tilingData_;
137+ int32_t blockIdx_{0};
138+ int32_t pBlock_{0};
139+ int32_t gBlock_{0};
140+ int32_t curGatherFactor_{0};
141+};
142+ 
143+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
144+__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::Init(
145+ GM_ADDR x, GM_ADDR indices, GM_ADDR axis, GM_ADDR y, const GatherV2LastTilingData* tilingData)
146+{
147+ tilingData_ = tilingData;
148+ blockIdx_ = static_cast<int32_t>(GetBlockIdx());
149+ pBlock_ = blockIdx_ / tilingData->coreInCols;
150+ gBlock_ = blockIdx_ % tilingData->coreInCols;
151+ xGm_.SetGlobalBuffer((__gm__ X_T*)x + pBlock_ * tilingData->blockFactor * tilingData->gatherDimSize);
152+ 
153+ indicesGm_.SetGlobalBuffer((__gm__ INDICES_T*)indices + gBlock_ * tilingData->gFactor);
154+ yGm_.SetGlobalBuffer((__gm__ X_T*)y + pBlock_ * tilingData->blockFactor * tilingData->gatherSize +
155+ gBlock_ * tilingData->gFactor);
156+ pipe_->InitBuffer(inQueue_, tilingData_->inputNum, tilingData_->inputUbSize);
157+ pipe_->InitBuffer(outQueue_, 2, tilingData_->outUbSize);
158+ pipe_->InitBuffer(indexQue_, tilingData_->indicesNum, tilingData_->indiceUbSize);
159+ pipe_->InitBuffer(indexBuf_, tilingData_->indiceCastUbSize);
160+ curGatherFactor_ = (gBlock_ == tilingData->coreInCols - 1) ?
161+ tilingData->gatherSize - (tilingData->coreInCols - 1) * tilingData->gFactor :
162+ tilingData->gFactor;
163+}
164+ 
165+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
166+__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::CopyInIndices(int64_t offset,
167+ uint32_t nBurst,
168+ uint32_t copyLen)
169+{
170+ DataCopyPadExtParams<INDICES_T> dataCopyPadExtParams;
171+ dataCopyPadExtParams.isPad = false;
172+ dataCopyPadExtParams.leftPadding = 0;
173+ dataCopyPadExtParams.rightPadding = 0;
174+ dataCopyPadExtParams.paddingValue = 0;
175+ LocalTensor<INDICES_T> indicesLocal = indexQue_.AllocTensor<INDICES_T>();
176+ DataCopyExtParams dataCoptExtParams;
177+ dataCoptExtParams.blockCount = nBurst;
178+ dataCoptExtParams.blockLen = copyLen * sizeof(INDICES_T);
179+ dataCoptExtParams.srcStride = 0;
180+ dataCoptExtParams.dstStride = 0;
181+ DataCopyPad(indicesLocal, indicesGm_[offset], dataCoptExtParams, dataCopyPadExtParams);
182+ indexQue_.EnQue<INDICES_T>(indicesLocal);
183+}
184+ 
185+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
186+__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::CopyInX(int64_t offset,
187+ uint32_t nBurst,
188+ uint32_t copyLen)
189+{
190+ DataCopyPadExtParams<X_T> dataCopyPadExtParams;
191+ dataCopyPadExtParams.isPad = false;
192+ dataCopyPadExtParams.leftPadding = 0;
193+ dataCopyPadExtParams.rightPadding = 0;
194+ dataCopyPadExtParams.paddingValue = 0;
195+ LocalTensor<X_T> xLocal = inQueue_.AllocTensor<X_T>();
196+ DataCopyExtParams dataCoptExtParams;
197+ dataCoptExtParams.blockCount = nBurst;
198+ dataCoptExtParams.blockLen = copyLen * sizeof(X_T);
199+ dataCoptExtParams.srcStride = 0;
200+ dataCoptExtParams.dstStride = 0;
201+ DataCopyPad(xLocal, xGm_[offset], dataCoptExtParams, dataCopyPadExtParams);
202+ inQueue_.EnQue<X_T>(xLocal);
203+}
204+ 
205+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
206+__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::CopyOut(int64_t offset,
207+ uint32_t nBurst,
208+ uint32_t copyLen)
209+{
210+ int32_t ubBlockSize = static_cast<int64_t>(Ops::Base::GetUbBlockSize());
211+ int32_t srcStride = (tilingData_->ubCols - copyLen) * sizeof(X_T) / ubBlockSize;
212+ int32_t dstStride = (tilingData_->gatherSize - copyLen) * sizeof(X_T);
213+ DataCopyExtParams dataCoptExtParams;
214+ dataCoptExtParams.blockCount = nBurst;
215+ dataCoptExtParams.blockLen = copyLen * sizeof(X_T);
216+ dataCoptExtParams.srcStride = srcStride;
217+ dataCoptExtParams.dstStride = dstStride;
218+ LocalTensor<X_T> yLocal = outQueue_.DeQue<X_T>();
219+ DataCopyPad(yGm_[offset], yLocal, dataCoptExtParams);
220+ outQueue_.FreeTensor(yLocal);
221+}
222+ 
223+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
224+__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::SplitIndicesProcess()
225+{
226+ int32_t inputFactor = tilingData_->ubRows;
227+ int64_t curBlockFactor = (GetBlockIdx() == tilingData_->needCoreNum - 1) ? tilingData_->tailBlockFactor :
228+ tilingData_->blockFactor;
229+ int64_t inputLoop = (curBlockFactor + inputFactor - 1) / inputFactor;
230+ int32_t tailInputFactor = curBlockFactor - (inputLoop - 1) * inputFactor;
231+ int32_t maxOutCols = tilingData_->ubCols;
232+ int64_t outLoopNum = (curGatherFactor_ + maxOutCols - 1) / maxOutCols;
233+ int32_t tailOutCols = curGatherFactor_ - (outLoopNum - 1) * maxOutCols;
234+ for (int64_t i = 0; i < inputLoop; i++) {
235+ int32_t curInputFactor = (i == inputLoop - 1) ? tailInputFactor : inputFactor;
236+ CopyInX(i * inputFactor * tilingData_->gatherDimSize, 1, curInputFactor * tilingData_->gatherDimSize);
237+ LocalTensor<X_T> xLocal = inQueue_.DeQue<X_T>();
238+ for (int64_t j = 0; j < outLoopNum; j++) {
239+ int32_t curOutCols = (j == outLoopNum - 1) ? tailOutCols : maxOutCols;
240+ CopyInIndices(j * maxOutCols, 1, curOutCols);
241+ LocalTensor<INDICES_T> indicesLocal = indexQue_.DeQue<INDICES_T>();
242+ if constexpr (sizeof(INDICES_T) == sizeof(int64_t)) {
243+ LocalTensor<int32_t> tmpLocal = indexBuf_.Get<int32_t>();
244+ Cast(tmpLocal, indicesLocal, AscendC::RoundMode::CAST_NONE, curOutCols);
245+ indexQue_.FreeTensor(indicesLocal);
246+ ConvertNegIndices(tmpLocal, curOutCols);
247+ GatherComputeMultiRows(xLocal, tmpLocal, curInputFactor, curOutCols,
248+ i * inputFactor * tilingData_->gatherSize + j * maxOutCols);
249+ } else {
250+ ConvertNegIndices(indicesLocal, curOutCols);
251+ GatherComputeMultiRows(xLocal, indicesLocal, curInputFactor, curOutCols,
252+ i * inputFactor * tilingData_->gatherSize + j * maxOutCols);
253+ indexQue_.FreeTensor(indicesLocal);
254+ }
255+ }
256+ inQueue_.FreeTensor(xLocal);
257+ }
258+}
259+ 
260+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
261+__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::NoSplitIndicesProcess()
262+{
263+ int32_t maxIndcieNum = tilingData_->outUbSize / sizeof(X_T);
264+ int64_t indiceLoopNum = (curGatherFactor_ + maxIndcieNum - 1) / maxIndcieNum;
265+ int32_t tailIndicesNum = curGatherFactor_ - (indiceLoopNum - 1) * maxIndcieNum;
266+ int32_t inputFactor = tilingData_->ubRows;
267+ int64_t curBlockFactor = (GetBlockIdx() == tilingData_->needCoreNum - 1) ? tilingData_->tailBlockFactor :
268+ tilingData_->blockFactor;
269+ int64_t inputLoop = (curBlockFactor + inputFactor - 1) / inputFactor;
270+ int32_t tailInputFactor = curBlockFactor - (inputLoop - 1) * inputFactor;
271+ CopyInIndices(0, 1, curGatherFactor_);
272+ LocalTensor<INDICES_T> indicesLocal = indexQue_.DeQue<INDICES_T>();
273+ LocalTensor<int32_t> tmpLocal = indexBuf_.Get<int32_t>();
274+ if constexpr (sizeof(INDICES_T) == sizeof(int64_t)) {
275+ Cast(tmpLocal, indicesLocal, AscendC::RoundMode::CAST_NONE, curGatherFactor_);
276+ ConvertNegIndices(tmpLocal, curGatherFactor_);
277+ } else {
278+ ConvertNegIndices(indicesLocal, curGatherFactor_);
279+ }
280+ 
281+ for (int64_t i = 0; i < inputLoop; i++) {
282+ int32_t curInputFactor = (i == inputLoop - 1) ? tailInputFactor : inputFactor;
283+ CopyInX(i * inputFactor * tilingData_->gatherDimSize, 1, curInputFactor * tilingData_->gatherDimSize);
284+ LocalTensor<X_T> xLocal = inQueue_.DeQue<X_T>();
285+ if constexpr (sizeof(INDICES_T) == sizeof(int64_t)) {
286+ GatherComputeMultiRows(xLocal, tmpLocal, curInputFactor, curGatherFactor_,
287+ i * inputFactor * tilingData_->gatherSize);
288+ } else {
289+ GatherComputeMultiRows(xLocal, indicesLocal, curInputFactor, curGatherFactor_,
290+ i * inputFactor * tilingData_->gatherSize);
291+ }
292+ inQueue_.FreeTensor(xLocal);
293+ }
294+ indexQue_.FreeTensor(indicesLocal);
295+}
296+ 
297+template <typename X_T, typename indiceType>
298+__simd_vf__ inline void GatherComputeMultiRowsVF(__ubuf__ int32_t* indiceAddr, __ubuf__ X_T* xAddr, __ubuf__ X_T* yAddr,
299+ int32_t curCols, int32_t indicesOffset, uint16_t vfLoopNum,
300+ int32_t dimSize, int16_t vfLen, uint16_t vfRowsLoop, int32_t colInUb)
301+{
302+ using RegDstT = typename std::conditional<sizeof(X_T) == sizeof(int64_t),
303+ MicroAPI::RegTensor<X_T, MicroAPI::RegTraitNumTwo>,
304+ MicroAPI::RegTensor<X_T>>::type;
305+ RegDstT vd0;
306+ MicroAPI::RegTensor<indiceType> vregIndcie;
307+ MicroAPI::MaskReg gatherMask;
308+ uint32_t size = curCols;
309+ __local_mem__ int32_t* curIndiceAddr = indiceAddr + indicesOffset;
310+ for (uint16_t i = 0; i < vfLoopNum; i++) {
311+ MicroAPI::MaskReg preg0 = AscendC::MicroAPI::UpdateMask<indiceType>(size);
312+ LoadIndices<indiceType, int32_t>(vregIndcie, gatherMask, curIndiceAddr, preg0, dimSize);
313+ curIndiceAddr += vfLen;
314+ __local_mem__ X_T* curyAddr = yAddr + i * vfLen;
315+ __local_mem__ X_T* curXaddr = xAddr;
316+ for (uint16_t j = 0; j < vfRowsLoop; j++) {
317+ if constexpr (sizeof(X_T) == 1) {
318+ MicroAPI::DataCopyGather((MicroAPI::RegTensor<int16_t>&)vd0, curXaddr, vregIndcie, gatherMask);
319+ MicroAPI::DataCopy<X_T, AscendC::MicroAPI::StoreDist::DIST_PACK_B16>(curyAddr, vd0, preg0);
320+ } else {
321+ MicroAPI::DataCopyGather(vd0, curXaddr, vregIndcie, gatherMask);
322+ MicroAPI::DataCopy(curyAddr, vd0, preg0);
323+ }
324+ curXaddr += dimSize;
325+ curyAddr += colInUb;
326+ }
327+ }
328+}
329+ 
330+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
331+__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::GatherComputeMultiRows(
332+ const LocalTensor<X_T>& xLocal, const LocalTensor<int32_t>& indicesLocal, int32_t rows, int32_t cols,
333+ int64_t yOffset)
334+{
335+ __local_mem__ X_T* xAddr = (__local_mem__ X_T*)xLocal.GetPhyAddr();
336+ __local_mem__ int32_t* indiceAddr = (__local_mem__ int32_t*)indicesLocal.GetPhyAddr();
337+ using indiceType = typename IndexTypeGet<X_T>::type;
338+ int32_t colInUb = tilingData_->ubCols;
339+ int32_t loopInCols = (cols + colInUb - 1) / colInUb;
340+ int32_t tailCols = cols - (loopInCols - 1) * colInUb;
341+ 
342+ for (int32_t colIdx = 0; colIdx < loopInCols; colIdx++) {
343+ LocalTensor<X_T> yLocal = outQueue_.AllocTensor<X_T>();
344+ __local_mem__ X_T* yAddr = (__local_mem__ X_T*)yLocal.GetPhyAddr();
345+ int32_t curCols = (colIdx == loopInCols - 1) ? tailCols : colInUb;
346+ constexpr int16_t vfLen = AscendC::VECTOR_REG_WIDTH / sizeof(indiceType);
347+ uint16_t vfLoopNum = (curCols + vfLen - 1) / vfLen;
348+ int32_t dimSize = tilingData_->gatherDimSize;
349+ uint16_t vfRowsLoop = rows;
350+ int32_t indicesOffset = colIdx * colInUb;
351+ GatherComputeMultiRowsVF<X_T, indiceType>(indiceAddr, xAddr, yAddr, curCols, indicesOffset, vfLoopNum, dimSize,
352+ vfLen, vfRowsLoop, colInUb);
353+ 
354+ outQueue_.EnQue(yLocal);
355+ CopyOut(yOffset + colIdx * colInUb, rows, curCols);
356+ }
357+}
358+ 
359+__simd_vf__ inline void ConvertNegIndicesVF(__ubuf__ int32_t* indiceAddr, int16_t vfLen, uint16_t vfLoopNum,
360+ int32_t dimSize, int32_t num)
361+{
362+ MicroAPI::RegTensor<int32_t> indice;
363+ MicroAPI::RegTensor<int32_t> dst;
364+ MicroAPI::MaskReg ltPreg;
365+ uint32_t size = num;
366+ __local_mem__ int32_t* curIndiceAddr = indiceAddr;
367+ for (uint16_t i = 0; i < vfLoopNum; i++) {
368+ MicroAPI::MaskReg preg = AscendC::MicroAPI::UpdateMask<int32_t>(size);
369+ MicroAPI::DataCopy(indice, curIndiceAddr);
370+ MicroAPI::CompareScalar<int32_t, CMPMODE::LT>(ltPreg, indice, 0, preg);
371+ MicroAPI::Adds(dst, indice, dimSize, ltPreg);
372+ MicroAPI::Copy<int32_t, MicroAPI::MaskMergeMode::MERGING>(indice, dst, ltPreg);
373+ MicroAPI::DataCopy(curIndiceAddr, indice, preg);
374+ curIndiceAddr += vfLen;
375+ }
376+}
377+ 
378+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
379+__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::ConvertNegIndices(
380+ const LocalTensor<int32_t>& indicesLocal, int32_t num)
381+{
382+ if constexpr (NEG_INDICE_SUPPORT) {
383+ __local_mem__ int32_t* indiceAddr = (__local_mem__ int32_t*)indicesLocal.GetPhyAddr();
384+ constexpr int16_t vfLen = AscendC::VECTOR_REG_WIDTH / sizeof(int32_t);
385+ uint16_t vfLoopNum = (num + vfLen - 1) / vfLen;
386+ int32_t dimSize = tilingData_->gatherDimSize;
387+ ConvertNegIndicesVF(indiceAddr, vfLen, vfLoopNum, dimSize, num);
388+ }
389+}
390+ 
391+template <typename X_T, typename INDICES_T, bool NEG_INDICE_SUPPORT>
392+__aicore__ inline void Gatherv2SimdLastGather<X_T, INDICES_T, NEG_INDICE_SUPPORT>::Process()
393+{
394+ if (blockIdx_ >= tilingData_->needCoreNum) {
395+ return;
396+ }
397+ 
398+ if (tilingData_->splitMode == 0) {
399+ NoSplitIndicesProcess();
400+ } else if (tilingData_->splitMode == 1) {
401+ SplitIndicesProcess();
402+ }
403+}
404+ 
405+#endif // KirinX90 不支持 MicroAPI
406+} // namespace gatherv2
407+#endif // GATHER_V2_SIMD_LAST_GATHER