Star801
279
代码
代码
Issues29
Pull Requests157
Actions
项目讨论
Wiki
分析
Star801
279
已合并
【feat】:v35 reduce全量迁移 #2118
guojianyang创建于 29 天前
【feat】:v35 reduce全量迁移 #2118
已合并
guojianyang创建于 29 天前
从Joey_guo/graph-autofusionredu合入到cann/graph-autofusiondevelop
共 26 个文件变更+4526-10
Mautofuse/tests/v35/st/backend_e2e_v2/load_compare_cast_sum_store_test/load_compare_cast_sum_store_backend_generator.cpp+1-1
@@ -54,6 +54,6 @@ TEST_F(TestBackendLoadCompareCastSumStoreE2e, LoadCompareCastSumStoreE2eCodegen)
54 EXPECT_NE(kernel.find("{static_cast<uint16_t>(((32 * Ceiling((Rational(1 , 32) * t->z2t_size))))/(1)), "54 EXPECT_NE(kernel.find("{static_cast<uint16_t>(((32 * Ceiling((Rational(1 , 32) * t->z2t_size))))/(1)), "
55 "static_cast<uint16_t>(1)}"),55 "static_cast<uint16_t>(1)}"),
56 std::string::npos);56 std::string::npos);
57- EXPECT_NE(kernel.find("ReduceSum<float, AscendC::Pattern::Reduce::RA, true>"), std::string::npos);57+ EXPECT_NE(kernel.find("ReduceSumExtend<float, AscendC::Pattern::Reduce::RA, true>"), std::string::npos);
58 });58 });
59}59}
Mautofuse/tests/v35/ut/ascendc/api_regbase/CMakeLists.txt+2-0
@@ -14,6 +14,7 @@ add_library(test_regbase_ascendc OBJECT
14 ./test_min.cpp14 ./test_min.cpp
15 ./test_where.cpp15 ./test_where.cpp
16 ./test_reduce_init.cpp16 ./test_reduce_init.cpp
17+ ./test_reduce_extend.cpp
17 ./test_argmax.cpp18 ./test_argmax.cpp
18 ./test_add.cpp19 ./test_add.cpp
19 ./test_airy_ai.cpp20 ./test_airy_ai.cpp
@@ -88,4 +89,5 @@ target_link_libraries(test_regbase_ascendc
88target_include_directories(test_regbase_ascendc PUBLIC89target_include_directories(test_regbase_ascendc PUBLIC
89 ${CODE_ROOT_DIR}/ascendc90 ${CODE_ROOT_DIR}/ascendc
90 ${CODE_ROOT_DIR}/v35/ascendc91 ${CODE_ROOT_DIR}/v35/ascendc
92+ ${ASCEND_INSTALL_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/asc/include
91)93)
Aautofuse/tests/v35/ut/ascendc/api_regbase/test_reduce_extend.cpp+148-0
@@ -0,0 +1,148 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include <algorithm>
12+#include <cstdint>
13+#include <functional>
14+#include <numeric>
15+#include <vector>
16+ 
17+#include "gtest/gtest.h"
18+#include "tikicpulib.h"
19+#include "test_api_utils.h"
20+#include "api_regbase/reduce_extend.h"
21+ 
22+namespace {
23+constexpr uint32_t kTmpBufferSize = 32 * 1024;
24+ 
25+template <typename T, typename Pattern, typename ReduceFunc>
26+void RunReduceKernel(T *input, T *output, uint32_t inputCount, uint32_t outputCount, uint32_t dim0, uint32_t dim1,
27+ ReduceFunc reduce) {
28+ TPipe pipe;
29+ TBuf<TPosition::VECCALC> srcBuffer;
30+ TBuf<TPosition::VECCALC> dstBuffer;
31+ TBuf<TPosition::VECCALC> tmpBuffer;
32+ pipe.InitBuffer(srcBuffer, AlignUp(inputCount * sizeof(T), ONE_BLK_SIZE));
33+ pipe.InitBuffer(dstBuffer, AlignUp(outputCount * sizeof(T), ONE_BLK_SIZE));
34+ pipe.InitBuffer(tmpBuffer, kTmpBufferSize);
35+ 
36+ LocalTensor<T> src = srcBuffer.Get<T>();
37+ LocalTensor<T> dst = dstBuffer.Get<T>();
38+ LocalTensor<uint8_t> tmp = tmpBuffer.Get<uint8_t>();
39+ GmToUb(src, input, inputCount);
40+ 
41+ const uint32_t shape[] = {dim0, dim1};
42+ reduce(dst, src, tmp, shape);
43+ UbToGm(output, dst, outputCount);
44+}
45+ 
46+template <typename T, typename Pattern, typename ReduceFunc>
47+std::vector<T> RunReduce(const std::vector<T> &input, uint32_t outputCount, uint32_t dim0, uint32_t dim1,
48+ ReduceFunc reduce) {
49+ const uint32_t inputBytes = AlignUp(input.size() * sizeof(T), ONE_BLK_SIZE);
50+ const uint32_t outputBytes = AlignUp(outputCount * sizeof(T), ONE_BLK_SIZE);
51+ auto *inputGm = static_cast<T *>(AscendC::GmAlloc(inputBytes));
52+ auto *outputGm = static_cast<T *>(AscendC::GmAlloc(outputBytes));
53+ std::copy(input.begin(), input.end(), inputGm);
54+ std::fill_n(outputGm, outputBytes / sizeof(T), T{});
55+ 
56+ auto kernel = [reduce](T *kernelInput, T *kernelOutput, uint32_t inputSize, uint32_t outputSize, uint32_t shape0,
57+ uint32_t shape1) {
58+ RunReduceKernel<T, Pattern>(kernelInput, kernelOutput, inputSize, outputSize, shape0, shape1, reduce);
59+ };
60+ AscendC::SetKernelMode(KernelMode::AIV_MODE);
61+ ICPU_RUN_KF(kernel, 1, inputGm, outputGm, input.size(), outputCount, dim0, dim1);
62+ 
63+ std::vector<T> output(outputGm, outputGm + outputCount);
64+ AscendC::GmFree(inputGm);
65+ AscendC::GmFree(outputGm);
66+ return output;
67+}
68+} // namespace
69+ 
70+TEST(TestApiReduceExtend, ReduceSumFloatArAndRa) {
71+ std::vector<float> arInput(16);
72+ std::iota(arInput.begin(), arInput.end(), 1.0F);
73+ auto arOutput = RunReduce<float, AscendC::Pattern::Reduce::AR>(
74+ arInput, 2, 2, 8, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
75+ AscendC::ReduceSumExtend<float, AscendC::Pattern::Reduce::AR>(dst, src, tmp, shape, false);
76+ });
77+ EXPECT_EQ(arOutput, (std::vector<float>{36.0F, 100.0F}));
78+ 
79+ std::vector<float> raInput(32);
80+ std::iota(raInput.begin(), raInput.end(), 1.0F);
81+ auto raOutput = RunReduce<float, AscendC::Pattern::Reduce::RA>(
82+ raInput, 8, 4, 8, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
83+ AscendC::ReduceSumExtend<float, AscendC::Pattern::Reduce::RA>(dst, src, tmp, shape, false);
84+ });
85+ for (uint32_t i = 0; i < raOutput.size(); ++i) {
86+ EXPECT_FLOAT_EQ(raOutput[i], 52.0F + 4.0F * i);
87+ }
88+}
89+ 
90+TEST(TestApiReduceExtend, ReduceSumSupportsInt8AndInt16) {
91+ std::vector<int8_t> int8Input(64, 1);
92+ auto int8Output = RunReduce<int8_t, AscendC::Pattern::Reduce::AR>(
93+ int8Input, 2, 2, 32, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
94+ AscendC::ReduceSumExtend<int8_t, AscendC::Pattern::Reduce::AR>(dst, src, tmp, shape, false);
95+ });
96+ EXPECT_EQ(int8Output, (std::vector<int8_t>{32, 32}));
97+ 
98+ std::vector<int16_t> int16Input(32, 2);
99+ auto int16Output = RunReduce<int16_t, AscendC::Pattern::Reduce::AR>(
100+ int16Input, 2, 2, 16, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
101+ AscendC::ReduceSumExtend<int16_t, AscendC::Pattern::Reduce::AR>(dst, src, tmp, shape, false);
102+ });
103+ EXPECT_EQ(int16Output, (std::vector<int16_t>{32, 32}));
104+}
105+ 
106+TEST(TestApiReduceExtend, ArithmeticAndLogicalReduceEntries) {
107+ const std::vector<float> input = {1, 2, 3, 4, 5, 6, 7, 8, -1, -2, -3, -4, -5, -6, -7, -8};
108+ 
109+ auto maxOutput = RunReduce<float, AscendC::Pattern::Reduce::AR>(
110+ input, 2, 2, 8, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
111+ AscendC::ReduceMaxExtend<float, AscendC::Pattern::Reduce::AR>(dst, src, tmp, shape, false);
112+ });
113+ EXPECT_EQ(maxOutput, (std::vector<float>{8.0F, -1.0F}));
114+ 
115+ auto minOutput = RunReduce<float, AscendC::Pattern::Reduce::AR>(
116+ input, 2, 2, 8, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
117+ AscendC::ReduceMinExtend<float, AscendC::Pattern::Reduce::AR>(dst, src, tmp, shape, false);
118+ });
119+ EXPECT_EQ(minOutput, (std::vector<float>{1.0F, -8.0F}));
120+ 
121+ auto meanOutput = RunReduce<float, AscendC::Pattern::Reduce::AR>(
122+ input, 2, 2, 8, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
123+ AscendC::ReduceMeanExtend<float, AscendC::Pattern::Reduce::AR>(dst, src, tmp, shape, true);
124+ });
125+ EXPECT_EQ(meanOutput, (std::vector<float>{4.5F, -4.5F}));
126+ 
127+ const std::vector<float> prodInput(16, 1.0F);
128+ auto prodOutput = RunReduce<float, AscendC::Pattern::Reduce::AR>(
129+ prodInput, 2, 2, 8, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
130+ AscendC::ReduceProdExtend<float, AscendC::Pattern::Reduce::AR>(dst, src, tmp, shape, true);
131+ });
132+ EXPECT_EQ(prodOutput, (std::vector<float>{1.0F, 1.0F}));
133+ 
134+ std::vector<uint8_t> logicalInput(64, 1);
135+ logicalInput[7] = 0;
136+ logicalInput[32] = 0;
137+ auto anyOutput = RunReduce<uint8_t, AscendC::Pattern::Reduce::AR>(
138+ logicalInput, 2, 2, 32, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
139+ AscendC::ReduceAnyExtend<uint8_t, AscendC::Pattern::Reduce::AR>(dst, src, tmp, shape, true);
140+ });
141+ EXPECT_EQ(anyOutput, (std::vector<uint8_t>{1, 1}));
142+ 
143+ auto allOutput = RunReduce<uint8_t, AscendC::Pattern::Reduce::AR>(
144+ logicalInput, 2, 2, 32, [](auto &dst, auto &src, auto &tmp, const uint32_t *shape) {
145+ AscendC::ReduceAllExtend<uint8_t, AscendC::Pattern::Reduce::AR>(dst, src, tmp, shape, true);
146+ });
147+ EXPECT_EQ(allOutput, (std::vector<uint8_t>{0, 0}));
148+}
Mautofuse/tests/v35/ut/codegen/reg_api_call/test_codegen_reduce_reg_api_call.cpp+1-1
@@ -140,7 +140,7 @@ TEST_F(RegReduceApicallTest, RegReduceApi_Test_001) {
140 EXPECT_EQ(status, af::SUCCESS);140 EXPECT_EQ(status, af::SUCCESS);
141 EXPECT_NE(result.find("first_actual"), std::string::npos);141 EXPECT_NE(result.find("first_actual"), std::string::npos);
142 EXPECT_NE(result.find("tmp_reduce_shape"), std::string::npos);142 EXPECT_NE(result.find("tmp_reduce_shape"), std::string::npos);
143- EXPECT_NE(result.find("ReduceMax<"), std::string::npos);143+ EXPECT_NE(result.find("ReduceMaxExtend<"), std::string::npos) << result;
144 EXPECT_NE(result.find("AscendC::Pattern::Reduce::AR"), std::string::npos);144 EXPECT_NE(result.find("AscendC::Pattern::Reduce::AR"), std::string::npos);
145 EXPECT_EQ(result.find("AscirNodeParams"), std::string::npos);145 EXPECT_EQ(result.find("AscirNodeParams"), std::string::npos);
146}146}
Aautofuse/tests/v35/ut/codegen/test_ascendc_reg_base_api_register_reduce.cpp+31-0
@@ -0,0 +1,31 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include "gtest/gtest.h"
12+#include "codegen/ascendc_api_registry.h"
13+ 
14+namespace codegen {
15+TEST(AscendcRegBaseApiRegisterReduceTest, RegistersCompleteReduceExtendSource) {
16+ const auto &source = AscendCApiRegistry::GetInstance().GetFileContent("reduce_extend_reg_base.h");
17+ 
18+ ASSERT_FALSE(source.empty());
19+ EXPECT_NE(source.find("ReduceSumExtend"), std::string::npos);
20+ EXPECT_NE(source.find("ReduceMeanExtend"), std::string::npos);
21+ EXPECT_NE(source.find("ReduceMaxExtend"), std::string::npos);
22+ EXPECT_NE(source.find("ReduceMinExtend"), std::string::npos);
23+ EXPECT_NE(source.find("ReduceProdExtend"), std::string::npos);
24+ EXPECT_NE(source.find("ReduceAnyExtend"), std::string::npos);
25+ EXPECT_NE(source.find("ReduceAllExtend"), std::string::npos);
26+ EXPECT_NE(source.find("ReduceXorSumExtend"), std::string::npos);
27+ EXPECT_NE(source.find("MeanExtend"), std::string::npos);
28+ EXPECT_NE(source.find("SumExtend"), std::string::npos);
29+ EXPECT_EQ(source.find("#include \"reduce/"), std::string::npos);
30+}
31+} // namespace codegen
Mautofuse/v35/ascendc/api_regbase/CMakeLists.txt+17-0
@@ -32,6 +32,23 @@ set(ascendc_api_regbase_extend_src
32 sign.h32 sign.h
33 signbit.h33 signbit.h
34 reduce_init.h34 reduce_init.h
35+ reduce_extend.h
36+ reduce/reduce_common.h
37+ reduce/reduce_common_util_3510_impl.h
38+ reduce/reduce_common_ar_reuse_align_less_than_vl_3510_impl.h
39+ reduce/reduce_common_ar_reuse_align_3510_impl.h
40+ reduce/reduce_common_ra_reuse_align_3510_impl.h
41+ reduce/reduce_common_ar_ra_reuse_unalign_3510_impl.h
42+ reduce/reduce_sum_3510_impl.h
43+ reduce/reduce_mean_3510_impl.h
44+ reduce/reduce_max_3510_impl.h
45+ reduce/reduce_min_3510_impl.h
46+ reduce/reduce_prod_3510_impl.h
47+ reduce/reduce_any_3510_impl.h
48+ reduce/reduce_all_3510_impl.h
49+ reduce/mean_3510_impl.h
50+ reduce/sum_3510_impl.h
51+ reduce/reduce_xor_sum_3510_impl.h
35 argmax.h52 argmax.h
36 atan2.h53 atan2.h
37 airy_ai.h54 airy_ai.h
Aautofuse/v35/ascendc/api_regbase/reduce/mean_3510_impl.h+175-0
@@ -0,0 +1,175 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file mean_3510_impl.h
13+ * \brief
14+ */
15+ 
16+#ifndef AUTOFUSE_REDUCE_EXTEND_MEAN_C310_IMPL_H
17+#define AUTOFUSE_REDUCE_EXTEND_MEAN_C310_IMPL_H
18+ 
19+#include "kernel_tensor.h"
20+#include "kernel_basic_intf.h"
21+#include "adv_api/reduce/mean_utils.h"
22+#include "reduce/reduce_common.h"
23+ 
24+namespace AscendC {
25+namespace ReduceExtendInternal {
26+namespace Internal {
27+ 
28+template <typename T, typename accType>
29+struct GetConvType {
30+ using type = T;
31+};
32+ 
33+template <>
34+struct GetConvType<half, float> {
35+ using type = float;
36+};
37+ 
38+template <typename T, typename ConvType>
39+__simd_callee__ inline void LoadSrcData(Reg::RegTensor<ConvType> &srcReg, __ubuf__ T *src, uint16_t index,
40+ uint32_t offset, Reg::MaskReg &mask) {
41+ Reg::RegTensor<T> srcTmpReg;
42+ if constexpr (std::is_same<T, half>::value && std::is_same<ConvType, float>::value) {
43+ Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(srcTmpReg, src + index * offset);
44+ Reg::Cast<float, T, AscendC::Internal::castTraitB16ToB32>(srcReg, srcTmpReg, mask);
45+ } else {
46+ Reg::LoadAlign(srcReg, src + index * offset);
47+ }
48+}
49+ 
50+template <typename T, typename U, typename ConvType>
51+__simd_vf__ inline void MeanForOneRepeatTime(__ubuf__ T *dstUb, __ubuf__ U *srcUb, const MeanParams meanParams,
52+ uint32_t calCount, uint32_t offset) {
53+ uint32_t count;
54+ ConvType scalarValue = static_cast<ConvType>(1.0f / meanParams.n);
55+ Reg::MaskReg mask;
56+ Reg::UnalignReg uregOut;
57+ Reg::RegTensor<ConvType> srcTmpReg, dstTmpReg;
58+ Reg::RegTensor<T> dstReg;
59+ 
60+ for (int i = 0; i < meanParams.outter; i++) {
61+ count = calCount;
62+ mask = Reg::UpdateMask<ConvType>(count);
63+ LoadSrcData(srcTmpReg, srcUb, i, offset, mask);
64+ Reg::ReduceSum(dstTmpReg, srcTmpReg, mask);
65+ Reg::Muls(dstTmpReg, dstTmpReg, scalarValue, mask);
66+ if constexpr (sizeof(T) == sizeof(half) && sizeof(ConvType) == sizeof(float)) {
67+ Reg::Cast<T, float, AscendC::Internal::castTraitB32ToB16>(dstReg, dstTmpReg, mask);
68+ Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint16_t> &)dstReg,
69+ (Reg::RegTensor<uint32_t> &)dstReg);
70+ } else {
71+ dstReg = dstTmpReg;
72+ }
73+ Reg::StoreUnAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(dstUb, dstReg, uregOut, 1);
74+ }
75+ Reg::StoreUnAlignPost(dstUb, uregOut, 0);
76+}
77+ 
78+template <typename T, typename ConvType, bool isFirstRepeat>
79+__simd_vf__ inline void ReduceSumNextN(__ubuf__ ConvType *dstUb, __ubuf__ T *srcUb, const MeanParams meanParams,
80+ uint32_t calCount, uint32_t repeatTimes, uint32_t offset) {
81+ uint32_t count;
82+ Reg::MaskReg mask;
83+ Reg::UnalignReg uregIn;
84+ Reg::RegTensor<ConvType> srcReg, dstReg;
85+ constexpr int32_t eleCountPerVL = GetVecLen() / sizeof(ConvType);
86+ 
87+ for (uint16_t i = 0; i < meanParams.outter; i++) {
88+ count = calCount;
89+ auto dstTmpUb = dstUb + i * offset;
90+ for (uint16_t j = 0; j < repeatTimes; j++) {
91+ mask = Reg::UpdateMask<ConvType>(count);
92+ if constexpr (isFirstRepeat) {
93+ LoadSrcData(srcReg, srcUb + i * meanParams.inner, j, eleCountPerVL, mask);
94+ } else {
95+ LoadSrcData(srcReg, srcUb + i * offset, j, eleCountPerVL, mask);
96+ }
97+ Reg::ReduceSum(dstReg, srcReg, mask);
98+ Reg::StoreUnAlign<ConvType, Reg::PostLiteral::POST_MODE_UPDATE>(dstTmpUb, dstReg, uregIn, 1);
99+ }
100+ Reg::StoreUnAlignPost(dstTmpUb, uregIn, 0);
101+ }
102+ Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
103+}
104+} // namespace Internal
105+ 
106+template <typename T, typename accType, bool isReuseSource, bool isBasicBlock, int32_t reduceDim>
107+__aicore__ inline void MeanCheckParams(const LocalTensor<T> &dstTensor, const LocalTensor<T> &srcTensor,
108+ const LocalTensor<uint8_t> &sharedTmpBuffer, const MeanParams &meanParams) {
109+ static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
110+ CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN/VECCALC/VECOUT", "Mean");
111+ CheckTensorPos<T>(srcTensor, Hardware::UB, "srcTensor", "VECIN/VECCALC/VECOUT", "Mean");
112+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "Mean");
113+ constexpr uint32_t meanInnerAlignLen = 32;
114+ ASCENDC_ASSERT((1 <= meanParams.n) && (meanParams.n <= meanParams.inner), {
115+ KERNEL_LOG(KERNEL_ERROR, "The value of n must be greater than or equal to 1 and less than or equal to inner.");
116+ });
117+ ASCENDC_ASSERT((meanParams.inner * sizeof(T) % meanInnerAlignLen == 0),
118+ { KERNEL_LOG(KERNEL_ERROR, "The value of inner * sizeof(T) must be an integer multiple of 32."); });
119+}
120+ 
121+template <typename T, typename accType = T, bool isReuseSource = false, bool isBasicBlock = false,
122+ int32_t reduceDim = -1>
123+__aicore__ inline void MeanImpl(const LocalTensor<T> &dstTensor, const LocalTensor<T> &srcTensor,
124+ const LocalTensor<uint8_t> &sharedTmpBuffer, const MeanParams &meanParams) {
125+ // Only for AI Vector Core.
126+ if ASCEND_IS_AIC {
127+ return;
128+ }
129+ 
130+ MeanCheckParams<T, accType, isReuseSource, isBasicBlock, reduceDim>(dstTensor, srcTensor, sharedTmpBuffer,
131+ meanParams);
132+ 
133+ using ConvType = typename Internal::GetConvType<T, accType>::type;
134+ __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
135+ __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
136+ __ubuf__ ConvType *sharedTmpBufferUb = (__ubuf__ ConvType *)sharedTmpBuffer.GetPhyAddr();
137+ 
138+ constexpr int32_t eleCountPerVL = GetVecLen() / sizeof(ConvType);
139+ uint16_t repeatTimes = CeilDivision(meanParams.n, eleCountPerVL);
140+ uint32_t loopRepeatTimes;
141+ uint32_t calCount = meanParams.n;
142+ 
143+ uint32_t totalCnt = 1;
144+ uint32_t dataSize = repeatTimes;
145+ uint32_t offset = AlignUp(CeilDivision(meanParams.inner, eleCountPerVL), 32);
146+ while (dataSize > 1) {
147+ ++totalCnt;
148+ dataSize = CeilDivision(dataSize, eleCountPerVL);
149+ }
150+ 
151+ if (repeatTimes == 1) {
152+ Internal::MeanForOneRepeatTime<T, T, ConvType>(dstUb, srcUb, meanParams, meanParams.n, meanParams.inner);
153+ return;
154+ }
155+ 
156+ Internal::ReduceSumNextN<T, ConvType, true>(sharedTmpBufferUb, srcUb, meanParams, calCount, repeatTimes, offset);
157+ 
158+ --totalCnt;
159+ loopRepeatTimes = repeatTimes;
160+ while (totalCnt != 0) {
161+ calCount = loopRepeatTimes;
162+ loopRepeatTimes = CeilDivision(loopRepeatTimes, eleCountPerVL);
163+ if (totalCnt == 1) {
164+ Internal::MeanForOneRepeatTime<T, ConvType, ConvType>(dstUb, sharedTmpBufferUb, meanParams, calCount, offset);
165+ } else {
166+ Internal::ReduceSumNextN<ConvType, ConvType, false>(sharedTmpBufferUb, sharedTmpBufferUb, meanParams, calCount,
167+ loopRepeatTimes, offset);
168+ }
169+ --totalCnt;
170+ }
171+}
172+} // namespace ReduceExtendInternal
173+} // namespace AscendC
174+ 
175+#endif // AUTOFUSE_REDUCE_EXTEND_MEAN_C310_IMPL_H
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_all_3510_impl.h+71-0
@@ -0,0 +1,71 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_ALL_REDUCE_ALL_C310_IMPL_H_
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_ALL_REDUCE_ALL_C310_IMPL_H_
13+ 
14+#include "kernel_tensor.h"
15+#include "kernel_basic_intf.h"
16+#ifdef ASCENDC_CPU_DEBUG
17+#include "reduce/reduce_common_util_3510_impl.h"
18+#include "reduce/reduce_common_ar_reuse_align_3510_impl.h"
19+#include "reduce/reduce_common_ra_reuse_align_3510_impl.h"
20+#endif // ASCENDC_CPU_DEBUG
21+ 
22+namespace AscendC {
23+namespace ReduceExtendInternal {
24+namespace Internal {
25+ 
26+template <class T, bool isReuseSource = false>
27+__aicore__ inline void ReduceAllARImpl(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr, uint32_t dimA,
28+ uint32_t dimR) {
29+ constexpr uint16_t vlSize = GetVecLen() / sizeof(T);
30+ using CastType = typename ReduceOpInternal::template ExtractReduceCastType<T>::CastT;
31+ 
32+ if (dimR <= vlSize) {
33+ ReduceARReuseSourceLessThanVL<T, Reg::RegTraitNumOne, vlSize,
34+ Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
35+ Reg::ReduceMin<CastType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<CastType>>>(
36+ dstAddr, srcAddr, dimA, dimR);
37+ } else {
38+ ReduceAROverVLImpl<T, Reg::RegTraitNumOne, vlSize, Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
39+ Reg::ReduceMin<CastType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<CastType>>, isReuseSource>(
40+ dstAddr, srcAddr, tmpAddr, dimA, dimR);
41+ }
42+}
43+ 
44+template <typename T, typename pattern, bool isReuseSource = false>
45+__aicore__ inline void ReduceAllImpl(const LocalTensor<T> &dst, const LocalTensor<T> &src,
46+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[],
47+ bool srcInnerPad) {
48+ CHECK_FUNC_HIGHLEVEL_API(ReduceAll, (T, pattern), (dst, src, sharedTmpBuffer, srcShape, srcInnerPad, srcShape[1]));
49+ 
50+ CheckTensorPos<T>(dst, Hardware::UB, "dst", "VECIN/VECCALC/VECOUT", "ReduceAll");
51+ CheckTensorPos<T>(src, Hardware::UB, "src", "VECIN/VECCALC/VECOUT", "ReduceAll");
52+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "ReduceAll");
53+ static_assert(SupportType<T, uint8_t, float>(), "ReduceAll only support uint8_t/float data type on current device!");
54+ static_assert(std::is_same_v<pattern, Pattern::Reduce::AR> || std::is_same_v<pattern, Pattern::Reduce::RA>,
55+ "ReduceAll only support AR and RA pattern on current device!");
56+ 
57+ __ubuf__ T *dstAddr = (__ubuf__ T *)dst.GetPhyAddr();
58+ __ubuf__ T *srcAddr = (__ubuf__ T *)src.GetPhyAddr();
59+ LocalTensor<T> tmpBuf = sharedTmpBuffer.ReinterpretCast<T>();
60+ __ubuf__ T *tmpAddr = (__ubuf__ T *)tmpBuf.GetPhyAddr();
61+ if constexpr (std::is_same_v<pattern, Pattern::Reduce::AR>) {
62+ ReduceAllARImpl<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
63+ } else {
64+ ReduceRAImpl<T, Reg::RegTraitNumOne, Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>, isReuseSource>(
65+ dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
66+ }
67+}
68+} // namespace Internal
69+} // namespace ReduceExtendInternal
70+} // namespace AscendC
71+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_All_REDUCE_All_C310_IMPL_H_
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_any_3510_impl.h+71-0
@@ -0,0 +1,71 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_ANY_REDUCE_ANY_C310_IMPL_H_
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_ANY_REDUCE_ANY_C310_IMPL_H_
13+ 
14+#include "kernel_tensor.h"
15+#include "kernel_basic_intf.h"
16+#include "reduce/reduce_common_util_3510_impl.h"
17+#include "reduce/reduce_common_ar_reuse_align_3510_impl.h"
18+#include "reduce/reduce_common_ra_reuse_align_3510_impl.h"
19+#ifdef ASCENDC_CPU_DEBUG
20+#endif // ASCENDC_CPU_DEBUG
21+ 
22+namespace AscendC {
23+namespace ReduceExtendInternal {
24+namespace Internal {
25+ 
26+template <class T, bool isReuseSource = false>
27+__aicore__ inline void ReduceAnyARImpl(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr, uint32_t dimA,
28+ uint32_t dimR) {
29+ constexpr uint16_t vlSize = GetVecLen() / sizeof(T);
30+ using CastType = typename ReduceOpInternal::template ExtractReduceCastType<T>::CastT;
31+ 
32+ if (dimR <= vlSize) {
33+ ReduceARReuseSourceLessThanVL<T, Reg::RegTraitNumOne, vlSize,
34+ Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
35+ Reg::ReduceMax<CastType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<CastType>>>(
36+ dstAddr, srcAddr, dimA, dimR);
37+ } else {
38+ ReduceAROverVLImpl<T, Reg::RegTraitNumOne, vlSize, Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
39+ Reg::ReduceMax<CastType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<CastType>>, isReuseSource>(
40+ dstAddr, srcAddr, tmpAddr, dimA, dimR);
41+ }
42+}
43+ 
44+template <typename T, typename pattern, bool isReuseSource = false>
45+__aicore__ inline void ReduceAnyImpl(const LocalTensor<T> &dst, const LocalTensor<T> &src,
46+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[],
47+ bool srcInnerPad) {
48+ CHECK_FUNC_HIGHLEVEL_API(ReduceAny, (T, pattern), (dst, src, sharedTmpBuffer, srcShape, srcInnerPad, srcShape[1]));
49+ 
50+ CheckTensorPos<T>(dst, Hardware::UB, "dst", "VECIN/VECCALC/VECOUT", "ReduceAny");
51+ CheckTensorPos<T>(src, Hardware::UB, "src", "VECIN/VECCALC/VECOUT", "ReduceAny");
52+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "ReduceAny");
53+ static_assert(SupportType<T, uint8_t, float>(), "ReduceAny only support uint8_t/float data type on current device!");
54+ static_assert(std::is_same_v<pattern, Pattern::Reduce::AR> || std::is_same_v<pattern, Pattern::Reduce::RA>,
55+ "ReduceAny only support AR and RA pattern on current device!");
56+ 
57+ __ubuf__ T *dstAddr = (__ubuf__ T *)dst.GetPhyAddr();
58+ __ubuf__ T *srcAddr = (__ubuf__ T *)src.GetPhyAddr();
59+ LocalTensor<T> tmpBuf = sharedTmpBuffer.ReinterpretCast<T>();
60+ __ubuf__ T *tmpAddr = (__ubuf__ T *)tmpBuf.GetPhyAddr();
61+ if constexpr (std::is_same_v<pattern, Pattern::Reduce::AR>) {
62+ ReduceAnyARImpl<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
63+ } else {
64+ ReduceRAImpl<T, Reg::RegTraitNumOne, Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>, isReuseSource>(
65+ dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
66+ }
67+}
68+} // namespace Internal
69+} // namespace ReduceExtendInternal
70+} // namespace AscendC
71+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_ANY_REDUCE_ANY_C310_IMPL_H_
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_common.h+29-0
@@ -0,0 +1,29 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+// Use the same include guard as adv_api/detail/common/common.h to avoid duplicate CastTrait definitions when UT or ST
12+// includes both headers.
13+#ifndef IMPL_COMMON_COMMON_H
14+#define IMPL_COMMON_COMMON_H
15+ 
16+#include "kernel_basic_intf.h"
17+ 
18+namespace AscendC {
19+namespace Internal {
20+ 
21+constexpr Reg::CastTrait castTraitB16ToB32 = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN, Reg::MaskMergeMode::ZEROING,
22+ RoundMode::UNKNOWN};
23+constexpr Reg::CastTrait castTraitB32ToB16 = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT, Reg::MaskMergeMode::ZEROING,
24+ RoundMode::CAST_RINT};
25+ 
26+} // namespace Internal
27+} // namespace AscendC
28+ 
29+#endif // IMPL_COMMON_COMMON_H
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_common_ar_reuse_align_3510_impl.h+528-0
@@ -0,0 +1,528 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_COMMON_AR_REUSE_ALIGN_C310_IMPL_H
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_COMMON_AR_REUSE_ALIGN_C310_IMPL_H
13+ 
14+#include "kernel_basic_intf.h"
15+#include "kernel_tensor.h"
16+#include "reduce/reduce_common_util_3510_impl.h"
17+#include "reduce/reduce_common_ar_reuse_align_less_than_vl_3510_impl.h"
18+ 
19+namespace AscendC {
20+namespace ReduceExtendInternal {
21+template <class T, class U, const Reg::RegTrait &Trait, const Reg::CastTrait &CastTraitUppper,
22+ const Reg::CastTrait &CastTraitLower, const uint16_t vlSize, auto Binaryfunc, auto Reducefunc>
23+__simd_callee__ inline void ReduceARCastfoldZero(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t dimA, uint32_t dimR,
24+ Reg::MaskReg &fullMask) {
25+ using UnpackSrcT = typename ReduceOpInternal::ExtractUnsignedTypeBySize<sizeof(T)>::T;
26+ using UnpackDstT = typename ReduceOpInternal::ExtractUnsignedTypeBySize<sizeof(U)>::T;
27+ Reg::RegTensor<U, Trait> vreg0CastB32;
28+ Reg::RegTensor<U, Trait> vreg1CastB32;
29+ Reg::RegTensor<T, Trait> vreg0;
30+ Reg::RegTensor<T, Trait> vreg1;
31+ Reg::UnalignReg uDst;
32+ for (uint16_t loopA = 0; loopA < static_cast<uint16_t>(dimA); loopA++) {
33+ Reg::LoadAlign(vreg0, srcAddr + loopA * dimR);
34+ Reg::LoadAlign(vreg1, srcAddr + vlSize / 2 + loopA * dimR);
35+ Binaryfunc(vreg0, vreg0, vreg1, fullMask);
36+ Reg::UnPack((Reg::RegTensor<UnpackDstT, Trait> &)vreg0, (Reg::RegTensor<UnpackSrcT, Trait> &)vreg0);
37+ Reg::Cast<U, T, ReduceOpInternal::CastTraitBF16F32>(vreg0CastB32, vreg0, fullMask);
38+ Reducefunc(vreg1CastB32, vreg0CastB32, fullMask);
39+ Reg::Cast<T, U, ReduceOpInternal::CastTraitF32BF16>(vreg1, vreg1CastB32, fullMask);
40+ Reg::StoreUnAlign((__ubuf__ T *&)dstAddr, vreg1, uDst, 1);
41+ }
42+ Reg::StoreUnAlignPost((__ubuf__ T *&)dstAddr, uDst, 0);
43+}
44+ 
45+template <class T, const Reg::RegTrait &Trait, auto Binaryfunc, auto Reducefunc>
46+__simd_callee__ inline void ReduceARfoldOneToThree(Reg::RegTensor<T, Trait> &vreg0, Reg::RegTensor<T, Trait> &vreg2,
47+ Reg::MaskReg &fullMask) {
48+ if constexpr (IsSameType<T, bfloat16_t>::value) {
49+ ReduceOpInternal::ReduceARCastfoldOneToThree<T, float, Trait, ReduceOpInternal::CastTraitBF16F32,
50+ ReduceOpInternal::CastTraitF32BF16, Binaryfunc, Reducefunc>(
51+ vreg0, vreg2, fullMask);
52+ } else if constexpr (SupportBytes<T, 1>()) {
53+ ReduceOpInternal::ReduceARCastfoldOneToThree<T, half, Trait, ReduceOpInternal::CastTraitB8F16,
54+ ReduceOpInternal::CastTraitF16B8, Binaryfunc, Reducefunc>(vreg0, vreg2,
55+ fullMask);
56+ } else {
57+ Reducefunc(vreg2, vreg0, fullMask);
58+ }
59+}
60+ 
61+template <class T, const Reg::RegTrait &Trait, const uint16_t vlSize, auto Binaryfunc, auto Reducefunc,
62+ bool isReuseSource>
63+__simd_vf__ inline void ReduceARB64OverVLVF(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
64+ uint32_t dimA, uint32_t dimR, uint32_t mainR, uint32_t tailR,
65+ uint16_t dimRAxis, uint16_t inplaceRepeats, uint32_t inplaceTail,
66+ uint16_t needInplaceAdd, uint16_t copyRepeats, uint16_t base,
67+ uint16_t folds, uint16_t avgFolds, uint16_t mainTimes, uint16_t foldZero,
68+ uint16_t foldOne, uint16_t foldTwo) {
69+ __ubuf__ T *addr;
70+ Reg::MaskReg mask;
71+ Reg::MaskReg fullMask;
72+ fullMask = Reg::CreateMask<T, Reg::MaskPattern::ALL, Trait>();
73+ // Process mainR and tailR
74+ if constexpr (isReuseSource) {
75+ addr = srcAddr;
76+ Reg::RegTensor<T, Trait> b64VregMain;
77+ Reg::RegTensor<T, Trait> b64VregTail;
78+ for (uint16_t i = 0; i < needInplaceAdd; i++) {
79+ uint32_t sreg0 = tailR;
80+ for (uint16_t loopR = 0; loopR < inplaceRepeats; loopR++) {
81+ mask = Reg::UpdateMask<T, Trait>(sreg0);
82+ for (uint16_t loopA = 0; loopA < static_cast<uint16_t>(dimA); loopA++) {
83+ Reg::LoadAlign(b64VregMain, srcAddr + loopA * dimRAxis + loopR * vlSize);
84+ Reg::LoadAlign(b64VregTail, srcAddr + loopA * dimRAxis + mainR + loopR * vlSize);
85+ Binaryfunc(b64VregMain, b64VregMain, b64VregTail, mask);
86+ Reg::StoreAlign(addr + loopA * dimRAxis + loopR * vlSize, b64VregMain, mask);
87+ }
88+ }
89+ Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
90+ }
91+ } else {
92+ addr = tmpAddr;
93+ uint16_t dimRTmp = mainR;
94+ Reg::RegTensor<T, Trait> b64VregMain;
95+ Reg::RegTensor<T, Trait> b64VregTail;
96+ for (uint16_t loopA = 0; loopA < static_cast<uint16_t>(dimA); loopA++) {
97+ // do inplace add tail with main
98+ for (uint16_t loopR = 0; loopR < inplaceRepeats; loopR++) {
99+ Reg::LoadAlign(b64VregMain, srcAddr + loopA * dimRAxis + loopR * vlSize);
100+ Reg::LoadAlign(b64VregTail, srcAddr + loopA * dimRAxis + mainR + loopR * vlSize);
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_common_ar_reuse_align_less_than_vl_3510_impl.h+172-0
@@ -0,0 +1,172 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_COMMON_AR_REUSE_ALIGN_LESS_THAN_VL_C310_IMPL_H
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_COMMON_AR_REUSE_ALIGN_LESS_THAN_VL_C310_IMPL_H
13+ 
14+#include "kernel_basic_intf.h"
15+#include "kernel_tensor.h"
16+#include "reduce/reduce_common_util_3510_impl.h"
17+ 
18+namespace AscendC {
19+namespace ReduceExtendInternal {
20+template <class T, class U, const Reg::RegTrait &Trait, const Reg::CastTrait &CastTraitUppper,
21+ const Reg::CastTrait &CastTraitLower, const uint16_t vlSize, auto Binaryfunc, auto Reducefunc>
22+__simd_vf__ inline void ReduceARCastLessThanVL(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t dimA, uint32_t dimR) {
23+ if (dimR <= (vlSize / 2)) {
24+ Reg::RegTensor<T, Trait> vreg0;
25+ Reg::RegTensor<T, Trait> vreg1;
26+ Reg::RegTensor<U, Trait> vreg0CastUpper;
27+ Reg::RegTensor<U, Trait> vreg1CastUpper;
28+ Reg::UnalignReg uDst;
29+ uint32_t sreg1 = dimR;
30+ Reg::MaskReg mask = Reg::UpdateMask<U>(sreg1);
31+ for (uint16_t loopA = 0; loopA < static_cast<uint16_t>(dimA); loopA++) {
32+ if constexpr (IsSameType<T, bfloat16_t>::value) {
33+ Reg::LoadAlign<T, Reg::LoadDist::DIST_US_B16>(vreg0, srcAddr + loopA * dimR);
34+ } else {
35+ Reg::LoadAlign<T, Reg::LoadDist::DIST_US_B8>(vreg0, srcAddr + loopA * dimR);
36+ }
37+ Reg::Cast<U, T, CastTraitUppper>(vreg0CastUpper, vreg0, mask);
38+ Reducefunc(vreg1CastUpper, vreg0CastUpper, mask);
39+ Reg::Cast<T, U, CastTraitLower>(vreg1, vreg1CastUpper, mask);
40+ Reg::StoreUnAlign((__ubuf__ T *&)dstAddr, vreg1, uDst, 1);
41+ }
42+ Reg::StoreUnAlignPost((__ubuf__ T *&)dstAddr, uDst, 0);
43+ } else {
44+ Reg::RegTensor<T, Trait> vreg0;
45+ Reg::RegTensor<T, Trait> vreg1;
46+ Reg::RegTensor<T, Trait> vreg2;
47+ Reg::RegTensor<U, Trait> vreg0CastB32;
48+ Reg::RegTensor<U, Trait> vreg1CastB32;
49+ Reg::UnalignReg uDst;
50+ uint32_t sreg1 = dimR;
51+ Reg::MaskReg fullMask = Reg::CreateMask<T, Reg::MaskPattern::ALL, Trait>();
52+ Reg::MaskReg mask = Reg::UpdateMask<U>(sreg1);
53+ mask = Reg::UpdateMask<U>(sreg1);
54+ Reg::MaskPack(mask, mask);
55+ for (uint16_t loopA = 0; loopA < static_cast<uint16_t>(dimA); loopA++) {
56+ Reg::LoadAlign(vreg0, srcAddr + loopA * dimR);
57+ Reg::LoadAlign(vreg1, srcAddr + vlSize / 2 + loopA * dimR);
58+ Binaryfunc(vreg2, vreg0, vreg1, mask);
59+ Select(vreg2, vreg2, vreg0, mask);
60+ if constexpr (IsSameType<T, bfloat16_t>::value) {
61+ Reg::UnPack((Reg::RegTensor<uint32_t, Trait> &)vreg2, (Reg::RegTensor<uint16_t, Trait> &)vreg2);
62+ } else {
63+ Reg::UnPack((Reg::RegTensor<uint16_t, Trait> &)vreg2, (Reg::RegTensor<uint8_t, Trait> &)vreg2);
64+ }
65+ Reg::Cast<U, T, CastTraitUppper>(vreg0CastB32, vreg2, fullMask);
66+ Reducefunc(vreg1CastB32, vreg0CastB32, fullMask);
67+ Reg::Cast<T, U, CastTraitLower>(vreg1, vreg1CastB32, fullMask);
68+ Reg::StoreUnAlign((__ubuf__ T *&)dstAddr, vreg1, uDst, 1);
69+ }
70+ Reg::StoreUnAlignPost((__ubuf__ T *&)dstAddr, uDst, 0);
71+ }
72+}
73+ 
74+template <class T, const Reg::RegTrait &Trait, auto Reducefunc>
75+__simd_vf__ inline void ReduceARLessThanVL(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t dimA, uint32_t dimR) {
76+ Reg::RegTensor<T, Trait> vreg0;
77+ Reg::RegTensor<T, Trait> vreg1;
78+ Reg::UnalignReg uDst;
79+ uint32_t sreg1 = dimR;
80+ Reg::MaskReg mask = Reg::UpdateMask<T, Trait>(sreg1);
81+ for (uint16_t loopA = 0; loopA < static_cast<uint16_t>(dimA); loopA++) {
82+ Reg::LoadAlign(vreg0, srcAddr + loopA * dimR);
83+ Reducefunc(vreg1, vreg0, mask);
84+ Reg::StoreUnAlign((__ubuf__ T *&)dstAddr, vreg1, uDst, 1);
85+ }
86+ Reg::StoreUnAlignPost((__ubuf__ T *&)dstAddr, uDst, 0);
87+}
88+ 
89+template <class T, const Reg::RegTrait &Trait, ReduceType reduceType>
90+__simd_callee__ inline void GroupReduce(Reg::RegTensor<T, Trait> &dst, Reg::RegTensor<T, Trait> &src,
91+ Reg::MaskReg &mask) {
92+ if constexpr (reduceType == ReduceType::SUM) {
93+ Reg::ReduceSumWithDataBlock(dst, src, mask);
94+ } else if constexpr (reduceType == ReduceType::MAX) {
95+ Reg::ReduceMaxWithDataBlock(dst, src, mask);
96+ } else if constexpr (reduceType == ReduceType::MIN) {
97+ Reg::ReduceMinWithDataBlock(dst, src, mask);
98+ }
99+}
100+ 
101+template <class T, const Reg::RegTrait &Trait, const uint16_t vlSize, auto Binaryfunc, auto GroupReduceType,
102+ bool NeedFoldR = true>
103+__simd_vf__ inline void GroupReduceVf(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint16_t innerFoldNum, uint16_t fusedA,
104+ uint16_t strideA, uint32_t outerRepElement) {
105+ constexpr uint16_t blockDataLen = GetDataBlockSizeInBytes() / sizeof(T);
106+ constexpr uint16_t blockNumInVl = vlSize / blockDataLen;
107+ Reg::RegTensor<T, Trait> vreg0;
108+ Reg::RegTensor<T, Trait> vreg1;
109+ Reg::UnalignReg uDst;
110+ Reg::MaskReg mask;
111+ const uint16_t innerFoldBinaryNum = innerFoldNum - 1;
112+ 
113+ for (uint16_t loopA = 0; loopA < static_cast<uint16_t>(fusedA); ++loopA) {
114+ mask = Reg::UpdateMask<T, Trait>(outerRepElement);
115+ if constexpr (NeedFoldR) {
116+ auto srcAddrFold = srcAddr + blockDataLen;
117+ Reg::LoadAlign<T, Reg::DataCopyMode::DATA_BLOCK_COPY, Reg::PostLiteral::POST_MODE_UPDATE>(
118+ vreg0, srcAddr, innerFoldNum, strideA, mask);
119+ for (uint16_t loopR = 0; loopR < innerFoldBinaryNum; ++loopR) {
120+ Reg::LoadAlign<T, Reg::DataCopyMode::DATA_BLOCK_COPY, Reg::PostLiteral::POST_MODE_UPDATE>(
121+ vreg1, srcAddrFold, innerFoldNum, 1, mask);
122+ Binaryfunc(vreg0, vreg1, vreg0, mask);
123+ }
124+ } else {
125+ Reg::LoadAlign(vreg0, srcAddr + vlSize * loopA);
126+ }
127+ GroupReduce<T, Trait, GroupReduceType>(vreg1, vreg0, mask);
128+ Reg::StoreUnAlign((__ubuf__ T *&)dstAddr, vreg1, uDst, blockNumInVl);
129+ }
130+ Reg::StoreUnAlignPost((__ubuf__ T *&)dstAddr, uDst, blockNumInVl);
131+}
132+ 
133+template <class T, const Reg::RegTrait &Trait, const uint16_t vlSize, auto Binaryfunc, auto groupReduceType,
134+ bool NeedFoldR = true>
135+__aicore__ inline void GroupReduceARLessThanVL(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t dimA, uint32_t dimR) {
136+ constexpr uint16_t blockDataLen = GetDataBlockSizeInBytes() / sizeof(T);
137+ constexpr uint16_t blockNumInVl = vlSize / blockDataLen;
138+ const auto innerFoldNum = dimR / blockDataLen;
139+ const auto fusedA = CeilDivision(dimA, blockNumInVl);
140+ const auto strideA = (blockNumInVl * dimR) / blockDataLen;
141+ uint32_t outerRepElement = dimA * blockDataLen;
142+ GroupReduceVf<T, Trait, vlSize, Binaryfunc, groupReduceType, NeedFoldR>(dstAddr, srcAddr, innerFoldNum, fusedA,
143+ strideA, outerRepElement);
144+}
145+ 
146+template <class T, const Reg::RegTrait &Trait, const uint16_t vlSize, auto Binaryfunc, auto Reducefunc,
147+ ReduceType groupReduceType = ReduceType::NONE>
148+__aicore__ inline void ReduceARReuseSourceLessThanVL(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t dimA,
149+ uint32_t dimR) {
150+ if (dimR == 1) {
151+ ReduceOpInternal::ReduceCopyOutImpl<T>(dstAddr, srcAddr, dimA);
152+ } else if constexpr (IsSameType<T, bfloat16_t>::value) {
153+ ReduceARCastLessThanVL<T, float, Trait, ReduceOpInternal::CastTraitBF16F32, ReduceOpInternal::CastTraitF32BF16,
154+ vlSize, Binaryfunc, Reducefunc>(dstAddr, srcAddr, dimA, dimR);
155+ } else if constexpr (SupportBytes<T, 1>()) {
156+ ReduceARCastLessThanVL<T, half, Trait, ReduceOpInternal::CastTraitB8F16, ReduceOpInternal::CastTraitF16B8, vlSize,
157+ Binaryfunc, Reducefunc>(dstAddr, srcAddr, dimA, dimR);
158+ } else if constexpr (groupReduceType != ReduceType::NONE) {
159+ if (dimR == GetDataBlockSizeInBytes() / sizeof(T)) {
160+ GroupReduceARLessThanVL<T, Trait, vlSize, Binaryfunc, groupReduceType, false>(dstAddr, srcAddr, dimA, dimR);
161+ } else if (dimR < vlSize / 2) {
162+ GroupReduceARLessThanVL<T, Trait, vlSize, Binaryfunc, groupReduceType, true>(dstAddr, srcAddr, dimA, dimR);
163+ } else {
164+ ReduceARLessThanVL<T, Trait, Reducefunc>(dstAddr, srcAddr, dimA, dimR);
165+ }
166+ } else {
167+ ReduceARLessThanVL<T, Trait, Reducefunc>(dstAddr, srcAddr, dimA, dimR);
168+ }
169+}
170+} // namespace ReduceExtendInternal
171+} // namespace AscendC
172+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_COMMON_AR_REUSE_ALIGN_LESS_THAN_VL_C310_IMPL_H
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_common_util_3510_impl.h+149-0
@@ -0,0 +1,149 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_COMMON_UTIL_C310_IMPL_H
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_COMMON_UTIL_C310_IMPL_H
13+ 
14+#include "kernel_basic_intf.h"
15+#include "kernel_tensor.h"
16+ 
17+namespace AscendC {
18+namespace ReduceExtendInternal {
19+namespace ReduceOpInternal {
20+// Calculate the sum of two points based on count. The main block size is ex.2000->1024 900->512.
21+__aicore__ inline uint32_t CalculateRMainBlock(uint32_t count) {
22+ constexpr int32_t TWO_BITS = 2;
23+ constexpr int32_t FOUR_BITS = 4;
24+ constexpr int32_t EIGHT_BITS = 8;
25+ constexpr int32_t SIXTEEN_BITS = 16;
26+ count |= count >> 1;
27+ count |= count >> TWO_BITS;
28+ count |= count >> FOUR_BITS;
29+ count |= count >> EIGHT_BITS;
30+ count |= count >> SIXTEEN_BITS;
31+ return (count + 1) >> 1;
32+}
33+ 
34+constexpr uint16_t BASE_FOLD = 4;
35+constexpr uint16_t BASE_FOLD_B64 = 3;
36+constexpr uint16_t FOLD_THREE = 3;
37+constexpr uint16_t FOLD_TWO = 2;
38+constexpr uint16_t FOLD_ONE = 1;
39+constexpr uint32_t U16_STRIDE = 65535;
40+constexpr uint16_t REGULAR_FOLD_NUM = 2;
41+constexpr uint16_t NO_REUSE_FOLD_NUM = 2;
42+ 
43+static constexpr Reg::CastTrait CastTraitBF16F32 = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN,
44+ Reg::MaskMergeMode::ZEROING};
45+static constexpr Reg::CastTrait CastTraitF32BF16 = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT,
46+ Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
47+static constexpr Reg::CastTrait CastTraitB8F16 = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN,
48+ Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
49+static constexpr Reg::CastTrait CastTraitF16B8 = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT,
50+ Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
51+ 
52+__aicore__ inline uint32_t CalculateMainR(int64_t dimR, bool isAR, uint16_t vlSize) {
53+ constexpr uint16_t MAX_OFFSET = 16;
54+ int64_t mainR = 1;
55+ for (uint16_t i = 1; i < MAX_OFFSET; i++) {
56+ if ((dimR >> i) == 0) {
57+ break;
58+ }
59+ mainR = 1 << i;
60+ }
61+ 
62+ if (isAR && dimR < vlSize) {
63+ mainR = dimR;
64+ }
65+ return static_cast<uint32_t>(mainR);
66+}
67+ 
68+__aicore__ inline uint16_t CalculateFolds(const uint16_t count) {
69+ constexpr uint16_t MAX_OFFSET = 16;
70+ uint16_t folds = 0;
71+ uint16_t base = count;
72+ for (uint16_t i = 1; i < MAX_OFFSET; i++) {
73+ if ((base >> i) == 0) {
74+ break;
75+ }
76+ folds++;
77+ }
78+ return folds;
79+}
80+ 
81+template <class T, class U, const Reg::RegTrait &Trait, const Reg::CastTrait &CastTraitUppper,
82+ const Reg::CastTrait &CastTraitLower, auto Binaryfunc, auto Reducefunc>
83+__simd_callee__ inline void ReduceARCastfoldOneToThree(Reg::RegTensor<T, Trait> &vreg0, Reg::RegTensor<T, Trait> &vreg1,
84+ Reg::MaskReg &fullMask) {
85+ Reg::RegTensor<U, Trait> vreg0CastB32;
86+ Reg::RegTensor<U, Trait> vreg1CastB32;
87+ Reg::RegTensor<T, Trait> vreg2;
88+ Reg::RegTensor<T, Trait> vreg3;
89+ Reg::RegTensor<T, Trait> vreg4;
90+ Duplicate(vreg4, static_cast<T>(0), fullMask);
91+ Interleave(vreg2, vreg3, vreg0, vreg4);
92+ Binaryfunc(vreg0, vreg2, vreg3, fullMask);
93+ Reg::Cast<U, T, CastTraitUppper>(vreg0CastB32, vreg0, fullMask);
94+ Reducefunc(vreg1CastB32, vreg0CastB32, fullMask);
95+ Reg::Cast<T, U, CastTraitLower>(vreg1, vreg1CastB32, fullMask);
96+}
97+ 
98+template <class T>
99+__simd_vf__ inline void ReduceCopyOutImpl(__ubuf__ T *dst, __ubuf__ T *src, const uint32_t calCount) {
100+ uint16_t repeatElm = GetVecLen();
101+ uint32_t sreg = calCount * sizeof(T);
102+ uint16_t repeatTime = static_cast<uint16_t>(CeilDivision(sreg, repeatElm));
103+ Reg::MaskReg preg;
104+ Reg::RegTensor<uint8_t> srcReg;
105+ for (uint16_t i = 0; i < repeatTime; ++i) {
106+ preg = Reg::UpdateMask<uint8_t>(sreg);
107+ Reg::LoadAlign(srcReg, (__ubuf__ uint8_t *)src + i * repeatElm);
108+ Reg::StoreAlign((__ubuf__ uint8_t *)dst + i * repeatElm, srcReg, preg);
109+ }
110+}
111+ 
112+template <typename T>
113+struct ExtractReduceCastType {
114+ using CastT = T;
115+};
116+ 
117+template <>
118+struct ExtractReduceCastType<bfloat16_t> {
119+ using CastT = float;
120+};
121+ 
122+template <>
123+struct ExtractReduceCastType<uint8_t> {
124+ using CastT = half;
125+};
126+ 
127+template <>
128+struct ExtractReduceCastType<int8_t> {
129+ using CastT = half;
130+};
131+ 
132+template <uint32_t size = sizeof(uint8_t)>
133+struct ExtractUnsignedTypeBySize {
134+ using T = uint8_t;
135+};
136+ 
137+template <>
138+struct ExtractUnsignedTypeBySize<sizeof(uint16_t)> {
139+ using T = uint16_t;
140+};
141+ 
142+template <>
143+struct ExtractUnsignedTypeBySize<sizeof(uint32_t)> {
144+ using T = uint32_t;
145+};
146+} // namespace ReduceOpInternal
147+} // namespace ReduceExtendInternal
148+} // namespace AscendC
149+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_COMMON_UTIL_C310_IMPL_H
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_max_3510_impl.h+131-0
@@ -0,0 +1,131 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_MAX_REDUCE_MAX_C310_IMPL_H_
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_MAX_REDUCE_MAX_C310_IMPL_H_
13+ 
14+#include "kernel_tensor.h"
15+#include "kernel_basic_intf.h"
16+#include "kernel_tiling/kernel_tiling.h"
17+#include "reduce/reduce_common_util_3510_impl.h"
18+#include "reduce/reduce_common_ar_ra_reuse_unalign_3510_impl.h"
19+#include "reduce/reduce_common_ar_reuse_align_3510_impl.h"
20+#include "reduce/reduce_common_ra_reuse_align_3510_impl.h"
21+ 
22+namespace AscendC {
23+namespace ReduceExtendInternal {
24+namespace Internal {
25+template <class T, bool isReuseSource = false>
26+__aicore__ inline void ReduceMaxARB64ReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
27+ const uint32_t srcShape[]) {
28+ if ((srcShape[1] * sizeof(T)) % 32 == 0) {
29+ ReduceARImpl<T, Reg::RegTraitNumTwo,
30+ Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
31+ Reg::ReduceMax<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>, isReuseSource>(
32+ dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
33+ } else {
34+ ReduceARReuseSourceUnAligned<
35+ T, Reg::RegTraitNumTwo, Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
36+ Reg::ReduceMax<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>>(
37+ dstAddr, srcAddr, srcShape[0], srcShape[1]);
38+ }
39+}
40+ 
41+template <class T, bool isReuseSource = false>
42+__aicore__ inline void ReduceMaxARReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
43+ const uint32_t srcShape[]) {
44+ using castType = typename ReduceOpInternal::ExtractReduceCastType<T>::CastT;
45+ if ((srcShape[1] * sizeof(T)) % 32 == 0 || srcShape[1] == 1) {
46+ ReduceARImpl<T, Reg::RegTraitNumOne, Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
47+ Reg::ReduceMax<castType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<castType>>, isReuseSource,
48+ ReduceType::MAX>(dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
49+ } else {
50+ if constexpr (SupportBytes<T, 1>()) {
51+ constexpr uint32_t ONE_STRIDE = 32768;
52+ uint16_t repeatTimes = (srcShape[0] + ONE_STRIDE - 1) >> 15;
53+ for (uint16_t i = 0; i < repeatTimes; ++i) {
54+ uint32_t dimA = ONE_STRIDE < srcShape[0] - ONE_STRIDE * i ? ONE_STRIDE : srcShape[0] - ONE_STRIDE * i;
55+ ReduceARReuseSourceUnAligned<T, Reg::RegTraitNumOne,
56+ Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
57+ Reg::ReduceMax<castType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<castType>>>(
58+ dstAddr + ONE_STRIDE * i, srcAddr + ONE_STRIDE * i * srcShape[1], dimA, srcShape[1]);
59+ }
60+ } else {
61+ ReduceARReuseSourceUnAligned<T, Reg::RegTraitNumOne, Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
62+ Reg::ReduceMax<castType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<castType>>>(
63+ dstAddr, srcAddr, srcShape[0], srcShape[1]);
64+ }
65+ }
66+}
67+ 
68+template <class T, bool isReuseSource = false>
69+__aicore__ inline void ReduceMaxRAB64ReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
70+ const uint32_t srcShape[]) {
71+ if ((srcShape[1] * sizeof(T)) % 32 == 0) {
72+ ReduceRAB64ReuseSource<T, Reg::RegTraitNumTwo,
73+ Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
74+ isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
75+ } else {
76+ ReduceRAReuseSourceUnAlignedB64<T, Reg::RegTraitNumTwo,
77+ Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>>(
78+ dstAddr, srcAddr, srcShape[1], srcShape[0]);
79+ }
80+}
81+ 
82+template <class T, bool isReuseSource = false>
83+__aicore__ inline void ReduceMaxRAReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
84+ const uint32_t srcShape[]) {
85+ if ((srcShape[1] * sizeof(T)) % 32 == 0) {
86+ ReduceRAImpl<T, Reg::RegTraitNumOne, Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>, isReuseSource>(
87+ dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
88+ } else {
89+ ReduceRAReuseSourceUnAligned<T, Reg::RegTraitNumOne, Reg::Max<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>>(
90+ dstAddr, srcAddr, srcShape[1], srcShape[0]);
91+ }
92+}
93+ 
94+template <class T, class pattern, bool isReuseSource = false>
95+__aicore__ inline void ReduceMaxImpl(const LocalTensor<T> &dst, const LocalTensor<T> &src,
96+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[],
97+ bool srcInnerPad) {
98+ CheckTensorPos<T>(dst, Hardware::UB, "dstTensor", "VECIN/VECCALC/VECOUT", "ReduceMax");
99+ CheckTensorPos<T>(src, Hardware::UB, "srcTensor", "VECIN/VECCALC/VECOUT", "ReduceMax");
100+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "ReduceMax");
101+ static_assert(
102+ SupportType<T, int8_t, uint8_t, int16_t, uint16_t, half, bfloat16_t, int32_t, uint32_t, float, int64_t,
103+ uint64_t>(),
104+ "ReduceMax only support "
105+ "int8_t/uint8_t/int16_t/uint16_t/half/bfloat16_t/int32_t/uint32_t/float/int64_t/uint64_t data type on current "
106+ "device!");
107+ static_assert(std::is_same_v<pattern, Pattern::Reduce::AR> || std::is_same_v<pattern, Pattern::Reduce::RA>,
108+ "ReduceMax only support AR and RA pattern on current device!");
109+ 
110+ __ubuf__ T *dstAddr = (__ubuf__ T *)dst.GetPhyAddr();
111+ __ubuf__ T *srcAddr = (__ubuf__ T *)src.GetPhyAddr();
112+ LocalTensor<T> tmpBuf = sharedTmpBuffer.ReinterpretCast<T>();
113+ __ubuf__ T *tmpAddr = (__ubuf__ T *)tmpBuf.GetPhyAddr();
114+ if constexpr (std::is_same_v<pattern, Pattern::Reduce::AR>) {
115+ if constexpr (SupportBytes<T, 8>()) {
116+ ReduceMaxARB64ReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
117+ } else {
118+ ReduceMaxARReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
119+ }
120+ } else {
121+ if constexpr (SupportBytes<T, 8>()) {
122+ ReduceMaxRAB64ReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
123+ } else {
124+ ReduceMaxRAReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
125+ }
126+ }
127+}
128+} // namespace Internal
129+} // namespace ReduceExtendInternal
130+} // namespace AscendC
131+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_MAX_REDUCE_MAX_C310_IMPL_H_
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_mean_3510_impl.h+54-0
@@ -0,0 +1,54 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_MEAN_REDUCE_MEAN_C310_IMPL_H
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_MEAN_REDUCE_MEAN_C310_IMPL_H
13+ 
14+#include "kernel_tensor.h"
15+#include "kernel_basic_intf.h"
16+#include "kernel_tiling/kernel_tiling.h"
17+#include "reduce/reduce_common_util_3510_impl.h"
18+#ifdef ASCENDC_CPU_DEBUG
19+#endif // ASCENDC_CPU_DEBUG
20+ 
21+namespace AscendC {
22+namespace ReduceExtendInternal {
23+namespace Internal {
24+template <class T, class pattern, bool isReuseSource = false>
25+__aicore__ inline void ReduceMeanImpl(const LocalTensor<T> &dst, const LocalTensor<T> &src,
26+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[],
27+ bool srcInnerPad) {
28+ CHECK_FUNC_HIGHLEVEL_API(ReduceMean, (T, pattern), (dst, src, sharedTmpBuffer, srcShape, srcInnerPad, srcShape[1]));
29+ 
30+ CheckTensorPos<T>(dst, Hardware::UB, "dst", "VECIN/VECCALC/VECOUT", "ReduceMean");
31+ CheckTensorPos<T>(src, Hardware::UB, "src", "VECIN/VECCALC/VECOUT", "ReduceMean");
32+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "ReduceMean");
33+ static_assert(SupportType<T, float>(), "ReduceMean only support float data type on current device!");
34+ static_assert(std::is_same_v<pattern, Pattern::Reduce::AR> || std::is_same_v<pattern, Pattern::Reduce::RA>,
35+ "ReduceMean only support AR and RA pattern on current device!");
36+ 
37+ __ubuf__ T *dstAddr = (__ubuf__ T *)dst.GetPhyAddr();
38+ __ubuf__ T *srcAddr = (__ubuf__ T *)src.GetPhyAddr();
39+ LocalTensor<T> tmpBuf = sharedTmpBuffer.ReinterpretCast<T>();
40+ __ubuf__ T *tmpAddr = (__ubuf__ T *)tmpBuf.GetPhyAddr();
41+ 
42+ ReduceSumImpl<T, pattern, isReuseSource>(dst, src, sharedTmpBuffer, srcShape, srcInnerPad);
43+ if constexpr (IsSameType<pattern, Pattern::Reduce::AR>::value) {
44+ float lastAxisValReciprocal = 1.0f / static_cast<int32_t>(srcShape[1]);
45+ Muls<T>(dst, dst, lastAxisValReciprocal, srcShape[0]);
46+ } else if constexpr (IsSameType<pattern, Pattern::Reduce::RA>::value) {
47+ float firstAxisValReciprocal = 1.0f / static_cast<int32_t>(srcShape[0]);
48+ Muls<T>(dst, dst, firstAxisValReciprocal, srcShape[1]);
49+ }
50+}
51+} // namespace Internal
52+} // namespace ReduceExtendInternal
53+} // namespace AscendC
54+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_MEAN_REDUCE_MEAN_C310_IMPL_H
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_min_3510_impl.h+137-0
@@ -0,0 +1,137 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_MIN_REDUCE_MIN_C310_IMPL_H_
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_MIN_REDUCE_MIN_C310_IMPL_H_
13+ 
14+#include "kernel_tensor.h"
15+#include "kernel_basic_intf.h"
16+#include "kernel_tiling/kernel_tiling.h"
17+#include "reduce/reduce_common_util_3510_impl.h"
18+#include "reduce/reduce_common_ar_reuse_align_3510_impl.h"
19+#include "reduce/reduce_common_ra_reuse_align_3510_impl.h"
20+#include "reduce/reduce_common_ar_ra_reuse_unalign_3510_impl.h"
21+ 
22+namespace AscendC {
23+namespace ReduceExtendInternal {
24+namespace Internal {
25+template <class T, bool isReuseSource = false>
26+__aicore__ inline void ReduceMinARB64ReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
27+ const uint32_t srcShape[]) {
28+ if ((srcShape[1] * sizeof(T)) % 32 == 0) {
29+ ReduceARImpl<T, Reg::RegTraitNumTwo,
30+ Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
31+ Reg::ReduceMin<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>, isReuseSource>(
32+ dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
33+ } else {
34+ ReduceARReuseSourceUnAligned<
35+ T, Reg::RegTraitNumTwo, Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
36+ Reg::ReduceMin<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>>(
37+ dstAddr, srcAddr, srcShape[0], srcShape[1]);
38+ }
39+}
40+ 
41+template <class T, bool isReuseSource = false>
42+__aicore__ inline void ReduceMinARReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
43+ const uint32_t srcShape[]) {
44+ using castType = typename ReduceOpInternal::ExtractReduceCastType<T>::CastT;
45+ if ((srcShape[1] * sizeof(T)) % 32 == 0 || srcShape[1] == 1) {
46+ ReduceARImpl<T, Reg::RegTraitNumOne, Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
47+ Reg::ReduceMin<castType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<castType>>, isReuseSource,
48+ ReduceType::MIN>(dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
49+ } else {
50+ if constexpr (SupportBytes<T, 1>()) {
51+ constexpr uint32_t ONE_STRIDE = 32768;
52+ uint16_t repeatTimes = (srcShape[0] + ONE_STRIDE - 1) >> 15;
53+ for (uint16_t i = 0; i < repeatTimes; ++i) {
54+ uint32_t dimA = ONE_STRIDE < srcShape[0] - ONE_STRIDE * i ? ONE_STRIDE : srcShape[0] - ONE_STRIDE * i;
55+ ReduceARReuseSourceUnAligned<T, Reg::RegTraitNumOne,
56+ Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
57+ Reg::ReduceMin<castType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<castType>>>(
58+ dstAddr + ONE_STRIDE * i, srcAddr + ONE_STRIDE * i * srcShape[1], dimA, srcShape[1]);
59+ }
60+ } else {
61+ ReduceARReuseSourceUnAligned<T, Reg::RegTraitNumOne, Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
62+ Reg::ReduceMin<castType, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<castType>>>(
63+ dstAddr, srcAddr, srcShape[0], srcShape[1]);
64+ }
65+ }
66+}
67+ 
68+template <class T, bool isReuseSource = false>
69+__aicore__ inline void ReduceMinRAB64ReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
70+ const uint32_t srcShape[]) {
71+ if ((srcShape[1] * sizeof(T)) % 32 == 0) {
72+ if (srcShape[1] * sizeof(T) <= GetVecLen() / 2) {
73+ ReduceRAConcatImpl<T, Reg::RegTraitNumTwo,
74+ Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
75+ isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
76+ } else {
77+ ReduceRAB64ReuseSource<T, Reg::RegTraitNumTwo,
78+ Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
79+ isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
80+ }
81+ } else {
82+ ReduceRAReuseSourceUnAlignedB64<T, Reg::RegTraitNumTwo,
83+ Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>>(
84+ dstAddr, srcAddr, srcShape[1], srcShape[0]);
85+ }
86+}
87+ 
88+template <class T, bool isReuseSource = false>
89+__aicore__ inline void ReduceMinRAReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
90+ const uint32_t srcShape[]) {
91+ if ((srcShape[1] * sizeof(T)) % 32 == 0) {
92+ ReduceRAImpl<T, Reg::RegTraitNumOne, Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>, isReuseSource>(
93+ dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
94+ } else {
95+ ReduceRAReuseSourceUnAligned<T, Reg::RegTraitNumOne, Reg::Min<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>>(
96+ dstAddr, srcAddr, srcShape[1], srcShape[0]);
97+ }
98+}
99+ 
100+template <class T, class pattern, bool isReuseSource = false>
101+__aicore__ inline void ReduceMinImpl(const LocalTensor<T> &dst, const LocalTensor<T> &src,
102+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[],
103+ bool srcInnerPad) {
104+ CheckTensorPos<T>(dst, Hardware::UB, "dstTensor", "VECIN/VECCALC/VECOUT", "ReduceMin");
105+ CheckTensorPos<T>(src, Hardware::UB, "srcTensor", "VECIN/VECCALC/VECOUT", "ReduceMin");
106+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "ReduceMin");
107+ static_assert(
108+ SupportType<T, int8_t, uint8_t, int16_t, uint16_t, half, bfloat16_t, int32_t, uint32_t, float, int64_t,
109+ uint64_t>(),
110+ "ReduceMin only support "
111+ "int8_t/uint8_t/int16_t/uint16_t/half/bfloat16_t/int32_t/uint32_t/float/int64_t/uint64_t data type on current "
112+ "device!");
113+ static_assert(std::is_same_v<pattern, Pattern::Reduce::AR> || std::is_same_v<pattern, Pattern::Reduce::RA>,
114+ "ReduceMin only support AR and RA pattern on current device!");
115+ 
116+ __ubuf__ T *dstAddr = (__ubuf__ T *)dst.GetPhyAddr();
117+ __ubuf__ T *srcAddr = (__ubuf__ T *)src.GetPhyAddr();
118+ LocalTensor<T> tmpBuf = sharedTmpBuffer.ReinterpretCast<T>();
119+ __ubuf__ T *tmpAddr = (__ubuf__ T *)tmpBuf.GetPhyAddr();
120+ if constexpr (std::is_same_v<pattern, Pattern::Reduce::AR>) {
121+ if constexpr (SupportBytes<T, 8>()) {
122+ ReduceMinARB64ReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
123+ } else {
124+ ReduceMinARReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
125+ }
126+ } else {
127+ if constexpr (SupportBytes<T, 8>()) {
128+ ReduceMinRAB64ReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
129+ } else {
130+ ReduceMinRAReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
131+ }
132+ }
133+}
134+} // namespace Internal
135+} // namespace ReduceExtendInternal
136+} // namespace AscendC
137+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_MIN_REDUCE_MIN_C310_IMPL_H_
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_prod_3510_impl.h+137-0
@@ -0,0 +1,137 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_PROD_REDUCE_PROD_C310_IMPL_H_
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_PROD_REDUCE_PROD_C310_IMPL_H_
13+ 
14+#include "kernel_tensor.h"
15+#include "kernel_basic_intf.h"
16+#include "reduce/reduce_common_util_3510_impl.h"
17+#include "reduce/reduce_common_ar_reuse_align_3510_impl.h"
18+#include "reduce/reduce_common_ra_reuse_align_3510_impl.h"
19+#ifdef ASCENDC_CPU_DEBUG
20+#endif // ASCENDC_CPU_DEBUG
21+ 
22+namespace AscendC {
23+namespace ReduceExtendInternal {
24+namespace Internal {
25+template <typename T>
26+__aicore__ inline void ReduceProd(Reg::RegTensor<T> &dst, Reg::RegTensor<T> src, Reg::MaskReg mask) {
27+ Reg::RegTensor<T> tempOne;
28+ // mask invalid data in src to one
29+ Reg::Duplicate(tempOne, 1);
30+ Reg::Select(src, src, tempOne, mask);
31+ 
32+ if constexpr (sizeof(T) == 1) {
33+ // fold to 128
34+ Reg::DeInterleave(dst, src, src, tempOne);
35+ Reg::Mul(src, dst, src, mask);
36+ }
37+ if constexpr (sizeof(T) <= 2) {
38+ // fold to 64
39+ Reg::DeInterleave(dst, src, src, tempOne);
40+ Reg::Mul(src, dst, src, mask);
41+ }
42+ // fold from 64 to 2
43+ Reg::DeInterleave(dst, src, src, tempOne);
44+ Reg::Mul(src, dst, src, mask);
45+ Reg::DeInterleave(dst, src, src, tempOne);
46+ Reg::Mul(src, dst, src, mask);
47+ Reg::DeInterleave(dst, src, src, tempOne);
48+ Reg::Mul(src, dst, src, mask);
49+ Reg::DeInterleave(dst, src, src, tempOne);
50+ Reg::Mul(src, dst, src, mask);
51+ Reg::DeInterleave(dst, src, src, tempOne);
52+ Reg::Mul(src, dst, src, mask);
53+ // fold to 1
54+ Reg::DeInterleave(dst, src, src, tempOne);
55+ Reg::Mul(dst, dst, src, mask);
56+}
57+ 
58+template <class T, const Reg::RegTrait &Trait, const uint16_t vlSize>
59+__simd_vf__ inline void ReduceProdARLessThanVLVF(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t dimA, uint32_t dimR,
60+ const uint16_t repeatTime) {
61+ Reg::RegTensor<T, Trait> vreg0;
62+ Reg::RegTensor<T, Trait> vreg1;
63+ Reg::UnalignReg uDst;
64+ Reg::RegTensor<T> tempOne;
65+ // mask invalid data in src to one
66+ Reg::Duplicate(tempOne, 1);
67+ uint32_t sreg1 = dimR;
68+ Reg::MaskReg mask = Reg::UpdateMask<T, Trait>(sreg1);
69+ for (uint16_t loopA = 0; loopA < static_cast<uint16_t>(dimA); loopA++) {
70+ Reg::LoadAlign(vreg0, srcAddr + loopA * dimR);
71+ Reg::Select(vreg0, vreg0, tempOne, mask);
72+ if constexpr (sizeof(T) == 1) {
73+ // fold to 128
74+ Reg::DeInterleave(vreg1, vreg0, vreg0, tempOne);
75+ Reg::Mul(vreg0, vreg1, vreg0, mask);
76+ }
77+ if constexpr (sizeof(T) <= 2) {
78+ // fold to 64
79+ Reg::DeInterleave(vreg1, vreg0, vreg0, tempOne);
80+ Reg::Mul(vreg0, vreg1, vreg0, mask);
81+ }
82+ for (uint16_t i = 0; i < repeatTime; ++i) {
83+ Reg::DeInterleave(vreg1, vreg0, vreg0, tempOne);
84+ Reg::Mul(vreg0, vreg1, vreg0, mask);
85+ }
86+ // fold to 1
87+ Reg::DeInterleave(vreg1, vreg0, vreg0, tempOne);
88+ Reg::Mul(vreg1, vreg1, vreg0, mask);
89+ Reg::StoreUnAlign((__ubuf__ T *&)dstAddr, vreg1, uDst, 1);
90+ }
91+ Reg::StoreUnAlignPost((__ubuf__ T *&)dstAddr, uDst, 0);
92+}
93+ 
94+template <class T, bool isReuseSource = false>
95+__aicore__ inline void ReduceProdARImpl(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr, uint32_t dimA,
96+ uint32_t dimR) {
97+ constexpr uint16_t vlSize = GetVecLen() / sizeof(T);
98+ if (dimR == 1) {
99+ ReduceOpInternal::ReduceCopyOutImpl<T>(dstAddr, srcAddr, dimA);
100+ } else if (dimR <= vlSize) {
101+ uint16_t repeatTime = AscendC::Internal::FindClosestPowerOfTwo(dimR);
102+ repeatTime = (1 << repeatTime) < dimR ? repeatTime : repeatTime - 1;
103+ ReduceProdARLessThanVLVF<T, Reg::RegTraitNumOne, vlSize>(dstAddr, srcAddr, dimA, dimR, repeatTime);
104+ } else {
105+ ReduceAROverVLImpl<T, Reg::RegTraitNumOne, vlSize, Reg::Mul<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
106+ ReduceProd<T>, isReuseSource>(dstAddr, srcAddr, tmpAddr, dimA, dimR);
107+ }
108+}
109+ 
110+template <typename T, typename pattern, bool isReuseSource = false>
111+__aicore__ inline void ReduceProdImpl(const LocalTensor<T> &dst, const LocalTensor<T> &src,
112+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[],
113+ bool srcInnerPad) {
114+ CHECK_FUNC_HIGHLEVEL_API(ReduceProd, (T, pattern), (dst, src, sharedTmpBuffer, srcShape, srcInnerPad, srcShape[1]));
115+ 
116+ CheckTensorPos<T>(dst, Hardware::UB, "dst", "VECIN/VECCALC/VECOUT", "ReduceProd");
117+ CheckTensorPos<T>(src, Hardware::UB, "src", "VECIN/VECCALC/VECOUT", "ReduceProd");
118+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "ReduceProd");
119+ static_assert(SupportType<T, float>(), "ReduceProd only support float data type on current device!");
120+ static_assert(std::is_same_v<pattern, Pattern::Reduce::AR> || std::is_same_v<pattern, Pattern::Reduce::RA>,
121+ "ReduceProd only support AR and RA pattern on current device!");
122+ 
123+ __ubuf__ T *dstAddr = (__ubuf__ T *)dst.GetPhyAddr();
124+ __ubuf__ T *srcAddr = (__ubuf__ T *)src.GetPhyAddr();
125+ LocalTensor<T> tmpBuf = sharedTmpBuffer.ReinterpretCast<T>();
126+ __ubuf__ T *tmpAddr = (__ubuf__ T *)tmpBuf.GetPhyAddr();
127+ if constexpr (std::is_same_v<pattern, Pattern::Reduce::AR>) {
128+ ReduceProdARImpl<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
129+ } else {
130+ ReduceRAImpl<T, Reg::RegTraitNumOne, Reg::Mul<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>, isReuseSource>(
131+ dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
132+ }
133+}
134+} // namespace Internal
135+} // namespace ReduceExtendInternal
136+} // namespace AscendC
137+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_PROD_REDUCE_PROD_C310_IMPL_H_
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_sum_3510_impl.h+204-0
@@ -0,0 +1,204 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_SUM_REDUCE_SUM_C310_IMPL_H
12+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_SUM_REDUCE_SUM_C310_IMPL_H
13+ 
14+#include "kernel_tensor.h"
15+#include "kernel_basic_intf.h"
16+#include "kernel_tiling/kernel_tiling.h"
17+#include "reduce/reduce_common_util_3510_impl.h"
18+#include "reduce/reduce_common_ar_ra_reuse_unalign_3510_impl.h"
19+#include "reduce/reduce_common_ar_reuse_align_3510_impl.h"
20+#include "reduce/reduce_common_ra_reuse_align_3510_impl.h"
21+ 
22+namespace AscendC {
23+namespace ReduceExtendInternal {
24+namespace Internal {
25+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)
26+template <const Reg::RegTrait &Trait>
27+__simd_callee__ inline void ReduceSumInt16(Reg::RegTensor<int16_t, Trait> &dst, Reg::RegTensor<int16_t, Trait> src,
28+ Reg::MaskReg mask) {
29+ Reg::RegTensor<int32_t, Trait> sum;
30+ Reg::ReduceSum<int32_t, int16_t, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<int32_t, Trait>,
31+ Reg::RegTensor<int16_t, Trait>>(sum, src, mask);
32+ Reg::Pack<uint16_t, int32_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint16_t, Trait> &)dst, sum);
33+}
34+#endif
35+ 
36+template <class T, bool isReuseSource = false>
37+__aicore__ inline void ReduceSumARB64ReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
38+ const uint32_t srcShape[]) {
39+ if ((srcShape[1] * sizeof(T)) % 32 == 0) {
40+ ReduceARImpl<T, Reg::RegTraitNumTwo,
41+ Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
42+ Reg::ReduceSum<T, T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>,
43+ Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
44+ isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
45+ } else {
46+ ReduceARReuseSourceUnAligned<
47+ T, Reg::RegTraitNumTwo, Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
48+ Reg::ReduceSum<T, T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>,
49+ Reg::RegTensor<T, Reg::RegTraitNumTwo>>>(dstAddr, srcAddr, srcShape[0], srcShape[1]);
50+ }
51+}
52+ 
53+template <class T, bool isReuseSource = false>
54+__aicore__ inline void ReduceSumARReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
55+ const uint32_t srcShape[]) {
56+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)
57+ if constexpr (std::is_same_v<T, int16_t>) {
58+ if ((srcShape[1] * sizeof(T)) % 32 == 0 || srcShape[1] == 1) {
59+ ReduceARImpl<T, Reg::RegTraitNumOne, Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
60+ ReduceSumInt16<Reg::RegTraitNumOne>, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape[0],
61+ srcShape[1]);
62+ } else {
63+ ReduceARReuseSourceUnAligned<T, Reg::RegTraitNumOne, Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
64+ ReduceSumInt16<Reg::RegTraitNumOne>>(dstAddr, srcAddr, srcShape[0], srcShape[1]);
65+ }
66+ } else {
67+ if ((srcShape[1] * sizeof(T)) % 32 == 0 || srcShape[1] == 1) {
68+ ReduceARImpl<T, Reg::RegTraitNumOne, Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
69+ Reg::ReduceSum<T, T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>, Reg::RegTensor<T>>,
70+ isReuseSource, ReduceType::SUM>(dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
71+ } else {
72+ ReduceARReuseSourceUnAligned<
73+ T, Reg::RegTraitNumOne, Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
74+ Reg::ReduceSum<T, T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>, Reg::RegTensor<T>>>(
75+ dstAddr, srcAddr, srcShape[0], srcShape[1]);
76+ }
77+ }
78+#else
79+ if ((srcShape[1] * sizeof(T)) % 32 == 0 || srcShape[1] == 1) {
80+ ReduceARImpl<T, Reg::RegTraitNumOne, Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
81+ Reg::ReduceSum<T, T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>, Reg::RegTensor<T>>, isReuseSource,
82+ ReduceType::SUM>(dstAddr, srcAddr, tmpAddr, srcShape[0], srcShape[1]);
83+ } else {
84+ ReduceARReuseSourceUnAligned<
85+ T, Reg::RegTraitNumOne, Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>,
86+ Reg::ReduceSum<T, T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>, Reg::RegTensor<T>>>(
87+ dstAddr, srcAddr, srcShape[0], srcShape[1]);
88+ }
89+#endif
90+}
91+ 
92+template <class T, bool isReuseSource = false>
93+__aicore__ inline void ReduceSumRAB64ReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
94+ const uint32_t srcShape[]) {
95+ if ((srcShape[1] * sizeof(T)) % 32 == 0) {
96+ ReduceRAB64ReuseSource<T, Reg::RegTraitNumTwo,
97+ Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>,
98+ isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
99+ } else {
100+ ReduceRAReuseSourceUnAlignedB64<T, Reg::RegTraitNumTwo,
101+ Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T, Reg::RegTraitNumTwo>>>(
102+ dstAddr, srcAddr, srcShape[1], srcShape[0]);
103+ }
104+}
105+ 
106+template <class T, bool isReuseSource = false>
107+__aicore__ inline void ReduceSumRAReuseSourceCompute(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, __ubuf__ T *tmpAddr,
108+ const uint32_t srcShape[]) {
109+ if ((srcShape[1] * sizeof(T)) % 32 == 0) {
110+ ReduceRAImpl<T, Reg::RegTraitNumOne, Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>, isReuseSource>(
111+ dstAddr, srcAddr, tmpAddr, srcShape[1], srcShape[0]);
112+ } else {
113+ ReduceRAReuseSourceUnAligned<T, Reg::RegTraitNumOne, Reg::Add<T, Reg::MaskMergeMode::ZEROING, Reg::RegTensor<T>>>(
114+ dstAddr, srcAddr, srcShape[1], srcShape[0]);
115+ }
116+}
117+ 
118+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)
119+__simd_vf__ inline void NarrowReduceSumOutput(__ubuf__ int8_t *dstAddr, __ubuf__ int16_t *srcAddr, uint32_t count) {
120+ constexpr uint32_t elementsPerLoop = GetVecLen() / sizeof(int16_t);
121+ const uint32_t loops = CeilDivision(count, elementsPerLoop);
122+ uint32_t remaining = count;
123+ Reg::RegTensor<int16_t> srcReg;
124+ Reg::RegTensor<int8_t> dstReg;
125+ Reg::MaskReg mask;
126+ for (uint32_t i = 0; i < loops; ++i) {
127+ mask = Reg::UpdateMask<int16_t>(remaining);
128+ Reg::LoadAlign(srcReg, srcAddr + i * elementsPerLoop);
129+ Reg::Pack<uint8_t, uint16_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t> &)dstReg,
130+ (Reg::RegTensor<uint16_t> &)srcReg);
131+ Reg::MaskPack(mask, mask);
132+ Reg::StoreAlign(dstAddr + i * elementsPerLoop, dstReg, mask);
133+ }
134+}
135+ 
136+template <class pattern>
137+__aicore__ inline void ReduceSumInt8Compute(const LocalTensor<int8_t> &dst, const LocalTensor<int8_t> &src,
138+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[]) {
139+ constexpr uint32_t blockBytes = 32;
140+ const uint32_t inputCount = srcShape[0] * srcShape[1];
141+ const uint32_t outputCount = std::is_same_v<pattern, Pattern::Reduce::AR> ? srcShape[0] : srcShape[1];
142+ const uint32_t widenedInputBytes = CeilDivision(inputCount * sizeof(int16_t), blockBytes) * blockBytes;
143+ LocalTensor<int16_t> widenedSrcTensor = sharedTmpBuffer.ReinterpretCast<int16_t>();
144+ LocalTensor<int16_t> widenedDstTensor = widenedSrcTensor[widenedInputBytes / sizeof(int16_t)];
145+ Cast(widenedSrcTensor, src, RoundMode::CAST_NONE, inputCount);
146+ PipeBarrier<PIPE_V>();
147+ __ubuf__ int16_t *widenedSrc = (__ubuf__ int16_t *)widenedSrcTensor.GetPhyAddr();
148+ __ubuf__ int16_t *widenedDst = (__ubuf__ int16_t *)widenedDstTensor.GetPhyAddr();
149+ if constexpr (std::is_same_v<pattern, Pattern::Reduce::AR>) {
150+ ReduceSumARReuseSourceCompute<int16_t, true>(widenedDst, widenedSrc, widenedDst, srcShape);
151+ } else {
152+ ReduceSumRAReuseSourceCompute<int16_t, true>(widenedDst, widenedSrc, widenedDst, srcShape);
153+ }
154+ PipeBarrier<PIPE_V>();
155+ NarrowReduceSumOutput((__ubuf__ int8_t *)dst.GetPhyAddr(), widenedDst, outputCount);
156+}
157+#endif
158+ 
159+template <class T, class pattern, bool isReuseSource = false>
160+__aicore__ inline void ReduceSumImpl(const LocalTensor<T> &dst, const LocalTensor<T> &src,
161+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[],
162+ bool srcInnerPad) {
163+ CheckTensorPos<T>(dst, Hardware::UB, "dstTensor", "VECIN/VECCALC/VECOUT", "ReduceSum");
164+ CheckTensorPos<T>(src, Hardware::UB, "srcTensor", "VECIN/VECCALC/VECOUT", "ReduceSum");
165+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "ReduceSum");
166+ static_assert(std::is_same_v<pattern, Pattern::Reduce::AR> || std::is_same_v<pattern, Pattern::Reduce::RA>,
167+ "ReduceSum only support AR and RA pattern on current device!");
168+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)
169+ static_assert(SupportType<T, int8_t, int16_t, int32_t, uint32_t, float, int64_t, uint64_t>(),
170+ "ReduceSum only support int8_t/int16_t/int32_t/uint32_t/float/int64_t/uint64_t data type on current "
171+ "device!");
172+#else
173+ static_assert(SupportType<T, int32_t, uint32_t, float, int64_t, uint64_t>(),
174+ "ReduceSum only support int32_t/uint32_t/float/int64_t/uint64_t data type on current device!");
175+#endif
176+ __ubuf__ T *dstAddr = (__ubuf__ T *)dst.GetPhyAddr();
177+ __ubuf__ T *srcAddr = (__ubuf__ T *)src.GetPhyAddr();
178+ LocalTensor<T> tmpBuf = sharedTmpBuffer.ReinterpretCast<T>();
179+ __ubuf__ T *tmpAddr = (__ubuf__ T *)tmpBuf.GetPhyAddr();
180+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)
181+ if constexpr (std::is_same_v<T, int8_t>) {
182+ ReduceSumInt8Compute<pattern>(dst, src, sharedTmpBuffer, srcShape);
183+ } else
184+#endif
185+ {
186+ if constexpr (std::is_same_v<pattern, Pattern::Reduce::AR>) {
187+ if constexpr (SupportBytes<T, 8>()) {
188+ ReduceSumARB64ReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
189+ } else {
190+ ReduceSumARReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
191+ }
192+ } else {
193+ if constexpr (SupportBytes<T, 8>()) {
194+ ReduceSumRAB64ReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
195+ } else {
196+ ReduceSumRAReuseSourceCompute<T, isReuseSource>(dstAddr, srcAddr, tmpAddr, srcShape);
197+ }
198+ }
199+ }
200+}
201+} // namespace Internal
202+} // namespace ReduceExtendInternal
203+} // namespace AscendC
204+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_SUM_REDUCE_SUM_C310_IMPL_H
Aautofuse/v35/ascendc/api_regbase/reduce/reduce_xor_sum_3510_impl.h+98-0
@@ -0,0 +1,98 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file reduce_xor_sum_3510_impl.h
13+ * \brief
14+ */
15+ 
16+#ifndef AUTOFUSE_REDUCE_EXTEND_REDUCE_XOR_SUM_C310_IMPL_H
17+#define AUTOFUSE_REDUCE_EXTEND_REDUCE_XOR_SUM_C310_IMPL_H
18+#include "kernel_tensor.h"
19+#include "kernel_basic_intf.h"
20+#include "kernel_tiling/kernel_tiling.h"
21+ 
22+namespace AscendC {
23+namespace ReduceExtendInternal {
24+namespace ReduceXorSumAPI {
25+ 
26+constexpr Reg::CastTrait castTraitF16F32 = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN, Reg::MaskMergeMode::ZEROING,
27+ RoundMode::UNKNOWN};
28+constexpr Reg::CastTrait castTraitF32F16 = {Reg::RegLayout::ZERO, Reg::SatMode::SAT, Reg::MaskMergeMode::ZEROING,
29+ RoundMode::CAST_RINT};
30+ 
31+template <typename T>
32+__simd_callee__ inline void LoadSrcDataAndXor(Reg::RegTensor<float> &srcReg, __ubuf__ T *src0, __ubuf__ T *src1,
33+ uint16_t index, Reg::MaskReg &mask) {
34+ constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(float);
35+ Reg::RegTensor<T> src0TmpReg, src1TmpReg;
36+ Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(src0TmpReg, src0 + index * eleCountPerVL);
37+ Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(src1TmpReg, src1 + index * eleCountPerVL);
38+ Reg::Xor(src0TmpReg, src0TmpReg, src1TmpReg, mask);
39+ Reg::Cast<float, T, castTraitF16F32>(srcReg, src0TmpReg, mask);
40+}
41+ 
42+template <typename T>
43+__simd_vf__ inline void ReduceXorSumCoreImpl(__ubuf__ T *dstUb, __ubuf__ T *src0Ub, __ubuf__ T *src1Ub,
44+ uint32_t calCount) {
45+ Reg::MaskReg mask;
46+ Reg::UnalignReg ureg;
47+ Reg::RegTensor<T> dstReg;
48+ Reg::RegTensor<float> srcTmpReg, dstTmpReg, tmpReg, zeroReg;
49+ Reg::MaskReg fullMask = Reg::CreateMask<float>();
50+ constexpr int32_t eleCountPerVL = GetVecLen() / sizeof(float);
51+ uint16_t repeatTimes = CeilDivision(calCount, eleCountPerVL);
52+ Reg::Duplicate(zeroReg, 0, fullMask);
53+ for (uint16_t i = 0; i < repeatTimes; i++) {
54+ mask = Reg::UpdateMask<float>(calCount);
55+ LoadSrcDataAndXor(srcTmpReg, src0Ub, src1Ub, i, mask);
56+ Reg::ReduceSum(dstTmpReg, srcTmpReg, mask);
57+ Reg::Add(zeroReg, zeroReg, dstTmpReg, mask);
58+ }
59+ Reg::Cast<T, float, castTraitF32F16>(dstReg, zeroReg, fullMask);
60+ Reg::Pack<uint16_t, uint32_t, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint16_t> &)dstReg,
61+ (Reg::RegTensor<uint32_t> &)dstReg);
62+ Reg::StoreUnAlign(dstUb, dstReg, ureg, 1);
63+ Reg::StoreUnAlignPost(dstUb, ureg, 0);
64+}
65+} // namespace ReduceXorSumAPI
66+ 
67+template <typename T, bool isReuseSource = false>
68+__aicore__ inline void ReduceXorSumCheckParams(LocalTensor<T> &dstTensor, const LocalTensor<T> &src0Tensor,
69+ const LocalTensor<T> &src1Tensor, LocalTensor<uint8_t> &sharedTmpBuffer,
70+ const uint32_t calCount) {
71+ static_assert(std::is_same<T, int16_t>::value, "ReduceXorSum only support int16_t data type on current device!");
72+ CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN/VECCALC/VECOUT", "ReduceXorSum");
73+ CheckTensorPos<T>(src0Tensor, Hardware::UB, "src0Tensor", "VECIN/VECCALC/VECOUT", "ReduceXorSum");
74+ CheckTensorPos<T>(src1Tensor, Hardware::UB, "src1Tensor", "VECIN/VECCALC/VECOUT", "ReduceXorSum");
75+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "ReduceXorSum");
76+ CheckCalCount(calCount, "calCount", src0Tensor, "src0Tensor", "ReduceXorSum");
77+ CheckCalCount(calCount, "calCount", src1Tensor, "src1Tensor", "ReduceXorSum");
78+}
79+ 
80+template <typename T, bool isReuseSource = false>
81+__aicore__ inline void ReduceXorSumCompute(LocalTensor<T> &dstTensor, const LocalTensor<T> &src0Tensor,
82+ const LocalTensor<T> &src1Tensor, LocalTensor<uint8_t> &sharedTmpBuffer,
83+ const uint32_t calCount) {
84+ // Only for AI Vector Core.
85+ if ASCEND_IS_AIC {
86+ return;
87+ }
88+ 
89+ ReduceXorSumCheckParams<T, isReuseSource>(dstTensor, src0Tensor, src1Tensor, sharedTmpBuffer, calCount);
90+ __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
91+ __ubuf__ T *src0Ub = (__ubuf__ T *)src0Tensor.GetPhyAddr();
92+ __ubuf__ T *src1Ub = (__ubuf__ T *)src1Tensor.GetPhyAddr();
93+ ReduceXorSumAPI::ReduceXorSumCoreImpl<T>(dstUb, src0Ub, src1Ub, calCount);
94+}
95+} // namespace ReduceExtendInternal
96+} // namespace AscendC
97+ 
98+#endif // AUTOFUSE_REDUCE_EXTEND_REDUCE_XOR_SUM_C310_IMPL_H
Aautofuse/v35/ascendc/api_regbase/reduce/sum_3510_impl.h+136-0
@@ -0,0 +1,136 @@
1+/**
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file sum_3510_impl.h
13+ * \brief
14+ */
15+ 
16+#ifndef AUTOFUSE_REDUCE_EXTEND_SUM_C310_IMPL_H
17+#define AUTOFUSE_REDUCE_EXTEND_SUM_C310_IMPL_H
18+ 
19+#include "kernel_basic_intf.h"
20+#include "kernel_tensor.h"
21+#include "adv_api/reduce/sum_utils.h"
22+ 
23+namespace AscendC {
24+namespace ReduceExtendInternal {
25+namespace SumInternal {
26+template <typename T>
27+__simd_vf__ inline void SumForOneRepeatTime(__ubuf__ T *dstUb, __ubuf__ T *srcUb, const SumParams sumParams,
28+ uint32_t count, uint32_t offset) {
29+ uint32_t calCount;
30+ Reg::MaskReg mask;
31+ Reg::UnalignReg uregOut;
32+ Reg::RegTensor<T> srcReg, dstReg;
33+ 
34+ for (int i = 0; i < sumParams.outter; i++) {
35+ calCount = count;
36+ mask = Reg::UpdateMask<T>(calCount);
37+ Reg::LoadAlign(srcReg, srcUb + i * offset);
38+ Reg::ReduceSum(dstReg, srcReg, mask);
39+ Reg::StoreUnAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(dstUb, dstReg, uregOut, 1);
40+ }
41+ Reg::StoreUnAlignPost(dstUb, uregOut, 0);
42+}
43+ 
44+template <typename T, bool isFirstRepeat>
45+__simd_vf__ inline void ReduceSumNextN(__ubuf__ T *dstUb, __ubuf__ T *srcUb, const SumParams sumParams,
46+ uint32_t calCount, uint32_t repeatTimes, uint32_t offset) {
47+ uint32_t count;
48+ Reg::MaskReg mask;
49+ Reg::UnalignReg uregIn;
50+ Reg::RegTensor<T> srcReg, dstReg;
51+ constexpr int32_t eleCountPerVL = GetVecLen() / sizeof(T);
52+ for (uint16_t i = 0; i < sumParams.outter; i++) {
53+ count = calCount;
54+ auto dstTmpUb = dstUb + i * offset;
55+ for (uint16_t j = 0; j < repeatTimes; j++) {
56+ mask = Reg::UpdateMask<T>(count);
57+ if constexpr (isFirstRepeat) {
58+ Reg::LoadAlign(srcReg, srcUb + i * sumParams.inner + j * eleCountPerVL);
59+ } else {
60+ Reg::LoadAlign(srcReg, srcUb + i * offset + j * eleCountPerVL);
61+ }
62+ Reg::ReduceSum(dstReg, srcReg, mask);
63+ Reg::StoreUnAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(dstTmpUb, dstReg, uregIn, 1);
64+ }
65+ Reg::StoreUnAlignPost(dstTmpUb, uregIn, 0);
66+ }
67+ Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
68+}
69+} // namespace SumInternal
70+ 
71+template <typename T, int32_t reduceDim = -1, bool isReuseSource = false, bool isBasicBlock = false>
72+__aicore__ inline void SumCheckParams(const LocalTensor<T> &dstTensor, const LocalTensor<T> &srcTensor,
73+ const LocalTensor<uint8_t> &sharedTmpBuffer, const SumParams &sumParams) {
74+ static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
75+ CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN/VECCALC/VECOUT", "Sum");
76+ CheckTensorPos<T>(srcTensor, Hardware::UB, "srcTensor", "VECIN/VECCALC/VECOUT", "Sum");
77+ CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN/VECCALC/VECOUT", "Sum");
78+ constexpr uint32_t sumInnerAlignLen = 32;
79+ ASCENDC_ASSERT((1 <= sumParams.n) && (sumParams.n <= sumParams.inner), {
80+ KERNEL_LOG(KERNEL_ERROR, "The value of n must be greater than or equal to 1 and less than or equal to inner.");
81+ });
82+ ASCENDC_ASSERT((sumParams.inner * sizeof(T) % sumInnerAlignLen == 0),
83+ { KERNEL_LOG(KERNEL_ERROR, "The value of inner * sizeof(T) must be an integer multiple of 32."); });
84+}
85+ 
86+template <typename T, int32_t reduceDim = -1, bool isReuseSource = false, bool isBasicBlock = false>
87+__aicore__ inline void SumCompute(const LocalTensor<T> &dstTensor, const LocalTensor<T> &srcTensor,
88+ const LocalTensor<uint8_t> &sharedTmpBuffer, const SumParams &sumParams) {
89+ // Only for AI Vector Core.
90+ if ASCEND_IS_AIC {
91+ return;
92+ }
93+ 
94+ SumCheckParams<T, reduceDim, isReuseSource, isBasicBlock>(dstTensor, srcTensor, sharedTmpBuffer, sumParams);
95+ __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
96+ __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
97+ __ubuf__ T *sharedTmpBufferUb = (__ubuf__ T *)sharedTmpBuffer.GetPhyAddr();
98+ 
99+ constexpr int32_t eleCountPerVL = GetVecLen() / sizeof(T);
100+ uint16_t repeatTimes = CeilDivision(sumParams.n, eleCountPerVL);
101+ uint32_t loopRepeatTimes;
102+ uint32_t calCount = sumParams.n;
103+ 
104+ uint32_t totalCnt = 1;
105+ uint32_t dataSize = repeatTimes;
106+ uint32_t offset = AlignUp(CeilDivision(sumParams.inner, eleCountPerVL), 32);
107+ while (dataSize > 1) {
108+ ++totalCnt;
109+ dataSize = CeilDivision(dataSize, eleCountPerVL);
110+ }
111+ 
112+ if (repeatTimes == 1) {
113+ SumInternal::SumForOneRepeatTime<T>(dstUb, srcUb, sumParams, sumParams.n, sumParams.inner);
114+ return;
115+ }
116+ 
117+ SumInternal::ReduceSumNextN<T, true>(sharedTmpBufferUb, srcUb, sumParams, calCount, repeatTimes, offset);
118+ 
119+ --totalCnt;
120+ loopRepeatTimes = repeatTimes;
121+ while (totalCnt != 0) {
122+ calCount = loopRepeatTimes;
123+ loopRepeatTimes = CeilDivision(loopRepeatTimes, eleCountPerVL);
124+ if (totalCnt == 1) {
125+ SumInternal::SumForOneRepeatTime<T>(dstUb, sharedTmpBufferUb, sumParams, calCount, offset);
126+ } else {
127+ SumInternal::ReduceSumNextN<T, false>(sharedTmpBufferUb, sharedTmpBufferUb, sumParams, calCount, loopRepeatTimes,
128+ offset);
129+ }
130+ --totalCnt;
131+ }
132+}
133+} // namespace ReduceExtendInternal
134+} // namespace AscendC
135+ 
136+#endif // AUTOFUSE_REDUCE_EXTEND_SUM_C310_IMPL_H
Aautofuse/v35/ascendc/api_regbase/reduce_extend.h+148-0
@@ -0,0 +1,148 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#ifndef AUTOFUSE_V35_ASCENDC_API_REGBASE_REDUCE_EXTEND_H
12+#define AUTOFUSE_V35_ASCENDC_API_REGBASE_REDUCE_EXTEND_H
13+ 
14+#include "adv_api/reduce/reduce.h"
15+ 
16+#include "reduce/reduce_common_util_3510_impl.h"
17+#include "reduce/reduce_common_ar_reuse_align_less_than_vl_3510_impl.h"
18+#include "reduce/reduce_common_ar_reuse_align_3510_impl.h"
19+#include "reduce/reduce_common_ra_reuse_align_3510_impl.h"
20+#include "reduce/reduce_common_ar_ra_reuse_unalign_3510_impl.h"
21+#include "reduce/reduce_sum_3510_impl.h"
22+#include "reduce/reduce_mean_3510_impl.h"
23+#include "reduce/reduce_max_3510_impl.h"
24+#include "reduce/reduce_min_3510_impl.h"
25+#include "reduce/reduce_prod_3510_impl.h"
26+#include "reduce/reduce_any_3510_impl.h"
27+#include "reduce/reduce_all_3510_impl.h"
28+#include "reduce/mean_3510_impl.h"
29+#include "reduce/sum_3510_impl.h"
30+#include "reduce/reduce_xor_sum_3510_impl.h"
31+ 
32+namespace AscendC {
33+template <class T, class pattern, bool isReuseSource = false>
34+__aicore__ inline void ReduceSumExtend(const LocalTensor<T> &dst, const LocalTensor<T> &src,
35+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[],
36+ bool srcInnerPad) {
37+ if ASCEND_IS_AIC {
38+ return;
39+ }
40+ ReduceExtendInternal::Internal::ReduceSumImpl<T, pattern, isReuseSource>(dst, src, sharedTmpBuffer, srcShape,
41+ srcInnerPad);
42+}
43+ 
44+template <class T, class pattern, bool isReuseSource = false>
45+__aicore__ inline void ReduceSumExtend(const LocalTensor<T> &dst, const LocalTensor<T> &src, const uint32_t srcShape[],
46+ bool srcInnerPad) {
47+ LocalTensor<uint8_t> sharedTmpBuffer;
48+ bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer);
49+ ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
50+ ReduceSumExtend<T, pattern, isReuseSource>(dst, src, sharedTmpBuffer, srcShape, srcInnerPad);
51+}
52+ 
53+template <class T, class pattern, bool isReuseSource = false>
54+__aicore__ inline void ReduceMeanExtend(const LocalTensor<T> &dst, const LocalTensor<T> &src,
55+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[],
56+ bool srcInnerPad) {
57+ if ASCEND_IS_AIC {
58+ return;
59+ }
60+ ReduceExtendInternal::Internal::ReduceMeanImpl<T, pattern, isReuseSource>(dst, src, sharedTmpBuffer, srcShape,
61+ srcInnerPad);
62+}
63+ 
64+template <class T, class pattern, bool isReuseSource = false>
65+__aicore__ inline void ReduceMeanExtend(const LocalTensor<T> &dst, const LocalTensor<T> &src, const uint32_t srcShape[],
66+ bool srcInnerPad) {
67+ LocalTensor<uint8_t> sharedTmpBuffer;
68+ bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer);
69+ ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
70+ ReduceMeanExtend<T, pattern, isReuseSource>(dst, src, sharedTmpBuffer, srcShape, srcInnerPad);
71+}
72+ 
73+#define AUTOFUSE_DEFINE_REDUCE_EXTEND(apiName, implName) \
74+ template <class T, class pattern, bool isReuseSource = false> \
75+ __aicore__ inline void apiName(const LocalTensor<T> &dst, const LocalTensor<T> &src, \
76+ const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t srcShape[], \
77+ bool srcInnerPad) { \
78+ if ASCEND_IS_AIC { \
79+ return; \
80+ } \
81+ ReduceExtendInternal::Internal::implName##Impl<T, pattern, isReuseSource>(dst, src, sharedTmpBuffer, srcShape, \
82+ srcInnerPad); \
83+ } \
84+ template <class T, class pattern, bool isReuseSource = false> \
85+ __aicore__ inline void apiName(const LocalTensor<T> &dst, const LocalTensor<T> &src, const uint32_t srcShape[], \
86+ bool srcInnerPad) { \
87+ LocalTensor<uint8_t> sharedTmpBuffer; \
88+ bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer); \
89+ ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); }); \
90+ apiName<T, pattern, isReuseSource>(dst, src, sharedTmpBuffer, srcShape, srcInnerPad); \
91+ }
92+ 
93+AUTOFUSE_DEFINE_REDUCE_EXTEND(ReduceMaxExtend, ReduceMax)
94+AUTOFUSE_DEFINE_REDUCE_EXTEND(ReduceMinExtend, ReduceMin)
95+AUTOFUSE_DEFINE_REDUCE_EXTEND(ReduceProdExtend, ReduceProd)
96+AUTOFUSE_DEFINE_REDUCE_EXTEND(ReduceAnyExtend, ReduceAny)
97+AUTOFUSE_DEFINE_REDUCE_EXTEND(ReduceAllExtend, ReduceAll)
98+ 
99+#undef AUTOFUSE_DEFINE_REDUCE_EXTEND
100+ 
101+template <typename T, typename accType = T, bool isReuseSource = false, bool isBasicBlock = false,
102+ int32_t reduceDim = -1>
103+__aicore__ inline void MeanExtend(const LocalTensor<T> &dst, const LocalTensor<T> &src,
104+ const LocalTensor<uint8_t> &sharedTmpBuffer, const MeanParams &meanParams) {
105+ ReduceExtendInternal::MeanImpl<T, accType, isReuseSource, isBasicBlock, reduceDim>(dst, src, sharedTmpBuffer,
106+ meanParams);
107+}
108+ 
109+template <typename T, typename accType = T, bool isReuseSource = false, bool isBasicBlock = false,
110+ int32_t reduceDim = -1>
111+__aicore__ inline void MeanExtend(const LocalTensor<T> &dst, const LocalTensor<T> &src, const MeanParams &meanParams) {
112+ LocalTensor<uint8_t> sharedTmpBuffer;
113+ bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer);
114+ ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
115+ MeanExtend<T, accType, isReuseSource, isBasicBlock, reduceDim>(dst, src, sharedTmpBuffer, meanParams);
116+}
117+ 
118+template <typename T, int32_t reduceDim = -1, bool isReuseSource = false, bool isBasicBlock = false>
119+__aicore__ inline void SumExtend(const LocalTensor<T> &dst, const LocalTensor<T> &src,
120+ const LocalTensor<uint8_t> &sharedTmpBuffer, const SumParams &sumParams) {
121+ ReduceExtendInternal::SumCompute<T, reduceDim, isReuseSource, isBasicBlock>(dst, src, sharedTmpBuffer, sumParams);
122+}
123+ 
124+template <typename T, int32_t reduceDim = -1, bool isReuseSource = false, bool isBasicBlock = false>
125+__aicore__ inline void SumExtend(const LocalTensor<T> &dst, const LocalTensor<T> &src, const SumParams &sumParams) {
126+ LocalTensor<uint8_t> sharedTmpBuffer;
127+ bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer);
128+ ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
129+ SumExtend<T, reduceDim, isReuseSource, isBasicBlock>(dst, src, sharedTmpBuffer, sumParams);
130+}
131+ 
132+template <typename T, bool isReuseSource = false>
133+__aicore__ inline void ReduceXorSumExtend(LocalTensor<T> &dst, const LocalTensor<T> &src0, const LocalTensor<T> &src1,
134+ LocalTensor<uint8_t> &sharedTmpBuffer, uint32_t calCount) {
135+ ReduceExtendInternal::ReduceXorSumCompute<T, isReuseSource>(dst, src0, src1, sharedTmpBuffer, calCount);
136+}
137+ 
138+template <typename T, bool isReuseSource = false>
139+__aicore__ inline void ReduceXorSumExtend(LocalTensor<T> &dst, const LocalTensor<T> &src0, const LocalTensor<T> &src1,
140+ uint32_t calCount) {
141+ LocalTensor<uint8_t> sharedTmpBuffer;
142+ bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer);
143+ ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
144+ ReduceXorSumExtend<T, isReuseSource>(dst, src0, src1, sharedTmpBuffer, calCount);
145+}
146+} // namespace AscendC
147+ 
148+#endif // AUTOFUSE_V35_ASCENDC_API_REGBASE_REDUCE_EXTEND_H
Mautofuse/v35/ascir/generator/v2_ascir_codegen_impl.h+7-7
@@ -2527,7 +2527,7 @@ class MaxAscIrCodegenImplV2 : public AscIrCodegenV2 {
2527 return "ReduceMax";2527 return "ReduceMax";
2528 }2528 }
2529 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {2529 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {
2530- return {"reduce_init_reg_base.h"};2530+ return {"reduce_init_reg_base.h", "reduce_extend_reg_base.h"};
2531 }2531 }
2532 [[nodiscard]] std::pair<std::vector<ge::DataType>, std::vector<ge::DataType>> GetConversionDtype(2532 [[nodiscard]] std::pair<std::vector<ge::DataType>, std::vector<ge::DataType>> GetConversionDtype(
2533 const AscNode &node) {2533 const AscNode &node) {
@@ -2561,7 +2561,7 @@ class SumAscIrCodegenImplV2 : public AscIrCodegenV2 {
2561 return "ReduceSum";2561 return "ReduceSum";
2562 }2562 }
2563 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {2563 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {
2564- return {"reduce_init_reg_base.h"};2564+ return {"reduce_init_reg_base.h", "reduce_extend_reg_base.h"};
2565 }2565 }
2566 [[nodiscard]] std::pair<std::vector<ge::DataType>, std::vector<ge::DataType>> GetConversionDtype(2566 [[nodiscard]] std::pair<std::vector<ge::DataType>, std::vector<ge::DataType>> GetConversionDtype(
2567 const AscNode &node) {2567 const AscNode &node) {
@@ -2598,7 +2598,7 @@ class MinAscIrCodegenImplV2 : public AscIrCodegenV2 {
2598 return "ReduceMin";2598 return "ReduceMin";
2599 }2599 }
2600 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {2600 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {
2601- return {"reduce_init_reg_base.h"};2601+ return {"reduce_init_reg_base.h", "reduce_extend_reg_base.h"};
2602 }2602 }
2603 [[nodiscard]] std::pair<std::vector<ge::DataType>, std::vector<ge::DataType>> GetConversionDtype(2603 [[nodiscard]] std::pair<std::vector<ge::DataType>, std::vector<ge::DataType>> GetConversionDtype(
2604 const AscNode &node) {2604 const AscNode &node) {
@@ -2632,7 +2632,7 @@ class MeanAscIrCodegenImplV2 : public AscIrCodegenV2 {
2632 return "ReduceMean";2632 return "ReduceMean";
2633 }2633 }
2634 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {2634 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {
2635- return {"reduce_init_reg_base.h"};2635+ return {"reduce_init_reg_base.h", "reduce_extend_reg_base.h"};
2636 }2636 }
2637 [[nodiscard]] std::vector<std::string> IncludeApiHeaderFiles() const override {2637 [[nodiscard]] std::vector<std::string> IncludeApiHeaderFiles() const override {
2638 return {2638 return {
@@ -2661,7 +2661,7 @@ class ProdAscIrCodegenImplV2 : public AscIrCodegenV2 {
2661 return "ReduceProd";2661 return "ReduceProd";
2662 }2662 }
2663 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {2663 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {
2664- return {"reduce_init_reg_base.h"};2664+ return {"reduce_init_reg_base.h", "reduce_extend_reg_base.h"};
2665 }2665 }
2666 [[nodiscard]] std::vector<std::string> IncludeApiHeaderFiles() const override {2666 [[nodiscard]] std::vector<std::string> IncludeApiHeaderFiles() const override {
2667 return {2667 return {
@@ -2690,7 +2690,7 @@ class AnyAscIrCodegenImplV2 : public AscIrCodegenV2 {
2690 return "ReduceAny";2690 return "ReduceAny";
2691 }2691 }
2692 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {2692 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {
2693- return {"reduce_init_reg_base.h"};2693+ return {"reduce_init_reg_base.h", "reduce_extend_reg_base.h"};
2694 }2694 }
2695 [[nodiscard]] std::vector<std::string> IncludeApiHeaderFiles() const override {2695 [[nodiscard]] std::vector<std::string> IncludeApiHeaderFiles() const override {
2696 return {2696 return {
@@ -2719,7 +2719,7 @@ class AllAscIrCodegenImplV2 : public AscIrCodegenV2 {
2719 return "ReduceAll";2719 return "ReduceAll";
2720 }2720 }
2721 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {2721 [[nodiscard]] std::vector<std::string> LoadApiHeaderFiles([[maybe_unused]] bool is_dynamic) const override {
2722- return {"reduce_init_reg_base.h"};2722+ return {"reduce_init_reg_base.h", "reduce_extend_reg_base.h"};
2723 }2723 }
2724 [[nodiscard]] std::vector<std::string> IncludeApiHeaderFiles() const override {2724 [[nodiscard]] std::vector<std::string> IncludeApiHeaderFiles() const override {
2725 return {2725 return {
Mautofuse/v35/codegen/ascendc_reg_base_api_register.cpp+67-0
@@ -32,6 +32,21 @@ static std::string StripBroadcastSelfIncludes(const std::string &src) {
32 return result;32 return result;
33}33}
34 34 
35+static std::string StripReduceSelfIncludes(const std::string &src) {
36+ std::string result;
37+ size_t pos = 0;
38+ while (pos < src.size()) {
39+ size_t end = src.find('\n', pos);
40+ if (end == std::string::npos) end = src.size();
41+ std::string line = src.substr(pos, end - pos);
42+ if (line.find("#include \"reduce/") == std::string::npos) {
43+ result += line + "\n";
44+ }
45+ pos = end + 1;
46+ }
47+ return result;
48+}
49+ 
35Register::Register() {50Register::Register() {
36 const std::string kAscendcCastRegStr = {51 const std::string kAscendcCastRegStr = {
37#include "cast_reg_base.h"52#include "cast_reg_base.h"
@@ -63,6 +78,57 @@ Register::Register() {
63 const std::string kAscendcReduce_initRegBase = {78 const std::string kAscendcReduce_initRegBase = {
64#include "reduce_init_reg_base.h"79#include "reduce_init_reg_base.h"
65 };80 };
81+ const std::string kAscendcReduceExtendRegBase = StripReduceSelfIncludes(std::string{
82+#include "reduce/reduce_common_reg_base.h"
83+ }) +
84+ StripReduceSelfIncludes(std::string{
85+#include "reduce/reduce_common_util_3510_impl_reg_base.h"
86+ }) +
87+ StripReduceSelfIncludes(std::string{
88+#include "reduce/reduce_common_ar_reuse_align_less_than_vl_3510_impl_reg_base.h"
89+ }) +
90+ StripReduceSelfIncludes(std::string{
91+#include "reduce/reduce_common_ar_reuse_align_3510_impl_reg_base.h"
92+ }) +
93+ StripReduceSelfIncludes(std::string{
94+#include "reduce/reduce_common_ra_reuse_align_3510_impl_reg_base.h"
95+ }) +
96+ StripReduceSelfIncludes(std::string{
97+#include "reduce/reduce_common_ar_ra_reuse_unalign_3510_impl_reg_base.h"
98+ }) +
99+ StripReduceSelfIncludes(std::string{
100+#include "reduce/reduce_sum_3510_impl_reg_base.h"
101+ }) +
102+ StripReduceSelfIncludes(std::string{
103+#include "reduce/reduce_mean_3510_impl_reg_base.h"
104+ }) +
105+ StripReduceSelfIncludes(std::string{
106+#include "reduce/reduce_max_3510_impl_reg_base.h"
107+ }) +
108+ StripReduceSelfIncludes(std::string{
109+#include "reduce/reduce_min_3510_impl_reg_base.h"
110+ }) +
111+ StripReduceSelfIncludes(std::string{
112+#include "reduce/reduce_prod_3510_impl_reg_base.h"
113+ }) +
114+ StripReduceSelfIncludes(std::string{
115+#include "reduce/reduce_any_3510_impl_reg_base.h"
116+ }) +
117+ StripReduceSelfIncludes(std::string{
118+#include "reduce/reduce_all_3510_impl_reg_base.h"
119+ }) +
120+ StripReduceSelfIncludes(std::string{
121+#include "reduce/mean_3510_impl_reg_base.h"
122+ }) +
123+ StripReduceSelfIncludes(std::string{
124+#include "reduce/sum_3510_impl_reg_base.h"
125+ }) +
126+ StripReduceSelfIncludes(std::string{
127+#include "reduce/reduce_xor_sum_3510_impl_reg_base.h"
128+ }) +
129+ StripReduceSelfIncludes(std::string{
130+#include "reduce_extend_reg_base.h"
131+ });
66 const std::string kAscendcFloorDivRegBaseStr = {132 const std::string kAscendcFloorDivRegBaseStr = {
67#include "floor_div_reg_base.h"133#include "floor_div_reg_base.h"
68 };134 };
@@ -352,6 +418,7 @@ Register::Register() {
352 {"erf_reg_base.h", kAscendcErfRegBaseStr},418 {"erf_reg_base.h", kAscendcErfRegBaseStr},
353 {"tanh_reg_base.h", kAscendcTanhRegBaseStr},419 {"tanh_reg_base.h", kAscendcTanhRegBaseStr},
354 {"reduce_init_reg_base.h", kAscendcReduce_initRegBase},420 {"reduce_init_reg_base.h", kAscendcReduce_initRegBase},
421+ {"reduce_extend_reg_base.h", kAscendcReduceExtendRegBase},
355 {"floor_div_reg_base.h", kAscendcFloorDivRegBaseStr},422 {"floor_div_reg_base.h", kAscendcFloorDivRegBaseStr},
356 {"sign_reg_base.h", kAscendcSignRegBaseStr},423 {"sign_reg_base.h", kAscendcSignRegBaseStr},
357 {"where_reg_base.h", kAscendcWhereRegBaseStr},424 {"where_reg_base.h", kAscendcWhereRegBaseStr},
Mautofuse/v35/codegen/reg_api_call/reg_reduce_api_call.cpp+1-1
@@ -78,7 +78,7 @@ Status RegReduceApiCall::BuildApiParam(const TPipe &tpipe, const std::vector<asc
78 78 
79 ReduceInitCodeGen(x, y, type_value, api_param->api_pre_process, tpipe, dtype_name);79 ReduceInitCodeGen(x, y, type_value, api_param->api_pre_process, tpipe, dtype_name);
80 api_param->api_pre_process.emplace_back("uint32_t tmp_reduce_shape[] = {first_actual, last};\n");80 api_param->api_pre_process.emplace_back("uint32_t tmp_reduce_shape[] = {first_actual, last};\n");
81- std::string new_api_name = this->api_name_ == "ReduceMean" ? "ReduceSum" : this->api_name_;81+ std::string new_api_name = this->api_name_ == "ReduceMean" ? "ReduceSumExtend" : this->api_name_ + "Extend";
82 api_param->api_name = new_api_name;82 api_param->api_name = new_api_name;
83 api_param->template_params.emplace_back(dtype_name);83 api_param->template_params.emplace_back(dtype_name);
84 api_param->template_params.emplace_back(reduce_pattern);84 api_param->template_params.emplace_back(reduce_pattern);