Merged
feature/text_gated_residual_grad_pipeline_v2 #15084
feature/text_gated_residual_grad_pipeline_v2 #15084
Merged
azyuliAdd on 15 days ago
Total: 47 files changed+8388-0
@@ -0,0 +1,19 @@
1+# -----------------------------------------------------------------------------------------------------------
2+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+# This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+# CANN Open Software License Agreement Version 2.0 (the "License").
5+# Please refer to the License for details. You may not use this file except in compliance with the License.
6+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+# See LICENSE in the root of the software repository for the full text of the License.
9+# -----------------------------------------------------------------------------------------------------------
10+ 
11+file(GLOB CURRENT_DIRS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*)
12+if(NOT ENABLE_TEST AND NOT BENCHMARK)
13+ list(REMOVE_ITEM CURRENT_DIRS tests)
14+endif()
15+foreach(SUB_DIR ${CURRENT_DIRS})
16+ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${SUB_DIR}/CMakeLists.txt")
17+ add_subdirectory(${SUB_DIR})
18+ endif()
19+endforeach()
@@ -0,0 +1,381 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ * TextGatedResidualGrad aclnn example: read binary inputs from a directory and write binary outputs.
10+ *
11+ * Usage: test_aclnn_text_gated_residual_grad [data_dir] [deviceId] (defaults: ./data, device 0).
12+ * Data directory files:
13+ * meta.txt: dtype(bf16|fp16) n c h r has_gamma [has_gate] (n=B*S; tensors use B=1, S=n;
14+ * has_gate defaults to 1; n=0 empty token input is supported).
15+ * grad_mixed.bin [n,h]T grad_inject.bin [n,c]T x.bin [n,d]T
16+ * w_down.bin [r,d]T w_up.bin [d,r]T w_gate.bin [c,d]T
17+ * gamma.bin [c*h]fp32 (original weight, used as 1+gamma) inv_rms.bin [n,c]fp32
18+ * inject.bin [n,c]T down_pre_act.bin [n,r]T (xn@w_down^T, before division by C) up.bin [n,d]T (upPreAct)
19+ * Outputs: dx/dw_down/dw_up/dw_gate.bin use T; dgamma.bin uses fp32. Omit dw_gate/dgamma when has_gate/has_gamma is 0.
20+ */
21+ 
22+#include <algorithm>
23+#include <array>
24+#include <cstdint>
25+#include <cstdlib>
26+#include <cstdio>
27+#include <cstring>
28+#include <fstream>
29+#include <iostream>
30+#include <limits>
31+#include <string>
32+#include <vector>
33+#include "acl/acl.h"
34+#include "aclnnop/aclnn_text_gated_residual_grad.h"
35+ 
36+#define CHECK_RET(cond) ((cond) ? true : (false))
37+#define LOG_PRINT(message, ...) printf(message, ##__VA_ARGS__)
38+ 
39+namespace {
40+constexpr size_t HALF_BYTES = sizeof(uint16_t);
41+constexpr int64_t METADATA_CHANNEL_LIMIT = 4096; // Tiling additionally checks the actual AIV count.
42+constexpr int DATA_DIR_ARG = 1;
43+constexpr int DEVICE_ARG = 2;
44+enum TensorIndex : size_t {
45+ GRAD_MIXED,
46+ GRAD_INJECT,
47+ X,
48+ W_DOWN,
49+ W_UP,
50+ W_GATE,
51+ GAMMA,
52+ INV_RMS,
53+ INJECT,
54+ DOWN_PRE,
55+ UP,
56+ DX,
57+ DW_DOWN,
58+ DW_UP,
59+ DW_GATE,
60+ DGAMMA,
61+ TENSOR_COUNT
62+};
63+ 
64+struct Metadata {
65+ std::string dtype;
66+ int64_t n = 0;
67+ int64_t c = 0;
68+ int64_t h = 0;
69+ int64_t r = 0;
70+ int64_t d = 0;
71+ int hasGamma = 0;
72+ int hasGate = 1;
73+};
74+ 
75+struct TensorSpec {
76+ std::string file;
77+ std::vector<int64_t> shape;
78+ aclDataType dtype;
79+ size_t elemBytes;
80+ bool enabled = true;
81+};
82+ 
83+struct Resources {
84+ std::array<aclTensor*, TENSOR_COUNT> tensors{};
85+ std::array<void*, TENSOR_COUNT> addresses{};
86+ void* workspace = nullptr;
87+ aclrtStream stream = nullptr;
88+ int32_t device = 0;
89+ bool initialized = false;
90+ bool deviceSet = false;
91+ 
92+ ~Resources()
93+ {
94+ for (auto tensor : tensors) {
95+ if (tensor != nullptr) {
96+ aclDestroyTensor(tensor);
97+ }
98+ }
99+ for (auto address : addresses) {
100+ if (address != nullptr) {
101+ aclrtFree(address);
102+ }
103+ }
104+ if (workspace != nullptr) {
105+ aclrtFree(workspace);
106+ }
107+ if (stream != nullptr) {
108+ aclrtDestroyStream(stream);
109+ }
110+ if (deviceSet) {
111+ aclrtResetDevice(device);
112+ }
113+ if (initialized) {
114+ aclFinalize();
115+ }
116+ }
117+};
118+ 
119+bool GetByteSize(const std::vector<int64_t>& shape, size_t elemSize, size_t& bytes)
120+{
121+ bytes = elemSize;
122+ for (auto i : shape) {
123+ if (i < 0 || (i != 0 && bytes > std::numeric_limits<size_t>::max() / static_cast<size_t>(i))) {
124+ LOG_PRINT("invalid or overflowing tensor shape\n");
125+ return false;
126+ }
127+ bytes *= static_cast<size_t>(i);
128+ }
129+ return true;
130+}
131+ 
132+int Init(Resources& resources)
133+{
134+ auto ret = aclInit(nullptr);
135+ if (!CHECK_RET(ret == ACL_SUCCESS)) {
136+ LOG_PRINT("aclInit failed. ERROR: %d\n", ret);
137+ return ret;
138+ }
139+ resources.initialized = true;
140+ ret = aclrtSetDevice(resources.device);
141+ if (!CHECK_RET(ret == ACL_SUCCESS)) {
142+ LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret);
143+ return ret;
144+ }
145+ resources.deviceSet = true;
146+ ret = aclrtCreateStream(&resources.stream);
147+ if (!CHECK_RET(ret == ACL_SUCCESS)) {
148+ LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret);
149+ return ret;
150+ }
151+ return 0;
152+}
153+ 
154+bool ReadBin(const std::string& path, std::string& data)
155+{
156+ std::ifstream ifs(path, std::ios::binary | std::ios::ate);
157+ if (!ifs) {
158+ LOG_PRINT("open %s failed\n", path.c_str());
159+ return false;
160+ }
161+ auto size = ifs.tellg();
162+ if (size < 0) {
163+ return false;
164+ }
165+ data.resize(size);
166+ ifs.seekg(0);
167+ ifs.read(data.data(), size);
168+ return ifs.good();
169+}
170+ 
171+bool WriteBin(const std::string& path, const std::string& data)
172+{
173+ std::ofstream ofs(path, std::ios::binary);
174+ ofs.write(data.data(), data.size());
175+ return ofs.good();
176+}
177+ 
178+int CreateTensorFromBin(const std::string& path, const std::vector<int64_t>& shape, aclDataType dtype, size_t elemSize,
179+ aclTensor** tensor, void** deviceAddr, std::string& hostBuf)
180+{
181+ if (!ReadBin(path, hostBuf)) {
182+ return -1;
183+ }
184+ size_t size = 0;
185+ if (!GetByteSize(shape, elemSize, size)) {
186+ return -1;
187+ }
188+ if (hostBuf.size() != size) {
189+ LOG_PRINT("size mismatch %s: file %zu expect %zu\n", path.c_str(), hostBuf.size(), size);
190+ return -1;
191+ }
192+ auto ret = aclrtMalloc(deviceAddr, std::max<size_t>(size, 1), ACL_MEM_MALLOC_HUGE_FIRST);
193+ if (!CHECK_RET(ret == ACL_SUCCESS)) {
194+ return -1;
195+ }
196+ if (size != 0) {
197+ ret = aclrtMemcpy(*deviceAddr, size, hostBuf.data(), size, ACL_MEMCPY_HOST_TO_DEVICE);
198+ if (!CHECK_RET(ret == ACL_SUCCESS)) {
199+ return -1;
200+ }
201+ }
202+ std::vector<int64_t> strides(shape.size(), 1);
203+ for (size_t i = shape.size(); i > 1; --i) {
204+ const size_t last = i - 1;
205+ strides[last - 1] = shape[last] * strides[last];
206+ }
207+ *tensor = aclCreateTensor(shape.data(), shape.size(), dtype, strides.data(), 0, aclFormat::ACL_FORMAT_ND,
208+ shape.data(), shape.size(), *deviceAddr);
209+ return *tensor != nullptr ? 0 : -1;
210+}
211+ 
212+int CreateOutTensor(const std::vector<int64_t>& shape, aclDataType dtype, size_t elemSize, aclTensor** tensor,
213+ void** deviceAddr)
214+{
215+ size_t size = 0;
216+ if (!GetByteSize(shape, elemSize, size)) {
217+ return -1;
218+ }
219+ auto ret = aclrtMalloc(deviceAddr, std::max<size_t>(size, 1), ACL_MEM_MALLOC_HUGE_FIRST);
220+ if (!CHECK_RET(ret == ACL_SUCCESS)) {
221+ return -1;
222+ }
223+ std::vector<int64_t> strides(shape.size(), 1);
224+ for (size_t i = shape.size(); i > 1; --i) {
225+ const size_t last = i - 1;
226+ strides[last - 1] = shape[last] * strides[last];
227+ }
228+ *tensor = aclCreateTensor(shape.data(), shape.size(), dtype, strides.data(), 0, aclFormat::ACL_FORMAT_ND,
229+ shape.data(), shape.size(), *deviceAddr);
230+ return *tensor != nullptr ? 0 : -1;
231+}
232+ 
233+int DumpOut(const std::string& path, void* deviceAddr, const std::vector<int64_t>& shape, size_t elemSize)
234+{
235+ size_t size = 0;
236+ if (!GetByteSize(shape, elemSize, size)) {
237+ return -1;
238+ }
239+ std::string host(size, '\0');
240+ if (size != 0) {
241+ auto ret = aclrtMemcpy(host.data(), size, deviceAddr, size, ACL_MEMCPY_DEVICE_TO_HOST);
242+ if (!CHECK_RET(ret == ACL_SUCCESS)) {
243+ return -1;
244+ }
245+ }
246+ return WriteBin(path, host) ? 0 : -1;
247+}
248+ 
249+bool ValidateMetadata(Metadata& m)
250+{
251+ const auto maxDim = std::numeric_limits<uint32_t>::max();
252+ if ((m.dtype != "bf16" && m.dtype != "fp16") || m.n < 0 || m.n > maxDim || m.c <= 0 ||
253+ m.c > METADATA_CHANNEL_LIMIT || m.h <= 0 || m.h > maxDim / m.c || m.r <= 0 || m.r > maxDim) {
254+ LOG_PRINT("invalid dtype or dimensions in meta.txt\n");
255+ return false;
256+ }
257+ if ((m.hasGamma != 0 && m.hasGamma != 1) || (m.hasGate != 0 && m.hasGate != 1)) {
258+ LOG_PRINT("invalid optional flags in meta.txt\n");
259+ return false;
260+ }
261+ m.d = m.c * m.h;
262+ const int64_t maxFloatElements = std::numeric_limits<int64_t>::max() / sizeof(float);
263+ if (m.n > maxFloatElements / m.d || m.n > maxFloatElements / m.r || m.r > maxFloatElements / m.d ||
264+ m.c > maxFloatElements / m.d) {
265+ LOG_PRINT("Tensor byte size overflows int64 in meta.txt\n");
266+ return false;
267+ }
268+ return true;
269+}
270+ 
271+bool ReadMetadata(const std::string& dir, Metadata& m)
272+{
273+ std::ifstream meta(dir + "/meta.txt");
274+ if (!(meta >> m.dtype >> m.n >> m.c >> m.h >> m.r >> m.hasGamma)) {
275+ LOG_PRINT("invalid or missing %s/meta.txt\n", dir.c_str());
276+ return false;
277+ }
278+ meta >> std::ws;
279+ if (!meta.eof() && !(meta >> m.hasGate)) {
280+ LOG_PRINT("invalid has_gate in meta.txt\n");
281+ return false;
282+ }
283+ return ValidateMetadata(m);
284+}
285+ 
286+std::array<TensorSpec, TENSOR_COUNT> MakeTensorSpecs(const Metadata& m)
287+{
288+ const auto type = m.dtype == "fp16" ? ACL_FLOAT16 : ACL_BF16;
289+ return {{{"grad_mixed.bin", {1, m.n, m.h}, type, HALF_BYTES},
290+ {"grad_inject.bin", {1, m.n, m.c}, type, HALF_BYTES, m.hasGate != 0},
291+ {"x.bin", {1, m.n, m.d}, type, HALF_BYTES},
292+ {"w_down.bin", {m.r, m.d}, type, HALF_BYTES},
293+ {"w_up.bin", {m.d, m.r}, type, HALF_BYTES},
294+ {"w_gate.bin", {m.c, m.d}, type, HALF_BYTES, m.hasGate != 0},
295+ {"gamma.bin", {m.d}, ACL_FLOAT, sizeof(float), m.hasGamma != 0},
296+ {"inv_rms.bin", {1, m.n, m.c}, ACL_FLOAT, sizeof(float)},
297+ {"inject.bin", {1, m.n, m.c}, type, HALF_BYTES, m.hasGate != 0},
298+ {"down_pre_act.bin", {1, m.n, m.r}, type, HALF_BYTES},
299+ {"up.bin", {1, m.n, m.d}, type, HALF_BYTES},
300+ {"dx.bin", {1, m.n, m.d}, type, HALF_BYTES},
301+ {"dw_down.bin", {m.r, m.d}, type, HALF_BYTES},
302+ {"dw_up.bin", {m.d, m.r}, type, HALF_BYTES},
303+ {"dw_gate.bin", {m.c, m.d}, type, HALF_BYTES, m.hasGate != 0},
304+ {"dgamma.bin", {m.d}, ACL_FLOAT, sizeof(float), m.hasGamma != 0}}};
305+}
306+ 
307+int CreateTensors(const std::string& dir, const std::array<TensorSpec, TENSOR_COUNT>& specs, Resources& resources)
308+{
309+ std::string hostBuf;
310+ for (size_t i = 0; i < specs.size(); ++i) {
311+ const auto& spec = specs[i];
312+ if (!spec.enabled) {
313+ continue;
314+ }
315+ auto* tensor = &resources.tensors[i];
316+ auto* address = &resources.addresses[i];
317+ const auto status = i < DX ? CreateTensorFromBin(dir + "/" + spec.file, spec.shape, spec.dtype, spec.elemBytes,
318+ tensor, address, hostBuf) :
319+ CreateOutTensor(spec.shape, spec.dtype, spec.elemBytes, tensor, address);
320+ if (status != 0) {
321+ return -1;
322+ }
323+ }
324+ return 0;
325+}
326+ 
327+int RunOperator(Resources& resources, int64_t channels)
328+{
329+ auto& t = resources.tensors;
330+ uint64_t workspaceSize = 0;
331+ aclOpExecutor* executor = nullptr;
332+ auto ret = aclnnTextGatedResidualGradGetWorkspaceSize(
333+ t[GRAD_MIXED], t[X], t[W_DOWN], t[W_UP], t[INV_RMS], t[DOWN_PRE], t[UP], t[GRAD_INJECT], t[W_GATE], t[GAMMA],
334+ t[INJECT], channels, t[DX], t[DW_DOWN], t[DW_UP], t[DW_GATE], t[DGAMMA], &workspaceSize, &executor);
335+ if (ret != ACL_SUCCESS) {
336+ LOG_PRINT("aclnnTextGatedResidualGradGetWorkspaceSize failed. ERROR: %d\n", ret);
337+ return -1;
338+ }
339+ if (workspaceSize > 0 &&
340+ aclrtMalloc(&resources.workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) {
341+ return -1;
342+ }
343+ ret = aclnnTextGatedResidualGrad(resources.workspace, workspaceSize, executor, resources.stream);
344+ if (ret != ACL_SUCCESS) {
345+ LOG_PRINT("aclnnTextGatedResidualGrad failed. ERROR: %d\n", ret);
346+ return -1;
347+ }
348+ return aclrtSynchronizeStream(resources.stream) == ACL_SUCCESS ? 0 : -1;
349+}
350+ 
351+int DumpOutputs(const std::string& dir, const std::array<TensorSpec, TENSOR_COUNT>& specs, const Resources& resources)
352+{
353+ for (size_t i = DX; i < specs.size(); ++i) {
354+ const auto& spec = specs[i];
355+ if (spec.enabled && DumpOut(dir + "/" + spec.file, resources.addresses[i], spec.shape, spec.elemBytes) != 0) {
356+ return -1;
357+ }
358+ }
359+ return 0;
360+}
361+} // namespace
362+ 
363+int main(int argc, char** argv)
364+{
365+ const std::string dir = argc > DATA_DIR_ARG ? argv[DATA_DIR_ARG] : "./data";
X
Xxuejinghui6 days ago

C02|高|ci-smoke|FAIL|置信度高

    const std::string dir = argc > DATA_DIR_ARG ? argv[DATA_DIR_ARG] : "./data";

默认 example 从 ./data 读取 meta.txt 和多份 .bin,但 PR 未提供这些文件或生成步骤。build.sh 会从干净的构建目录无参数启动程序,因此 ReadMetadata 在 aclInit 前直接失败。该算子在 tests/test_config.yaml 中配置了 examples: True,changed-file smoke 会选中它。

**建议:**在 example 内生成确定性的输入并校验输出,或提供数据生成脚本并由构建入口显式传入数据目录。

likedislike
azyuli
azyuli
6 days ago Comments:
366+ Metadata metadata;
367+ if (!ReadMetadata(dir, metadata)) {
368+ return -1;
369+ }
370+ LOG_PRINT("meta: dtype=%s n=%ld c=%ld h=%ld r=%ld hasGamma=%d\n", metadata.dtype.c_str(), metadata.n, metadata.c,
371+ metadata.h, metadata.r, metadata.hasGamma);
372+ Resources resources;
373+ resources.device = argc > DEVICE_ARG ? atoi(argv[DEVICE_ARG]) : 0;
374+ const auto specs = MakeTensorSpecs(metadata);
375+ if (Init(resources) != 0 || CreateTensors(dir, specs, resources) != 0 || RunOperator(resources, metadata.c) != 0 ||
376+ DumpOutputs(dir, specs, resources) != 0) {
377+ return -1;
378+ }
379+ LOG_PRINT("run success, outputs dumped to %s\n", dir.c_str());
380+ return 0;
381+}
@@ -0,0 +1,11 @@
1+# -----------------------------------------------------------------------------------------------------------
2+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+# This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+# CANN Open Software License Agreement Version 2.0 (the "License").
5+# Please refer to the License for details. You may not use this file except in compliance with the License.
6+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+# See LICENSE in the root of the software repository for the full text of the License.
9+# -----------------------------------------------------------------------------------------------------------
10+ 
11+add_graph_plugin_sources()
@@ -0,0 +1,89 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file text_gated_residual_grad_proto.h
13+ * \brief TextGatedResidualGrad graph operator registration.
14+ */
15+ 
16+#ifndef OPS_OP_PROTO_INC_TEXT_GATED_RESIDUAL_GRAD_H_
17+#define OPS_OP_PROTO_INC_TEXT_GATED_RESIDUAL_GRAD_H_
18+ 
19+#include "graph/operator_reg.h"
20+#include "graph/types.h"
21+ 
22+namespace ge {
23+/**
24+ * @brief Computes gradients for the RMS normalization, mixed output and injection
25+ * branch of TextGatedResidual.
26+ * B is the batch size, S is the sequence length, C = channels, H is the hidden
27+ * size per channel, D = C * H, and R is the projection size. All tensors use ND
28+ * format. All FLOAT16 or BFLOAT16 inputs must have the same dtype as x.
29+ *
30+ * @par Inputs:
31+ * @li grad_mixed: Gradient of the mixed output. A 3D tensor of shape [B, S, H].
32+ * Type supports FLOAT16 and BFLOAT16.
33+ * @li x: Forward input. A 3D tensor of shape [B, S, D]. Type supports FLOAT16
34+ * and BFLOAT16.
35+ * @li w_down: Down-projection weight. A 2D tensor of shape [R, D]. Type supports
36+ * FLOAT16 and BFLOAT16.
37+ * @li w_up: Up-projection weight. A 2D tensor of shape [D, R]. Type supports
38+ * FLOAT16 and BFLOAT16.
39+ * @li inv_rms: Saved inverse RMS. A 3D tensor of shape [B, S, C]. Type is FLOAT32.
40+ * @li down_pre_act: Saved down-projection result before division by C. A 3D
41+ * tensor of shape [B, S, R]. Type supports FLOAT16 and BFLOAT16.
42+ * @li up: Saved up-projection result. A 3D tensor of shape [B, S, D]. Type
43+ * supports FLOAT16 and BFLOAT16.
44+ * @li grad_inject: Optional gradient of the injection output. A 3D tensor of
45+ * shape [B, S, C]. Type supports FLOAT16 and BFLOAT16.
46+ * @li w_gate: Optional gate-projection weight. A 2D tensor of shape [C, D].
47+ * Type supports FLOAT16 and BFLOAT16. When provided, grad_inject and inject
48+ * are required.
49+ * @li gamma: Optional RMS weight. A 1D tensor of shape [D]. Type is FLOAT32.
50+ * Normalization uses 1 + gamma; omitting gamma uses a scale of 1.
51+ * @li inject: Optional saved injection output. A 3D tensor of shape [B, S, C].
52+ * Type supports FLOAT16 and BFLOAT16.
53+ *
54+ * @par Outputs:
55+ * @li dx: Gradient of x. A 3D tensor of shape [B, S, D], with the dtype of x.
56+ * @li dw_down: Gradient of w_down. A 2D tensor of shape [R, D], with the dtype
57+ * of w_down.
58+ * @li dw_up: Gradient of w_up. A 2D tensor of shape [D, R], with the dtype of w_up.
59+ * @li dw_gate: Gradient of w_gate. When w_gate is provided, a 2D tensor of shape
60+ * [C, D], with the dtype of w_gate. Otherwise, an empty tensor of shape [0].
61+ * @li dgamma: Gradient of gamma. When gamma is provided, a 1D tensor of shape
62+ * [D], with type FLOAT32. Otherwise, an empty tensor of shape [0].
63+ *
64+ * @par Attributes:
65+ * @li channels: A required positive int attribute specifying the number of
66+ * channels C. The last dimension of x must be divisible by channels.
67+ */
68+REG_OP(TextGatedResidualGrad)
69+ .INPUT(grad_mixed, TensorType({DT_FLOAT16, DT_BF16}))
T
Ttang-lei016 days ago

缺少参数的资料描述

likedislike
azyuli
azyuli
6 days ago Comments:
70+ .INPUT(x, TensorType({DT_FLOAT16, DT_BF16}))
71+ .INPUT(w_down, TensorType({DT_FLOAT16, DT_BF16}))
72+ .INPUT(w_up, TensorType({DT_FLOAT16, DT_BF16}))
73+ .INPUT(inv_rms, TensorType({DT_FLOAT}))
74+ .INPUT(down_pre_act, TensorType({DT_FLOAT16, DT_BF16}))
75+ .INPUT(up, TensorType({DT_FLOAT16, DT_BF16}))
76+ .OPTIONAL_INPUT(grad_inject, TensorType({DT_FLOAT16, DT_BF16}))
77+ .OPTIONAL_INPUT(w_gate, TensorType({DT_FLOAT16, DT_BF16}))
78+ .OPTIONAL_INPUT(gamma, TensorType({DT_FLOAT}))
79+ .OPTIONAL_INPUT(inject, TensorType({DT_FLOAT16, DT_BF16}))
80+ .REQUIRED_ATTR(channels, Int)
81+ .OUTPUT(dx, TensorType({DT_FLOAT16, DT_BF16}))
82+ .OUTPUT(dw_down, TensorType({DT_FLOAT16, DT_BF16}))
83+ .OUTPUT(dw_up, TensorType({DT_FLOAT16, DT_BF16}))
84+ .OUTPUT(dw_gate, TensorType({DT_FLOAT16, DT_BF16}))
85+ .OUTPUT(dgamma, TensorType({DT_FLOAT}))
86+ .OP_END_FACTORY_REG(TextGatedResidualGrad)
87+} // namespace ge
88+ 
89+#endif // OPS_OP_PROTO_INC_TEXT_GATED_RESIDUAL_GRAD_H_
@@ -0,0 +1,35 @@
1+# -----------------------------------------------------------------------------------------------------------
2+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+# This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+# CANN Open Software License Agreement Version 2.0 (the "License").
5+# Please refer to the License for details. You may not use this file except in compliance with the License.
6+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+# See LICENSE in the root of the software repository for the full text of the License.
9+# -----------------------------------------------------------------------------------------------------------
10+ 
11+if (BUILD_OPEN_PROJECT)
12+ target_sources(op_host_aclnnExc PRIVATE
13+ text_gated_residual_grad_def.cpp
14+ )
15+ 
16+ if("${ASCEND_COMPUTE_UNIT}" STREQUAL "ascend950")
17+ add_ops_compile_options(
18+ OP_NAME TextGatedResidualGrad
19+ COMPUTE_UNIT Ascend950PR_9599
20+ OPTIONS -mllvm -cce-aicore-dcci-before-kernel-end=false
21+ -DCATLASS_ARCH=3510
22+ )
23+ else()
24+ add_ops_compile_options(
25+ OP_NAME TextGatedResidualGrad
26+ OPTIONS --cce-auto-sync=on
27+ -Wno-deprecated-declarations
28+ )
29+ endif()
30+endif()
31+ 
32+if(NOT BUILD_OPS_RTY_KERNEL)
33+ add_op_to_compiled_list()
34+ add_modules_sources(OPTYPE text_gated_residual_grad ACLNNTYPE aclnn_exclude)
35+endif()
@@ -0,0 +1,353 @@
1+{
2+ "op_type": "TextGatedResidualGrad",
3+ "op_list": [
4+ {
5+ "bin_filename": "TextGatedResidualGrad_BF16",
6+ "inputs": [
7+ {
8+ "name": "grad_mixed",
9+ "index": 0,
10+ "dtype": "bfloat16",
11+ "format": "ND",
12+ "paramType": "required",
13+ "shape": [
14+ -2
15+ ]
16+ },
17+ {
18+ "name": "x",
19+ "index": 1,
20+ "dtype": "bfloat16",
21+ "format": "ND",
22+ "paramType": "required",
23+ "shape": [
24+ -2
25+ ]
26+ },
27+ {
28+ "name": "w_down",
29+ "index": 2,
30+ "dtype": "bfloat16",
31+ "format": "ND",
32+ "paramType": "required",
33+ "shape": [
34+ -2
35+ ]
36+ },
37+ {
38+ "name": "w_up",
39+ "index": 3,
40+ "dtype": "bfloat16",
41+ "format": "ND",
42+ "paramType": "required",
43+ "shape": [
44+ -2
45+ ]
46+ },
47+ {
48+ "name": "inv_rms",
49+ "index": 4,
50+ "dtype": "float32",
51+ "format": "ND",
52+ "paramType": "required",
53+ "shape": [
54+ -2
55+ ]
56+ },
57+ {
58+ "name": "down_pre_act",
59+ "index": 5,
60+ "dtype": "bfloat16",
61+ "format": "ND",
62+ "paramType": "required",
63+ "shape": [
64+ -2
65+ ]
66+ },
67+ {
68+ "name": "up",
69+ "index": 6,
70+ "dtype": "bfloat16",
71+ "format": "ND",
72+ "paramType": "required",
73+ "shape": [
74+ -2
75+ ]
76+ },
77+ {
78+ "name": "grad_inject",
79+ "index": 7,
80+ "dtype": "bfloat16",
81+ "format": "ND",
82+ "paramType": "optional",
83+ "shape": [
84+ -2
85+ ]
86+ },
87+ {
88+ "name": "w_gate",
89+ "index": 8,
90+ "dtype": "bfloat16",
91+ "format": "ND",
92+ "paramType": "optional",
93+ "shape": [
94+ -2
95+ ]
96+ },
97+ {
98+ "name": "gamma",
99+ "index": 9,
100+ "dtype": "float32",
101+ "format": "ND",
102+ "paramType": "optional",
103+ "shape": [
104+ -2
105+ ]
106+ },
107+ {
108+ "name": "inject",
109+ "index": 10,
110+ "dtype": "bfloat16",
111+ "format": "ND",
112+ "paramType": "optional",
113+ "shape": [
114+ -2
115+ ]
116+ }
117+ ],
118+ "outputs": [
119+ {
120+ "name": "dx",
121+ "index": 0,
122+ "dtype": "bfloat16",
123+ "format": "ND",
124+ "paramType": "required",
125+ "shape": [
126+ -2
127+ ]
128+ },
129+ {
130+ "name": "dw_down",
131+ "index": 1,
132+ "dtype": "bfloat16",
133+ "format": "ND",
134+ "paramType": "required",
135+ "shape": [
136+ -2
137+ ]
138+ },
139+ {
140+ "name": "dw_up",
141+ "index": 2,
142+ "dtype": "bfloat16",
143+ "format": "ND",
144+ "paramType": "required",
145+ "shape": [
146+ -2
147+ ]
148+ },
149+ {
150+ "name": "dw_gate",
151+ "index": 3,
152+ "dtype": "bfloat16",
153+ "format": "ND",
154+ "paramType": "optional",
155+ "shape": [
156+ -2
157+ ]
158+ },
159+ {
160+ "name": "dgamma",
161+ "index": 4,
162+ "dtype": "float32",
163+ "format": "ND",
164+ "paramType": "optional",
165+ "shape": [
166+ -2
167+ ]
168+ }
169+ ],
170+ "attrs": [
171+ {
172+ "name": "channels",
173+ "dtype": "int",
174+ "value": 4
175+ }
176+ ]
177+ },
178+ {
179+ "bin_filename": "TextGatedResidualGrad_FP16",
180+ "inputs": [
181+ {
182+ "name": "grad_mixed",
183+ "index": 0,
184+ "dtype": "float16",
185+ "format": "ND",
186+ "paramType": "required",
187+ "shape": [
188+ -2
189+ ]
190+ },
191+ {
192+ "name": "x",
193+ "index": 1,
194+ "dtype": "float16",
195+ "format": "ND",
196+ "paramType": "required",
197+ "shape": [
198+ -2
199+ ]
200+ },
201+ {
202+ "name": "w_down",
203+ "index": 2,
204+ "dtype": "float16",
205+ "format": "ND",
206+ "paramType": "required",
207+ "shape": [
208+ -2
209+ ]
210+ },
211+ {
212+ "name": "w_up",
213+ "index": 3,
214+ "dtype": "float16",
215+ "format": "ND",
216+ "paramType": "required",
217+ "shape": [
218+ -2
219+ ]
220+ },
221+ {
222+ "name": "inv_rms",
223+ "index": 4,
224+ "dtype": "float32",
225+ "format": "ND",
226+ "paramType": "required",
227+ "shape": [
228+ -2
229+ ]
230+ },
231+ {
232+ "name": "down_pre_act",
233+ "index": 5,
234+ "dtype": "float16",
235+ "format": "ND",
236+ "paramType": "required",
237+ "shape": [
238+ -2
239+ ]
240+ },
241+ {
242+ "name": "up",
243+ "index": 6,
244+ "dtype": "float16",
245+ "format": "ND",
246+ "paramType": "required",
247+ "shape": [
248+ -2
249+ ]
250+ },
251+ {
252+ "name": "grad_inject",
253+ "index": 7,
254+ "dtype": "float16",
255+ "format": "ND",
256+ "paramType": "optional",
257+ "shape": [
258+ -2
259+ ]
260+ },
261+ {
262+ "name": "w_gate",
263+ "index": 8,
264+ "dtype": "float16",
265+ "format": "ND",
266+ "paramType": "optional",
267+ "shape": [
268+ -2
269+ ]
270+ },
271+ {
272+ "name": "gamma",
273+ "index": 9,
274+ "dtype": "float32",
275+ "format": "ND",
276+ "paramType": "optional",
277+ "shape": [
278+ -2
279+ ]
280+ },
281+ {
282+ "name": "inject",
283+ "index": 10,
284+ "dtype": "float16",
285+ "format": "ND",
286+ "paramType": "optional",
287+ "shape": [
288+ -2
289+ ]
290+ }
291+ ],
292+ "outputs": [
293+ {
294+ "name": "dx",
295+ "index": 0,
296+ "dtype": "float16",
297+ "format": "ND",
298+ "paramType": "required",
299+ "shape": [
300+ -2
301+ ]
302+ },
303+ {
304+ "name": "dw_down",
305+ "index": 1,
306+ "dtype": "float16",
307+ "format": "ND",
308+ "paramType": "required",
309+ "shape": [
310+ -2
311+ ]
312+ },
313+ {
314+ "name": "dw_up",
315+ "index": 2,
316+ "dtype": "float16",
317+ "format": "ND",
318+ "paramType": "required",
319+ "shape": [
320+ -2
321+ ]
322+ },
323+ {
324+ "name": "dw_gate",
325+ "index": 3,
326+ "dtype": "float16",
327+ "format": "ND",
328+ "paramType": "optional",
329+ "shape": [
330+ -2
331+ ]
332+ },
333+ {
334+ "name": "dgamma",
335+ "index": 4,
336+ "dtype": "float32",
337+ "format": "ND",
338+ "paramType": "optional",
339+ "shape": [
340+ -2
341+ ]
342+ }
343+ ],
344+ "attrs": [
345+ {
346+ "name": "channels",
347+ "dtype": "int",
348+ "value": 4
349+ }
350+ ]
351+ }
352+ ]
353+}
@@ -0,0 +1,2 @@
1+[TextGatedResidualGrad]
2+default=0
@@ -0,0 +1,319 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include "aclnn_text_gated_residual_grad.h"
12+#include "text_gated_residual_grad.h"
13+ 
14+#include <limits>
15+ 
16+#include "aclnn_kernels/contiguous.h"
17+#include "external/aclnn_kernels/aclnn_platform.h"
18+#include "aclnn/aclnn_base.h"
19+#include "acl/acl.h"
20+#include "aclnn_kernels/common/op_error_check.h"
21+#include "opdev/common_types.h"
22+#include "opdev/data_type_utils.h"
23+#include "opdev/format_utils.h"
24+#include "opdev/op_dfx.h"
25+#include "opdev/op_executor.h"
26+#include "opdev/op_log.h"
27+#include "opdev/platform.h"
28+#include "opdev/shape_utils.h"
29+#include "opdev/tensor_view_utils.h"
30+#include "opdev/make_op_executor.h"
31+ 
32+using namespace op;
33+ 
34+#ifdef __cplusplus
35+extern "C" {
36+#endif
37+ 
38+namespace {
39+constexpr int64_t INPUT_RANK = 3;
40+constexpr int64_t MATRIX_RANK = 2;
41+constexpr int64_t BATCH_DIM = 0;
42+constexpr int64_t SEQUENCE_DIM = 1;
43+constexpr int64_t FEATURE_DIM = 2;
44+constexpr int64_t MATRIX_ROW_DIM = 0;
45+constexpr size_t OUT_DX = 0;
46+constexpr size_t OUT_DW_DOWN = 1;
47+constexpr size_t OUT_DW_UP = 2;
48+constexpr size_t OUT_DW_GATE = 3;
49+constexpr size_t OUT_DGAMMA = 4;
50+ 
51+struct TextGatedResidualGradParams {
52+ const aclTensor* gradMixed = nullptr;
53+ const aclTensor* x = nullptr;
54+ const aclTensor* wDown = nullptr;
55+ const aclTensor* wUp = nullptr;
56+ const aclTensor* invRms = nullptr;
57+ const aclTensor* downPreAct = nullptr;
58+ const aclTensor* up = nullptr;
59+ const aclTensor* gradInject = nullptr;
60+ const aclTensor* wGate = nullptr;
61+ const aclTensor* gamma = nullptr;
62+ const aclTensor* inject = nullptr;
63+ int64_t channels = 0;
64+ aclTensor* dx = nullptr;
65+ aclTensor* dwDown = nullptr;
66+ aclTensor* dwUp = nullptr;
67+ aclTensor* dwGate = nullptr;
68+ aclTensor* dgamma = nullptr;
69+};
70+ 
71+aclnnStatus CheckNotNull(const TextGatedResidualGradParams& p)
72+{
73+ CHECK_COND(p.gradMixed != nullptr, ACLNN_ERR_PARAM_NULLPTR, "gradMixed must not be nullptr.");
74+ CHECK_COND(p.x != nullptr, ACLNN_ERR_PARAM_NULLPTR, "x must not be nullptr.");
75+ CHECK_COND(p.wDown != nullptr, ACLNN_ERR_PARAM_NULLPTR, "wDown must not be nullptr.");
76+ CHECK_COND(p.wUp != nullptr, ACLNN_ERR_PARAM_NULLPTR, "wUp must not be nullptr.");
77+ CHECK_COND(p.invRms != nullptr, ACLNN_ERR_PARAM_NULLPTR, "invRms must not be nullptr.");
78+ CHECK_COND(p.downPreAct != nullptr, ACLNN_ERR_PARAM_NULLPTR, "downPreAct must not be nullptr.");
79+ CHECK_COND(p.up != nullptr, ACLNN_ERR_PARAM_NULLPTR, "up must not be nullptr.");
80+ CHECK_COND(p.dx != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dx must not be nullptr.");
81+ CHECK_COND(p.dwDown != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dwDown must not be nullptr.");
82+ CHECK_COND(p.dwUp != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dwUp must not be nullptr.");
83+ CHECK_COND(p.wGate != nullptr || (p.gradInject == nullptr && p.inject == nullptr && p.dwGate == nullptr),
84+ ACLNN_ERR_PARAM_NULLPTR, "wGate is required when gradInject, inject or dwGate is given.");
85+ CHECK_COND(p.gamma != nullptr || p.dgamma == nullptr, ACLNN_ERR_PARAM_NULLPTR,
86+ "gamma is required when dgamma is given.");
87+ if (p.wGate != nullptr) {
88+ CHECK_COND(p.gradInject != nullptr, ACLNN_ERR_PARAM_NULLPTR,
89+ "gradInject must not be nullptr when wGate is given.");
90+ CHECK_COND(p.inject != nullptr, ACLNN_ERR_PARAM_NULLPTR, "inject must not be nullptr when wGate is given.");
91+ CHECK_COND(p.dwGate != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dwGate must not be nullptr when wGate is given.");
92+ }
93+ if (p.gamma != nullptr) {
94+ CHECK_COND(p.dgamma != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dgamma must not be nullptr when gamma is given.");
95+ }
96+ return ACLNN_SUCCESS;
97+}
98+ 
99+aclnnStatus CheckDtype(const TextGatedResidualGradParams& p)
100+{
101+ const std::initializer_list<op::DataType> supportDtype = {op::DataType::DT_FLOAT16, op::DataType::DT_BF16};
102+ OP_CHECK_DTYPE_NOT_SUPPORT(p.x, supportDtype, return ACLNN_ERR_PARAM_INVALID);
103+ OP_CHECK_DTYPE_NOT_SUPPORT(p.gradMixed, supportDtype, return ACLNN_ERR_PARAM_INVALID);
104+ OP_CHECK_DTYPE_NOT_SUPPORT(p.wDown, supportDtype, return ACLNN_ERR_PARAM_INVALID);
105+ OP_CHECK_DTYPE_NOT_SUPPORT(p.wUp, supportDtype, return ACLNN_ERR_PARAM_INVALID);
106+ OP_CHECK_DTYPE_NOT_SUPPORT(p.downPreAct, supportDtype, return ACLNN_ERR_PARAM_INVALID);
107+ OP_CHECK_DTYPE_NOT_SAME(p.x, p.gradMixed, return ACLNN_ERR_PARAM_INVALID);
108+ OP_CHECK_DTYPE_NOT_SAME(p.x, p.wDown, return ACLNN_ERR_PARAM_INVALID);
109+ OP_CHECK_DTYPE_NOT_SAME(p.x, p.wUp, return ACLNN_ERR_PARAM_INVALID);
110+ OP_CHECK_DTYPE_NOT_SAME(p.x, p.downPreAct, return ACLNN_ERR_PARAM_INVALID);
111+ OP_CHECK_DTYPE_NOT_SAME(p.x, p.up, return ACLNN_ERR_PARAM_INVALID);
112+ OP_CHECK_DTYPE_NOT_SAME(p.x, p.dx, return ACLNN_ERR_PARAM_INVALID);
113+ OP_CHECK_DTYPE_NOT_MATCH(p.invRms, op::DataType::DT_FLOAT, return ACLNN_ERR_PARAM_INVALID);
114+ OP_CHECK_DTYPE_NOT_SAME(p.wDown, p.dwDown, return ACLNN_ERR_PARAM_INVALID);
115+ OP_CHECK_DTYPE_NOT_SAME(p.wUp, p.dwUp, return ACLNN_ERR_PARAM_INVALID);
116+ if (p.wGate != nullptr) {
117+ OP_CHECK_DTYPE_NOT_SAME(p.x, p.gradInject, return ACLNN_ERR_PARAM_INVALID);
118+ OP_CHECK_DTYPE_NOT_SAME(p.x, p.wGate, return ACLNN_ERR_PARAM_INVALID);
119+ OP_CHECK_DTYPE_NOT_SAME(p.x, p.inject, return ACLNN_ERR_PARAM_INVALID);
120+ OP_CHECK_DTYPE_NOT_SAME(p.wGate, p.dwGate, return ACLNN_ERR_PARAM_INVALID);
121+ }
122+ if (p.gamma != nullptr) {
123+ OP_CHECK_DTYPE_NOT_MATCH(p.gamma, op::DataType::DT_FLOAT, return ACLNN_ERR_PARAM_INVALID);
124+ OP_CHECK_DTYPE_NOT_MATCH(p.dgamma, op::DataType::DT_FLOAT, return ACLNN_ERR_PARAM_INVALID);
125+ }
126+ return ACLNN_SUCCESS;
127+}
128+ 
129+aclnnStatus CheckTensorShape(const aclTensor* t, std::initializer_list<int64_t> expect, const char* name)
130+{
131+ auto shape = t->GetViewShape();
132+ CHECK_COND(shape.GetDimNum() == static_cast<int64_t>(expect.size()), ACLNN_ERR_PARAM_INVALID,
133+ "%s dim num mismatch, expect %zu but got %zu", name, expect.size(), shape.GetDimNum());
134+ size_t i = 0;
135+ for (auto e : expect) {
136+ CHECK_COND(shape.GetDim(i) == e, ACLNN_ERR_PARAM_INVALID, "%s dim[%zu] expect %ld but got %ld", name, i, e,
137+ shape.GetDim(i));
138+ i++;
139+ }
140+ return ACLNN_SUCCESS;
141+}
142+ 
143+aclnnStatus CheckShape(const TextGatedResidualGradParams& p)
144+{
145+ auto xShape = p.x->GetViewShape();
146+ CHECK_COND(xShape.GetDimNum() == INPUT_RANK, ACLNN_ERR_PARAM_INVALID, "x dim num should be 3, but got %zu",
147+ xShape.GetDimNum());
148+ int64_t b = xShape.GetDim(BATCH_DIM);
149+ int64_t s = xShape.GetDim(SEQUENCE_DIM);
150+ int64_t d = xShape.GetDim(FEATURE_DIM);
151+ CHECK_COND(b >= 0 && s >= 0, ACLNN_ERR_PARAM_INVALID, "B/S must be non-negative, but got B=%ld, S=%ld", b, s);
152+ CHECK_COND(s == 0 || b <= std::numeric_limits<int64_t>::max() / s, ACLNN_ERR_PARAM_INVALID,
153+ "B*S overflows int64, got B=%ld, S=%ld", b, s);
154+ CHECK_COND(d > 0, ACLNN_ERR_PARAM_INVALID, "x last dim must be positive, but got %ld", d);
155+ CHECK_COND(p.channels > 0 && d % p.channels == 0, ACLNN_ERR_PARAM_INVALID,
156+ "channels %ld invalid for x last dim %ld", p.channels, d);
157+ int64_t h = d / p.channels;
158+ CHECK_COND(p.wDown->GetViewShape().GetDimNum() == MATRIX_RANK, ACLNN_ERR_PARAM_INVALID,
159+ "wDown dim num should be 2, but got %zu", p.wDown->GetViewShape().GetDimNum());
160+ int64_t r = p.wDown->GetViewShape().GetDim(MATRIX_ROW_DIM);
161+ CHECK_COND(h > 0 && r > 0, ACLNN_ERR_PARAM_INVALID, "invalid channels/H or R.");
162+ 
163+ CHECK_RET(CheckTensorShape(p.gradMixed, {b, s, h}, "gradMixed") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
164+ CHECK_RET(CheckTensorShape(p.x, {b, s, d}, "x") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
165+ CHECK_RET(CheckTensorShape(p.wDown, {r, d}, "wDown") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
166+ CHECK_RET(CheckTensorShape(p.wUp, {d, r}, "wUp") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
167+ CHECK_RET(CheckTensorShape(p.invRms, {b, s, p.channels}, "invRms") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
168+ CHECK_RET(CheckTensorShape(p.downPreAct, {b, s, r}, "downPreAct") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
169+ CHECK_RET(CheckTensorShape(p.up, {b, s, d}, "up") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
170+ CHECK_RET(CheckTensorShape(p.dx, {b, s, d}, "dx") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
171+ CHECK_RET(CheckTensorShape(p.dwDown, {r, d}, "dwDown") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
172+ CHECK_RET(CheckTensorShape(p.dwUp, {d, r}, "dwUp") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
173+ if (p.wGate != nullptr) {
174+ CHECK_RET(CheckTensorShape(p.wGate, {p.channels, d}, "wGate") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
175+ CHECK_RET(CheckTensorShape(p.gradInject, {b, s, p.channels}, "gradInject") == ACLNN_SUCCESS,
176+ ACLNN_ERR_PARAM_INVALID);
177+ CHECK_RET(CheckTensorShape(p.inject, {b, s, p.channels}, "inject") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
178+ CHECK_RET(CheckTensorShape(p.dwGate, {p.channels, d}, "dwGate") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
179+ }
180+ if (p.gamma != nullptr) {
181+ CHECK_RET(CheckTensorShape(p.gamma, {d}, "gamma") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
182+ CHECK_RET(CheckTensorShape(p.dgamma, {d}, "dgamma") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
183+ }
184+ return ACLNN_SUCCESS;
185+}
186+ 
187+aclnnStatus CheckParams(const TextGatedResidualGradParams& p)
188+{
189+ CHECK_COND(CheckNotNull(p) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_NULLPTR, "required param is nullptr.");
190+ CHECK_COND(CheckDtype(p) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID, "invalid dtype.");
191+ CHECK_COND(CheckShape(p) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID, "invalid shape.");
192+ return ACLNN_SUCCESS;
193+}
194+ 
195+const aclTensor* ContiguousOrNull(const aclTensor* t, aclOpExecutor* executor)
196+{
197+ if (t == nullptr) {
198+ return nullptr;
199+ }
200+ return l0op::Contiguous(t, executor);
201+}
202+ 
203+aclnnStatus MakeContiguousInputs(const TextGatedResidualGradParams& p, TextGatedResidualGradParams& contiguous,
204+ aclOpExecutor* executor)
205+{
206+ // Make inputs contiguous.
207+ contiguous.gradMixed = l0op::Contiguous(p.gradMixed, executor);
208+ CHECK_RET(contiguous.gradMixed != nullptr, ACLNN_ERR_INNER_NULLPTR);
209+ contiguous.gradInject = ContiguousOrNull(p.gradInject, executor);
210+ contiguous.x = l0op::Contiguous(p.x, executor);
211+ CHECK_RET(contiguous.x != nullptr, ACLNN_ERR_INNER_NULLPTR);
212+ contiguous.wDown = l0op::Contiguous(p.wDown, executor);
213+ CHECK_RET(contiguous.wDown != nullptr, ACLNN_ERR_INNER_NULLPTR);
214+ contiguous.wUp = l0op::Contiguous(p.wUp, executor);
215+ CHECK_RET(contiguous.wUp != nullptr, ACLNN_ERR_INNER_NULLPTR);
216+ contiguous.wGate = ContiguousOrNull(p.wGate, executor);
217+ contiguous.gamma = ContiguousOrNull(p.gamma, executor);
218+ CHECK_RET(p.wGate == nullptr || contiguous.wGate != nullptr, ACLNN_ERR_INNER_NULLPTR);
219+ CHECK_RET(p.gamma == nullptr || contiguous.gamma != nullptr, ACLNN_ERR_INNER_NULLPTR);
220+ CHECK_RET(p.gradInject == nullptr || contiguous.gradInject != nullptr, ACLNN_ERR_INNER_NULLPTR);
221+ contiguous.invRms = l0op::Contiguous(p.invRms, executor);
222+ CHECK_RET(contiguous.invRms != nullptr, ACLNN_ERR_INNER_NULLPTR);
223+ contiguous.inject = ContiguousOrNull(p.inject, executor);
224+ CHECK_RET(p.inject == nullptr || contiguous.inject != nullptr, ACLNN_ERR_INNER_NULLPTR);
225+ contiguous.downPreAct = l0op::Contiguous(p.downPreAct, executor);
226+ CHECK_RET(contiguous.downPreAct != nullptr, ACLNN_ERR_INNER_NULLPTR);
227+ contiguous.up = l0op::Contiguous(p.up, executor);
228+ CHECK_RET(contiguous.up != nullptr, ACLNN_ERR_INNER_NULLPTR);
229+ 
230+ return ACLNN_SUCCESS;
231+}
232+} // namespace
233+ 
234+aclnnStatus aclnnTextGatedResidualGradGetWorkspaceSize(
235+ const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp,
236+ const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInjectOptional,
237+ const aclTensor* wGateOptional, const aclTensor* gammaOptional, const aclTensor* injectOptional, int64_t channels,
238+ aclTensor* dx, aclTensor* dwDown, aclTensor* dwUp, aclTensor* dwGateOptional, aclTensor* dgammaOptional,
239+ uint64_t* workspaceSize, aclOpExecutor** executor)
240+{
241+ L2_DFX_PHASE_1(aclnnTextGatedResidualGrad,
242+ DFX_IN(gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInjectOptional, wGateOptional,
243+ gammaOptional, injectOptional),
244+ DFX_OUT(dx, dwDown, dwUp, dwGateOptional, dgammaOptional));
245+ 
246+ CHECK_COND(workspaceSize != nullptr, ACLNN_ERR_PARAM_NULLPTR, "workspaceSize must not be nullptr.");
247+ CHECK_COND(executor != nullptr, ACLNN_ERR_PARAM_NULLPTR, "executor must not be nullptr.");
248+ 
249+ TextGatedResidualGradParams params{gradMixed,
250+ x,
251+ wDown,
252+ wUp,
253+ invRms,
254+ downPreAct,
255+ up,
256+ gradInjectOptional,
257+ wGateOptional,
258+ gammaOptional,
259+ injectOptional,
260+ channels,
261+ dx,
262+ dwDown,
263+ dwUp,
264+ dwGateOptional,
265+ dgammaOptional};
266+ aclnnStatus ret = CheckParams(params);
267+ CHECK_RET(ret == ACLNN_SUCCESS, ret);
268+ 
269+ auto uniqueExecutor = CREATE_EXECUTOR();
270+ CHECK_RET(uniqueExecutor.get() != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR);
271+ 
272+ auto contiguous = params;
273+ ret = MakeContiguousInputs(params, contiguous, uniqueExecutor.get());
274+ CHECK_RET(ret == ACLNN_SUCCESS, ret);
275+ 
276+ auto result = l0op::TextGatedResidualGrad(contiguous.gradMixed, contiguous.x, contiguous.wDown, contiguous.wUp,
277+ contiguous.invRms, contiguous.downPreAct, contiguous.up,
278+ contiguous.gradInject, contiguous.wGate, contiguous.gamma,
279+ contiguous.inject, channels, uniqueExecutor.get());
280+ CHECK_RET(std::get<OUT_DX>(result) != nullptr, ACLNN_ERR_INNER_NULLPTR);
281+ 
282+ auto viewCopyIfNotNull = [&uniqueExecutor](aclTensor* out, aclTensor* userOut) -> aclnnStatus {
283+ if (userOut == nullptr || userOut->IsEmpty()) {
284+ return ACLNN_SUCCESS;
285+ }
286+ auto copiedOutput = l0op::ViewCopy(out, userOut, uniqueExecutor.get());
287+ CHECK_RET(copiedOutput != nullptr, ACLNN_ERR_INNER_NULLPTR);
288+ return ACLNN_SUCCESS;
289+ };
290+ CHECK_RET(viewCopyIfNotNull(std::get<OUT_DX>(result), dx) == ACLNN_SUCCESS, ACLNN_ERR_INNER_NULLPTR);
291+ CHECK_RET(viewCopyIfNotNull(std::get<OUT_DW_DOWN>(result), dwDown) == ACLNN_SUCCESS, ACLNN_ERR_INNER_NULLPTR);
292+ CHECK_RET(viewCopyIfNotNull(std::get<OUT_DW_UP>(result), dwUp) == ACLNN_SUCCESS, ACLNN_ERR_INNER_NULLPTR);
293+ CHECK_RET(viewCopyIfNotNull(std::get<OUT_DW_GATE>(result), dwGateOptional) == ACLNN_SUCCESS,
294+ ACLNN_ERR_INNER_NULLPTR);
295+ if (gammaOptional != nullptr && dgammaOptional != nullptr) {
296+ CHECK_RET(viewCopyIfNotNull(std::get<OUT_DGAMMA>(result), dgammaOptional) == ACLNN_SUCCESS,
297+ ACLNN_ERR_INNER_NULLPTR);
298+ }
299+ 
300+ *workspaceSize = uniqueExecutor->GetWorkspaceSize();
301+ uniqueExecutor.ReleaseTo(executor);
302+ return ACLNN_SUCCESS;
303+}
304+ 
305+aclnnStatus aclnnTextGatedResidualGrad(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor,
306+ aclrtStream stream)
307+{
308+ L2_DFX_PHASE_2(aclnnTextGatedResidualGrad);
309+ auto ret = CommonOpExecutorRun(workspace, workspaceSize, executor, stream);
310+ if (ret != ACLNN_SUCCESS) {
311+ OP_LOGE(ACLNN_ERR_INNER, "This is an error in TextGatedResidualGrad launch aicore");
312+ return ACLNN_ERR_INNER;
313+ }
314+ return ACLNN_SUCCESS;
315+}
316+ 
317+#ifdef __cplusplus
318+}
319+#endif
@@ -0,0 +1,71 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file aclnn_text_gated_residual_grad.h
13+ * \brief TextGatedResidualGrad ACLNN workspace-query and execution interfaces.
14+ */
15+ 
16+#ifndef OP_API_INC_ACLNN_TEXT_GATED_RESIDUAL_GRAD_H
17+#define OP_API_INC_ACLNN_TEXT_GATED_RESIDUAL_GRAD_H
18+ 
19+#include "aclnn/aclnn_base.h"
20+#include "aclnnop/aclnn_util.h"
21+ 
22+#ifdef __cplusplus
23+extern "C" {
24+#endif
25+ 
26+/**
27+ * @brief First-stage aclnnTextGatedResidualGrad API: calculate workspace size.
28+ * Compute gradients for the RMS normalization, mixed output and injection branch of TextGatedResidual.
29+ * Inputs follow OpDef order; C = channels, D = C * H, and R is the projection size.
30+ * The activation and projection-weight tensors use one common FLOAT16 or BFLOAT16 dtype.
31+ * @param [in] gradMixed Gradient of mixed output, shape [B, S, H].
32+ * @param [in] x Forward input, shape [B, S, D].
33+ * @param [in] wDown Down-projection weight, shape [R, D].
34+ * @param [in] wUp Up-projection weight, shape [D, R].
35+ * @param [in] invRms Saved inverse RMS, shape [B, S, C], FLOAT32.
36+ * @param [in] downPreAct Saved down-projection result before division by C, shape [B, S, R].
37+ * @param [in] up Saved up-projection result, shape [B, S, D].
38+ * @param [in] gradInjectOptional Injection-output gradient, shape [B, S, C].
39+ * @param [in] wGateOptional Gate-projection weight, shape [C, D].
40+ * @param [in] gammaOptional RMS weight, shape [D], FLOAT32; normalization uses 1 + gamma.
41+ * @param [in] injectOptional Saved injection output, shape [B, S, C].
42+ * @param [in] channels Number of channels.
43+ * @param [out] dx Input gradient, with the shape and dtype of x.
44+ * @param [out] dwDown Down-weight gradient, with the shape and dtype of wDown.
45+ * @param [out] dwUp Up-weight gradient, with the shape and dtype of wUp.
46+ * @param [out] dwGateOptional Gate-weight gradient, with the shape and dtype of wGateOptional.
47+ * @param [out] dgammaOptional RMS-weight gradient, shape [D], FLOAT32.
48+ * @param [out] workspaceSize Required workspace size in bytes.
49+ * @param [out] executor Executor for the second-stage API.
50+ * gradInjectOptional, wGateOptional, injectOptional and dwGateOptional must be provided or omitted together.
51+ * gammaOptional and dgammaOptional must be provided or omitted together.
52+ * @domain aclnn_ops_infer
53+ */
54+ACLNN_API aclnnStatus aclnnTextGatedResidualGradGetWorkspaceSize(
55+ const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp,
56+ const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInjectOptional,
57+ const aclTensor* wGateOptional, const aclTensor* gammaOptional, const aclTensor* injectOptional, int64_t channels,
58+ aclTensor* dx, aclTensor* dwDown, aclTensor* dwUp, aclTensor* dwGateOptional, aclTensor* dgammaOptional,
59+ uint64_t* workspaceSize, aclOpExecutor** executor);
60+ 
61+/**
62+ * @brief Second-stage aclnnTextGatedResidualGrad API: execute the operator.
63+ */
64+ACLNN_API aclnnStatus aclnnTextGatedResidualGrad(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor,
65+ aclrtStream stream);
66+ 
67+#ifdef __cplusplus
68+}
69+#endif
70+ 
71+#endif // OP_API_INC_ACLNN_TEXT_GATED_RESIDUAL_GRAD_H
@@ -0,0 +1,90 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include "text_gated_residual_grad.h"
12+#include "opdev/op_log.h"
13+#include "opdev/op_executor.h"
14+#include "opdev/make_op_executor.h"
15+#include "opdev/op_def.h"
16+#include "opdev/op_dfx.h"
17+#include "aclnn_kernels/common/op_error_check.h"
18+ 
19+using namespace op;
20+ 
21+namespace l0op {
22+ 
23+OP_TYPE_REGISTER(TextGatedResidualGrad);
24+ 
25+namespace {
26+ 
27+const std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*> TextGatedResidualGradAICore(
28+ const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp,
29+ const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInject,
30+ const aclTensor* wGate, const aclTensor* gamma, const aclTensor* inject, int64_t channels, aclTensor* dx,
31+ aclTensor* dwDown, aclTensor* dwUp, aclTensor* dwGate, aclTensor* dgamma, aclOpExecutor* executor)
32+{
33+ L0_DFX(TextGatedResidualGradAICore, gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInject, wGate, gamma,
34+ inject, dx, dwDown, dwUp, dwGate, dgamma);
35+ 
36+ auto retAicore = ADD_TO_LAUNCHER_LIST_AICORE(
37+ TextGatedResidualGrad,
38+ OP_INPUT(gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInject, wGate, gamma, inject),
39+ OP_OUTPUT(dx, dwDown, dwUp, dwGate, dgamma), OP_ATTR(channels));
40+ if (retAicore != ACLNN_SUCCESS) {
41+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "TextGatedResidualGrad launch kernel failed.");
42+ return std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*>(nullptr, nullptr, nullptr,
43+ nullptr, nullptr);
44+ }
45+ return std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*>(dx, dwDown, dwUp, dwGate, dgamma);
46+}
47+ 
48+} // namespace
49+ 
50+const std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*> TextGatedResidualGrad(
51+ const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp,
52+ const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInject,
53+ const aclTensor* wGate, const aclTensor* gamma, const aclTensor* inject, int64_t channels, aclOpExecutor* executor)
54+{
55+ L0_DFX(TextGatedResidualGrad, gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInject, wGate, gamma, inject);
56+ 
57+ // Empty tensor placeholders for omitted optional inputs/outputs.
58+ op::Shape emptyShape({0});
59+ auto allocateEmptyTensor = [&](op::DataType dtype, op::Format format) {
60+ return executor->AllocTensor(emptyShape, dtype, format);
61+ };
62+ 
63+ auto gammaInput = gamma != nullptr ? gamma : allocateEmptyTensor(op::DataType::DT_FLOAT, op::Format::FORMAT_ND);
64+ auto gradInjectInput =
65+ gradInject != nullptr ? gradInject : allocateEmptyTensor(x->GetDataType(), x->GetViewFormat());
66+ auto wGateInput = wGate != nullptr ? wGate : allocateEmptyTensor(x->GetDataType(), x->GetViewFormat());
67+ auto injectInput = inject != nullptr ? inject : allocateEmptyTensor(x->GetDataType(), x->GetViewFormat());
68+ 
69+ // Allocate outputs.
70+ auto dx = executor->AllocTensor(x->GetViewShape(), x->GetDataType(), x->GetViewFormat());
71+ auto dwDown = executor->AllocTensor(wDown->GetViewShape(), wDown->GetDataType(), wDown->GetViewFormat());
72+ auto dwUp = executor->AllocTensor(wUp->GetViewShape(), wUp->GetDataType(), wUp->GetViewFormat());
73+ aclTensor* dwGate = nullptr;
74+ if (wGate != nullptr) {
75+ dwGate = executor->AllocTensor(wGate->GetViewShape(), wGate->GetDataType(), wGate->GetViewFormat());
76+ } else {
77+ dwGate = allocateEmptyTensor(x->GetDataType(), op::Format::FORMAT_ND);
78+ }
79+ aclTensor* dgamma = nullptr;
80+ if (gamma != nullptr) {
81+ dgamma = executor->AllocTensor(gamma->GetViewShape(), op::DataType::DT_FLOAT, gamma->GetViewFormat());
82+ } else {
83+ dgamma = allocateEmptyTensor(op::DataType::DT_FLOAT, op::Format::FORMAT_ND);
84+ }
85+ 
86+ return TextGatedResidualGradAICore(gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInjectInput, wGateInput,
87+ gammaInput, injectInput, channels, dx, dwDown, dwUp, dwGate, dgamma, executor);
88+}
89+ 
90+} // namespace l0op
@@ -0,0 +1,39 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file text_gated_residual_grad.h
13+ * \brief TextGatedResidualGrad L0 kernel interface.
14+ */
15+ 
16+#ifndef OP_API_INC_LEVEL0_OP_TEXT_GATED_RESIDUAL_GRAD_H
17+#define OP_API_INC_LEVEL0_OP_TEXT_GATED_RESIDUAL_GRAD_H
18+ 
19+#include <tuple>
20+ 
21+#include "opdev/op_executor.h"
22+ 
23+namespace l0op {
24+ 
25+/**
26+ * @brief Allocate output tensors and enqueue TextGatedResidualGrad on the executor.
27+ * Inputs follow the OpDef order: required tensors first, followed by optional tensors.
28+ * The gate group (gradInject, wGate, inject) is provided or omitted together.
29+ * @return (dx, dwDown, dwUp, dwGate, dgamma), with empty tensors for omitted outputs.
30+ * Weight gradients use the corresponding weight dtype; dgamma uses FLOAT32.
31+ */
32+const std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*> TextGatedResidualGrad(
33+ const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp,
34+ const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInject,
35+ const aclTensor* wGate, const aclTensor* gamma, const aclTensor* inject, int64_t channels, aclOpExecutor* executor);
36+ 
37+} // namespace l0op
38+ 
39+#endif // OP_API_INC_LEVEL0_OP_TEXT_GATED_RESIDUAL_GRAD_H
@@ -0,0 +1,573 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+#include <cstdint>
11+#include <initializer_list>
12+#include <limits>
13+#include <vector>
14+#include "register/op_def_registry.h"
15+#include "op_host/tiling_templates_registry.h"
16+#include "op_host/tiling_base.h"
17+#include "platform/platform_info.h"
18+#include "log/log.h"
19+#include "../text_gated_residual_grad_tiling.h"
20+#include "../../../op_kernel/arch35/text_gated_residual_grad_tiling_data_arch35.h"
21+#include "../../../op_kernel/arch35/text_gated_residual_grad_tiling_key_arch35.h"
22+#include "../../../op_kernel/arch35/text_gated_residual_grad_pipeline_arch35.h"
23+ 
24+namespace optiling {
25+namespace {
26+namespace hcgrad = TextGatedResidualGrad;
27+constexpr size_t INPUT_RANK = 3;
28+constexpr size_t MATRIX_RANK = 2;
29+constexpr size_t BATCH_DIM = 0;
30+constexpr size_t SEQUENCE_DIM = 1;
31+constexpr size_t FEATURE_DIM = 2;
32+constexpr size_t MATRIX_ROW_DIM = 0;
33+constexpr size_t MATRIX_COLUMN_DIM = 1;
34+constexpr int64_t CHANNEL_SHAPE_LIMIT = 4096;
35+constexpr int64_t FLOAT_BYTES = sizeof(float);
36+constexpr int64_t SPLIT_K_CORE_WAVES = 4;
37+constexpr int64_t SPLIT_K_TARGET_ELEMENTS = 1152;
38+constexpr uint32_t BATCH_SCHEDULE_MODE = 1U;
39+constexpr size_t WORKSPACE_COUNT = 1;
40+constexpr size_t USER_WORKSPACE_INDEX = 0;
41+constexpr uint32_t TILING_TEMPLATE_PRIORITY = 0;
42+ 
43+constexpr size_t IDX_GRAD_MIXED = 0;
44+constexpr size_t IDX_X = 1;
45+constexpr size_t IDX_W_DOWN = 2;
46+constexpr size_t IDX_W_UP = 3;
47+constexpr size_t IDX_INV_RMS = 4;
48+constexpr size_t IDX_DOWN_PRE = 5;
49+constexpr size_t IDX_UP = 6;
50+constexpr size_t IDX_GRAD_INJECT = 7;
51+constexpr size_t IDX_W_GATE = 8;
52+constexpr size_t IDX_GAMMA = 9;
53+constexpr size_t IDX_INJECT = 10;
54+constexpr size_t IDX_ATTR_CHANNELS = 0;
55+static_assert(IDX_X == HC_GRAD_TPL_INPUT_X, "Template dtype binding must use the registered x input.");
56+constexpr size_t OUT_DW_GATE = 3;
57+constexpr size_t OUT_DGAMMA = 4;
58+ 
59+// Shared tiling for A5 (DAV_3510) and A2/A3 (DAV_2201), with platform-specific Cube tiles, UB budgets and transpose
60+// scratch.
61+const std::vector<int32_t> supportedNpuArch = {static_cast<int32_t>(NpuArch::DAV_2201),
62+ static_cast<int32_t>(NpuArch::DAV_3510)};
63+ 
64+ge::graphStatus CheckInputShape(gert::TilingContext* context, size_t irIndex, std::initializer_list<int64_t> dims,
65+ const char* name, bool optional = false)
66+{
67+ auto shape = optional ? context->GetOptionalInputShape(irIndex) : context->GetRequiredInputShape(irIndex);
68+ if (shape == nullptr) {
69+ OP_LOGE(context, "%s shape is null.", name);
70+ return ge::GRAPH_FAILED;
71+ }
72+ const auto& storage = shape->GetStorageShape();
73+ if (storage.GetDimNum() != dims.size()) {
74+ OP_LOGE(context, "%s rank should be %zu, but got %zu.", name, dims.size(), storage.GetDimNum());
75+ return ge::GRAPH_FAILED;
76+ }
77+ size_t i = 0;
78+ for (int64_t expect : dims) {
79+ if (storage.GetDim(i) != expect) {
80+ OP_LOGE(context, "%s dim[%zu] should be %ld, but got %ld.", name, i, expect, storage.GetDim(i));
81+ return ge::GRAPH_FAILED;
82+ }
83+ ++i;
84+ }
85+ return ge::GRAPH_SUCCESS;
86+}
87+ 
88+ge::graphStatus CheckInputDtype(gert::TilingContext* context, size_t irIndex, ge::DataType expect, const char* name,
89+ bool optional = false)
90+{
91+ auto desc = optional ? context->GetOptionalInputDesc(irIndex) : context->GetRequiredInputDesc(irIndex);
92+ if (desc == nullptr) {
93+ OP_LOGE(context, "%s desc is null.", name);
94+ return ge::GRAPH_FAILED;
95+ }
96+ if (desc->GetDataType() != expect) {
97+ OP_LOGE(context, "%s dtype mismatch, expect %d but got %d.", name, static_cast<int32_t>(expect),
98+ static_cast<int32_t>(desc->GetDataType()));
99+ return ge::GRAPH_FAILED;
100+ }
@@ -0,0 +1,418 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include <cstdint>
12+#include <initializer_list>
13+ 
14+#include "log/log.h"
15+#include "register/op_impl_registry.h"
16+ 
17+using namespace ge;
18+ 
19+namespace ops {
20+namespace {
21+constexpr size_t INPUT_RANK = 3;
22+constexpr size_t MATRIX_RANK = 2;
23+constexpr size_t VECTOR_RANK = 1;
24+constexpr size_t FIRST_DIM = 0;
25+constexpr size_t SECOND_DIM = 1;
26+constexpr size_t FEATURE_DIM = 2;
27+constexpr size_t CHANNELS_ATTR_INDEX = 0;
28+constexpr size_t IDX_GRAD_MIXED = 0;
29+constexpr size_t IDX_X = 1;
30+constexpr size_t IDX_W_DOWN = 2;
31+constexpr size_t IDX_W_UP = 3;
32+constexpr size_t IDX_INV_RMS = 4;
33+constexpr size_t IDX_DOWN_PRE = 5;
34+constexpr size_t IDX_UP = 6;
35+constexpr size_t IDX_GRAD_INJECT = 7;
36+constexpr size_t IDX_W_GATE = 8;
37+constexpr size_t IDX_GAMMA = 9;
38+constexpr size_t IDX_INJECT = 10;
39+ 
40+constexpr size_t OUT_DX = 0;
41+constexpr size_t OUT_DW_DOWN = 1;
42+constexpr size_t OUT_DW_UP = 2;
43+constexpr size_t OUT_DW_GATE = 3;
44+constexpr size_t OUT_DGAMMA = 4;
45+ 
46+constexpr int64_t UNKNOWN_DIM = -1LL;
47+constexpr int64_t UNKNOWN_RANK = -2LL;
48+ 
49+bool IsUnknownRank(const gert::Shape* shape)
50+{
51+ return shape != nullptr && shape->GetDimNum() == VECTOR_RANK && shape->GetDim(FIRST_DIM) == UNKNOWN_RANK;
52+}
53+ 
54+bool IsEmptyShape(const gert::Shape* shape)
55+{
56+ return shape == nullptr || (shape->GetDimNum() == VECTOR_RANK && shape->GetDim(FIRST_DIM) == 0);
57+}
58+ 
59+bool DimEqualOrUnknown(int64_t actual, int64_t expect)
60+{
61+ // Only known, unequal dimensions are incompatible; an unknown dimension remains available for dynamic inference.
62+ return actual == expect || actual == UNKNOWN_DIM || expect == UNKNOWN_DIM;
63+}
64+ 
65+gert::Shape* GetOutputShapeByIrIndex(gert::InferShapeContext* context, size_t irIndex)
66+{
67+ const auto* instance = context->GetIrOutputInstanceInfo(irIndex);
68+ if (instance == nullptr || instance->GetInstanceNum() == 0) {
69+ return nullptr;
70+ }
71+ return context->GetOutputShape(instance->GetInstanceStart());
72+}
73+ 
74+void SetUnknownRank(gert::Shape* shape)
75+{
76+ if (shape == nullptr) {
77+ return;
78+ }
79+ shape->SetDimNum(0);
80+ shape->AppendDim(UNKNOWN_RANK);
81+}
82+ 
83+void CopyShape(const gert::Shape* src, gert::Shape* dst)
84+{
85+ if (src == nullptr || dst == nullptr) {
86+ return;
87+ }
88+ dst->SetDimNum(src->GetDimNum());
89+ for (size_t i = 0; i < src->GetDimNum(); ++i) {
90+ dst->SetDim(i, src->GetDim(i));
91+ }
92+}
93+ 
94+graphStatus CheckRank(const gert::Shape* shape, size_t rank, const char* name, gert::InferShapeContext* context)
95+{
96+ if (shape->GetDimNum() != rank) {
97+ OP_LOGE(context->GetNodeName(), "%s rank should be %zu, but got %zu.", name, rank, shape->GetDimNum());
98+ return GRAPH_FAILED;
99+ }
100+ return GRAPH_SUCCESS;
101+}
102+ 
103+graphStatus CheckDim(const gert::Shape* shape, size_t idx, int64_t expect, const char* name,
104+ gert::InferShapeContext* context)
105+{
106+ const int64_t actual = shape->GetDim(idx);
107+ if (!DimEqualOrUnknown(actual, expect)) {
108+ OP_LOGE(context->GetNodeName(), "%s dim[%zu] should be %ld, but got %ld.", name, idx, expect, actual);
109+ return GRAPH_FAILED;
110+ }
111+ return GRAPH_SUCCESS;
112+}
113+ 
114+graphStatus CheckShape3(const gert::Shape* shape, int64_t d0, int64_t d1, int64_t d2, const char* name,
115+ gert::InferShapeContext* context)
116+{
117+ if (CheckRank(shape, INPUT_RANK, name, context) != GRAPH_SUCCESS) {
118+ return GRAPH_FAILED;
119+ }
120+ if (CheckDim(shape, FIRST_DIM, d0, name, context) != GRAPH_SUCCESS ||
121+ CheckDim(shape, SECOND_DIM, d1, name, context) != GRAPH_SUCCESS ||
122+ CheckDim(shape, FEATURE_DIM, d2, name, context) != GRAPH_SUCCESS) {
123+ return GRAPH_FAILED;
124+ }
125+ return GRAPH_SUCCESS;
126+}
127+ 
128+graphStatus CheckShape2(const gert::Shape* shape, int64_t d0, int64_t d1, const char* name,
129+ gert::InferShapeContext* context)
130+{
131+ if (CheckRank(shape, MATRIX_RANK, name, context) != GRAPH_SUCCESS) {
132+ return GRAPH_FAILED;
133+ }
134+ if (CheckDim(shape, FIRST_DIM, d0, name, context) != GRAPH_SUCCESS ||
135+ CheckDim(shape, SECOND_DIM, d1, name, context) != GRAPH_SUCCESS) {
136+ return GRAPH_FAILED;
137+ }
138+ return GRAPH_SUCCESS;
139+}
140+ 
141+class TextGatedResidualGradShapeInference {
142+public:
143+ explicit TextGatedResidualGradShapeInference(gert::InferShapeContext* ctx)
144+ : context(ctx)
145+ {}
146+ 
147+ graphStatus Infer()
148+ {
149+ if (LoadShapes() != GRAPH_SUCCESS || CheckAttributes() != GRAPH_SUCCESS) {
150+ return GRAPH_FAILED;
151+ }
152+ if (PropagateUnknownRank()) {
153+ return GRAPH_SUCCESS;
154+ }
155+ if (ReadDimensions() != GRAPH_SUCCESS || CheckKnownShapes() != GRAPH_SUCCESS) {
156+ return GRAPH_FAILED;
157+ }
158+ return SetOutputShapes();
159+ }
160+ 
161+private:
162+ graphStatus LoadShapes()
163+ {
164+ gradMixedShape = context->GetRequiredInputShape(IDX_GRAD_MIXED);
165+ xShape = context->GetRequiredInputShape(IDX_X);
166+ wDownShape = context->GetRequiredInputShape(IDX_W_DOWN);
167+ wUpShape = context->GetRequiredInputShape(IDX_W_UP);
168+ invRmsShape = context->GetRequiredInputShape(IDX_INV_RMS);
169+ downPreShape = context->GetRequiredInputShape(IDX_DOWN_PRE);
170+ upShape = context->GetRequiredInputShape(IDX_UP);
171+ gradInjectShape = context->GetOptionalInputShape(IDX_GRAD_INJECT);
172+ wGateShape = context->GetOptionalInputShape(IDX_W_GATE);
173+ gammaShape = context->GetOptionalInputShape(IDX_GAMMA);
174+ injectShape = context->GetOptionalInputShape(IDX_INJECT);
175+ 
176+ OP_CHECK_NULL_WITH_CONTEXT(context, gradMixedShape);
177+ OP_CHECK_NULL_WITH_CONTEXT(context, xShape);
178+ OP_CHECK_NULL_WITH_CONTEXT(context, wDownShape);
179+ OP_CHECK_NULL_WITH_CONTEXT(context, wUpShape);
180+ OP_CHECK_NULL_WITH_CONTEXT(context, invRmsShape);
181+ OP_CHECK_NULL_WITH_CONTEXT(context, downPreShape);
182+ OP_CHECK_NULL_WITH_CONTEXT(context, upShape);
183+ 
184+ dxShape = GetOutputShapeByIrIndex(context, OUT_DX);
185+ dwDownShape = GetOutputShapeByIrIndex(context, OUT_DW_DOWN);
186+ dwUpShape = GetOutputShapeByIrIndex(context, OUT_DW_UP);
187+ dwGateShape = GetOutputShapeByIrIndex(context, OUT_DW_GATE);
188+ dgammaShape = GetOutputShapeByIrIndex(context, OUT_DGAMMA);
189+ OP_CHECK_NULL_WITH_CONTEXT(context, dxShape);
190+ OP_CHECK_NULL_WITH_CONTEXT(context, dwDownShape);
191+ OP_CHECK_NULL_WITH_CONTEXT(context, dwUpShape);
192+ 
193+ return GRAPH_SUCCESS;
194+ }
195+ 
196+ graphStatus CheckAttributes()
197+ {
198+ const auto* attrs = context->GetAttrs();
199+ OP_CHECK_NULL_WITH_CONTEXT(context, attrs);
200+ const int64_t* channelsPtr = attrs->GetAttrPointer<int64_t>(CHANNELS_ATTR_INDEX);
201+ if (channelsPtr == nullptr) {
202+ OP_LOGE(context->GetNodeName(), "channels attr is null.");
203+ return GRAPH_FAILED;
204+ }
205+ c = *channelsPtr;
206+ if (c <= 0) {
207+ OP_LOGE(context->GetNodeName(), "channels should be positive, but got %ld.", c);
208+ return GRAPH_FAILED;
209+ }
210+ hasGate = !IsEmptyShape(wGateShape);
X
Xxuejinghui6 days ago

C01|高|graph-contract|FAIL|置信度高

        hasGate = !IsEmptyShape(wGateShape);

缺少 w_gate 但提供 grad_inject 和/或 inject 时,hasGate=false,后续只检查 hasGate=true 的组合,导致该输入组合被接受。Tiling 的 CheckGate 也只由 wGateTensor 决定开关,因此非空注入梯度会被静默忽略,图模式返回成功但梯度不完整;ACLNN 则会正确拒绝同类组合。

**建议:**在 InferShape 和 Tiling 中同时校验 gate 输入组:w_gate 缺失时,grad_inject/inject 必须也为空;补充“缺少 w_gate、保留 companion 输入”的负向 UT。

likedislike
azyuli
azyuli
6 days ago Comments:
211+ if (hasGate && dwGateShape == nullptr) {
212+ OP_LOGE(context->GetNodeName(), "dw_gate output is required when w_gate is present.");
213+ return GRAPH_FAILED;
214+ }
215+ if (hasGate && (gradInjectShape == nullptr || injectShape == nullptr)) {
216+ OP_LOGE(context->GetNodeName(), "grad_inject and inject are required when w_gate is present.");
217+ return GRAPH_FAILED;
218+ }
219+ 
220+ hasGamma = !IsEmptyShape(gammaShape);
221+ if (hasGamma && dgammaShape == nullptr) {
222+ OP_LOGE(context->GetNodeName(), "dgamma output is required when gamma is present.");
223+ return GRAPH_FAILED;
224+ }
225+ 
226+ return GRAPH_SUCCESS;
227+ }
228+ 
229+ bool PropagateUnknownRank()
230+ {
231+ const gert::Shape* allInputs[] = {gradMixedShape, gradInjectShape, xShape, wDownShape,
232+ wUpShape, wGateShape, gammaShape, invRmsShape,
233+ injectShape, downPreShape, upShape};
234+ for (const auto* shape : allInputs) {
235+ if (shape != nullptr && IsUnknownRank(shape)) {
236+ SetUnknownRank(dxShape);
237+ SetUnknownRank(dwDownShape);
238+ SetUnknownRank(dwUpShape);
239+ SetUnknownRank(dwGateShape);
240+ if (dgammaShape != nullptr) {
241+ SetUnknownRank(dgammaShape);
242+ }
243+ return true;
244+ }
245+ }
246+ 
247+ return false;
248+ }
249+ 
250+ graphStatus ReadDimensions()
251+ {
252+ if (CheckRank(xShape, INPUT_RANK, "x", context) != GRAPH_SUCCESS ||
253+ CheckRank(wDownShape, MATRIX_RANK, "w_down", context) != GRAPH_SUCCESS) {
254+ return GRAPH_FAILED;
255+ }
256+ d = xShape->GetDim(FEATURE_DIM);
257+ if (d != UNKNOWN_DIM && (d <= 0 || d % c != 0)) {
258+ OP_LOGE(context->GetNodeName(), "x last dim %ld is not divisible by channels %ld.", d, c);
259+ return GRAPH_FAILED;
260+ }
261+ h = (d == UNKNOWN_DIM) ? UNKNOWN_DIM : d / c;
262+ r = wDownShape->GetDim(FIRST_DIM);
263+ if (r != UNKNOWN_DIM && r <= 0) {
264+ OP_LOGE(context->GetNodeName(), "R must be positive or unknown, but got %ld.", r);
265+ return GRAPH_FAILED;
266+ }
267+ b = xShape->GetDim(FIRST_DIM);
268+ s = xShape->GetDim(SECOND_DIM);
269+ if ((b < 0 && b != UNKNOWN_DIM) || (s < 0 && s != UNKNOWN_DIM)) {
270+ OP_LOGE(context->GetNodeName(), "B/S must be non-negative or unknown, but got B=%ld, S=%ld.", b, s);
271+ return GRAPH_FAILED;
272+ }
273+ return GRAPH_SUCCESS;
274+ }
275+ 
276+ graphStatus CheckKnownShapes()
277+ {
278+ if (CheckShape3(gradMixedShape, b, s, h, "grad_mixed", context) != GRAPH_SUCCESS ||
279+ CheckShape3(xShape, b, s, d, "x", context) != GRAPH_SUCCESS ||
280+ CheckShape2(wDownShape, r, d, "w_down", context) != GRAPH_SUCCESS ||
281+ CheckShape2(wUpShape, d, r, "w_up", context) != GRAPH_SUCCESS ||
282+ CheckShape3(invRmsShape, b, s, c, "inv_rms", context) != GRAPH_SUCCESS ||
283+ CheckShape3(downPreShape, b, s, r, "down_pre_act", context) != GRAPH_SUCCESS ||
284+ CheckShape3(upShape, b, s, d, "up", context) != GRAPH_SUCCESS) {
285+ return GRAPH_FAILED;
286+ }
287+ 
288+ if (hasGate && (CheckShape2(wGateShape, c, d, "w_gate", context) != GRAPH_SUCCESS ||
289+ CheckShape3(gradInjectShape, b, s, c, "grad_inject", context) != GRAPH_SUCCESS ||
290+ CheckShape3(injectShape, b, s, c, "inject", context) != GRAPH_SUCCESS)) {
291+ return GRAPH_FAILED;
292+ }
293+ 
294+ if (hasGamma && (CheckRank(gammaShape, VECTOR_RANK, "gamma", context) != GRAPH_SUCCESS ||
295+ CheckDim(gammaShape, FIRST_DIM, d, "gamma", context) != GRAPH_SUCCESS)) {
296+ return GRAPH_FAILED;
297+ }
298+ 
299+ return GRAPH_SUCCESS;
300+ }
301+ 
302+ graphStatus SetOutputShapes()
303+ {
304+ CopyShape(xShape, dxShape);
305+ CopyShape(wDownShape, dwDownShape);
306+ CopyShape(wUpShape, dwUpShape);
307+ if (dwGateShape != nullptr) {
308+ if (hasGate) {
309+ CopyShape(wGateShape, dwGateShape);
310+ } else {
311+ dwGateShape->SetDimNum(1);
312+ dwGateShape->SetDim(0, 0);
313+ }
314+ }
315+ if (dgammaShape != nullptr) {
316+ if (hasGamma) {
317+ CopyShape(gammaShape, dgammaShape);
318+ } else {
319+ dgammaShape->SetDimNum(1);
320+ dgammaShape->SetDim(0, 0);
321+ }
322+ }
323+ return GRAPH_SUCCESS;
324+ }
325+ 
326+ gert::InferShapeContext* context;
327+ const gert::Shape* gradMixedShape = nullptr;
328+ const gert::Shape* xShape = nullptr;
329+ const gert::Shape* wDownShape = nullptr;
330+ const gert::Shape* wUpShape = nullptr;
331+ const gert::Shape* invRmsShape = nullptr;
332+ const gert::Shape* downPreShape = nullptr;
333+ const gert::Shape* upShape = nullptr;
334+ const gert::Shape* gradInjectShape = nullptr;
335+ const gert::Shape* wGateShape = nullptr;
336+ const gert::Shape* gammaShape = nullptr;
337+ const gert::Shape* injectShape = nullptr;
338+ gert::Shape* dxShape = nullptr;
339+ gert::Shape* dwDownShape = nullptr;
340+ gert::Shape* dwUpShape = nullptr;
341+ gert::Shape* dwGateShape = nullptr;
342+ gert::Shape* dgammaShape = nullptr;
343+ int64_t b = 0;
344+ int64_t s = 0;
345+ int64_t c = 0;
346+ int64_t d = 0;
347+ int64_t h = 0;
348+ int64_t r = 0;
349+ bool hasGate = false;
350+ bool hasGamma = false;
351+};
352+ 
353+ge::graphStatus InferShapeForTextGatedResidualGrad(gert::InferShapeContext* context)
354+{
355+ return TextGatedResidualGradShapeInference(context).Infer();
356+}
357+ 
358+graphStatus SetOutputDtype(gert::InferDataTypeContext* context, size_t irIndex, ge::DataType dtype)
359+{
360+ const auto* instance = context->GetIrOutputInstanceInfo(irIndex);
361+ if (instance == nullptr || instance->GetInstanceNum() == 0) {
362+ return ge::GRAPH_SUCCESS;
363+ }
364+ return context->SetOutputDataType(instance->GetInstanceStart(), dtype);
365+}
366+ 
367+ge::graphStatus InferDataTypeForTextGatedResidualGrad(gert::InferDataTypeContext* context)
368+{
369+ if (context == nullptr) {
370+ return GRAPH_FAILED;
371+ }
372+ const ge::DataType xDtype = context->GetRequiredInputDataType(IDX_X);
373+ if (xDtype != ge::DT_BF16 && xDtype != ge::DT_FLOAT16) {
374+ OP_LOGE(context->GetNodeName(), "x dtype only supports bf16/fp16, but got %d.", static_cast<int32_t>(xDtype));
375+ return GRAPH_FAILED;
376+ }
377+ const size_t requiredSameDtypeInputs[] = {IDX_GRAD_MIXED, IDX_W_DOWN, IDX_W_UP, IDX_DOWN_PRE, IDX_UP};
378+ for (size_t idx : requiredSameDtypeInputs) {
379+ if (context->GetRequiredInputDataType(idx) != xDtype) {
380+ OP_LOGE(context->GetNodeName(), "input %zu dtype should match x.", idx);
381+ return GRAPH_FAILED;
382+ }
383+ }
384+ const size_t optionalSameDtypeInputs[] = {IDX_GRAD_INJECT, IDX_W_GATE, IDX_INJECT};
385+ for (size_t idx : optionalSameDtypeInputs) {
386+ const ge::DataType dtype = context->GetOptionalInputDataType(idx);
387+ if (dtype != ge::DT_UNDEFINED && dtype != xDtype) {
388+ OP_LOGE(context->GetNodeName(), "input %zu dtype should match x.", idx);
389+ return GRAPH_FAILED;
390+ }
391+ }
392+ const ge::DataType gammaDtype = context->GetOptionalInputDataType(IDX_GAMMA);
393+ if (gammaDtype != ge::DT_UNDEFINED && gammaDtype != ge::DT_FLOAT) {
394+ OP_LOGE(context->GetNodeName(), "gamma dtype only supports float32.");
395+ return GRAPH_FAILED;
396+ }
397+ if (context->GetRequiredInputDataType(IDX_INV_RMS) != ge::DT_FLOAT) {
398+ OP_LOGE(context->GetNodeName(), "inv_rms dtype only supports float32.");
399+ return GRAPH_FAILED;
400+ }
401+ 
402+ if (SetOutputDtype(context, OUT_DX, xDtype) != ge::GRAPH_SUCCESS ||
403+ SetOutputDtype(context, OUT_DW_DOWN, xDtype) != ge::GRAPH_SUCCESS ||
404+ SetOutputDtype(context, OUT_DW_UP, xDtype) != ge::GRAPH_SUCCESS ||
405+ SetOutputDtype(context, OUT_DW_GATE, xDtype) != ge::GRAPH_SUCCESS ||
406+ SetOutputDtype(context, OUT_DGAMMA, ge::DT_FLOAT) != ge::GRAPH_SUCCESS) {
407+ OP_LOGE(context->GetNodeName(), "failed to set output dtype.");
408+ return GRAPH_FAILED;
409+ }
410+ return GRAPH_SUCCESS;
411+}
412+} // namespace
413+ 
414+IMPL_OP_INFERSHAPE(TextGatedResidualGrad)
415+ .InferShape(InferShapeForTextGatedResidualGrad)
416+ .InferDataType(InferDataTypeForTextGatedResidualGrad);
417+ 
418+} // namespace ops
@@ -0,0 +1,47 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include "text_gated_residual_grad_tiling.h"
12+#include "register/op_def_registry.h"
13+#include "op_host/tiling_base.h"
14+#include "op_host/tiling_templates_registry.h"
15+#include "log/log.h"
16+#include "kernel_tiling/kernel_tiling.h"
17+ 
18+using Ops::Transformer::OpTiling::TilingRegistryNew;
19+using Ops::Transformer::OpTiling::TilingRegistryArch;
20+ 
21+namespace optiling {
22+ 
23+static ge::graphStatus Tiling4TextGatedResidualGrad(gert::TilingContext* context)
24+{
25+ auto platformInfo = context->GetPlatformInfo();
26+ OP_CHECK_NULL_WITH_CONTEXT(context, platformInfo);
27+ platform_ascendc::PlatformAscendC ascendcPlatform(platformInfo);
28+ NpuArch npuArch = ascendcPlatform.GetCurNpuArch();
29+ if (npuArch == NpuArch::DAV_3510 || npuArch == NpuArch::DAV_2201) {
30+ return TilingRegistryArch::GetInstance().DoTilingImpl(context);
31+ }
32+ OP_LOGE(context,
33+ "TextGatedResidualGrad: only DAV_3510 (ascend950) and DAV_2201 (ascend910b/910_93) are supported.");
34+ return ge::GRAPH_FAILED;
35+}
36+ 
37+static ge::graphStatus TilingPrepareForTextGatedResidualGrad(gert::TilingParseContext* context)
38+{
39+ (void)context;
40+ return ge::GRAPH_SUCCESS;
41+}
42+ 
43+IMPL_OP_OPTILING(TextGatedResidualGrad)
44+ .Tiling(Tiling4TextGatedResidualGrad)
45+ .TilingParse<TextGatedResidualGradCompileInfo>(TilingPrepareForTextGatedResidualGrad);
46+ 
47+} // namespace optiling
@@ -0,0 +1,25 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file text_gated_residual_grad_tiling.h
13+ * \brief TextGatedResidualGrad compile-time tiling information.
14+ */
15+ 
16+#pragma once
17+ 
18+#include <cstdint>
19+ 
20+namespace optiling {
21+struct TextGatedResidualGradCompileInfo {
22+ uint32_t aicNum{0};
23+ uint32_t aivNum{0};
24+};
25+} // namespace optiling
@@ -0,0 +1,129 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software; you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+#include "register/op_def_registry.h"
12+ 
13+namespace ops {
14+class TextGatedResidualGrad : public OpDef {
15+public:
16+ explicit TextGatedResidualGrad(const char* name)
17+ : OpDef(name)
18+ {
azyuli
azyuliazyuli9 days ago

改一下写法

likedislike
azyuli
azyuli
9 days ago Comments:
19+ this->Input("grad_mixed")
20+ .ParamType(REQUIRED)
21+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
22+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
23+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
24+ .AutoContiguous();
25+ this->Input("x")
26+ .ParamType(REQUIRED)
27+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
28+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
29+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
30+ .AutoContiguous();
31+ this->Input("w_down")
32+ .ParamType(REQUIRED)
33+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
34+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
35+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
36+ .AutoContiguous();
37+ this->Input("w_up")
38+ .ParamType(REQUIRED)
39+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
40+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
41+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
42+ .AutoContiguous();
43+ this->Input("inv_rms")
44+ .ParamType(REQUIRED)
45+ .DataType({ge::DT_FLOAT, ge::DT_FLOAT})
46+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
47+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
48+ .AutoContiguous();
49+ this->Input("down_pre_act")
50+ .ParamType(REQUIRED)
51+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
52+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
53+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
54+ .AutoContiguous();
55+ this->Input("up")
56+ .ParamType(REQUIRED)
57+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
58+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
59+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
60+ .AutoContiguous();
61+ this->Input("grad_inject")
62+ .ParamType(OPTIONAL)
63+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
64+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
65+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
66+ .AutoContiguous();
67+ this->Input("w_gate")
68+ .ParamType(OPTIONAL)
69+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
70+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
71+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
72+ .AutoContiguous();
73+ this->Input("gamma")
74+ .ParamType(OPTIONAL)
75+ .DataType({ge::DT_FLOAT, ge::DT_FLOAT})
76+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
77+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
78+ .AutoContiguous();
79+ this->Input("inject")
80+ .ParamType(OPTIONAL)
81+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
82+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
83+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND})
84+ .AutoContiguous();
85+ this->Attr("channels").AttrType(REQUIRED).Int();
86+ 
87+ this->Output("dx")
88+ .ParamType(REQUIRED)
89+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
90+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
91+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND});
92+ this->Output("dw_down")
93+ .ParamType(REQUIRED)
94+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
95+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
96+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND});
97+ this->Output("dw_up")
98+ .ParamType(REQUIRED)
99+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
100+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
101+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND});
102+ this->Output("dw_gate")
103+ .ParamType(OPTIONAL)
104+ .DataType({ge::DT_FLOAT16, ge::DT_BF16})
105+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
106+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND});
107+ this->Output("dgamma")
108+ .ParamType(OPTIONAL)
109+ .DataType({ge::DT_FLOAT, ge::DT_FLOAT})
110+ .Format({ge::FORMAT_ND, ge::FORMAT_ND})
111+ .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND});
112+ 
113+ OpAICoreConfig config;
114+ config.DynamicCompileStaticFlag(true)
115+ .DynamicFormatFlag(true)
116+ .DynamicRankSupportFlag(true)
117+ .DynamicShapeSupportFlag(true)
118+ .NeedCheckSupportFlag(false)
119+ .PrecisionReduceFlag(true)
120+ .ExtendCfgInfo("aclnnSupport.value", "support_aclnn")
121+ .ExtendCfgInfo("opFile.value", "text_gated_residual_grad");
122+ this->AICore().AddConfig("ascend950", config);
123+ this->AICore().AddConfig("ascend910b", config);
124+ this->AICore().AddConfig("ascend910_93", config);
125+ }
126+};
127+ 
128+OP_ADD(TextGatedResidualGrad);
129+} // namespace ops
@@ -0,0 +1,280 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file text_gated_residual_grad_gemm_arch22.h
13+ * \brief Atlas A2/A3 CATLASS GEMM backend for TextGatedResidualGrad.
14+ */
15+#ifndef TEXT_GATED_RESIDUAL_GRAD_GEMM_ARCH22_H
16+#define TEXT_GATED_RESIDUAL_GRAD_GEMM_ARCH22_H
17+ 
18+#include "../text_gated_residual_grad_constants.h"
19+ 
20+#if !defined(CATLASS_ARCH) || CATLASS_ARCH != HC_GRAD_ARCH_A2
21+#error "text_gated_residual_grad_gemm_arch22.h requires CATLASS_ARCH=2201"
22+#endif
23+ 
24+#ifndef K_MAX_SHAPE_DIM
25+#define K_MAX_SHAPE_DIM 0
26+#endif
27+ 
28+#include <type_traits>
29+ 
30+#include "kernel_operator.h"
31+#include "catlass/catlass.hpp"
32+#include "catlass/arch/local_tensor_buffer.hpp"
33+ 
34+#if HC_GRAD_NPU_ARCH == HC_GRAD_ARCH_A2
35+// Limit AIC VECCALC UB allocation only for this operator's A2/A3 kernel.
36+// CATLASS's full-UB default blocks L0C-to-GM Fixpipe; use a local specialization instead of patching shared code.
37+namespace Catlass::Arch {
38+template <>
39+struct LocalTensorBuffer<AtlasA2, AscendC::TPosition::VECCALC> : LocalTensorBufferBase {
40+public:
41+ static constexpr AscendC::TPosition Position = AscendC::TPosition::VECCALC;
42+ 
43+ CATLASS_DEVICE LocalTensorBuffer()
44+ {
45+ AscendC::TBuf<AscendC::TPosition::VECCALC> tbufVECCALC;
46+ uint32_t ubBytes = AtlasA2::UB_SIZE;
47+ if ASCEND_IS_AIC {
48+ ubBytes = TextGatedResidualGrad::HC_GRAD_A2_CUBE_UB_BYTES;
49+ }
50+ GetTPipePtr()->InitBuffer(tbufVECCALC, ubBytes);
51+ tensor = tbufVECCALC.Get<uint8_t>();
52+ }
53+};
54+} // namespace Catlass::Arch
55+#endif
56+ 
57+#include "catlass/gemm/block/block_mmad.hpp"
58+#include "catlass/gemm/block/block_swizzle.hpp"
59+#include "catlass/gemm/dispatch_policy.hpp"
60+#include "catlass/gemm/gemm_type.hpp"
61+#include "catlass/gemm_coord.hpp"
62+#include "catlass/layout/layout.hpp"
63+#include "../arch35/text_gated_residual_grad_pipeline_arch35.h"
64+ 
65+namespace TextGatedResidualGrad {
66+ 
67+// A2/A3 (arch22) uses the same algorithm as arch35, with these platform differences:
68+// 1) Cube uses AtlasA2 BlockMmad (Gemm::Block::BlockMmad + MmadAtlasA2Pingpong),
69+// as other A2 operators do; A5 MmadPingpongMutex supports Ascend950 only.
70+// 2) C1/DX use 128x128x64 tiles for A2's 128 KiB L0C, consuming 64 KiB per tile.
71+// BlockMmad uses one L0C stage and double buffering for L1/L0A/L0B.
72+// 3) MmadAtlasA2Pingpong<false> disables unit flags and uses explicit M_FIX/FIX_M synchronization.
73+// 4) Block strides use the tiling AIC count; GetBlockNum() is not the AIC count in a mixed kernel.
74+// As on A5, C1b and G56 read Vector results directly as ColumnMajor without materializing transposes.
75+template <typename D_T>
76+class HcGradGemmArch22 {
77+ static constexpr uint32_t C1TileM = HC_GRAD_CUBE_TILE_M_A2;
78+ static constexpr uint32_t C1TileN = HC_GRAD_C1A_TILE_N;
79+ static constexpr uint32_t C1TileK = HC_GRAD_CUBE_TILE_K;
80+ 
81+public:
82+ __aicore__ inline void Init(uint32_t aicNum)
83+ {
84+ aicNum_ = aicNum;
85+ }
86+ 
87+ // C1a: dact = dup @ wUp. As on A5, distribute all (m,n,slice) blocks across AICs in one loop,
88+ // treating split-K slices as virtual cores. Each slice writes its own FP32 partial,
89+ // which Vector REDUCE_DACT merges; slices do not run as separate serial GEMMs.
90+ __aicore__ inline void RunC1aSplitK(GM_ADDR dup, GM_ADDR wUp, GM_ADDR partial, uint32_t rows, uint32_t r,
91+ uint32_t d, uint32_t splitkFactor)
92+ {
93+ using BlockScheduler =
94+ Catlass::Gemm::Block::SplitkGemmIdentityBlockSwizzle<HC_GRAD_SWIZZLE_OFFSET, HC_GRAD_SWIZZLE_M_FIRST>;
95+ Catlass::GemmCoord problemShape{rows, r, d};
96+ BlockScheduler scheduler(problemShape, Catlass::GemmCoord(C1TileM, C1TileN, C1TileK), splitkFactor);
97+ const uint32_t coreLoops = scheduler.GetCoreLoops();
98+ 
99+ Catlass::Arch::Resource<ArchTag> resource;
100+ C1BlockMmad blockMmad(resource);
101+ 
102+ AscendC::GlobalTensor<D_T> gmA;
103+ gmA.SetGlobalBuffer(reinterpret_cast<__gm__ D_T*>(dup));
104+ AscendC::GlobalTensor<D_T> gmB;
105+ gmB.SetGlobalBuffer(reinterpret_cast<__gm__ D_T*>(wUp));
106+ AscendC::GlobalTensor<float> gmW;
107+ gmW.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(partial));
108+ 
109+ RowMajor layoutA{rows, d, d};
110+ RowMajor layoutB{d, r, r};
111+ RowMajor layoutC{rows, r, r};
112+ const size_t lenC = static_cast<size_t>(rows) * r;
113+ for (uint32_t loopIdx = AscendC::GetBlockIdx(); loopIdx < coreLoops; loopIdx += aicNum_) {
114+ const Catlass::GemmCoord blockCoord = scheduler.GetBlockCoord(loopIdx);
115+ const uint32_t slice = scheduler.GetSplitkSliceIdx(loopIdx);
116+ const Catlass::GemmCoord actual = scheduler.GetActualBlockShape(blockCoord, slice);
117+ const MatrixCoord offsetA{blockCoord.m() * C1TileM, blockCoord.k() * C1TileK};
118+ const MatrixCoord offsetB{blockCoord.k() * C1TileK, blockCoord.n() * C1TileN};
119+ const MatrixCoord offsetC{blockCoord.m() * C1TileM, blockCoord.n() * C1TileN};
120+ blockMmad(gmA[layoutA.GetOffset(offsetA)], layoutA, gmB[layoutB.GetOffset(offsetB)], layoutB,
121+ gmW[lenC * slice + layoutC.GetOffset(offsetC)], layoutC, actual);
122+ }
123+ AscendC::PipeBarrier<PIPE_ALL>();
124+ AscendC::DataSyncBarrier<AscendC::MemDsbT::DDR>();
125+ }
126+ 
127+ // C1b: dwUpPartial = dup^T @ act, reading dup as ColumnMajor [D,rows], as on A5.
128+ __aicore__ inline void RunC1b(GM_ADDR dup, GM_ADDR act, GM_ADDR dwUpPartial, uint32_t rows, uint32_t r, uint32_t d)
129+ {
130+ RunC1GemmColumnMajor(d, r, rows, dup, d, act, r, dwUpPartial, r);
131+ }
132+ 
133+ // For multiple tasks, use one M=R+gateRows GEMM to avoid rereading xn in a separate small gate GEMM.
134+ __aicore__ inline void RunG56Merged(GM_ADDR packedT, GM_ADDR xn, GM_ADDR dwOut, uint32_t rows, uint32_t r,
135+ uint32_t d, uint32_t gateRows, bool hasGate)
136+ {
137+ const uint32_t m = hasGate ? (r + gateRows) : r;
138+ RunC1Gemm(m, d, rows, packedT, rows, xn, d, dwOut, d);
139+ }
140+ 
141+ __aicore__ inline void RunG56MergedPacked(GM_ADDR packed, GM_ADDR xn, GM_ADDR dwOut, uint32_t rows, uint32_t r,
142+ uint32_t d, uint32_t packedStride, uint32_t gateRows, bool hasGate)
143+ {
144+ const uint32_t m = hasGate ? (r + gateRows) : r;
145+ RunC1GemmColumnMajor(m, d, rows, packed, packedStride, xn, d, dwOut, d);
146+ }
147+ 
148+ __aicore__ inline void RunG56DirectPacked(GM_ADDR packed, GM_ADDR xn, GM_ADDR dwDown, GM_ADDR dwGate, uint32_t rows,
149+ uint32_t r, uint32_t d, uint32_t packedStride, uint32_t gateRows,
150+ bool hasGate)
151+ {
152+ RunC1GemmColumnMajor(r, d, rows, packed, packedStride, xn, d, dwDown, d);
153+ if (hasGate) {
154+ RunC1GemmColumnMajor(gateRows, d, rows, packed + r * sizeof(D_T), packedStride, xn, d, dwGate, d);
155+ }
156+ }
157+ 
158+ // Single-task fast paths: accumulate straight into the FP32 outputs.
159+ __aicore__ inline void RunC1bDirect(GM_ADDR dup, GM_ADDR act, GM_ADDR dwUp, uint32_t rows, uint32_t r, uint32_t d)
160+ {
161+ RunC1GemmColumnMajor(d, r, rows, dup, d, act, r, dwUp, r);
162+ }
163+ 
164+ __aicore__ inline void RunG56Direct(GM_ADDR packedT, GM_ADDR xn, GM_ADDR dwDown, GM_ADDR dwGate, uint32_t rows,
165+ uint32_t r, uint32_t d, uint32_t gateRows, bool hasGate)
166+ {
167+ RunC1Gemm(r, d, rows, packedT, rows, xn, d, dwDown, d);
168+ if (hasGate) {
169+ // As on A5, write gateRows aligned to 16 into dwGatePartial, which has room for all padded rows.
170+ // Vector REDUCE_WD copies only the first c*d elements into the dwGate FP32 accumulator.
171+ RunC1Gemm(gateRows, d, rows, packedT + r * rows * sizeof(D_T), rows, xn, d, dwGate, d);
172+ }
173+ }
174+ 
175+ // dxnorm = ddown @ w_down + dlogits @ w_gate: concatenate weights along K into wConcat for one GEMM.
176+ __aicore__ inline void RunDx(GM_ADDR packed, GM_ADDR wDown, GM_ADDR wGate, GM_ADDR wConcat, GM_ADDR dxOut,
177+ uint32_t rows, uint32_t d, uint32_t r, uint32_t c, uint32_t packedStride, bool hasGate)
178+ {
179+ if (hasGate) {
180+ RunDxGemm(rows, d, r + c, packed, packedStride, wConcat, d, dxOut, d);
181+ } else {
182+ RunDxGemm(rows, d, r, packed, packedStride, wDown, d, dxOut, d);
183+ }
184+ }
185+ 
186+private:
187+ using ArchTag = Catlass::Arch::AtlasA2;
188+ using DispatchPolicy = Catlass::Gemm::MmadAtlasA2Pingpong<false>;
189+ using RowMajor = Catlass::layout::RowMajor;
190+ using ColumnMajor = Catlass::layout::ColumnMajor;
191+ using MatrixCoord = Catlass::MatrixCoord;
192+ 
193+ static_assert(C1TileM * C1TileN * sizeof(float) <= Catlass::Arch::AtlasA2::L0C_SIZE,
194+ "A2/A3 L0C tile exceeds 128KB");
195+ 
196+ using AType = Catlass::Gemm::GemmType<D_T, RowMajor>;
197+ using BType = Catlass::Gemm::GemmType<D_T, RowMajor>;
198+ using CType = Catlass::Gemm::GemmType<float, RowMajor>;
199+ 
200+ using C1BlockMmad =
201+ Catlass::Gemm::Block::BlockMmad<DispatchPolicy, Catlass::GemmShape<C1TileM, C1TileN, C1TileK>,
202+ Catlass::GemmShape<C1TileM, C1TileN, C1TileK>, AType, BType, CType>;
203+ 
204+ using ATypeColumnMajor = Catlass::Gemm::GemmType<D_T, Catlass::layout::ColumnMajor>;
205+ using C1BlockMmadColumnMajor =
206+ Catlass::Gemm::Block::BlockMmad<DispatchPolicy, Catlass::GemmShape<C1TileM, C1TileN, C1TileK>,
207+ Catlass::GemmShape<C1TileM, C1TileN, C1TileK>, ATypeColumnMajor, BType, CType>;
208+ 
209+ template <class BlockMmad, uint32_t Direction, class LayoutAT = RowMajor>
210+ __aicore__ inline void RunBlockLoop(uint32_t m, uint32_t n, uint32_t k, GM_ADDR ptrA, int64_t lda, GM_ADDR ptrB,
211+ int64_t ldb, GM_ADDR ptrC, int64_t ldc)
212+ {
213+ using BlockScheduler = Catlass::Gemm::Block::GemmIdentityBlockSwizzle<HC_GRAD_SWIZZLE_OFFSET, Direction>;
214+ constexpr uint32_t tileM = BlockMmad::L1TileShape::M;
215+ constexpr uint32_t tileN = BlockMmad::L1TileShape::N;
216+ constexpr uint32_t tileK = BlockMmad::L1TileShape::K;
217+ Catlass::GemmCoord problemShape{m, n, k};
218+ BlockScheduler scheduler(problemShape, MatrixCoord(tileM, tileN));
219+ const uint32_t coreLoops = scheduler.GetCoreLoops();
220+ 
221+ Catlass::Arch::Resource<ArchTag> resource;
222+ BlockMmad blockMmad(resource);
223+ 
224+ AscendC::GlobalTensor<D_T> gmA;
225+ gmA.SetGlobalBuffer(reinterpret_cast<__gm__ D_T*>(ptrA));
226+ AscendC::GlobalTensor<D_T> gmB;
227+ gmB.SetGlobalBuffer(reinterpret_cast<__gm__ D_T*>(ptrB));
228+ AscendC::GlobalTensor<float> gmC;
229+ gmC.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(ptrC));
230+ 
231+ LayoutAT layoutA{m, k, lda};
232+ RowMajor layoutB{k, n, ldb};
233+ RowMajor layoutC{m, n, ldc};
234+ for (uint32_t loopIdx = AscendC::GetBlockIdx(); loopIdx < coreLoops; loopIdx += aicNum_) {
235+ const Catlass::GemmCoord blockCoord = scheduler.GetBlockCoord(loopIdx);
236+ const Catlass::GemmCoord actual = scheduler.GetActualBlockShape(blockCoord);
237+ const MatrixCoord offsetA{blockCoord.m() * tileM, blockCoord.k() * tileK};
238+ const MatrixCoord offsetB{blockCoord.k() * tileK, blockCoord.n() * tileN};
239+ const MatrixCoord offsetC{blockCoord.m() * tileM, blockCoord.n() * tileN};
240+ blockMmad(gmA[layoutA.GetOffset(offsetA)], layoutA, gmB[layoutB.GetOffset(offsetB)], layoutB,
241+ gmC[layoutC.GetOffset(offsetC)], layoutC, actual);
242+ }
243+ AscendC::PipeBarrier<PIPE_ALL>();
244+ AscendC::DataSyncBarrier<AscendC::MemDsbT::DDR>();
245+ }
246+ 
247+ __aicore__ inline void RunC1Gemm(uint32_t m, uint32_t n, uint32_t k, GM_ADDR ptrA, int64_t lda, GM_ADDR ptrB,
248+ int64_t ldb, GM_ADDR ptrC, int64_t ldc)
249+ {
250+ if (m >= n) {
251+ RunBlockLoop<C1BlockMmad, HC_GRAD_SWIZZLE_M_FIRST>(m, n, k, ptrA, lda, ptrB, ldb, ptrC, ldc);
252+ } else {
253+ RunBlockLoop<C1BlockMmad, HC_GRAD_SWIZZLE_N_FIRST>(m, n, k, ptrA, lda, ptrB, ldb, ptrC, ldc);
254+ }
255+ }
256+ 
257+ __aicore__ inline void RunC1GemmColumnMajor(uint32_t m, uint32_t n, uint32_t k, GM_ADDR ptrA, int64_t lda,
258+ GM_ADDR ptrB, int64_t ldb, GM_ADDR ptrC, int64_t ldc)
259+ {
260+ if (m >= n) {
261+ RunBlockLoop<C1BlockMmadColumnMajor, HC_GRAD_SWIZZLE_M_FIRST, ColumnMajor>(m, n, k, ptrA, lda, ptrB, ldb,
262+ ptrC, ldc);
263+ } else {
264+ RunBlockLoop<C1BlockMmadColumnMajor, HC_GRAD_SWIZZLE_N_FIRST, ColumnMajor>(m, n, k, ptrA, lda, ptrB, ldb,
265+ ptrC, ldc);
266+ }
267+ }
268+ 
269+ __aicore__ inline void RunDxGemm(uint32_t m, uint32_t n, uint32_t k, GM_ADDR ptrA, int64_t lda, GM_ADDR ptrB,
270+ int64_t ldb, GM_ADDR ptrC, int64_t ldc)
271+ {
272+ RunC1Gemm(m, n, k, ptrA, lda, ptrB, ldb, ptrC, ldc);
273+ }
274+ 
275+ uint32_t aicNum_{1};
276+};
277+ 
278+} // namespace TextGatedResidualGrad
279+ 
280+#endif // TEXT_GATED_RESIDUAL_GRAD_GEMM_ARCH22_H
@@ -0,0 +1,31 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file text_gated_residual_grad_kernel_arch22.h
13+ * \brief Atlas A2/A3 specialization of the shared TextGatedResidualGrad kernel.
14+ */
15+#ifndef TEXT_GATED_RESIDUAL_GRAD_KERNEL_ARCH22_H
16+#define TEXT_GATED_RESIDUAL_GRAD_KERNEL_ARCH22_H
17+ 
18+#include "../arch35/text_gated_residual_grad_kernel_common.h"
19+#include "text_gated_residual_grad_gemm_arch22.h"
20+ 
21+namespace TextGatedResidualGrad {
22+ 
23+// Atlas A2/A3: CATLASS AtlasA2 BlockMmad; G56 and C1b read Vector results directly as ColumnMajor,
24+// as on A5. Init/Process/RunCube use the shared pipeline.
25+template <typename D_T, int HAS_GAMMA>
26+using KernelTextGatedResidualGradArch22 =
27+ KernelTextGatedResidualGrad<D_T, HAS_GAMMA, HcGradGemmArch22<D_T>, HC_GRAD_USE_DUP_T, false>;
28+ 
29+} // namespace TextGatedResidualGrad
30+ 
31+#endif // TEXT_GATED_RESIDUAL_GRAD_KERNEL_ARCH22_H