Merged
feature/text_gated_residual_grad_pipeline_v2 #15084
azyuliAdd on 15 days ago
feature/text_gated_residual_grad_pipeline_v2 #15084
Merged
Total: 47 files changed+8388-0
| @@ -0,0 +1,19 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | + | ||
| 11 | +file(GLOB CURRENT_DIRS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*) | ||
| 12 | +if(NOT ENABLE_TEST AND NOT BENCHMARK) | ||
| 13 | + list(REMOVE_ITEM CURRENT_DIRS tests) | ||
| 14 | +endif() | ||
| 15 | +foreach(SUB_DIR ${CURRENT_DIRS}) | ||
| 16 | + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${SUB_DIR}/CMakeLists.txt") | ||
| 17 | + add_subdirectory(${SUB_DIR}) | ||
| 18 | + endif() | ||
| 19 | +endforeach() | ||
| @@ -0,0 +1,381 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + * TextGatedResidualGrad aclnn example: read binary inputs from a directory and write binary outputs. | ||
| 10 | + * | ||
| 11 | + * Usage: test_aclnn_text_gated_residual_grad [data_dir] [deviceId] (defaults: ./data, device 0). | ||
| 12 | + * Data directory files: | ||
| 13 | + * meta.txt: dtype(bf16|fp16) n c h r has_gamma [has_gate] (n=B*S; tensors use B=1, S=n; | ||
| 14 | + * has_gate defaults to 1; n=0 empty token input is supported). | ||
| 15 | + * grad_mixed.bin [n,h]T grad_inject.bin [n,c]T x.bin [n,d]T | ||
| 16 | + * w_down.bin [r,d]T w_up.bin [d,r]T w_gate.bin [c,d]T | ||
| 17 | + * gamma.bin [c*h]fp32 (original weight, used as 1+gamma) inv_rms.bin [n,c]fp32 | ||
| 18 | + * inject.bin [n,c]T down_pre_act.bin [n,r]T (xn@w_down^T, before division by C) up.bin [n,d]T (upPreAct) | ||
| 19 | + * Outputs: dx/dw_down/dw_up/dw_gate.bin use T; dgamma.bin uses fp32. Omit dw_gate/dgamma when has_gate/has_gamma is 0. | ||
| 20 | + */ | ||
| 21 | + | ||
| 22 | +#include <algorithm> | ||
| 23 | +#include <array> | ||
| 24 | +#include <cstdint> | ||
| 25 | +#include <cstdlib> | ||
| 26 | +#include <cstdio> | ||
| 27 | +#include <cstring> | ||
| 28 | +#include <fstream> | ||
| 29 | +#include <iostream> | ||
| 30 | +#include <limits> | ||
| 31 | +#include <string> | ||
| 32 | +#include <vector> | ||
| 33 | +#include "acl/acl.h" | ||
| 34 | +#include "aclnnop/aclnn_text_gated_residual_grad.h" | ||
| 35 | + | ||
| 36 | +#define CHECK_RET(cond) ((cond) ? true : (false)) | ||
| 37 | +#define LOG_PRINT(message, ...) printf(message, ##__VA_ARGS__) | ||
| 38 | + | ||
| 39 | +namespace { | ||
| 40 | +constexpr size_t HALF_BYTES = sizeof(uint16_t); | ||
| 41 | +constexpr int64_t METADATA_CHANNEL_LIMIT = 4096; // Tiling additionally checks the actual AIV count. | ||
| 42 | +constexpr int DATA_DIR_ARG = 1; | ||
| 43 | +constexpr int DEVICE_ARG = 2; | ||
| 44 | +enum TensorIndex : size_t { | ||
| 45 | + GRAD_MIXED, | ||
| 46 | + GRAD_INJECT, | ||
| 47 | + X, | ||
| 48 | + W_DOWN, | ||
| 49 | + W_UP, | ||
| 50 | + W_GATE, | ||
| 51 | + GAMMA, | ||
| 52 | + INV_RMS, | ||
| 53 | + INJECT, | ||
| 54 | + DOWN_PRE, | ||
| 55 | + UP, | ||
| 56 | + DX, | ||
| 57 | + DW_DOWN, | ||
| 58 | + DW_UP, | ||
| 59 | + DW_GATE, | ||
| 60 | + DGAMMA, | ||
| 61 | + TENSOR_COUNT | ||
| 62 | +}; | ||
| 63 | + | ||
| 64 | +struct Metadata { | ||
| 65 | + std::string dtype; | ||
| 66 | + int64_t n = 0; | ||
| 67 | + int64_t c = 0; | ||
| 68 | + int64_t h = 0; | ||
| 69 | + int64_t r = 0; | ||
| 70 | + int64_t d = 0; | ||
| 71 | + int hasGamma = 0; | ||
| 72 | + int hasGate = 1; | ||
| 73 | +}; | ||
| 74 | + | ||
| 75 | +struct TensorSpec { | ||
| 76 | + std::string file; | ||
| 77 | + std::vector<int64_t> shape; | ||
| 78 | + aclDataType dtype; | ||
| 79 | + size_t elemBytes; | ||
| 80 | + bool enabled = true; | ||
| 81 | +}; | ||
| 82 | + | ||
| 83 | +struct Resources { | ||
| 84 | + std::array<aclTensor*, TENSOR_COUNT> tensors{}; | ||
| 85 | + std::array<void*, TENSOR_COUNT> addresses{}; | ||
| 86 | + void* workspace = nullptr; | ||
| 87 | + aclrtStream stream = nullptr; | ||
| 88 | + int32_t device = 0; | ||
| 89 | + bool initialized = false; | ||
| 90 | + bool deviceSet = false; | ||
| 91 | + | ||
| 92 | + ~Resources() | ||
| 93 | + { | ||
| 94 | + for (auto tensor : tensors) { | ||
| 95 | + if (tensor != nullptr) { | ||
| 96 | + aclDestroyTensor(tensor); | ||
| 97 | + } | ||
| 98 | + } | ||
| 99 | + for (auto address : addresses) { | ||
| 100 | + if (address != nullptr) { | ||
| 101 | + aclrtFree(address); | ||
| 102 | + } | ||
| 103 | + } | ||
| 104 | + if (workspace != nullptr) { | ||
| 105 | + aclrtFree(workspace); | ||
| 106 | + } | ||
| 107 | + if (stream != nullptr) { | ||
| 108 | + aclrtDestroyStream(stream); | ||
| 109 | + } | ||
| 110 | + if (deviceSet) { | ||
| 111 | + aclrtResetDevice(device); | ||
| 112 | + } | ||
| 113 | + if (initialized) { | ||
| 114 | + aclFinalize(); | ||
| 115 | + } | ||
| 116 | + } | ||
| 117 | +}; | ||
| 118 | + | ||
| 119 | +bool GetByteSize(const std::vector<int64_t>& shape, size_t elemSize, size_t& bytes) | ||
| 120 | +{ | ||
| 121 | + bytes = elemSize; | ||
| 122 | + for (auto i : shape) { | ||
| 123 | + if (i < 0 || (i != 0 && bytes > std::numeric_limits<size_t>::max() / static_cast<size_t>(i))) { | ||
| 124 | + LOG_PRINT("invalid or overflowing tensor shape\n"); | ||
| 125 | + return false; | ||
| 126 | + } | ||
| 127 | + bytes *= static_cast<size_t>(i); | ||
| 128 | + } | ||
| 129 | + return true; | ||
| 130 | +} | ||
| 131 | + | ||
| 132 | +int Init(Resources& resources) | ||
| 133 | +{ | ||
| 134 | + auto ret = aclInit(nullptr); | ||
| 135 | + if (!CHECK_RET(ret == ACL_SUCCESS)) { | ||
| 136 | + LOG_PRINT("aclInit failed. ERROR: %d\n", ret); | ||
| 137 | + return ret; | ||
| 138 | + } | ||
| 139 | + resources.initialized = true; | ||
| 140 | + ret = aclrtSetDevice(resources.device); | ||
| 141 | + if (!CHECK_RET(ret == ACL_SUCCESS)) { | ||
| 142 | + LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret); | ||
| 143 | + return ret; | ||
| 144 | + } | ||
| 145 | + resources.deviceSet = true; | ||
| 146 | + ret = aclrtCreateStream(&resources.stream); | ||
| 147 | + if (!CHECK_RET(ret == ACL_SUCCESS)) { | ||
| 148 | + LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret); | ||
| 149 | + return ret; | ||
| 150 | + } | ||
| 151 | + return 0; | ||
| 152 | +} | ||
| 153 | + | ||
| 154 | +bool ReadBin(const std::string& path, std::string& data) | ||
| 155 | +{ | ||
| 156 | + std::ifstream ifs(path, std::ios::binary | std::ios::ate); | ||
| 157 | + if (!ifs) { | ||
| 158 | + LOG_PRINT("open %s failed\n", path.c_str()); | ||
| 159 | + return false; | ||
| 160 | + } | ||
| 161 | + auto size = ifs.tellg(); | ||
| 162 | + if (size < 0) { | ||
| 163 | + return false; | ||
| 164 | + } | ||
| 165 | + data.resize(size); | ||
| 166 | + ifs.seekg(0); | ||
| 167 | + ifs.read(data.data(), size); | ||
| 168 | + return ifs.good(); | ||
| 169 | +} | ||
| 170 | + | ||
| 171 | +bool WriteBin(const std::string& path, const std::string& data) | ||
| 172 | +{ | ||
| 173 | + std::ofstream ofs(path, std::ios::binary); | ||
| 174 | + ofs.write(data.data(), data.size()); | ||
| 175 | + return ofs.good(); | ||
| 176 | +} | ||
| 177 | + | ||
| 178 | +int CreateTensorFromBin(const std::string& path, const std::vector<int64_t>& shape, aclDataType dtype, size_t elemSize, | ||
| 179 | + aclTensor** tensor, void** deviceAddr, std::string& hostBuf) | ||
| 180 | +{ | ||
| 181 | + if (!ReadBin(path, hostBuf)) { | ||
| 182 | + return -1; | ||
| 183 | + } | ||
| 184 | + size_t size = 0; | ||
| 185 | + if (!GetByteSize(shape, elemSize, size)) { | ||
| 186 | + return -1; | ||
| 187 | + } | ||
| 188 | + if (hostBuf.size() != size) { | ||
| 189 | + LOG_PRINT("size mismatch %s: file %zu expect %zu\n", path.c_str(), hostBuf.size(), size); | ||
| 190 | + return -1; | ||
| 191 | + } | ||
| 192 | + auto ret = aclrtMalloc(deviceAddr, std::max<size_t>(size, 1), ACL_MEM_MALLOC_HUGE_FIRST); | ||
| 193 | + if (!CHECK_RET(ret == ACL_SUCCESS)) { | ||
| 194 | + return -1; | ||
| 195 | + } | ||
| 196 | + if (size != 0) { | ||
| 197 | + ret = aclrtMemcpy(*deviceAddr, size, hostBuf.data(), size, ACL_MEMCPY_HOST_TO_DEVICE); | ||
| 198 | + if (!CHECK_RET(ret == ACL_SUCCESS)) { | ||
| 199 | + return -1; | ||
| 200 | + } | ||
| 201 | + } | ||
| 202 | + std::vector<int64_t> strides(shape.size(), 1); | ||
| 203 | + for (size_t i = shape.size(); i > 1; --i) { | ||
| 204 | + const size_t last = i - 1; | ||
| 205 | + strides[last - 1] = shape[last] * strides[last]; | ||
| 206 | + } | ||
| 207 | + *tensor = aclCreateTensor(shape.data(), shape.size(), dtype, strides.data(), 0, aclFormat::ACL_FORMAT_ND, | ||
| 208 | + shape.data(), shape.size(), *deviceAddr); | ||
| 209 | + return *tensor != nullptr ? 0 : -1; | ||
| 210 | +} | ||
| 211 | + | ||
| 212 | +int CreateOutTensor(const std::vector<int64_t>& shape, aclDataType dtype, size_t elemSize, aclTensor** tensor, | ||
| 213 | + void** deviceAddr) | ||
| 214 | +{ | ||
| 215 | + size_t size = 0; | ||
| 216 | + if (!GetByteSize(shape, elemSize, size)) { | ||
| 217 | + return -1; | ||
| 218 | + } | ||
| 219 | + auto ret = aclrtMalloc(deviceAddr, std::max<size_t>(size, 1), ACL_MEM_MALLOC_HUGE_FIRST); | ||
| 220 | + if (!CHECK_RET(ret == ACL_SUCCESS)) { | ||
| 221 | + return -1; | ||
| 222 | + } | ||
| 223 | + std::vector<int64_t> strides(shape.size(), 1); | ||
| 224 | + for (size_t i = shape.size(); i > 1; --i) { | ||
| 225 | + const size_t last = i - 1; | ||
| 226 | + strides[last - 1] = shape[last] * strides[last]; | ||
| 227 | + } | ||
| 228 | + *tensor = aclCreateTensor(shape.data(), shape.size(), dtype, strides.data(), 0, aclFormat::ACL_FORMAT_ND, | ||
| 229 | + shape.data(), shape.size(), *deviceAddr); | ||
| 230 | + return *tensor != nullptr ? 0 : -1; | ||
| 231 | +} | ||
| 232 | + | ||
| 233 | +int DumpOut(const std::string& path, void* deviceAddr, const std::vector<int64_t>& shape, size_t elemSize) | ||
| 234 | +{ | ||
| 235 | + size_t size = 0; | ||
| 236 | + if (!GetByteSize(shape, elemSize, size)) { | ||
| 237 | + return -1; | ||
| 238 | + } | ||
| 239 | + std::string host(size, '\0'); | ||
| 240 | + if (size != 0) { | ||
| 241 | + auto ret = aclrtMemcpy(host.data(), size, deviceAddr, size, ACL_MEMCPY_DEVICE_TO_HOST); | ||
| 242 | + if (!CHECK_RET(ret == ACL_SUCCESS)) { | ||
| 243 | + return -1; | ||
| 244 | + } | ||
| 245 | + } | ||
| 246 | + return WriteBin(path, host) ? 0 : -1; | ||
| 247 | +} | ||
| 248 | + | ||
| 249 | +bool ValidateMetadata(Metadata& m) | ||
| 250 | +{ | ||
| 251 | + const auto maxDim = std::numeric_limits<uint32_t>::max(); | ||
| 252 | + if ((m.dtype != "bf16" && m.dtype != "fp16") || m.n < 0 || m.n > maxDim || m.c <= 0 || | ||
| 253 | + m.c > METADATA_CHANNEL_LIMIT || m.h <= 0 || m.h > maxDim / m.c || m.r <= 0 || m.r > maxDim) { | ||
| 254 | + LOG_PRINT("invalid dtype or dimensions in meta.txt\n"); | ||
| 255 | + return false; | ||
| 256 | + } | ||
| 257 | + if ((m.hasGamma != 0 && m.hasGamma != 1) || (m.hasGate != 0 && m.hasGate != 1)) { | ||
| 258 | + LOG_PRINT("invalid optional flags in meta.txt\n"); | ||
| 259 | + return false; | ||
| 260 | + } | ||
| 261 | + m.d = m.c * m.h; | ||
| 262 | + const int64_t maxFloatElements = std::numeric_limits<int64_t>::max() / sizeof(float); | ||
| 263 | + if (m.n > maxFloatElements / m.d || m.n > maxFloatElements / m.r || m.r > maxFloatElements / m.d || | ||
| 264 | + m.c > maxFloatElements / m.d) { | ||
| 265 | + LOG_PRINT("Tensor byte size overflows int64 in meta.txt\n"); | ||
| 266 | + return false; | ||
| 267 | + } | ||
| 268 | + return true; | ||
| 269 | +} | ||
| 270 | + | ||
| 271 | +bool ReadMetadata(const std::string& dir, Metadata& m) | ||
| 272 | +{ | ||
| 273 | + std::ifstream meta(dir + "/meta.txt"); | ||
| 274 | + if (!(meta >> m.dtype >> m.n >> m.c >> m.h >> m.r >> m.hasGamma)) { | ||
| 275 | + LOG_PRINT("invalid or missing %s/meta.txt\n", dir.c_str()); | ||
| 276 | + return false; | ||
| 277 | + } | ||
| 278 | + meta >> std::ws; | ||
| 279 | + if (!meta.eof() && !(meta >> m.hasGate)) { | ||
| 280 | + LOG_PRINT("invalid has_gate in meta.txt\n"); | ||
| 281 | + return false; | ||
| 282 | + } | ||
| 283 | + return ValidateMetadata(m); | ||
| 284 | +} | ||
| 285 | + | ||
| 286 | +std::array<TensorSpec, TENSOR_COUNT> MakeTensorSpecs(const Metadata& m) | ||
| 287 | +{ | ||
| 288 | + const auto type = m.dtype == "fp16" ? ACL_FLOAT16 : ACL_BF16; | ||
| 289 | + return {{{"grad_mixed.bin", {1, m.n, m.h}, type, HALF_BYTES}, | ||
| 290 | + {"grad_inject.bin", {1, m.n, m.c}, type, HALF_BYTES, m.hasGate != 0}, | ||
| 291 | + {"x.bin", {1, m.n, m.d}, type, HALF_BYTES}, | ||
| 292 | + {"w_down.bin", {m.r, m.d}, type, HALF_BYTES}, | ||
| 293 | + {"w_up.bin", {m.d, m.r}, type, HALF_BYTES}, | ||
| 294 | + {"w_gate.bin", {m.c, m.d}, type, HALF_BYTES, m.hasGate != 0}, | ||
| 295 | + {"gamma.bin", {m.d}, ACL_FLOAT, sizeof(float), m.hasGamma != 0}, | ||
| 296 | + {"inv_rms.bin", {1, m.n, m.c}, ACL_FLOAT, sizeof(float)}, | ||
| 297 | + {"inject.bin", {1, m.n, m.c}, type, HALF_BYTES, m.hasGate != 0}, | ||
| 298 | + {"down_pre_act.bin", {1, m.n, m.r}, type, HALF_BYTES}, | ||
| 299 | + {"up.bin", {1, m.n, m.d}, type, HALF_BYTES}, | ||
| 300 | + {"dx.bin", {1, m.n, m.d}, type, HALF_BYTES}, | ||
| 301 | + {"dw_down.bin", {m.r, m.d}, type, HALF_BYTES}, | ||
| 302 | + {"dw_up.bin", {m.d, m.r}, type, HALF_BYTES}, | ||
| 303 | + {"dw_gate.bin", {m.c, m.d}, type, HALF_BYTES, m.hasGate != 0}, | ||
| 304 | + {"dgamma.bin", {m.d}, ACL_FLOAT, sizeof(float), m.hasGamma != 0}}}; | ||
| 305 | +} | ||
| 306 | + | ||
| 307 | +int CreateTensors(const std::string& dir, const std::array<TensorSpec, TENSOR_COUNT>& specs, Resources& resources) | ||
| 308 | +{ | ||
| 309 | + std::string hostBuf; | ||
| 310 | + for (size_t i = 0; i < specs.size(); ++i) { | ||
| 311 | + const auto& spec = specs[i]; | ||
| 312 | + if (!spec.enabled) { | ||
| 313 | + continue; | ||
| 314 | + } | ||
| 315 | + auto* tensor = &resources.tensors[i]; | ||
| 316 | + auto* address = &resources.addresses[i]; | ||
| 317 | + const auto status = i < DX ? CreateTensorFromBin(dir + "/" + spec.file, spec.shape, spec.dtype, spec.elemBytes, | ||
| 318 | + tensor, address, hostBuf) : | ||
| 319 | + CreateOutTensor(spec.shape, spec.dtype, spec.elemBytes, tensor, address); | ||
| 320 | + if (status != 0) { | ||
| 321 | + return -1; | ||
| 322 | + } | ||
| 323 | + } | ||
| 324 | + return 0; | ||
| 325 | +} | ||
| 326 | + | ||
| 327 | +int RunOperator(Resources& resources, int64_t channels) | ||
| 328 | +{ | ||
| 329 | + auto& t = resources.tensors; | ||
| 330 | + uint64_t workspaceSize = 0; | ||
| 331 | + aclOpExecutor* executor = nullptr; | ||
| 332 | + auto ret = aclnnTextGatedResidualGradGetWorkspaceSize( | ||
| 333 | + t[GRAD_MIXED], t[X], t[W_DOWN], t[W_UP], t[INV_RMS], t[DOWN_PRE], t[UP], t[GRAD_INJECT], t[W_GATE], t[GAMMA], | ||
| 334 | + t[INJECT], channels, t[DX], t[DW_DOWN], t[DW_UP], t[DW_GATE], t[DGAMMA], &workspaceSize, &executor); | ||
| 335 | + if (ret != ACL_SUCCESS) { | ||
| 336 | + LOG_PRINT("aclnnTextGatedResidualGradGetWorkspaceSize failed. ERROR: %d\n", ret); | ||
| 337 | + return -1; | ||
| 338 | + } | ||
| 339 | + if (workspaceSize > 0 && | ||
| 340 | + aclrtMalloc(&resources.workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { | ||
| 341 | + return -1; | ||
| 342 | + } | ||
| 343 | + ret = aclnnTextGatedResidualGrad(resources.workspace, workspaceSize, executor, resources.stream); | ||
| 344 | + if (ret != ACL_SUCCESS) { | ||
| 345 | + LOG_PRINT("aclnnTextGatedResidualGrad failed. ERROR: %d\n", ret); | ||
| 346 | + return -1; | ||
| 347 | + } | ||
| 348 | + return aclrtSynchronizeStream(resources.stream) == ACL_SUCCESS ? 0 : -1; | ||
| 349 | +} | ||
| 350 | + | ||
| 351 | +int DumpOutputs(const std::string& dir, const std::array<TensorSpec, TENSOR_COUNT>& specs, const Resources& resources) | ||
| 352 | +{ | ||
| 353 | + for (size_t i = DX; i < specs.size(); ++i) { | ||
| 354 | + const auto& spec = specs[i]; | ||
| 355 | + if (spec.enabled && DumpOut(dir + "/" + spec.file, resources.addresses[i], spec.shape, spec.elemBytes) != 0) { | ||
| 356 | + return -1; | ||
| 357 | + } | ||
| 358 | + } | ||
| 359 | + return 0; | ||
| 360 | +} | ||
| 361 | +} // namespace | ||
| 362 | + | ||
| 363 | +int main(int argc, char** argv) | ||
| 364 | +{ | ||
| 365 | + const std::string dir = argc > DATA_DIR_ARG ? argv[DATA_DIR_ARG] : "./data"; | ||
X | |||
| 366 | + Metadata metadata; | ||
| 367 | + if (!ReadMetadata(dir, metadata)) { | ||
| 368 | + return -1; | ||
| 369 | + } | ||
| 370 | + LOG_PRINT("meta: dtype=%s n=%ld c=%ld h=%ld r=%ld hasGamma=%d\n", metadata.dtype.c_str(), metadata.n, metadata.c, | ||
| 371 | + metadata.h, metadata.r, metadata.hasGamma); | ||
| 372 | + Resources resources; | ||
| 373 | + resources.device = argc > DEVICE_ARG ? atoi(argv[DEVICE_ARG]) : 0; | ||
| 374 | + const auto specs = MakeTensorSpecs(metadata); | ||
| 375 | + if (Init(resources) != 0 || CreateTensors(dir, specs, resources) != 0 || RunOperator(resources, metadata.c) != 0 || | ||
| 376 | + DumpOutputs(dir, specs, resources) != 0) { | ||
| 377 | + return -1; | ||
| 378 | + } | ||
| 379 | + LOG_PRINT("run success, outputs dumped to %s\n", dir.c_str()); | ||
| 380 | + return 0; | ||
| 381 | +} | ||
| @@ -0,0 +1,11 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | + | ||
| 11 | +add_graph_plugin_sources() | ||
| @@ -0,0 +1,89 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file text_gated_residual_grad_proto.h | ||
| 13 | + * \brief TextGatedResidualGrad graph operator registration. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#ifndef OPS_OP_PROTO_INC_TEXT_GATED_RESIDUAL_GRAD_H_ | ||
| 17 | +#define OPS_OP_PROTO_INC_TEXT_GATED_RESIDUAL_GRAD_H_ | ||
| 18 | + | ||
| 19 | +#include "graph/operator_reg.h" | ||
| 20 | +#include "graph/types.h" | ||
| 21 | + | ||
| 22 | +namespace ge { | ||
| 23 | +/** | ||
| 24 | + * @brief Computes gradients for the RMS normalization, mixed output and injection | ||
| 25 | + * branch of TextGatedResidual. | ||
| 26 | + * B is the batch size, S is the sequence length, C = channels, H is the hidden | ||
| 27 | + * size per channel, D = C * H, and R is the projection size. All tensors use ND | ||
| 28 | + * format. All FLOAT16 or BFLOAT16 inputs must have the same dtype as x. | ||
| 29 | + * | ||
| 30 | + * @par Inputs: | ||
| 31 | + * @li grad_mixed: Gradient of the mixed output. A 3D tensor of shape [B, S, H]. | ||
| 32 | + * Type supports FLOAT16 and BFLOAT16. | ||
| 33 | + * @li x: Forward input. A 3D tensor of shape [B, S, D]. Type supports FLOAT16 | ||
| 34 | + * and BFLOAT16. | ||
| 35 | + * @li w_down: Down-projection weight. A 2D tensor of shape [R, D]. Type supports | ||
| 36 | + * FLOAT16 and BFLOAT16. | ||
| 37 | + * @li w_up: Up-projection weight. A 2D tensor of shape [D, R]. Type supports | ||
| 38 | + * FLOAT16 and BFLOAT16. | ||
| 39 | + * @li inv_rms: Saved inverse RMS. A 3D tensor of shape [B, S, C]. Type is FLOAT32. | ||
| 40 | + * @li down_pre_act: Saved down-projection result before division by C. A 3D | ||
| 41 | + * tensor of shape [B, S, R]. Type supports FLOAT16 and BFLOAT16. | ||
| 42 | + * @li up: Saved up-projection result. A 3D tensor of shape [B, S, D]. Type | ||
| 43 | + * supports FLOAT16 and BFLOAT16. | ||
| 44 | + * @li grad_inject: Optional gradient of the injection output. A 3D tensor of | ||
| 45 | + * shape [B, S, C]. Type supports FLOAT16 and BFLOAT16. | ||
| 46 | + * @li w_gate: Optional gate-projection weight. A 2D tensor of shape [C, D]. | ||
| 47 | + * Type supports FLOAT16 and BFLOAT16. When provided, grad_inject and inject | ||
| 48 | + * are required. | ||
| 49 | + * @li gamma: Optional RMS weight. A 1D tensor of shape [D]. Type is FLOAT32. | ||
| 50 | + * Normalization uses 1 + gamma; omitting gamma uses a scale of 1. | ||
| 51 | + * @li inject: Optional saved injection output. A 3D tensor of shape [B, S, C]. | ||
| 52 | + * Type supports FLOAT16 and BFLOAT16. | ||
| 53 | + * | ||
| 54 | + * @par Outputs: | ||
| 55 | + * @li dx: Gradient of x. A 3D tensor of shape [B, S, D], with the dtype of x. | ||
| 56 | + * @li dw_down: Gradient of w_down. A 2D tensor of shape [R, D], with the dtype | ||
| 57 | + * of w_down. | ||
| 58 | + * @li dw_up: Gradient of w_up. A 2D tensor of shape [D, R], with the dtype of w_up. | ||
| 59 | + * @li dw_gate: Gradient of w_gate. When w_gate is provided, a 2D tensor of shape | ||
| 60 | + * [C, D], with the dtype of w_gate. Otherwise, an empty tensor of shape [0]. | ||
| 61 | + * @li dgamma: Gradient of gamma. When gamma is provided, a 1D tensor of shape | ||
| 62 | + * [D], with type FLOAT32. Otherwise, an empty tensor of shape [0]. | ||
| 63 | + * | ||
| 64 | + * @par Attributes: | ||
| 65 | + * @li channels: A required positive int attribute specifying the number of | ||
| 66 | + * channels C. The last dimension of x must be divisible by channels. | ||
| 67 | + */ | ||
| 68 | +REG_OP(TextGatedResidualGrad) | ||
| 69 | + .INPUT(grad_mixed, TensorType({DT_FLOAT16, DT_BF16})) | ||
T 缺少参数的资料描述 ![]() ![]() | |||
| 70 | + .INPUT(x, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 71 | + .INPUT(w_down, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 72 | + .INPUT(w_up, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 73 | + .INPUT(inv_rms, TensorType({DT_FLOAT})) | ||
| 74 | + .INPUT(down_pre_act, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 75 | + .INPUT(up, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 76 | + .OPTIONAL_INPUT(grad_inject, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 77 | + .OPTIONAL_INPUT(w_gate, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 78 | + .OPTIONAL_INPUT(gamma, TensorType({DT_FLOAT})) | ||
| 79 | + .OPTIONAL_INPUT(inject, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 80 | + .REQUIRED_ATTR(channels, Int) | ||
| 81 | + .OUTPUT(dx, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 82 | + .OUTPUT(dw_down, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 83 | + .OUTPUT(dw_up, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 84 | + .OUTPUT(dw_gate, TensorType({DT_FLOAT16, DT_BF16})) | ||
| 85 | + .OUTPUT(dgamma, TensorType({DT_FLOAT})) | ||
| 86 | + .OP_END_FACTORY_REG(TextGatedResidualGrad) | ||
| 87 | +} // namespace ge | ||
| 88 | + | ||
| 89 | +#endif // OPS_OP_PROTO_INC_TEXT_GATED_RESIDUAL_GRAD_H_ | ||
| @@ -0,0 +1,35 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | + | ||
| 11 | +if (BUILD_OPEN_PROJECT) | ||
| 12 | + target_sources(op_host_aclnnExc PRIVATE | ||
| 13 | + text_gated_residual_grad_def.cpp | ||
| 14 | + ) | ||
| 15 | + | ||
| 16 | + if("${ASCEND_COMPUTE_UNIT}" STREQUAL "ascend950") | ||
| 17 | + add_ops_compile_options( | ||
| 18 | + OP_NAME TextGatedResidualGrad | ||
| 19 | + COMPUTE_UNIT Ascend950PR_9599 | ||
| 20 | + OPTIONS -mllvm -cce-aicore-dcci-before-kernel-end=false | ||
| 21 | + -DCATLASS_ARCH=3510 | ||
| 22 | + ) | ||
| 23 | + else() | ||
| 24 | + add_ops_compile_options( | ||
| 25 | + OP_NAME TextGatedResidualGrad | ||
| 26 | + OPTIONS --cce-auto-sync=on | ||
| 27 | + -Wno-deprecated-declarations | ||
| 28 | + ) | ||
| 29 | + endif() | ||
| 30 | +endif() | ||
| 31 | + | ||
| 32 | +if(NOT BUILD_OPS_RTY_KERNEL) | ||
| 33 | + add_op_to_compiled_list() | ||
| 34 | + add_modules_sources(OPTYPE text_gated_residual_grad ACLNNTYPE aclnn_exclude) | ||
| 35 | +endif() | ||
| @@ -0,0 +1,353 @@ | |||
| 1 | +{ | ||
| 2 | + "op_type": "TextGatedResidualGrad", | ||
| 3 | + "op_list": [ | ||
| 4 | + { | ||
| 5 | + "bin_filename": "TextGatedResidualGrad_BF16", | ||
| 6 | + "inputs": [ | ||
| 7 | + { | ||
| 8 | + "name": "grad_mixed", | ||
| 9 | + "index": 0, | ||
| 10 | + "dtype": "bfloat16", | ||
| 11 | + "format": "ND", | ||
| 12 | + "paramType": "required", | ||
| 13 | + "shape": [ | ||
| 14 | + -2 | ||
| 15 | + ] | ||
| 16 | + }, | ||
| 17 | + { | ||
| 18 | + "name": "x", | ||
| 19 | + "index": 1, | ||
| 20 | + "dtype": "bfloat16", | ||
| 21 | + "format": "ND", | ||
| 22 | + "paramType": "required", | ||
| 23 | + "shape": [ | ||
| 24 | + -2 | ||
| 25 | + ] | ||
| 26 | + }, | ||
| 27 | + { | ||
| 28 | + "name": "w_down", | ||
| 29 | + "index": 2, | ||
| 30 | + "dtype": "bfloat16", | ||
| 31 | + "format": "ND", | ||
| 32 | + "paramType": "required", | ||
| 33 | + "shape": [ | ||
| 34 | + -2 | ||
| 35 | + ] | ||
| 36 | + }, | ||
| 37 | + { | ||
| 38 | + "name": "w_up", | ||
| 39 | + "index": 3, | ||
| 40 | + "dtype": "bfloat16", | ||
| 41 | + "format": "ND", | ||
| 42 | + "paramType": "required", | ||
| 43 | + "shape": [ | ||
| 44 | + -2 | ||
| 45 | + ] | ||
| 46 | + }, | ||
| 47 | + { | ||
| 48 | + "name": "inv_rms", | ||
| 49 | + "index": 4, | ||
| 50 | + "dtype": "float32", | ||
| 51 | + "format": "ND", | ||
| 52 | + "paramType": "required", | ||
| 53 | + "shape": [ | ||
| 54 | + -2 | ||
| 55 | + ] | ||
| 56 | + }, | ||
| 57 | + { | ||
| 58 | + "name": "down_pre_act", | ||
| 59 | + "index": 5, | ||
| 60 | + "dtype": "bfloat16", | ||
| 61 | + "format": "ND", | ||
| 62 | + "paramType": "required", | ||
| 63 | + "shape": [ | ||
| 64 | + -2 | ||
| 65 | + ] | ||
| 66 | + }, | ||
| 67 | + { | ||
| 68 | + "name": "up", | ||
| 69 | + "index": 6, | ||
| 70 | + "dtype": "bfloat16", | ||
| 71 | + "format": "ND", | ||
| 72 | + "paramType": "required", | ||
| 73 | + "shape": [ | ||
| 74 | + -2 | ||
| 75 | + ] | ||
| 76 | + }, | ||
| 77 | + { | ||
| 78 | + "name": "grad_inject", | ||
| 79 | + "index": 7, | ||
| 80 | + "dtype": "bfloat16", | ||
| 81 | + "format": "ND", | ||
| 82 | + "paramType": "optional", | ||
| 83 | + "shape": [ | ||
| 84 | + -2 | ||
| 85 | + ] | ||
| 86 | + }, | ||
| 87 | + { | ||
| 88 | + "name": "w_gate", | ||
| 89 | + "index": 8, | ||
| 90 | + "dtype": "bfloat16", | ||
| 91 | + "format": "ND", | ||
| 92 | + "paramType": "optional", | ||
| 93 | + "shape": [ | ||
| 94 | + -2 | ||
| 95 | + ] | ||
| 96 | + }, | ||
| 97 | + { | ||
| 98 | + "name": "gamma", | ||
| 99 | + "index": 9, | ||
| 100 | + "dtype": "float32", | ||
| 101 | + "format": "ND", | ||
| 102 | + "paramType": "optional", | ||
| 103 | + "shape": [ | ||
| 104 | + -2 | ||
| 105 | + ] | ||
| 106 | + }, | ||
| 107 | + { | ||
| 108 | + "name": "inject", | ||
| 109 | + "index": 10, | ||
| 110 | + "dtype": "bfloat16", | ||
| 111 | + "format": "ND", | ||
| 112 | + "paramType": "optional", | ||
| 113 | + "shape": [ | ||
| 114 | + -2 | ||
| 115 | + ] | ||
| 116 | + } | ||
| 117 | + ], | ||
| 118 | + "outputs": [ | ||
| 119 | + { | ||
| 120 | + "name": "dx", | ||
| 121 | + "index": 0, | ||
| 122 | + "dtype": "bfloat16", | ||
| 123 | + "format": "ND", | ||
| 124 | + "paramType": "required", | ||
| 125 | + "shape": [ | ||
| 126 | + -2 | ||
| 127 | + ] | ||
| 128 | + }, | ||
| 129 | + { | ||
| 130 | + "name": "dw_down", | ||
| 131 | + "index": 1, | ||
| 132 | + "dtype": "bfloat16", | ||
| 133 | + "format": "ND", | ||
| 134 | + "paramType": "required", | ||
| 135 | + "shape": [ | ||
| 136 | + -2 | ||
| 137 | + ] | ||
| 138 | + }, | ||
| 139 | + { | ||
| 140 | + "name": "dw_up", | ||
| 141 | + "index": 2, | ||
| 142 | + "dtype": "bfloat16", | ||
| 143 | + "format": "ND", | ||
| 144 | + "paramType": "required", | ||
| 145 | + "shape": [ | ||
| 146 | + -2 | ||
| 147 | + ] | ||
| 148 | + }, | ||
| 149 | + { | ||
| 150 | + "name": "dw_gate", | ||
| 151 | + "index": 3, | ||
| 152 | + "dtype": "bfloat16", | ||
| 153 | + "format": "ND", | ||
| 154 | + "paramType": "optional", | ||
| 155 | + "shape": [ | ||
| 156 | + -2 | ||
| 157 | + ] | ||
| 158 | + }, | ||
| 159 | + { | ||
| 160 | + "name": "dgamma", | ||
| 161 | + "index": 4, | ||
| 162 | + "dtype": "float32", | ||
| 163 | + "format": "ND", | ||
| 164 | + "paramType": "optional", | ||
| 165 | + "shape": [ | ||
| 166 | + -2 | ||
| 167 | + ] | ||
| 168 | + } | ||
| 169 | + ], | ||
| 170 | + "attrs": [ | ||
| 171 | + { | ||
| 172 | + "name": "channels", | ||
| 173 | + "dtype": "int", | ||
| 174 | + "value": 4 | ||
| 175 | + } | ||
| 176 | + ] | ||
| 177 | + }, | ||
| 178 | + { | ||
| 179 | + "bin_filename": "TextGatedResidualGrad_FP16", | ||
| 180 | + "inputs": [ | ||
| 181 | + { | ||
| 182 | + "name": "grad_mixed", | ||
| 183 | + "index": 0, | ||
| 184 | + "dtype": "float16", | ||
| 185 | + "format": "ND", | ||
| 186 | + "paramType": "required", | ||
| 187 | + "shape": [ | ||
| 188 | + -2 | ||
| 189 | + ] | ||
| 190 | + }, | ||
| 191 | + { | ||
| 192 | + "name": "x", | ||
| 193 | + "index": 1, | ||
| 194 | + "dtype": "float16", | ||
| 195 | + "format": "ND", | ||
| 196 | + "paramType": "required", | ||
| 197 | + "shape": [ | ||
| 198 | + -2 | ||
| 199 | + ] | ||
| 200 | + }, | ||
| 201 | + { | ||
| 202 | + "name": "w_down", | ||
| 203 | + "index": 2, | ||
| 204 | + "dtype": "float16", | ||
| 205 | + "format": "ND", | ||
| 206 | + "paramType": "required", | ||
| 207 | + "shape": [ | ||
| 208 | + -2 | ||
| 209 | + ] | ||
| 210 | + }, | ||
| 211 | + { | ||
| 212 | + "name": "w_up", | ||
| 213 | + "index": 3, | ||
| 214 | + "dtype": "float16", | ||
| 215 | + "format": "ND", | ||
| 216 | + "paramType": "required", | ||
| 217 | + "shape": [ | ||
| 218 | + -2 | ||
| 219 | + ] | ||
| 220 | + }, | ||
| 221 | + { | ||
| 222 | + "name": "inv_rms", | ||
| 223 | + "index": 4, | ||
| 224 | + "dtype": "float32", | ||
| 225 | + "format": "ND", | ||
| 226 | + "paramType": "required", | ||
| 227 | + "shape": [ | ||
| 228 | + -2 | ||
| 229 | + ] | ||
| 230 | + }, | ||
| 231 | + { | ||
| 232 | + "name": "down_pre_act", | ||
| 233 | + "index": 5, | ||
| 234 | + "dtype": "float16", | ||
| 235 | + "format": "ND", | ||
| 236 | + "paramType": "required", | ||
| 237 | + "shape": [ | ||
| 238 | + -2 | ||
| 239 | + ] | ||
| 240 | + }, | ||
| 241 | + { | ||
| 242 | + "name": "up", | ||
| 243 | + "index": 6, | ||
| 244 | + "dtype": "float16", | ||
| 245 | + "format": "ND", | ||
| 246 | + "paramType": "required", | ||
| 247 | + "shape": [ | ||
| 248 | + -2 | ||
| 249 | + ] | ||
| 250 | + }, | ||
| 251 | + { | ||
| 252 | + "name": "grad_inject", | ||
| 253 | + "index": 7, | ||
| 254 | + "dtype": "float16", | ||
| 255 | + "format": "ND", | ||
| 256 | + "paramType": "optional", | ||
| 257 | + "shape": [ | ||
| 258 | + -2 | ||
| 259 | + ] | ||
| 260 | + }, | ||
| 261 | + { | ||
| 262 | + "name": "w_gate", | ||
| 263 | + "index": 8, | ||
| 264 | + "dtype": "float16", | ||
| 265 | + "format": "ND", | ||
| 266 | + "paramType": "optional", | ||
| 267 | + "shape": [ | ||
| 268 | + -2 | ||
| 269 | + ] | ||
| 270 | + }, | ||
| 271 | + { | ||
| 272 | + "name": "gamma", | ||
| 273 | + "index": 9, | ||
| 274 | + "dtype": "float32", | ||
| 275 | + "format": "ND", | ||
| 276 | + "paramType": "optional", | ||
| 277 | + "shape": [ | ||
| 278 | + -2 | ||
| 279 | + ] | ||
| 280 | + }, | ||
| 281 | + { | ||
| 282 | + "name": "inject", | ||
| 283 | + "index": 10, | ||
| 284 | + "dtype": "float16", | ||
| 285 | + "format": "ND", | ||
| 286 | + "paramType": "optional", | ||
| 287 | + "shape": [ | ||
| 288 | + -2 | ||
| 289 | + ] | ||
| 290 | + } | ||
| 291 | + ], | ||
| 292 | + "outputs": [ | ||
| 293 | + { | ||
| 294 | + "name": "dx", | ||
| 295 | + "index": 0, | ||
| 296 | + "dtype": "float16", | ||
| 297 | + "format": "ND", | ||
| 298 | + "paramType": "required", | ||
| 299 | + "shape": [ | ||
| 300 | + -2 | ||
| 301 | + ] | ||
| 302 | + }, | ||
| 303 | + { | ||
| 304 | + "name": "dw_down", | ||
| 305 | + "index": 1, | ||
| 306 | + "dtype": "float16", | ||
| 307 | + "format": "ND", | ||
| 308 | + "paramType": "required", | ||
| 309 | + "shape": [ | ||
| 310 | + -2 | ||
| 311 | + ] | ||
| 312 | + }, | ||
| 313 | + { | ||
| 314 | + "name": "dw_up", | ||
| 315 | + "index": 2, | ||
| 316 | + "dtype": "float16", | ||
| 317 | + "format": "ND", | ||
| 318 | + "paramType": "required", | ||
| 319 | + "shape": [ | ||
| 320 | + -2 | ||
| 321 | + ] | ||
| 322 | + }, | ||
| 323 | + { | ||
| 324 | + "name": "dw_gate", | ||
| 325 | + "index": 3, | ||
| 326 | + "dtype": "float16", | ||
| 327 | + "format": "ND", | ||
| 328 | + "paramType": "optional", | ||
| 329 | + "shape": [ | ||
| 330 | + -2 | ||
| 331 | + ] | ||
| 332 | + }, | ||
| 333 | + { | ||
| 334 | + "name": "dgamma", | ||
| 335 | + "index": 4, | ||
| 336 | + "dtype": "float32", | ||
| 337 | + "format": "ND", | ||
| 338 | + "paramType": "optional", | ||
| 339 | + "shape": [ | ||
| 340 | + -2 | ||
| 341 | + ] | ||
| 342 | + } | ||
| 343 | + ], | ||
| 344 | + "attrs": [ | ||
| 345 | + { | ||
| 346 | + "name": "channels", | ||
| 347 | + "dtype": "int", | ||
| 348 | + "value": 4 | ||
| 349 | + } | ||
| 350 | + ] | ||
| 351 | + } | ||
| 352 | + ] | ||
| 353 | +} | ||
Amhc/text_gated_residual_grad/op_host/config/ascend950/text_gated_residual_grad_simplified_key.ini+2-0
| @@ -0,0 +1,2 @@ | |||
| 1 | +[TextGatedResidualGrad] | ||
| 2 | +default=0 | ||
| @@ -0,0 +1,319 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +#include "aclnn_text_gated_residual_grad.h" | ||
| 12 | +#include "text_gated_residual_grad.h" | ||
| 13 | + | ||
| 14 | +#include <limits> | ||
| 15 | + | ||
| 16 | +#include "aclnn_kernels/contiguous.h" | ||
| 17 | +#include "external/aclnn_kernels/aclnn_platform.h" | ||
| 18 | +#include "aclnn/aclnn_base.h" | ||
| 19 | +#include "acl/acl.h" | ||
| 20 | +#include "aclnn_kernels/common/op_error_check.h" | ||
| 21 | +#include "opdev/common_types.h" | ||
| 22 | +#include "opdev/data_type_utils.h" | ||
| 23 | +#include "opdev/format_utils.h" | ||
| 24 | +#include "opdev/op_dfx.h" | ||
| 25 | +#include "opdev/op_executor.h" | ||
| 26 | +#include "opdev/op_log.h" | ||
| 27 | +#include "opdev/platform.h" | ||
| 28 | +#include "opdev/shape_utils.h" | ||
| 29 | +#include "opdev/tensor_view_utils.h" | ||
| 30 | +#include "opdev/make_op_executor.h" | ||
| 31 | + | ||
| 32 | +using namespace op; | ||
| 33 | + | ||
| 34 | +#ifdef __cplusplus | ||
| 35 | +extern "C" { | ||
| 36 | +#endif | ||
| 37 | + | ||
| 38 | +namespace { | ||
| 39 | +constexpr int64_t INPUT_RANK = 3; | ||
| 40 | +constexpr int64_t MATRIX_RANK = 2; | ||
| 41 | +constexpr int64_t BATCH_DIM = 0; | ||
| 42 | +constexpr int64_t SEQUENCE_DIM = 1; | ||
| 43 | +constexpr int64_t FEATURE_DIM = 2; | ||
| 44 | +constexpr int64_t MATRIX_ROW_DIM = 0; | ||
| 45 | +constexpr size_t OUT_DX = 0; | ||
| 46 | +constexpr size_t OUT_DW_DOWN = 1; | ||
| 47 | +constexpr size_t OUT_DW_UP = 2; | ||
| 48 | +constexpr size_t OUT_DW_GATE = 3; | ||
| 49 | +constexpr size_t OUT_DGAMMA = 4; | ||
| 50 | + | ||
| 51 | +struct TextGatedResidualGradParams { | ||
| 52 | + const aclTensor* gradMixed = nullptr; | ||
| 53 | + const aclTensor* x = nullptr; | ||
| 54 | + const aclTensor* wDown = nullptr; | ||
| 55 | + const aclTensor* wUp = nullptr; | ||
| 56 | + const aclTensor* invRms = nullptr; | ||
| 57 | + const aclTensor* downPreAct = nullptr; | ||
| 58 | + const aclTensor* up = nullptr; | ||
| 59 | + const aclTensor* gradInject = nullptr; | ||
| 60 | + const aclTensor* wGate = nullptr; | ||
| 61 | + const aclTensor* gamma = nullptr; | ||
| 62 | + const aclTensor* inject = nullptr; | ||
| 63 | + int64_t channels = 0; | ||
| 64 | + aclTensor* dx = nullptr; | ||
| 65 | + aclTensor* dwDown = nullptr; | ||
| 66 | + aclTensor* dwUp = nullptr; | ||
| 67 | + aclTensor* dwGate = nullptr; | ||
| 68 | + aclTensor* dgamma = nullptr; | ||
| 69 | +}; | ||
| 70 | + | ||
| 71 | +aclnnStatus CheckNotNull(const TextGatedResidualGradParams& p) | ||
| 72 | +{ | ||
| 73 | + CHECK_COND(p.gradMixed != nullptr, ACLNN_ERR_PARAM_NULLPTR, "gradMixed must not be nullptr."); | ||
| 74 | + CHECK_COND(p.x != nullptr, ACLNN_ERR_PARAM_NULLPTR, "x must not be nullptr."); | ||
| 75 | + CHECK_COND(p.wDown != nullptr, ACLNN_ERR_PARAM_NULLPTR, "wDown must not be nullptr."); | ||
| 76 | + CHECK_COND(p.wUp != nullptr, ACLNN_ERR_PARAM_NULLPTR, "wUp must not be nullptr."); | ||
| 77 | + CHECK_COND(p.invRms != nullptr, ACLNN_ERR_PARAM_NULLPTR, "invRms must not be nullptr."); | ||
| 78 | + CHECK_COND(p.downPreAct != nullptr, ACLNN_ERR_PARAM_NULLPTR, "downPreAct must not be nullptr."); | ||
| 79 | + CHECK_COND(p.up != nullptr, ACLNN_ERR_PARAM_NULLPTR, "up must not be nullptr."); | ||
| 80 | + CHECK_COND(p.dx != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dx must not be nullptr."); | ||
| 81 | + CHECK_COND(p.dwDown != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dwDown must not be nullptr."); | ||
| 82 | + CHECK_COND(p.dwUp != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dwUp must not be nullptr."); | ||
| 83 | + CHECK_COND(p.wGate != nullptr || (p.gradInject == nullptr && p.inject == nullptr && p.dwGate == nullptr), | ||
| 84 | + ACLNN_ERR_PARAM_NULLPTR, "wGate is required when gradInject, inject or dwGate is given."); | ||
| 85 | + CHECK_COND(p.gamma != nullptr || p.dgamma == nullptr, ACLNN_ERR_PARAM_NULLPTR, | ||
| 86 | + "gamma is required when dgamma is given."); | ||
| 87 | + if (p.wGate != nullptr) { | ||
| 88 | + CHECK_COND(p.gradInject != nullptr, ACLNN_ERR_PARAM_NULLPTR, | ||
| 89 | + "gradInject must not be nullptr when wGate is given."); | ||
| 90 | + CHECK_COND(p.inject != nullptr, ACLNN_ERR_PARAM_NULLPTR, "inject must not be nullptr when wGate is given."); | ||
| 91 | + CHECK_COND(p.dwGate != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dwGate must not be nullptr when wGate is given."); | ||
| 92 | + } | ||
| 93 | + if (p.gamma != nullptr) { | ||
| 94 | + CHECK_COND(p.dgamma != nullptr, ACLNN_ERR_PARAM_NULLPTR, "dgamma must not be nullptr when gamma is given."); | ||
| 95 | + } | ||
| 96 | + return ACLNN_SUCCESS; | ||
| 97 | +} | ||
| 98 | + | ||
| 99 | +aclnnStatus CheckDtype(const TextGatedResidualGradParams& p) | ||
| 100 | +{ | ||
| 101 | + const std::initializer_list<op::DataType> supportDtype = {op::DataType::DT_FLOAT16, op::DataType::DT_BF16}; | ||
| 102 | + OP_CHECK_DTYPE_NOT_SUPPORT(p.x, supportDtype, return ACLNN_ERR_PARAM_INVALID); | ||
| 103 | + OP_CHECK_DTYPE_NOT_SUPPORT(p.gradMixed, supportDtype, return ACLNN_ERR_PARAM_INVALID); | ||
| 104 | + OP_CHECK_DTYPE_NOT_SUPPORT(p.wDown, supportDtype, return ACLNN_ERR_PARAM_INVALID); | ||
| 105 | + OP_CHECK_DTYPE_NOT_SUPPORT(p.wUp, supportDtype, return ACLNN_ERR_PARAM_INVALID); | ||
| 106 | + OP_CHECK_DTYPE_NOT_SUPPORT(p.downPreAct, supportDtype, return ACLNN_ERR_PARAM_INVALID); | ||
| 107 | + OP_CHECK_DTYPE_NOT_SAME(p.x, p.gradMixed, return ACLNN_ERR_PARAM_INVALID); | ||
| 108 | + OP_CHECK_DTYPE_NOT_SAME(p.x, p.wDown, return ACLNN_ERR_PARAM_INVALID); | ||
| 109 | + OP_CHECK_DTYPE_NOT_SAME(p.x, p.wUp, return ACLNN_ERR_PARAM_INVALID); | ||
| 110 | + OP_CHECK_DTYPE_NOT_SAME(p.x, p.downPreAct, return ACLNN_ERR_PARAM_INVALID); | ||
| 111 | + OP_CHECK_DTYPE_NOT_SAME(p.x, p.up, return ACLNN_ERR_PARAM_INVALID); | ||
| 112 | + OP_CHECK_DTYPE_NOT_SAME(p.x, p.dx, return ACLNN_ERR_PARAM_INVALID); | ||
| 113 | + OP_CHECK_DTYPE_NOT_MATCH(p.invRms, op::DataType::DT_FLOAT, return ACLNN_ERR_PARAM_INVALID); | ||
| 114 | + OP_CHECK_DTYPE_NOT_SAME(p.wDown, p.dwDown, return ACLNN_ERR_PARAM_INVALID); | ||
| 115 | + OP_CHECK_DTYPE_NOT_SAME(p.wUp, p.dwUp, return ACLNN_ERR_PARAM_INVALID); | ||
| 116 | + if (p.wGate != nullptr) { | ||
| 117 | + OP_CHECK_DTYPE_NOT_SAME(p.x, p.gradInject, return ACLNN_ERR_PARAM_INVALID); | ||
| 118 | + OP_CHECK_DTYPE_NOT_SAME(p.x, p.wGate, return ACLNN_ERR_PARAM_INVALID); | ||
| 119 | + OP_CHECK_DTYPE_NOT_SAME(p.x, p.inject, return ACLNN_ERR_PARAM_INVALID); | ||
| 120 | + OP_CHECK_DTYPE_NOT_SAME(p.wGate, p.dwGate, return ACLNN_ERR_PARAM_INVALID); | ||
| 121 | + } | ||
| 122 | + if (p.gamma != nullptr) { | ||
| 123 | + OP_CHECK_DTYPE_NOT_MATCH(p.gamma, op::DataType::DT_FLOAT, return ACLNN_ERR_PARAM_INVALID); | ||
| 124 | + OP_CHECK_DTYPE_NOT_MATCH(p.dgamma, op::DataType::DT_FLOAT, return ACLNN_ERR_PARAM_INVALID); | ||
| 125 | + } | ||
| 126 | + return ACLNN_SUCCESS; | ||
| 127 | +} | ||
| 128 | + | ||
| 129 | +aclnnStatus CheckTensorShape(const aclTensor* t, std::initializer_list<int64_t> expect, const char* name) | ||
| 130 | +{ | ||
| 131 | + auto shape = t->GetViewShape(); | ||
| 132 | + CHECK_COND(shape.GetDimNum() == static_cast<int64_t>(expect.size()), ACLNN_ERR_PARAM_INVALID, | ||
| 133 | + "%s dim num mismatch, expect %zu but got %zu", name, expect.size(), shape.GetDimNum()); | ||
| 134 | + size_t i = 0; | ||
| 135 | + for (auto e : expect) { | ||
| 136 | + CHECK_COND(shape.GetDim(i) == e, ACLNN_ERR_PARAM_INVALID, "%s dim[%zu] expect %ld but got %ld", name, i, e, | ||
| 137 | + shape.GetDim(i)); | ||
| 138 | + i++; | ||
| 139 | + } | ||
| 140 | + return ACLNN_SUCCESS; | ||
| 141 | +} | ||
| 142 | + | ||
| 143 | +aclnnStatus CheckShape(const TextGatedResidualGradParams& p) | ||
| 144 | +{ | ||
| 145 | + auto xShape = p.x->GetViewShape(); | ||
| 146 | + CHECK_COND(xShape.GetDimNum() == INPUT_RANK, ACLNN_ERR_PARAM_INVALID, "x dim num should be 3, but got %zu", | ||
| 147 | + xShape.GetDimNum()); | ||
| 148 | + int64_t b = xShape.GetDim(BATCH_DIM); | ||
| 149 | + int64_t s = xShape.GetDim(SEQUENCE_DIM); | ||
| 150 | + int64_t d = xShape.GetDim(FEATURE_DIM); | ||
| 151 | + CHECK_COND(b >= 0 && s >= 0, ACLNN_ERR_PARAM_INVALID, "B/S must be non-negative, but got B=%ld, S=%ld", b, s); | ||
| 152 | + CHECK_COND(s == 0 || b <= std::numeric_limits<int64_t>::max() / s, ACLNN_ERR_PARAM_INVALID, | ||
| 153 | + "B*S overflows int64, got B=%ld, S=%ld", b, s); | ||
| 154 | + CHECK_COND(d > 0, ACLNN_ERR_PARAM_INVALID, "x last dim must be positive, but got %ld", d); | ||
| 155 | + CHECK_COND(p.channels > 0 && d % p.channels == 0, ACLNN_ERR_PARAM_INVALID, | ||
| 156 | + "channels %ld invalid for x last dim %ld", p.channels, d); | ||
| 157 | + int64_t h = d / p.channels; | ||
| 158 | + CHECK_COND(p.wDown->GetViewShape().GetDimNum() == MATRIX_RANK, ACLNN_ERR_PARAM_INVALID, | ||
| 159 | + "wDown dim num should be 2, but got %zu", p.wDown->GetViewShape().GetDimNum()); | ||
| 160 | + int64_t r = p.wDown->GetViewShape().GetDim(MATRIX_ROW_DIM); | ||
| 161 | + CHECK_COND(h > 0 && r > 0, ACLNN_ERR_PARAM_INVALID, "invalid channels/H or R."); | ||
| 162 | + | ||
| 163 | + CHECK_RET(CheckTensorShape(p.gradMixed, {b, s, h}, "gradMixed") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 164 | + CHECK_RET(CheckTensorShape(p.x, {b, s, d}, "x") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 165 | + CHECK_RET(CheckTensorShape(p.wDown, {r, d}, "wDown") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 166 | + CHECK_RET(CheckTensorShape(p.wUp, {d, r}, "wUp") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 167 | + CHECK_RET(CheckTensorShape(p.invRms, {b, s, p.channels}, "invRms") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 168 | + CHECK_RET(CheckTensorShape(p.downPreAct, {b, s, r}, "downPreAct") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 169 | + CHECK_RET(CheckTensorShape(p.up, {b, s, d}, "up") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 170 | + CHECK_RET(CheckTensorShape(p.dx, {b, s, d}, "dx") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 171 | + CHECK_RET(CheckTensorShape(p.dwDown, {r, d}, "dwDown") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 172 | + CHECK_RET(CheckTensorShape(p.dwUp, {d, r}, "dwUp") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 173 | + if (p.wGate != nullptr) { | ||
| 174 | + CHECK_RET(CheckTensorShape(p.wGate, {p.channels, d}, "wGate") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 175 | + CHECK_RET(CheckTensorShape(p.gradInject, {b, s, p.channels}, "gradInject") == ACLNN_SUCCESS, | ||
| 176 | + ACLNN_ERR_PARAM_INVALID); | ||
| 177 | + CHECK_RET(CheckTensorShape(p.inject, {b, s, p.channels}, "inject") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 178 | + CHECK_RET(CheckTensorShape(p.dwGate, {p.channels, d}, "dwGate") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 179 | + } | ||
| 180 | + if (p.gamma != nullptr) { | ||
| 181 | + CHECK_RET(CheckTensorShape(p.gamma, {d}, "gamma") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 182 | + CHECK_RET(CheckTensorShape(p.dgamma, {d}, "dgamma") == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID); | ||
| 183 | + } | ||
| 184 | + return ACLNN_SUCCESS; | ||
| 185 | +} | ||
| 186 | + | ||
| 187 | +aclnnStatus CheckParams(const TextGatedResidualGradParams& p) | ||
| 188 | +{ | ||
| 189 | + CHECK_COND(CheckNotNull(p) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_NULLPTR, "required param is nullptr."); | ||
| 190 | + CHECK_COND(CheckDtype(p) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID, "invalid dtype."); | ||
| 191 | + CHECK_COND(CheckShape(p) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID, "invalid shape."); | ||
| 192 | + return ACLNN_SUCCESS; | ||
| 193 | +} | ||
| 194 | + | ||
| 195 | +const aclTensor* ContiguousOrNull(const aclTensor* t, aclOpExecutor* executor) | ||
| 196 | +{ | ||
| 197 | + if (t == nullptr) { | ||
| 198 | + return nullptr; | ||
| 199 | + } | ||
| 200 | + return l0op::Contiguous(t, executor); | ||
| 201 | +} | ||
| 202 | + | ||
| 203 | +aclnnStatus MakeContiguousInputs(const TextGatedResidualGradParams& p, TextGatedResidualGradParams& contiguous, | ||
| 204 | + aclOpExecutor* executor) | ||
| 205 | +{ | ||
| 206 | + // Make inputs contiguous. | ||
| 207 | + contiguous.gradMixed = l0op::Contiguous(p.gradMixed, executor); | ||
| 208 | + CHECK_RET(contiguous.gradMixed != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 209 | + contiguous.gradInject = ContiguousOrNull(p.gradInject, executor); | ||
| 210 | + contiguous.x = l0op::Contiguous(p.x, executor); | ||
| 211 | + CHECK_RET(contiguous.x != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 212 | + contiguous.wDown = l0op::Contiguous(p.wDown, executor); | ||
| 213 | + CHECK_RET(contiguous.wDown != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 214 | + contiguous.wUp = l0op::Contiguous(p.wUp, executor); | ||
| 215 | + CHECK_RET(contiguous.wUp != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 216 | + contiguous.wGate = ContiguousOrNull(p.wGate, executor); | ||
| 217 | + contiguous.gamma = ContiguousOrNull(p.gamma, executor); | ||
| 218 | + CHECK_RET(p.wGate == nullptr || contiguous.wGate != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 219 | + CHECK_RET(p.gamma == nullptr || contiguous.gamma != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 220 | + CHECK_RET(p.gradInject == nullptr || contiguous.gradInject != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 221 | + contiguous.invRms = l0op::Contiguous(p.invRms, executor); | ||
| 222 | + CHECK_RET(contiguous.invRms != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 223 | + contiguous.inject = ContiguousOrNull(p.inject, executor); | ||
| 224 | + CHECK_RET(p.inject == nullptr || contiguous.inject != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 225 | + contiguous.downPreAct = l0op::Contiguous(p.downPreAct, executor); | ||
| 226 | + CHECK_RET(contiguous.downPreAct != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 227 | + contiguous.up = l0op::Contiguous(p.up, executor); | ||
| 228 | + CHECK_RET(contiguous.up != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 229 | + | ||
| 230 | + return ACLNN_SUCCESS; | ||
| 231 | +} | ||
| 232 | +} // namespace | ||
| 233 | + | ||
| 234 | +aclnnStatus aclnnTextGatedResidualGradGetWorkspaceSize( | ||
| 235 | + const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp, | ||
| 236 | + const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInjectOptional, | ||
| 237 | + const aclTensor* wGateOptional, const aclTensor* gammaOptional, const aclTensor* injectOptional, int64_t channels, | ||
| 238 | + aclTensor* dx, aclTensor* dwDown, aclTensor* dwUp, aclTensor* dwGateOptional, aclTensor* dgammaOptional, | ||
| 239 | + uint64_t* workspaceSize, aclOpExecutor** executor) | ||
| 240 | +{ | ||
| 241 | + L2_DFX_PHASE_1(aclnnTextGatedResidualGrad, | ||
| 242 | + DFX_IN(gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInjectOptional, wGateOptional, | ||
| 243 | + gammaOptional, injectOptional), | ||
| 244 | + DFX_OUT(dx, dwDown, dwUp, dwGateOptional, dgammaOptional)); | ||
| 245 | + | ||
| 246 | + CHECK_COND(workspaceSize != nullptr, ACLNN_ERR_PARAM_NULLPTR, "workspaceSize must not be nullptr."); | ||
| 247 | + CHECK_COND(executor != nullptr, ACLNN_ERR_PARAM_NULLPTR, "executor must not be nullptr."); | ||
| 248 | + | ||
| 249 | + TextGatedResidualGradParams params{gradMixed, | ||
| 250 | + x, | ||
| 251 | + wDown, | ||
| 252 | + wUp, | ||
| 253 | + invRms, | ||
| 254 | + downPreAct, | ||
| 255 | + up, | ||
| 256 | + gradInjectOptional, | ||
| 257 | + wGateOptional, | ||
| 258 | + gammaOptional, | ||
| 259 | + injectOptional, | ||
| 260 | + channels, | ||
| 261 | + dx, | ||
| 262 | + dwDown, | ||
| 263 | + dwUp, | ||
| 264 | + dwGateOptional, | ||
| 265 | + dgammaOptional}; | ||
| 266 | + aclnnStatus ret = CheckParams(params); | ||
| 267 | + CHECK_RET(ret == ACLNN_SUCCESS, ret); | ||
| 268 | + | ||
| 269 | + auto uniqueExecutor = CREATE_EXECUTOR(); | ||
| 270 | + CHECK_RET(uniqueExecutor.get() != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR); | ||
| 271 | + | ||
| 272 | + auto contiguous = params; | ||
| 273 | + ret = MakeContiguousInputs(params, contiguous, uniqueExecutor.get()); | ||
| 274 | + CHECK_RET(ret == ACLNN_SUCCESS, ret); | ||
| 275 | + | ||
| 276 | + auto result = l0op::TextGatedResidualGrad(contiguous.gradMixed, contiguous.x, contiguous.wDown, contiguous.wUp, | ||
| 277 | + contiguous.invRms, contiguous.downPreAct, contiguous.up, | ||
| 278 | + contiguous.gradInject, contiguous.wGate, contiguous.gamma, | ||
| 279 | + contiguous.inject, channels, uniqueExecutor.get()); | ||
| 280 | + CHECK_RET(std::get<OUT_DX>(result) != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 281 | + | ||
| 282 | + auto viewCopyIfNotNull = [&uniqueExecutor](aclTensor* out, aclTensor* userOut) -> aclnnStatus { | ||
| 283 | + if (userOut == nullptr || userOut->IsEmpty()) { | ||
| 284 | + return ACLNN_SUCCESS; | ||
| 285 | + } | ||
| 286 | + auto copiedOutput = l0op::ViewCopy(out, userOut, uniqueExecutor.get()); | ||
| 287 | + CHECK_RET(copiedOutput != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 288 | + return ACLNN_SUCCESS; | ||
| 289 | + }; | ||
| 290 | + CHECK_RET(viewCopyIfNotNull(std::get<OUT_DX>(result), dx) == ACLNN_SUCCESS, ACLNN_ERR_INNER_NULLPTR); | ||
| 291 | + CHECK_RET(viewCopyIfNotNull(std::get<OUT_DW_DOWN>(result), dwDown) == ACLNN_SUCCESS, ACLNN_ERR_INNER_NULLPTR); | ||
| 292 | + CHECK_RET(viewCopyIfNotNull(std::get<OUT_DW_UP>(result), dwUp) == ACLNN_SUCCESS, ACLNN_ERR_INNER_NULLPTR); | ||
| 293 | + CHECK_RET(viewCopyIfNotNull(std::get<OUT_DW_GATE>(result), dwGateOptional) == ACLNN_SUCCESS, | ||
| 294 | + ACLNN_ERR_INNER_NULLPTR); | ||
| 295 | + if (gammaOptional != nullptr && dgammaOptional != nullptr) { | ||
| 296 | + CHECK_RET(viewCopyIfNotNull(std::get<OUT_DGAMMA>(result), dgammaOptional) == ACLNN_SUCCESS, | ||
| 297 | + ACLNN_ERR_INNER_NULLPTR); | ||
| 298 | + } | ||
| 299 | + | ||
| 300 | + *workspaceSize = uniqueExecutor->GetWorkspaceSize(); | ||
| 301 | + uniqueExecutor.ReleaseTo(executor); | ||
| 302 | + return ACLNN_SUCCESS; | ||
| 303 | +} | ||
| 304 | + | ||
| 305 | +aclnnStatus aclnnTextGatedResidualGrad(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, | ||
| 306 | + aclrtStream stream) | ||
| 307 | +{ | ||
| 308 | + L2_DFX_PHASE_2(aclnnTextGatedResidualGrad); | ||
| 309 | + auto ret = CommonOpExecutorRun(workspace, workspaceSize, executor, stream); | ||
| 310 | + if (ret != ACLNN_SUCCESS) { | ||
| 311 | + OP_LOGE(ACLNN_ERR_INNER, "This is an error in TextGatedResidualGrad launch aicore"); | ||
| 312 | + return ACLNN_ERR_INNER; | ||
| 313 | + } | ||
| 314 | + return ACLNN_SUCCESS; | ||
| 315 | +} | ||
| 316 | + | ||
| 317 | +#ifdef __cplusplus | ||
| 318 | +} | ||
| 319 | +#endif | ||
| @@ -0,0 +1,71 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclnn_text_gated_residual_grad.h | ||
| 13 | + * \brief TextGatedResidualGrad ACLNN workspace-query and execution interfaces. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#ifndef OP_API_INC_ACLNN_TEXT_GATED_RESIDUAL_GRAD_H | ||
| 17 | +#define OP_API_INC_ACLNN_TEXT_GATED_RESIDUAL_GRAD_H | ||
| 18 | + | ||
| 19 | +#include "aclnn/aclnn_base.h" | ||
| 20 | +#include "aclnnop/aclnn_util.h" | ||
| 21 | + | ||
| 22 | +#ifdef __cplusplus | ||
| 23 | +extern "C" { | ||
| 24 | +#endif | ||
| 25 | + | ||
| 26 | +/** | ||
| 27 | + * @brief First-stage aclnnTextGatedResidualGrad API: calculate workspace size. | ||
| 28 | + * Compute gradients for the RMS normalization, mixed output and injection branch of TextGatedResidual. | ||
| 29 | + * Inputs follow OpDef order; C = channels, D = C * H, and R is the projection size. | ||
| 30 | + * The activation and projection-weight tensors use one common FLOAT16 or BFLOAT16 dtype. | ||
| 31 | + * @param [in] gradMixed Gradient of mixed output, shape [B, S, H]. | ||
| 32 | + * @param [in] x Forward input, shape [B, S, D]. | ||
| 33 | + * @param [in] wDown Down-projection weight, shape [R, D]. | ||
| 34 | + * @param [in] wUp Up-projection weight, shape [D, R]. | ||
| 35 | + * @param [in] invRms Saved inverse RMS, shape [B, S, C], FLOAT32. | ||
| 36 | + * @param [in] downPreAct Saved down-projection result before division by C, shape [B, S, R]. | ||
| 37 | + * @param [in] up Saved up-projection result, shape [B, S, D]. | ||
| 38 | + * @param [in] gradInjectOptional Injection-output gradient, shape [B, S, C]. | ||
| 39 | + * @param [in] wGateOptional Gate-projection weight, shape [C, D]. | ||
| 40 | + * @param [in] gammaOptional RMS weight, shape [D], FLOAT32; normalization uses 1 + gamma. | ||
| 41 | + * @param [in] injectOptional Saved injection output, shape [B, S, C]. | ||
| 42 | + * @param [in] channels Number of channels. | ||
| 43 | + * @param [out] dx Input gradient, with the shape and dtype of x. | ||
| 44 | + * @param [out] dwDown Down-weight gradient, with the shape and dtype of wDown. | ||
| 45 | + * @param [out] dwUp Up-weight gradient, with the shape and dtype of wUp. | ||
| 46 | + * @param [out] dwGateOptional Gate-weight gradient, with the shape and dtype of wGateOptional. | ||
| 47 | + * @param [out] dgammaOptional RMS-weight gradient, shape [D], FLOAT32. | ||
| 48 | + * @param [out] workspaceSize Required workspace size in bytes. | ||
| 49 | + * @param [out] executor Executor for the second-stage API. | ||
| 50 | + * gradInjectOptional, wGateOptional, injectOptional and dwGateOptional must be provided or omitted together. | ||
| 51 | + * gammaOptional and dgammaOptional must be provided or omitted together. | ||
| 52 | + * @domain aclnn_ops_infer | ||
| 53 | + */ | ||
| 54 | +ACLNN_API aclnnStatus aclnnTextGatedResidualGradGetWorkspaceSize( | ||
| 55 | + const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp, | ||
| 56 | + const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInjectOptional, | ||
| 57 | + const aclTensor* wGateOptional, const aclTensor* gammaOptional, const aclTensor* injectOptional, int64_t channels, | ||
| 58 | + aclTensor* dx, aclTensor* dwDown, aclTensor* dwUp, aclTensor* dwGateOptional, aclTensor* dgammaOptional, | ||
| 59 | + uint64_t* workspaceSize, aclOpExecutor** executor); | ||
| 60 | + | ||
| 61 | +/** | ||
| 62 | + * @brief Second-stage aclnnTextGatedResidualGrad API: execute the operator. | ||
| 63 | + */ | ||
| 64 | +ACLNN_API aclnnStatus aclnnTextGatedResidualGrad(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, | ||
| 65 | + aclrtStream stream); | ||
| 66 | + | ||
| 67 | +#ifdef __cplusplus | ||
| 68 | +} | ||
| 69 | +#endif | ||
| 70 | + | ||
| 71 | +#endif // OP_API_INC_ACLNN_TEXT_GATED_RESIDUAL_GRAD_H | ||
| @@ -0,0 +1,90 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +#include "text_gated_residual_grad.h" | ||
| 12 | +#include "opdev/op_log.h" | ||
| 13 | +#include "opdev/op_executor.h" | ||
| 14 | +#include "opdev/make_op_executor.h" | ||
| 15 | +#include "opdev/op_def.h" | ||
| 16 | +#include "opdev/op_dfx.h" | ||
| 17 | +#include "aclnn_kernels/common/op_error_check.h" | ||
| 18 | + | ||
| 19 | +using namespace op; | ||
| 20 | + | ||
| 21 | +namespace l0op { | ||
| 22 | + | ||
| 23 | +OP_TYPE_REGISTER(TextGatedResidualGrad); | ||
| 24 | + | ||
| 25 | +namespace { | ||
| 26 | + | ||
| 27 | +const std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*> TextGatedResidualGradAICore( | ||
| 28 | + const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp, | ||
| 29 | + const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInject, | ||
| 30 | + const aclTensor* wGate, const aclTensor* gamma, const aclTensor* inject, int64_t channels, aclTensor* dx, | ||
| 31 | + aclTensor* dwDown, aclTensor* dwUp, aclTensor* dwGate, aclTensor* dgamma, aclOpExecutor* executor) | ||
| 32 | +{ | ||
| 33 | + L0_DFX(TextGatedResidualGradAICore, gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInject, wGate, gamma, | ||
| 34 | + inject, dx, dwDown, dwUp, dwGate, dgamma); | ||
| 35 | + | ||
| 36 | + auto retAicore = ADD_TO_LAUNCHER_LIST_AICORE( | ||
| 37 | + TextGatedResidualGrad, | ||
| 38 | + OP_INPUT(gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInject, wGate, gamma, inject), | ||
| 39 | + OP_OUTPUT(dx, dwDown, dwUp, dwGate, dgamma), OP_ATTR(channels)); | ||
| 40 | + if (retAicore != ACLNN_SUCCESS) { | ||
| 41 | + OP_LOGE(ACLNN_ERR_PARAM_INVALID, "TextGatedResidualGrad launch kernel failed."); | ||
| 42 | + return std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*>(nullptr, nullptr, nullptr, | ||
| 43 | + nullptr, nullptr); | ||
| 44 | + } | ||
| 45 | + return std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*>(dx, dwDown, dwUp, dwGate, dgamma); | ||
| 46 | +} | ||
| 47 | + | ||
| 48 | +} // namespace | ||
| 49 | + | ||
| 50 | +const std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*> TextGatedResidualGrad( | ||
| 51 | + const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp, | ||
| 52 | + const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInject, | ||
| 53 | + const aclTensor* wGate, const aclTensor* gamma, const aclTensor* inject, int64_t channels, aclOpExecutor* executor) | ||
| 54 | +{ | ||
| 55 | + L0_DFX(TextGatedResidualGrad, gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInject, wGate, gamma, inject); | ||
| 56 | + | ||
| 57 | + // Empty tensor placeholders for omitted optional inputs/outputs. | ||
| 58 | + op::Shape emptyShape({0}); | ||
| 59 | + auto allocateEmptyTensor = [&](op::DataType dtype, op::Format format) { | ||
| 60 | + return executor->AllocTensor(emptyShape, dtype, format); | ||
| 61 | + }; | ||
| 62 | + | ||
| 63 | + auto gammaInput = gamma != nullptr ? gamma : allocateEmptyTensor(op::DataType::DT_FLOAT, op::Format::FORMAT_ND); | ||
| 64 | + auto gradInjectInput = | ||
| 65 | + gradInject != nullptr ? gradInject : allocateEmptyTensor(x->GetDataType(), x->GetViewFormat()); | ||
| 66 | + auto wGateInput = wGate != nullptr ? wGate : allocateEmptyTensor(x->GetDataType(), x->GetViewFormat()); | ||
| 67 | + auto injectInput = inject != nullptr ? inject : allocateEmptyTensor(x->GetDataType(), x->GetViewFormat()); | ||
| 68 | + | ||
| 69 | + // Allocate outputs. | ||
| 70 | + auto dx = executor->AllocTensor(x->GetViewShape(), x->GetDataType(), x->GetViewFormat()); | ||
| 71 | + auto dwDown = executor->AllocTensor(wDown->GetViewShape(), wDown->GetDataType(), wDown->GetViewFormat()); | ||
| 72 | + auto dwUp = executor->AllocTensor(wUp->GetViewShape(), wUp->GetDataType(), wUp->GetViewFormat()); | ||
| 73 | + aclTensor* dwGate = nullptr; | ||
| 74 | + if (wGate != nullptr) { | ||
| 75 | + dwGate = executor->AllocTensor(wGate->GetViewShape(), wGate->GetDataType(), wGate->GetViewFormat()); | ||
| 76 | + } else { | ||
| 77 | + dwGate = allocateEmptyTensor(x->GetDataType(), op::Format::FORMAT_ND); | ||
| 78 | + } | ||
| 79 | + aclTensor* dgamma = nullptr; | ||
| 80 | + if (gamma != nullptr) { | ||
| 81 | + dgamma = executor->AllocTensor(gamma->GetViewShape(), op::DataType::DT_FLOAT, gamma->GetViewFormat()); | ||
| 82 | + } else { | ||
| 83 | + dgamma = allocateEmptyTensor(op::DataType::DT_FLOAT, op::Format::FORMAT_ND); | ||
| 84 | + } | ||
| 85 | + | ||
| 86 | + return TextGatedResidualGradAICore(gradMixed, x, wDown, wUp, invRms, downPreAct, up, gradInjectInput, wGateInput, | ||
| 87 | + gammaInput, injectInput, channels, dx, dwDown, dwUp, dwGate, dgamma, executor); | ||
| 88 | +} | ||
| 89 | + | ||
| 90 | +} // namespace l0op | ||
| @@ -0,0 +1,39 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file text_gated_residual_grad.h | ||
| 13 | + * \brief TextGatedResidualGrad L0 kernel interface. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#ifndef OP_API_INC_LEVEL0_OP_TEXT_GATED_RESIDUAL_GRAD_H | ||
| 17 | +#define OP_API_INC_LEVEL0_OP_TEXT_GATED_RESIDUAL_GRAD_H | ||
| 18 | + | ||
| 19 | +#include <tuple> | ||
| 20 | + | ||
| 21 | +#include "opdev/op_executor.h" | ||
| 22 | + | ||
| 23 | +namespace l0op { | ||
| 24 | + | ||
| 25 | +/** | ||
| 26 | + * @brief Allocate output tensors and enqueue TextGatedResidualGrad on the executor. | ||
| 27 | + * Inputs follow the OpDef order: required tensors first, followed by optional tensors. | ||
| 28 | + * The gate group (gradInject, wGate, inject) is provided or omitted together. | ||
| 29 | + * @return (dx, dwDown, dwUp, dwGate, dgamma), with empty tensors for omitted outputs. | ||
| 30 | + * Weight gradients use the corresponding weight dtype; dgamma uses FLOAT32. | ||
| 31 | + */ | ||
| 32 | +const std::tuple<aclTensor*, aclTensor*, aclTensor*, aclTensor*, aclTensor*> TextGatedResidualGrad( | ||
| 33 | + const aclTensor* gradMixed, const aclTensor* x, const aclTensor* wDown, const aclTensor* wUp, | ||
| 34 | + const aclTensor* invRms, const aclTensor* downPreAct, const aclTensor* up, const aclTensor* gradInject, | ||
| 35 | + const aclTensor* wGate, const aclTensor* gamma, const aclTensor* inject, int64_t channels, aclOpExecutor* executor); | ||
| 36 | + | ||
| 37 | +} // namespace l0op | ||
| 38 | + | ||
| 39 | +#endif // OP_API_INC_LEVEL0_OP_TEXT_GATED_RESIDUAL_GRAD_H | ||
Amhc/text_gated_residual_grad/op_host/op_tiling/arch35/text_gated_residual_grad_tiling_arch35.cpp+573-0
| @@ -0,0 +1,573 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#include <cstdint> | ||
| 11 | +#include <initializer_list> | ||
| 12 | +#include <limits> | ||
| 13 | +#include <vector> | ||
| 14 | +#include "register/op_def_registry.h" | ||
| 15 | +#include "op_host/tiling_templates_registry.h" | ||
| 16 | +#include "op_host/tiling_base.h" | ||
| 17 | +#include "platform/platform_info.h" | ||
| 18 | +#include "log/log.h" | ||
| 19 | +#include "../text_gated_residual_grad_tiling.h" | ||
| 20 | +#include "../../../op_kernel/arch35/text_gated_residual_grad_tiling_data_arch35.h" | ||
| 21 | +#include "../../../op_kernel/arch35/text_gated_residual_grad_tiling_key_arch35.h" | ||
| 22 | +#include "../../../op_kernel/arch35/text_gated_residual_grad_pipeline_arch35.h" | ||
| 23 | + | ||
| 24 | +namespace optiling { | ||
| 25 | +namespace { | ||
| 26 | +namespace hcgrad = TextGatedResidualGrad; | ||
| 27 | +constexpr size_t INPUT_RANK = 3; | ||
| 28 | +constexpr size_t MATRIX_RANK = 2; | ||
| 29 | +constexpr size_t BATCH_DIM = 0; | ||
| 30 | +constexpr size_t SEQUENCE_DIM = 1; | ||
| 31 | +constexpr size_t FEATURE_DIM = 2; | ||
| 32 | +constexpr size_t MATRIX_ROW_DIM = 0; | ||
| 33 | +constexpr size_t MATRIX_COLUMN_DIM = 1; | ||
| 34 | +constexpr int64_t CHANNEL_SHAPE_LIMIT = 4096; | ||
| 35 | +constexpr int64_t FLOAT_BYTES = sizeof(float); | ||
| 36 | +constexpr int64_t SPLIT_K_CORE_WAVES = 4; | ||
| 37 | +constexpr int64_t SPLIT_K_TARGET_ELEMENTS = 1152; | ||
| 38 | +constexpr uint32_t BATCH_SCHEDULE_MODE = 1U; | ||
| 39 | +constexpr size_t WORKSPACE_COUNT = 1; | ||
| 40 | +constexpr size_t USER_WORKSPACE_INDEX = 0; | ||
| 41 | +constexpr uint32_t TILING_TEMPLATE_PRIORITY = 0; | ||
| 42 | + | ||
| 43 | +constexpr size_t IDX_GRAD_MIXED = 0; | ||
| 44 | +constexpr size_t IDX_X = 1; | ||
| 45 | +constexpr size_t IDX_W_DOWN = 2; | ||
| 46 | +constexpr size_t IDX_W_UP = 3; | ||
| 47 | +constexpr size_t IDX_INV_RMS = 4; | ||
| 48 | +constexpr size_t IDX_DOWN_PRE = 5; | ||
| 49 | +constexpr size_t IDX_UP = 6; | ||
| 50 | +constexpr size_t IDX_GRAD_INJECT = 7; | ||
| 51 | +constexpr size_t IDX_W_GATE = 8; | ||
| 52 | +constexpr size_t IDX_GAMMA = 9; | ||
| 53 | +constexpr size_t IDX_INJECT = 10; | ||
| 54 | +constexpr size_t IDX_ATTR_CHANNELS = 0; | ||
| 55 | +static_assert(IDX_X == HC_GRAD_TPL_INPUT_X, "Template dtype binding must use the registered x input."); | ||
| 56 | +constexpr size_t OUT_DW_GATE = 3; | ||
| 57 | +constexpr size_t OUT_DGAMMA = 4; | ||
| 58 | + | ||
| 59 | +// Shared tiling for A5 (DAV_3510) and A2/A3 (DAV_2201), with platform-specific Cube tiles, UB budgets and transpose | ||
| 60 | +// scratch. | ||
| 61 | +const std::vector<int32_t> supportedNpuArch = {static_cast<int32_t>(NpuArch::DAV_2201), | ||
| 62 | + static_cast<int32_t>(NpuArch::DAV_3510)}; | ||
| 63 | + | ||
| 64 | +ge::graphStatus CheckInputShape(gert::TilingContext* context, size_t irIndex, std::initializer_list<int64_t> dims, | ||
| 65 | + const char* name, bool optional = false) | ||
| 66 | +{ | ||
| 67 | + auto shape = optional ? context->GetOptionalInputShape(irIndex) : context->GetRequiredInputShape(irIndex); | ||
| 68 | + if (shape == nullptr) { | ||
| 69 | + OP_LOGE(context, "%s shape is null.", name); | ||
| 70 | + return ge::GRAPH_FAILED; | ||
| 71 | + } | ||
| 72 | + const auto& storage = shape->GetStorageShape(); | ||
| 73 | + if (storage.GetDimNum() != dims.size()) { | ||
| 74 | + OP_LOGE(context, "%s rank should be %zu, but got %zu.", name, dims.size(), storage.GetDimNum()); | ||
| 75 | + return ge::GRAPH_FAILED; | ||
| 76 | + } | ||
| 77 | + size_t i = 0; | ||
| 78 | + for (int64_t expect : dims) { | ||
| 79 | + if (storage.GetDim(i) != expect) { | ||
| 80 | + OP_LOGE(context, "%s dim[%zu] should be %ld, but got %ld.", name, i, expect, storage.GetDim(i)); | ||
| 81 | + return ge::GRAPH_FAILED; | ||
| 82 | + } | ||
| 83 | + ++i; | ||
| 84 | + } | ||
| 85 | + return ge::GRAPH_SUCCESS; | ||
| 86 | +} | ||
| 87 | + | ||
| 88 | +ge::graphStatus CheckInputDtype(gert::TilingContext* context, size_t irIndex, ge::DataType expect, const char* name, | ||
| 89 | + bool optional = false) | ||
| 90 | +{ | ||
| 91 | + auto desc = optional ? context->GetOptionalInputDesc(irIndex) : context->GetRequiredInputDesc(irIndex); | ||
| 92 | + if (desc == nullptr) { | ||
| 93 | + OP_LOGE(context, "%s desc is null.", name); | ||
| 94 | + return ge::GRAPH_FAILED; | ||
| 95 | + } | ||
| 96 | + if (desc->GetDataType() != expect) { | ||
| 97 | + OP_LOGE(context, "%s dtype mismatch, expect %d but got %d.", name, static_cast<int32_t>(expect), | ||
| 98 | + static_cast<int32_t>(desc->GetDataType())); | ||
| 99 | + return ge::GRAPH_FAILED; | ||
| 100 | + } | ||
| @@ -0,0 +1,418 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +#include <cstdint> | ||
| 12 | +#include <initializer_list> | ||
| 13 | + | ||
| 14 | +#include "log/log.h" | ||
| 15 | +#include "register/op_impl_registry.h" | ||
| 16 | + | ||
| 17 | +using namespace ge; | ||
| 18 | + | ||
| 19 | +namespace ops { | ||
| 20 | +namespace { | ||
| 21 | +constexpr size_t INPUT_RANK = 3; | ||
| 22 | +constexpr size_t MATRIX_RANK = 2; | ||
| 23 | +constexpr size_t VECTOR_RANK = 1; | ||
| 24 | +constexpr size_t FIRST_DIM = 0; | ||
| 25 | +constexpr size_t SECOND_DIM = 1; | ||
| 26 | +constexpr size_t FEATURE_DIM = 2; | ||
| 27 | +constexpr size_t CHANNELS_ATTR_INDEX = 0; | ||
| 28 | +constexpr size_t IDX_GRAD_MIXED = 0; | ||
| 29 | +constexpr size_t IDX_X = 1; | ||
| 30 | +constexpr size_t IDX_W_DOWN = 2; | ||
| 31 | +constexpr size_t IDX_W_UP = 3; | ||
| 32 | +constexpr size_t IDX_INV_RMS = 4; | ||
| 33 | +constexpr size_t IDX_DOWN_PRE = 5; | ||
| 34 | +constexpr size_t IDX_UP = 6; | ||
| 35 | +constexpr size_t IDX_GRAD_INJECT = 7; | ||
| 36 | +constexpr size_t IDX_W_GATE = 8; | ||
| 37 | +constexpr size_t IDX_GAMMA = 9; | ||
| 38 | +constexpr size_t IDX_INJECT = 10; | ||
| 39 | + | ||
| 40 | +constexpr size_t OUT_DX = 0; | ||
| 41 | +constexpr size_t OUT_DW_DOWN = 1; | ||
| 42 | +constexpr size_t OUT_DW_UP = 2; | ||
| 43 | +constexpr size_t OUT_DW_GATE = 3; | ||
| 44 | +constexpr size_t OUT_DGAMMA = 4; | ||
| 45 | + | ||
| 46 | +constexpr int64_t UNKNOWN_DIM = -1LL; | ||
| 47 | +constexpr int64_t UNKNOWN_RANK = -2LL; | ||
| 48 | + | ||
| 49 | +bool IsUnknownRank(const gert::Shape* shape) | ||
| 50 | +{ | ||
| 51 | + return shape != nullptr && shape->GetDimNum() == VECTOR_RANK && shape->GetDim(FIRST_DIM) == UNKNOWN_RANK; | ||
| 52 | +} | ||
| 53 | + | ||
| 54 | +bool IsEmptyShape(const gert::Shape* shape) | ||
| 55 | +{ | ||
| 56 | + return shape == nullptr || (shape->GetDimNum() == VECTOR_RANK && shape->GetDim(FIRST_DIM) == 0); | ||
| 57 | +} | ||
| 58 | + | ||
| 59 | +bool DimEqualOrUnknown(int64_t actual, int64_t expect) | ||
| 60 | +{ | ||
| 61 | + // Only known, unequal dimensions are incompatible; an unknown dimension remains available for dynamic inference. | ||
| 62 | + return actual == expect || actual == UNKNOWN_DIM || expect == UNKNOWN_DIM; | ||
| 63 | +} | ||
| 64 | + | ||
| 65 | +gert::Shape* GetOutputShapeByIrIndex(gert::InferShapeContext* context, size_t irIndex) | ||
| 66 | +{ | ||
| 67 | + const auto* instance = context->GetIrOutputInstanceInfo(irIndex); | ||
| 68 | + if (instance == nullptr || instance->GetInstanceNum() == 0) { | ||
| 69 | + return nullptr; | ||
| 70 | + } | ||
| 71 | + return context->GetOutputShape(instance->GetInstanceStart()); | ||
| 72 | +} | ||
| 73 | + | ||
| 74 | +void SetUnknownRank(gert::Shape* shape) | ||
| 75 | +{ | ||
| 76 | + if (shape == nullptr) { | ||
| 77 | + return; | ||
| 78 | + } | ||
| 79 | + shape->SetDimNum(0); | ||
| 80 | + shape->AppendDim(UNKNOWN_RANK); | ||
| 81 | +} | ||
| 82 | + | ||
| 83 | +void CopyShape(const gert::Shape* src, gert::Shape* dst) | ||
| 84 | +{ | ||
| 85 | + if (src == nullptr || dst == nullptr) { | ||
| 86 | + return; | ||
| 87 | + } | ||
| 88 | + dst->SetDimNum(src->GetDimNum()); | ||
| 89 | + for (size_t i = 0; i < src->GetDimNum(); ++i) { | ||
| 90 | + dst->SetDim(i, src->GetDim(i)); | ||
| 91 | + } | ||
| 92 | +} | ||
| 93 | + | ||
| 94 | +graphStatus CheckRank(const gert::Shape* shape, size_t rank, const char* name, gert::InferShapeContext* context) | ||
| 95 | +{ | ||
| 96 | + if (shape->GetDimNum() != rank) { | ||
| 97 | + OP_LOGE(context->GetNodeName(), "%s rank should be %zu, but got %zu.", name, rank, shape->GetDimNum()); | ||
| 98 | + return GRAPH_FAILED; | ||
| 99 | + } | ||
| 100 | + return GRAPH_SUCCESS; | ||
| 101 | +} | ||
| 102 | + | ||
| 103 | +graphStatus CheckDim(const gert::Shape* shape, size_t idx, int64_t expect, const char* name, | ||
| 104 | + gert::InferShapeContext* context) | ||
| 105 | +{ | ||
| 106 | + const int64_t actual = shape->GetDim(idx); | ||
| 107 | + if (!DimEqualOrUnknown(actual, expect)) { | ||
| 108 | + OP_LOGE(context->GetNodeName(), "%s dim[%zu] should be %ld, but got %ld.", name, idx, expect, actual); | ||
| 109 | + return GRAPH_FAILED; | ||
| 110 | + } | ||
| 111 | + return GRAPH_SUCCESS; | ||
| 112 | +} | ||
| 113 | + | ||
| 114 | +graphStatus CheckShape3(const gert::Shape* shape, int64_t d0, int64_t d1, int64_t d2, const char* name, | ||
| 115 | + gert::InferShapeContext* context) | ||
| 116 | +{ | ||
| 117 | + if (CheckRank(shape, INPUT_RANK, name, context) != GRAPH_SUCCESS) { | ||
| 118 | + return GRAPH_FAILED; | ||
| 119 | + } | ||
| 120 | + if (CheckDim(shape, FIRST_DIM, d0, name, context) != GRAPH_SUCCESS || | ||
| 121 | + CheckDim(shape, SECOND_DIM, d1, name, context) != GRAPH_SUCCESS || | ||
| 122 | + CheckDim(shape, FEATURE_DIM, d2, name, context) != GRAPH_SUCCESS) { | ||
| 123 | + return GRAPH_FAILED; | ||
| 124 | + } | ||
| 125 | + return GRAPH_SUCCESS; | ||
| 126 | +} | ||
| 127 | + | ||
| 128 | +graphStatus CheckShape2(const gert::Shape* shape, int64_t d0, int64_t d1, const char* name, | ||
| 129 | + gert::InferShapeContext* context) | ||
| 130 | +{ | ||
| 131 | + if (CheckRank(shape, MATRIX_RANK, name, context) != GRAPH_SUCCESS) { | ||
| 132 | + return GRAPH_FAILED; | ||
| 133 | + } | ||
| 134 | + if (CheckDim(shape, FIRST_DIM, d0, name, context) != GRAPH_SUCCESS || | ||
| 135 | + CheckDim(shape, SECOND_DIM, d1, name, context) != GRAPH_SUCCESS) { | ||
| 136 | + return GRAPH_FAILED; | ||
| 137 | + } | ||
| 138 | + return GRAPH_SUCCESS; | ||
| 139 | +} | ||
| 140 | + | ||
| 141 | +class TextGatedResidualGradShapeInference { | ||
| 142 | +public: | ||
| 143 | + explicit TextGatedResidualGradShapeInference(gert::InferShapeContext* ctx) | ||
| 144 | + : context(ctx) | ||
| 145 | + {} | ||
| 146 | + | ||
| 147 | + graphStatus Infer() | ||
| 148 | + { | ||
| 149 | + if (LoadShapes() != GRAPH_SUCCESS || CheckAttributes() != GRAPH_SUCCESS) { | ||
| 150 | + return GRAPH_FAILED; | ||
| 151 | + } | ||
| 152 | + if (PropagateUnknownRank()) { | ||
| 153 | + return GRAPH_SUCCESS; | ||
| 154 | + } | ||
| 155 | + if (ReadDimensions() != GRAPH_SUCCESS || CheckKnownShapes() != GRAPH_SUCCESS) { | ||
| 156 | + return GRAPH_FAILED; | ||
| 157 | + } | ||
| 158 | + return SetOutputShapes(); | ||
| 159 | + } | ||
| 160 | + | ||
| 161 | +private: | ||
| 162 | + graphStatus LoadShapes() | ||
| 163 | + { | ||
| 164 | + gradMixedShape = context->GetRequiredInputShape(IDX_GRAD_MIXED); | ||
| 165 | + xShape = context->GetRequiredInputShape(IDX_X); | ||
| 166 | + wDownShape = context->GetRequiredInputShape(IDX_W_DOWN); | ||
| 167 | + wUpShape = context->GetRequiredInputShape(IDX_W_UP); | ||
| 168 | + invRmsShape = context->GetRequiredInputShape(IDX_INV_RMS); | ||
| 169 | + downPreShape = context->GetRequiredInputShape(IDX_DOWN_PRE); | ||
| 170 | + upShape = context->GetRequiredInputShape(IDX_UP); | ||
| 171 | + gradInjectShape = context->GetOptionalInputShape(IDX_GRAD_INJECT); | ||
| 172 | + wGateShape = context->GetOptionalInputShape(IDX_W_GATE); | ||
| 173 | + gammaShape = context->GetOptionalInputShape(IDX_GAMMA); | ||
| 174 | + injectShape = context->GetOptionalInputShape(IDX_INJECT); | ||
| 175 | + | ||
| 176 | + OP_CHECK_NULL_WITH_CONTEXT(context, gradMixedShape); | ||
| 177 | + OP_CHECK_NULL_WITH_CONTEXT(context, xShape); | ||
| 178 | + OP_CHECK_NULL_WITH_CONTEXT(context, wDownShape); | ||
| 179 | + OP_CHECK_NULL_WITH_CONTEXT(context, wUpShape); | ||
| 180 | + OP_CHECK_NULL_WITH_CONTEXT(context, invRmsShape); | ||
| 181 | + OP_CHECK_NULL_WITH_CONTEXT(context, downPreShape); | ||
| 182 | + OP_CHECK_NULL_WITH_CONTEXT(context, upShape); | ||
| 183 | + | ||
| 184 | + dxShape = GetOutputShapeByIrIndex(context, OUT_DX); | ||
| 185 | + dwDownShape = GetOutputShapeByIrIndex(context, OUT_DW_DOWN); | ||
| 186 | + dwUpShape = GetOutputShapeByIrIndex(context, OUT_DW_UP); | ||
| 187 | + dwGateShape = GetOutputShapeByIrIndex(context, OUT_DW_GATE); | ||
| 188 | + dgammaShape = GetOutputShapeByIrIndex(context, OUT_DGAMMA); | ||
| 189 | + OP_CHECK_NULL_WITH_CONTEXT(context, dxShape); | ||
| 190 | + OP_CHECK_NULL_WITH_CONTEXT(context, dwDownShape); | ||
| 191 | + OP_CHECK_NULL_WITH_CONTEXT(context, dwUpShape); | ||
| 192 | + | ||
| 193 | + return GRAPH_SUCCESS; | ||
| 194 | + } | ||
| 195 | + | ||
| 196 | + graphStatus CheckAttributes() | ||
| 197 | + { | ||
| 198 | + const auto* attrs = context->GetAttrs(); | ||
| 199 | + OP_CHECK_NULL_WITH_CONTEXT(context, attrs); | ||
| 200 | + const int64_t* channelsPtr = attrs->GetAttrPointer<int64_t>(CHANNELS_ATTR_INDEX); | ||
| 201 | + if (channelsPtr == nullptr) { | ||
| 202 | + OP_LOGE(context->GetNodeName(), "channels attr is null."); | ||
| 203 | + return GRAPH_FAILED; | ||
| 204 | + } | ||
| 205 | + c = *channelsPtr; | ||
| 206 | + if (c <= 0) { | ||
| 207 | + OP_LOGE(context->GetNodeName(), "channels should be positive, but got %ld.", c); | ||
| 208 | + return GRAPH_FAILED; | ||
| 209 | + } | ||
| 210 | + hasGate = !IsEmptyShape(wGateShape); | ||
X C01|高|
缺少 **建议:**在 InferShape 和 Tiling 中同时校验 gate 输入组: ![]() ![]() | |||
| 211 | + if (hasGate && dwGateShape == nullptr) { | ||
| 212 | + OP_LOGE(context->GetNodeName(), "dw_gate output is required when w_gate is present."); | ||
| 213 | + return GRAPH_FAILED; | ||
| 214 | + } | ||
| 215 | + if (hasGate && (gradInjectShape == nullptr || injectShape == nullptr)) { | ||
| 216 | + OP_LOGE(context->GetNodeName(), "grad_inject and inject are required when w_gate is present."); | ||
| 217 | + return GRAPH_FAILED; | ||
| 218 | + } | ||
| 219 | + | ||
| 220 | + hasGamma = !IsEmptyShape(gammaShape); | ||
| 221 | + if (hasGamma && dgammaShape == nullptr) { | ||
| 222 | + OP_LOGE(context->GetNodeName(), "dgamma output is required when gamma is present."); | ||
| 223 | + return GRAPH_FAILED; | ||
| 224 | + } | ||
| 225 | + | ||
| 226 | + return GRAPH_SUCCESS; | ||
| 227 | + } | ||
| 228 | + | ||
| 229 | + bool PropagateUnknownRank() | ||
| 230 | + { | ||
| 231 | + const gert::Shape* allInputs[] = {gradMixedShape, gradInjectShape, xShape, wDownShape, | ||
| 232 | + wUpShape, wGateShape, gammaShape, invRmsShape, | ||
| 233 | + injectShape, downPreShape, upShape}; | ||
| 234 | + for (const auto* shape : allInputs) { | ||
| 235 | + if (shape != nullptr && IsUnknownRank(shape)) { | ||
| 236 | + SetUnknownRank(dxShape); | ||
| 237 | + SetUnknownRank(dwDownShape); | ||
| 238 | + SetUnknownRank(dwUpShape); | ||
| 239 | + SetUnknownRank(dwGateShape); | ||
| 240 | + if (dgammaShape != nullptr) { | ||
| 241 | + SetUnknownRank(dgammaShape); | ||
| 242 | + } | ||
| 243 | + return true; | ||
| 244 | + } | ||
| 245 | + } | ||
| 246 | + | ||
| 247 | + return false; | ||
| 248 | + } | ||
| 249 | + | ||
| 250 | + graphStatus ReadDimensions() | ||
| 251 | + { | ||
| 252 | + if (CheckRank(xShape, INPUT_RANK, "x", context) != GRAPH_SUCCESS || | ||
| 253 | + CheckRank(wDownShape, MATRIX_RANK, "w_down", context) != GRAPH_SUCCESS) { | ||
| 254 | + return GRAPH_FAILED; | ||
| 255 | + } | ||
| 256 | + d = xShape->GetDim(FEATURE_DIM); | ||
| 257 | + if (d != UNKNOWN_DIM && (d <= 0 || d % c != 0)) { | ||
| 258 | + OP_LOGE(context->GetNodeName(), "x last dim %ld is not divisible by channels %ld.", d, c); | ||
| 259 | + return GRAPH_FAILED; | ||
| 260 | + } | ||
| 261 | + h = (d == UNKNOWN_DIM) ? UNKNOWN_DIM : d / c; | ||
| 262 | + r = wDownShape->GetDim(FIRST_DIM); | ||
| 263 | + if (r != UNKNOWN_DIM && r <= 0) { | ||
| 264 | + OP_LOGE(context->GetNodeName(), "R must be positive or unknown, but got %ld.", r); | ||
| 265 | + return GRAPH_FAILED; | ||
| 266 | + } | ||
| 267 | + b = xShape->GetDim(FIRST_DIM); | ||
| 268 | + s = xShape->GetDim(SECOND_DIM); | ||
| 269 | + if ((b < 0 && b != UNKNOWN_DIM) || (s < 0 && s != UNKNOWN_DIM)) { | ||
| 270 | + OP_LOGE(context->GetNodeName(), "B/S must be non-negative or unknown, but got B=%ld, S=%ld.", b, s); | ||
| 271 | + return GRAPH_FAILED; | ||
| 272 | + } | ||
| 273 | + return GRAPH_SUCCESS; | ||
| 274 | + } | ||
| 275 | + | ||
| 276 | + graphStatus CheckKnownShapes() | ||
| 277 | + { | ||
| 278 | + if (CheckShape3(gradMixedShape, b, s, h, "grad_mixed", context) != GRAPH_SUCCESS || | ||
| 279 | + CheckShape3(xShape, b, s, d, "x", context) != GRAPH_SUCCESS || | ||
| 280 | + CheckShape2(wDownShape, r, d, "w_down", context) != GRAPH_SUCCESS || | ||
| 281 | + CheckShape2(wUpShape, d, r, "w_up", context) != GRAPH_SUCCESS || | ||
| 282 | + CheckShape3(invRmsShape, b, s, c, "inv_rms", context) != GRAPH_SUCCESS || | ||
| 283 | + CheckShape3(downPreShape, b, s, r, "down_pre_act", context) != GRAPH_SUCCESS || | ||
| 284 | + CheckShape3(upShape, b, s, d, "up", context) != GRAPH_SUCCESS) { | ||
| 285 | + return GRAPH_FAILED; | ||
| 286 | + } | ||
| 287 | + | ||
| 288 | + if (hasGate && (CheckShape2(wGateShape, c, d, "w_gate", context) != GRAPH_SUCCESS || | ||
| 289 | + CheckShape3(gradInjectShape, b, s, c, "grad_inject", context) != GRAPH_SUCCESS || | ||
| 290 | + CheckShape3(injectShape, b, s, c, "inject", context) != GRAPH_SUCCESS)) { | ||
| 291 | + return GRAPH_FAILED; | ||
| 292 | + } | ||
| 293 | + | ||
| 294 | + if (hasGamma && (CheckRank(gammaShape, VECTOR_RANK, "gamma", context) != GRAPH_SUCCESS || | ||
| 295 | + CheckDim(gammaShape, FIRST_DIM, d, "gamma", context) != GRAPH_SUCCESS)) { | ||
| 296 | + return GRAPH_FAILED; | ||
| 297 | + } | ||
| 298 | + | ||
| 299 | + return GRAPH_SUCCESS; | ||
| 300 | + } | ||
| 301 | + | ||
| 302 | + graphStatus SetOutputShapes() | ||
| 303 | + { | ||
| 304 | + CopyShape(xShape, dxShape); | ||
| 305 | + CopyShape(wDownShape, dwDownShape); | ||
| 306 | + CopyShape(wUpShape, dwUpShape); | ||
| 307 | + if (dwGateShape != nullptr) { | ||
| 308 | + if (hasGate) { | ||
| 309 | + CopyShape(wGateShape, dwGateShape); | ||
| 310 | + } else { | ||
| 311 | + dwGateShape->SetDimNum(1); | ||
| 312 | + dwGateShape->SetDim(0, 0); | ||
| 313 | + } | ||
| 314 | + } | ||
| 315 | + if (dgammaShape != nullptr) { | ||
| 316 | + if (hasGamma) { | ||
| 317 | + CopyShape(gammaShape, dgammaShape); | ||
| 318 | + } else { | ||
| 319 | + dgammaShape->SetDimNum(1); | ||
| 320 | + dgammaShape->SetDim(0, 0); | ||
| 321 | + } | ||
| 322 | + } | ||
| 323 | + return GRAPH_SUCCESS; | ||
| 324 | + } | ||
| 325 | + | ||
| 326 | + gert::InferShapeContext* context; | ||
| 327 | + const gert::Shape* gradMixedShape = nullptr; | ||
| 328 | + const gert::Shape* xShape = nullptr; | ||
| 329 | + const gert::Shape* wDownShape = nullptr; | ||
| 330 | + const gert::Shape* wUpShape = nullptr; | ||
| 331 | + const gert::Shape* invRmsShape = nullptr; | ||
| 332 | + const gert::Shape* downPreShape = nullptr; | ||
| 333 | + const gert::Shape* upShape = nullptr; | ||
| 334 | + const gert::Shape* gradInjectShape = nullptr; | ||
| 335 | + const gert::Shape* wGateShape = nullptr; | ||
| 336 | + const gert::Shape* gammaShape = nullptr; | ||
| 337 | + const gert::Shape* injectShape = nullptr; | ||
| 338 | + gert::Shape* dxShape = nullptr; | ||
| 339 | + gert::Shape* dwDownShape = nullptr; | ||
| 340 | + gert::Shape* dwUpShape = nullptr; | ||
| 341 | + gert::Shape* dwGateShape = nullptr; | ||
| 342 | + gert::Shape* dgammaShape = nullptr; | ||
| 343 | + int64_t b = 0; | ||
| 344 | + int64_t s = 0; | ||
| 345 | + int64_t c = 0; | ||
| 346 | + int64_t d = 0; | ||
| 347 | + int64_t h = 0; | ||
| 348 | + int64_t r = 0; | ||
| 349 | + bool hasGate = false; | ||
| 350 | + bool hasGamma = false; | ||
| 351 | +}; | ||
| 352 | + | ||
| 353 | +ge::graphStatus InferShapeForTextGatedResidualGrad(gert::InferShapeContext* context) | ||
| 354 | +{ | ||
| 355 | + return TextGatedResidualGradShapeInference(context).Infer(); | ||
| 356 | +} | ||
| 357 | + | ||
| 358 | +graphStatus SetOutputDtype(gert::InferDataTypeContext* context, size_t irIndex, ge::DataType dtype) | ||
| 359 | +{ | ||
| 360 | + const auto* instance = context->GetIrOutputInstanceInfo(irIndex); | ||
| 361 | + if (instance == nullptr || instance->GetInstanceNum() == 0) { | ||
| 362 | + return ge::GRAPH_SUCCESS; | ||
| 363 | + } | ||
| 364 | + return context->SetOutputDataType(instance->GetInstanceStart(), dtype); | ||
| 365 | +} | ||
| 366 | + | ||
| 367 | +ge::graphStatus InferDataTypeForTextGatedResidualGrad(gert::InferDataTypeContext* context) | ||
| 368 | +{ | ||
| 369 | + if (context == nullptr) { | ||
| 370 | + return GRAPH_FAILED; | ||
| 371 | + } | ||
| 372 | + const ge::DataType xDtype = context->GetRequiredInputDataType(IDX_X); | ||
| 373 | + if (xDtype != ge::DT_BF16 && xDtype != ge::DT_FLOAT16) { | ||
| 374 | + OP_LOGE(context->GetNodeName(), "x dtype only supports bf16/fp16, but got %d.", static_cast<int32_t>(xDtype)); | ||
| 375 | + return GRAPH_FAILED; | ||
| 376 | + } | ||
| 377 | + const size_t requiredSameDtypeInputs[] = {IDX_GRAD_MIXED, IDX_W_DOWN, IDX_W_UP, IDX_DOWN_PRE, IDX_UP}; | ||
| 378 | + for (size_t idx : requiredSameDtypeInputs) { | ||
| 379 | + if (context->GetRequiredInputDataType(idx) != xDtype) { | ||
| 380 | + OP_LOGE(context->GetNodeName(), "input %zu dtype should match x.", idx); | ||
| 381 | + return GRAPH_FAILED; | ||
| 382 | + } | ||
| 383 | + } | ||
| 384 | + const size_t optionalSameDtypeInputs[] = {IDX_GRAD_INJECT, IDX_W_GATE, IDX_INJECT}; | ||
| 385 | + for (size_t idx : optionalSameDtypeInputs) { | ||
| 386 | + const ge::DataType dtype = context->GetOptionalInputDataType(idx); | ||
| 387 | + if (dtype != ge::DT_UNDEFINED && dtype != xDtype) { | ||
| 388 | + OP_LOGE(context->GetNodeName(), "input %zu dtype should match x.", idx); | ||
| 389 | + return GRAPH_FAILED; | ||
| 390 | + } | ||
| 391 | + } | ||
| 392 | + const ge::DataType gammaDtype = context->GetOptionalInputDataType(IDX_GAMMA); | ||
| 393 | + if (gammaDtype != ge::DT_UNDEFINED && gammaDtype != ge::DT_FLOAT) { | ||
| 394 | + OP_LOGE(context->GetNodeName(), "gamma dtype only supports float32."); | ||
| 395 | + return GRAPH_FAILED; | ||
| 396 | + } | ||
| 397 | + if (context->GetRequiredInputDataType(IDX_INV_RMS) != ge::DT_FLOAT) { | ||
| 398 | + OP_LOGE(context->GetNodeName(), "inv_rms dtype only supports float32."); | ||
| 399 | + return GRAPH_FAILED; | ||
| 400 | + } | ||
| 401 | + | ||
| 402 | + if (SetOutputDtype(context, OUT_DX, xDtype) != ge::GRAPH_SUCCESS || | ||
| 403 | + SetOutputDtype(context, OUT_DW_DOWN, xDtype) != ge::GRAPH_SUCCESS || | ||
| 404 | + SetOutputDtype(context, OUT_DW_UP, xDtype) != ge::GRAPH_SUCCESS || | ||
| 405 | + SetOutputDtype(context, OUT_DW_GATE, xDtype) != ge::GRAPH_SUCCESS || | ||
| 406 | + SetOutputDtype(context, OUT_DGAMMA, ge::DT_FLOAT) != ge::GRAPH_SUCCESS) { | ||
| 407 | + OP_LOGE(context->GetNodeName(), "failed to set output dtype."); | ||
| 408 | + return GRAPH_FAILED; | ||
| 409 | + } | ||
| 410 | + return GRAPH_SUCCESS; | ||
| 411 | +} | ||
| 412 | +} // namespace | ||
| 413 | + | ||
| 414 | +IMPL_OP_INFERSHAPE(TextGatedResidualGrad) | ||
| 415 | + .InferShape(InferShapeForTextGatedResidualGrad) | ||
| 416 | + .InferDataType(InferDataTypeForTextGatedResidualGrad); | ||
| 417 | + | ||
| 418 | +} // namespace ops | ||
| @@ -0,0 +1,47 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +#include "text_gated_residual_grad_tiling.h" | ||
| 12 | +#include "register/op_def_registry.h" | ||
| 13 | +#include "op_host/tiling_base.h" | ||
| 14 | +#include "op_host/tiling_templates_registry.h" | ||
| 15 | +#include "log/log.h" | ||
| 16 | +#include "kernel_tiling/kernel_tiling.h" | ||
| 17 | + | ||
| 18 | +using Ops::Transformer::OpTiling::TilingRegistryNew; | ||
| 19 | +using Ops::Transformer::OpTiling::TilingRegistryArch; | ||
| 20 | + | ||
| 21 | +namespace optiling { | ||
| 22 | + | ||
| 23 | +static ge::graphStatus Tiling4TextGatedResidualGrad(gert::TilingContext* context) | ||
| 24 | +{ | ||
| 25 | + auto platformInfo = context->GetPlatformInfo(); | ||
| 26 | + OP_CHECK_NULL_WITH_CONTEXT(context, platformInfo); | ||
| 27 | + platform_ascendc::PlatformAscendC ascendcPlatform(platformInfo); | ||
| 28 | + NpuArch npuArch = ascendcPlatform.GetCurNpuArch(); | ||
| 29 | + if (npuArch == NpuArch::DAV_3510 || npuArch == NpuArch::DAV_2201) { | ||
| 30 | + return TilingRegistryArch::GetInstance().DoTilingImpl(context); | ||
| 31 | + } | ||
| 32 | + OP_LOGE(context, | ||
| 33 | + "TextGatedResidualGrad: only DAV_3510 (ascend950) and DAV_2201 (ascend910b/910_93) are supported."); | ||
| 34 | + return ge::GRAPH_FAILED; | ||
| 35 | +} | ||
| 36 | + | ||
| 37 | +static ge::graphStatus TilingPrepareForTextGatedResidualGrad(gert::TilingParseContext* context) | ||
| 38 | +{ | ||
| 39 | + (void)context; | ||
| 40 | + return ge::GRAPH_SUCCESS; | ||
| 41 | +} | ||
| 42 | + | ||
| 43 | +IMPL_OP_OPTILING(TextGatedResidualGrad) | ||
| 44 | + .Tiling(Tiling4TextGatedResidualGrad) | ||
| 45 | + .TilingParse<TextGatedResidualGradCompileInfo>(TilingPrepareForTextGatedResidualGrad); | ||
| 46 | + | ||
| 47 | +} // namespace optiling | ||
| @@ -0,0 +1,25 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file text_gated_residual_grad_tiling.h | ||
| 13 | + * \brief TextGatedResidualGrad compile-time tiling information. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#pragma once | ||
| 17 | + | ||
| 18 | +#include <cstdint> | ||
| 19 | + | ||
| 20 | +namespace optiling { | ||
| 21 | +struct TextGatedResidualGradCompileInfo { | ||
| 22 | + uint32_t aicNum{0}; | ||
| 23 | + uint32_t aivNum{0}; | ||
| 24 | +}; | ||
| 25 | +} // namespace optiling | ||
| @@ -0,0 +1,129 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +#include "register/op_def_registry.h" | ||
| 12 | + | ||
| 13 | +namespace ops { | ||
| 14 | +class TextGatedResidualGrad : public OpDef { | ||
| 15 | +public: | ||
| 16 | + explicit TextGatedResidualGrad(const char* name) | ||
| 17 | + : OpDef(name) | ||
| 18 | + { | ||
改一下写法 ![]() ![]() | |||
| 19 | + this->Input("grad_mixed") | ||
| 20 | + .ParamType(REQUIRED) | ||
| 21 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 22 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 23 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 24 | + .AutoContiguous(); | ||
| 25 | + this->Input("x") | ||
| 26 | + .ParamType(REQUIRED) | ||
| 27 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 28 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 29 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 30 | + .AutoContiguous(); | ||
| 31 | + this->Input("w_down") | ||
| 32 | + .ParamType(REQUIRED) | ||
| 33 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 34 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 35 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 36 | + .AutoContiguous(); | ||
| 37 | + this->Input("w_up") | ||
| 38 | + .ParamType(REQUIRED) | ||
| 39 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 40 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 41 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 42 | + .AutoContiguous(); | ||
| 43 | + this->Input("inv_rms") | ||
| 44 | + .ParamType(REQUIRED) | ||
| 45 | + .DataType({ge::DT_FLOAT, ge::DT_FLOAT}) | ||
| 46 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 47 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 48 | + .AutoContiguous(); | ||
| 49 | + this->Input("down_pre_act") | ||
| 50 | + .ParamType(REQUIRED) | ||
| 51 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 52 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 53 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 54 | + .AutoContiguous(); | ||
| 55 | + this->Input("up") | ||
| 56 | + .ParamType(REQUIRED) | ||
| 57 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 58 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 59 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 60 | + .AutoContiguous(); | ||
| 61 | + this->Input("grad_inject") | ||
| 62 | + .ParamType(OPTIONAL) | ||
| 63 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 64 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 65 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 66 | + .AutoContiguous(); | ||
| 67 | + this->Input("w_gate") | ||
| 68 | + .ParamType(OPTIONAL) | ||
| 69 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 70 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 71 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 72 | + .AutoContiguous(); | ||
| 73 | + this->Input("gamma") | ||
| 74 | + .ParamType(OPTIONAL) | ||
| 75 | + .DataType({ge::DT_FLOAT, ge::DT_FLOAT}) | ||
| 76 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 77 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 78 | + .AutoContiguous(); | ||
| 79 | + this->Input("inject") | ||
| 80 | + .ParamType(OPTIONAL) | ||
| 81 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 82 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 83 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 84 | + .AutoContiguous(); | ||
| 85 | + this->Attr("channels").AttrType(REQUIRED).Int(); | ||
| 86 | + | ||
| 87 | + this->Output("dx") | ||
| 88 | + .ParamType(REQUIRED) | ||
| 89 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 90 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 91 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}); | ||
| 92 | + this->Output("dw_down") | ||
| 93 | + .ParamType(REQUIRED) | ||
| 94 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 95 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 96 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}); | ||
| 97 | + this->Output("dw_up") | ||
| 98 | + .ParamType(REQUIRED) | ||
| 99 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 100 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 101 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}); | ||
| 102 | + this->Output("dw_gate") | ||
| 103 | + .ParamType(OPTIONAL) | ||
| 104 | + .DataType({ge::DT_FLOAT16, ge::DT_BF16}) | ||
| 105 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 106 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}); | ||
| 107 | + this->Output("dgamma") | ||
| 108 | + .ParamType(OPTIONAL) | ||
| 109 | + .DataType({ge::DT_FLOAT, ge::DT_FLOAT}) | ||
| 110 | + .Format({ge::FORMAT_ND, ge::FORMAT_ND}) | ||
| 111 | + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND}); | ||
| 112 | + | ||
| 113 | + OpAICoreConfig config; | ||
| 114 | + config.DynamicCompileStaticFlag(true) | ||
| 115 | + .DynamicFormatFlag(true) | ||
| 116 | + .DynamicRankSupportFlag(true) | ||
| 117 | + .DynamicShapeSupportFlag(true) | ||
| 118 | + .NeedCheckSupportFlag(false) | ||
| 119 | + .PrecisionReduceFlag(true) | ||
| 120 | + .ExtendCfgInfo("aclnnSupport.value", "support_aclnn") | ||
| 121 | + .ExtendCfgInfo("opFile.value", "text_gated_residual_grad"); | ||
| 122 | + this->AICore().AddConfig("ascend950", config); | ||
| 123 | + this->AICore().AddConfig("ascend910b", config); | ||
| 124 | + this->AICore().AddConfig("ascend910_93", config); | ||
| 125 | + } | ||
| 126 | +}; | ||
| 127 | + | ||
| 128 | +OP_ADD(TextGatedResidualGrad); | ||
| 129 | +} // namespace ops | ||
| @@ -0,0 +1,280 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file text_gated_residual_grad_gemm_arch22.h | ||
| 13 | + * \brief Atlas A2/A3 CATLASS GEMM backend for TextGatedResidualGrad. | ||
| 14 | + */ | ||
| 15 | +#ifndef TEXT_GATED_RESIDUAL_GRAD_GEMM_ARCH22_H | ||
| 16 | +#define TEXT_GATED_RESIDUAL_GRAD_GEMM_ARCH22_H | ||
| 17 | + | ||
| 18 | +#include "../text_gated_residual_grad_constants.h" | ||
| 19 | + | ||
| 20 | +#if !defined(CATLASS_ARCH) || CATLASS_ARCH != HC_GRAD_ARCH_A2 | ||
| 21 | +#error "text_gated_residual_grad_gemm_arch22.h requires CATLASS_ARCH=2201" | ||
| 22 | +#endif | ||
| 23 | + | ||
| 24 | +#ifndef K_MAX_SHAPE_DIM | ||
| 25 | +#define K_MAX_SHAPE_DIM 0 | ||
| 26 | +#endif | ||
| 27 | + | ||
| 28 | +#include <type_traits> | ||
| 29 | + | ||
| 30 | +#include "kernel_operator.h" | ||
| 31 | +#include "catlass/catlass.hpp" | ||
| 32 | +#include "catlass/arch/local_tensor_buffer.hpp" | ||
| 33 | + | ||
| 34 | +#if HC_GRAD_NPU_ARCH == HC_GRAD_ARCH_A2 | ||
| 35 | +// Limit AIC VECCALC UB allocation only for this operator's A2/A3 kernel. | ||
| 36 | +// CATLASS's full-UB default blocks L0C-to-GM Fixpipe; use a local specialization instead of patching shared code. | ||
| 37 | +namespace Catlass::Arch { | ||
| 38 | +template <> | ||
| 39 | +struct LocalTensorBuffer<AtlasA2, AscendC::TPosition::VECCALC> : LocalTensorBufferBase { | ||
| 40 | +public: | ||
| 41 | + static constexpr AscendC::TPosition Position = AscendC::TPosition::VECCALC; | ||
| 42 | + | ||
| 43 | + CATLASS_DEVICE LocalTensorBuffer() | ||
| 44 | + { | ||
| 45 | + AscendC::TBuf<AscendC::TPosition::VECCALC> tbufVECCALC; | ||
| 46 | + uint32_t ubBytes = AtlasA2::UB_SIZE; | ||
| 47 | + if ASCEND_IS_AIC { | ||
| 48 | + ubBytes = TextGatedResidualGrad::HC_GRAD_A2_CUBE_UB_BYTES; | ||
| 49 | + } | ||
| 50 | + GetTPipePtr()->InitBuffer(tbufVECCALC, ubBytes); | ||
| 51 | + tensor = tbufVECCALC.Get<uint8_t>(); | ||
| 52 | + } | ||
| 53 | +}; | ||
| 54 | +} // namespace Catlass::Arch | ||
| 55 | +#endif | ||
| 56 | + | ||
| 57 | +#include "catlass/gemm/block/block_mmad.hpp" | ||
| 58 | +#include "catlass/gemm/block/block_swizzle.hpp" | ||
| 59 | +#include "catlass/gemm/dispatch_policy.hpp" | ||
| 60 | +#include "catlass/gemm/gemm_type.hpp" | ||
| 61 | +#include "catlass/gemm_coord.hpp" | ||
| 62 | +#include "catlass/layout/layout.hpp" | ||
| 63 | +#include "../arch35/text_gated_residual_grad_pipeline_arch35.h" | ||
| 64 | + | ||
| 65 | +namespace TextGatedResidualGrad { | ||
| 66 | + | ||
| 67 | +// A2/A3 (arch22) uses the same algorithm as arch35, with these platform differences: | ||
| 68 | +// 1) Cube uses AtlasA2 BlockMmad (Gemm::Block::BlockMmad + MmadAtlasA2Pingpong), | ||
| 69 | +// as other A2 operators do; A5 MmadPingpongMutex supports Ascend950 only. | ||
| 70 | +// 2) C1/DX use 128x128x64 tiles for A2's 128 KiB L0C, consuming 64 KiB per tile. | ||
| 71 | +// BlockMmad uses one L0C stage and double buffering for L1/L0A/L0B. | ||
| 72 | +// 3) MmadAtlasA2Pingpong<false> disables unit flags and uses explicit M_FIX/FIX_M synchronization. | ||
| 73 | +// 4) Block strides use the tiling AIC count; GetBlockNum() is not the AIC count in a mixed kernel. | ||
| 74 | +// As on A5, C1b and G56 read Vector results directly as ColumnMajor without materializing transposes. | ||
| 75 | +template <typename D_T> | ||
| 76 | +class HcGradGemmArch22 { | ||
| 77 | + static constexpr uint32_t C1TileM = HC_GRAD_CUBE_TILE_M_A2; | ||
| 78 | + static constexpr uint32_t C1TileN = HC_GRAD_C1A_TILE_N; | ||
| 79 | + static constexpr uint32_t C1TileK = HC_GRAD_CUBE_TILE_K; | ||
| 80 | + | ||
| 81 | +public: | ||
| 82 | + __aicore__ inline void Init(uint32_t aicNum) | ||
| 83 | + { | ||
| 84 | + aicNum_ = aicNum; | ||
| 85 | + } | ||
| 86 | + | ||
| 87 | + // C1a: dact = dup @ wUp. As on A5, distribute all (m,n,slice) blocks across AICs in one loop, | ||
| 88 | + // treating split-K slices as virtual cores. Each slice writes its own FP32 partial, | ||
| 89 | + // which Vector REDUCE_DACT merges; slices do not run as separate serial GEMMs. | ||
| 90 | + __aicore__ inline void RunC1aSplitK(GM_ADDR dup, GM_ADDR wUp, GM_ADDR partial, uint32_t rows, uint32_t r, | ||
| 91 | + uint32_t d, uint32_t splitkFactor) | ||
| 92 | + { | ||
| 93 | + using BlockScheduler = | ||
| 94 | + Catlass::Gemm::Block::SplitkGemmIdentityBlockSwizzle<HC_GRAD_SWIZZLE_OFFSET, HC_GRAD_SWIZZLE_M_FIRST>; | ||
| 95 | + Catlass::GemmCoord problemShape{rows, r, d}; | ||
| 96 | + BlockScheduler scheduler(problemShape, Catlass::GemmCoord(C1TileM, C1TileN, C1TileK), splitkFactor); | ||
| 97 | + const uint32_t coreLoops = scheduler.GetCoreLoops(); | ||
| 98 | + | ||
| 99 | + Catlass::Arch::Resource<ArchTag> resource; | ||
| 100 | + C1BlockMmad blockMmad(resource); | ||
| 101 | + | ||
| 102 | + AscendC::GlobalTensor<D_T> gmA; | ||
| 103 | + gmA.SetGlobalBuffer(reinterpret_cast<__gm__ D_T*>(dup)); | ||
| 104 | + AscendC::GlobalTensor<D_T> gmB; | ||
| 105 | + gmB.SetGlobalBuffer(reinterpret_cast<__gm__ D_T*>(wUp)); | ||
| 106 | + AscendC::GlobalTensor<float> gmW; | ||
| 107 | + gmW.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(partial)); | ||
| 108 | + | ||
| 109 | + RowMajor layoutA{rows, d, d}; | ||
| 110 | + RowMajor layoutB{d, r, r}; | ||
| 111 | + RowMajor layoutC{rows, r, r}; | ||
| 112 | + const size_t lenC = static_cast<size_t>(rows) * r; | ||
| 113 | + for (uint32_t loopIdx = AscendC::GetBlockIdx(); loopIdx < coreLoops; loopIdx += aicNum_) { | ||
| 114 | + const Catlass::GemmCoord blockCoord = scheduler.GetBlockCoord(loopIdx); | ||
| 115 | + const uint32_t slice = scheduler.GetSplitkSliceIdx(loopIdx); | ||
| 116 | + const Catlass::GemmCoord actual = scheduler.GetActualBlockShape(blockCoord, slice); | ||
| 117 | + const MatrixCoord offsetA{blockCoord.m() * C1TileM, blockCoord.k() * C1TileK}; | ||
| 118 | + const MatrixCoord offsetB{blockCoord.k() * C1TileK, blockCoord.n() * C1TileN}; | ||
| 119 | + const MatrixCoord offsetC{blockCoord.m() * C1TileM, blockCoord.n() * C1TileN}; | ||
| 120 | + blockMmad(gmA[layoutA.GetOffset(offsetA)], layoutA, gmB[layoutB.GetOffset(offsetB)], layoutB, | ||
| 121 | + gmW[lenC * slice + layoutC.GetOffset(offsetC)], layoutC, actual); | ||
| 122 | + } | ||
| 123 | + AscendC::PipeBarrier<PIPE_ALL>(); | ||
| 124 | + AscendC::DataSyncBarrier<AscendC::MemDsbT::DDR>(); | ||
| 125 | + } | ||
| 126 | + | ||
| 127 | + // C1b: dwUpPartial = dup^T @ act, reading dup as ColumnMajor [D,rows], as on A5. | ||
| 128 | + __aicore__ inline void RunC1b(GM_ADDR dup, GM_ADDR act, GM_ADDR dwUpPartial, uint32_t rows, uint32_t r, uint32_t d) | ||
| 129 | + { | ||
| 130 | + RunC1GemmColumnMajor(d, r, rows, dup, d, act, r, dwUpPartial, r); | ||
| 131 | + } | ||
| 132 | + | ||
| 133 | + // For multiple tasks, use one M=R+gateRows GEMM to avoid rereading xn in a separate small gate GEMM. | ||
| 134 | + __aicore__ inline void RunG56Merged(GM_ADDR packedT, GM_ADDR xn, GM_ADDR dwOut, uint32_t rows, uint32_t r, | ||
| 135 | + uint32_t d, uint32_t gateRows, bool hasGate) | ||
| 136 | + { | ||
| 137 | + const uint32_t m = hasGate ? (r + gateRows) : r; | ||
| 138 | + RunC1Gemm(m, d, rows, packedT, rows, xn, d, dwOut, d); | ||
| 139 | + } | ||
| 140 | + | ||
| 141 | + __aicore__ inline void RunG56MergedPacked(GM_ADDR packed, GM_ADDR xn, GM_ADDR dwOut, uint32_t rows, uint32_t r, | ||
| 142 | + uint32_t d, uint32_t packedStride, uint32_t gateRows, bool hasGate) | ||
| 143 | + { | ||
| 144 | + const uint32_t m = hasGate ? (r + gateRows) : r; | ||
| 145 | + RunC1GemmColumnMajor(m, d, rows, packed, packedStride, xn, d, dwOut, d); | ||
| 146 | + } | ||
| 147 | + | ||
| 148 | + __aicore__ inline void RunG56DirectPacked(GM_ADDR packed, GM_ADDR xn, GM_ADDR dwDown, GM_ADDR dwGate, uint32_t rows, | ||
| 149 | + uint32_t r, uint32_t d, uint32_t packedStride, uint32_t gateRows, | ||
| 150 | + bool hasGate) | ||
| 151 | + { | ||
| 152 | + RunC1GemmColumnMajor(r, d, rows, packed, packedStride, xn, d, dwDown, d); | ||
| 153 | + if (hasGate) { | ||
| 154 | + RunC1GemmColumnMajor(gateRows, d, rows, packed + r * sizeof(D_T), packedStride, xn, d, dwGate, d); | ||
| 155 | + } | ||
| 156 | + } | ||
| 157 | + | ||
| 158 | + // Single-task fast paths: accumulate straight into the FP32 outputs. | ||
| 159 | + __aicore__ inline void RunC1bDirect(GM_ADDR dup, GM_ADDR act, GM_ADDR dwUp, uint32_t rows, uint32_t r, uint32_t d) | ||
| 160 | + { | ||
| 161 | + RunC1GemmColumnMajor(d, r, rows, dup, d, act, r, dwUp, r); | ||
| 162 | + } | ||
| 163 | + | ||
| 164 | + __aicore__ inline void RunG56Direct(GM_ADDR packedT, GM_ADDR xn, GM_ADDR dwDown, GM_ADDR dwGate, uint32_t rows, | ||
| 165 | + uint32_t r, uint32_t d, uint32_t gateRows, bool hasGate) | ||
| 166 | + { | ||
| 167 | + RunC1Gemm(r, d, rows, packedT, rows, xn, d, dwDown, d); | ||
| 168 | + if (hasGate) { | ||
| 169 | + // As on A5, write gateRows aligned to 16 into dwGatePartial, which has room for all padded rows. | ||
| 170 | + // Vector REDUCE_WD copies only the first c*d elements into the dwGate FP32 accumulator. | ||
| 171 | + RunC1Gemm(gateRows, d, rows, packedT + r * rows * sizeof(D_T), rows, xn, d, dwGate, d); | ||
| 172 | + } | ||
| 173 | + } | ||
| 174 | + | ||
| 175 | + // dxnorm = ddown @ w_down + dlogits @ w_gate: concatenate weights along K into wConcat for one GEMM. | ||
| 176 | + __aicore__ inline void RunDx(GM_ADDR packed, GM_ADDR wDown, GM_ADDR wGate, GM_ADDR wConcat, GM_ADDR dxOut, | ||
| 177 | + uint32_t rows, uint32_t d, uint32_t r, uint32_t c, uint32_t packedStride, bool hasGate) | ||
| 178 | + { | ||
| 179 | + if (hasGate) { | ||
| 180 | + RunDxGemm(rows, d, r + c, packed, packedStride, wConcat, d, dxOut, d); | ||
| 181 | + } else { | ||
| 182 | + RunDxGemm(rows, d, r, packed, packedStride, wDown, d, dxOut, d); | ||
| 183 | + } | ||
| 184 | + } | ||
| 185 | + | ||
| 186 | +private: | ||
| 187 | + using ArchTag = Catlass::Arch::AtlasA2; | ||
| 188 | + using DispatchPolicy = Catlass::Gemm::MmadAtlasA2Pingpong<false>; | ||
| 189 | + using RowMajor = Catlass::layout::RowMajor; | ||
| 190 | + using ColumnMajor = Catlass::layout::ColumnMajor; | ||
| 191 | + using MatrixCoord = Catlass::MatrixCoord; | ||
| 192 | + | ||
| 193 | + static_assert(C1TileM * C1TileN * sizeof(float) <= Catlass::Arch::AtlasA2::L0C_SIZE, | ||
| 194 | + "A2/A3 L0C tile exceeds 128KB"); | ||
| 195 | + | ||
| 196 | + using AType = Catlass::Gemm::GemmType<D_T, RowMajor>; | ||
| 197 | + using BType = Catlass::Gemm::GemmType<D_T, RowMajor>; | ||
| 198 | + using CType = Catlass::Gemm::GemmType<float, RowMajor>; | ||
| 199 | + | ||
| 200 | + using C1BlockMmad = | ||
| 201 | + Catlass::Gemm::Block::BlockMmad<DispatchPolicy, Catlass::GemmShape<C1TileM, C1TileN, C1TileK>, | ||
| 202 | + Catlass::GemmShape<C1TileM, C1TileN, C1TileK>, AType, BType, CType>; | ||
| 203 | + | ||
| 204 | + using ATypeColumnMajor = Catlass::Gemm::GemmType<D_T, Catlass::layout::ColumnMajor>; | ||
| 205 | + using C1BlockMmadColumnMajor = | ||
| 206 | + Catlass::Gemm::Block::BlockMmad<DispatchPolicy, Catlass::GemmShape<C1TileM, C1TileN, C1TileK>, | ||
| 207 | + Catlass::GemmShape<C1TileM, C1TileN, C1TileK>, ATypeColumnMajor, BType, CType>; | ||
| 208 | + | ||
| 209 | + template <class BlockMmad, uint32_t Direction, class LayoutAT = RowMajor> | ||
| 210 | + __aicore__ inline void RunBlockLoop(uint32_t m, uint32_t n, uint32_t k, GM_ADDR ptrA, int64_t lda, GM_ADDR ptrB, | ||
| 211 | + int64_t ldb, GM_ADDR ptrC, int64_t ldc) | ||
| 212 | + { | ||
| 213 | + using BlockScheduler = Catlass::Gemm::Block::GemmIdentityBlockSwizzle<HC_GRAD_SWIZZLE_OFFSET, Direction>; | ||
| 214 | + constexpr uint32_t tileM = BlockMmad::L1TileShape::M; | ||
| 215 | + constexpr uint32_t tileN = BlockMmad::L1TileShape::N; | ||
| 216 | + constexpr uint32_t tileK = BlockMmad::L1TileShape::K; | ||
| 217 | + Catlass::GemmCoord problemShape{m, n, k}; | ||
| 218 | + BlockScheduler scheduler(problemShape, MatrixCoord(tileM, tileN)); | ||
| 219 | + const uint32_t coreLoops = scheduler.GetCoreLoops(); | ||
| 220 | + | ||
| 221 | + Catlass::Arch::Resource<ArchTag> resource; | ||
| 222 | + BlockMmad blockMmad(resource); | ||
| 223 | + | ||
| 224 | + AscendC::GlobalTensor<D_T> gmA; | ||
| 225 | + gmA.SetGlobalBuffer(reinterpret_cast<__gm__ D_T*>(ptrA)); | ||
| 226 | + AscendC::GlobalTensor<D_T> gmB; | ||
| 227 | + gmB.SetGlobalBuffer(reinterpret_cast<__gm__ D_T*>(ptrB)); | ||
| 228 | + AscendC::GlobalTensor<float> gmC; | ||
| 229 | + gmC.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(ptrC)); | ||
| 230 | + | ||
| 231 | + LayoutAT layoutA{m, k, lda}; | ||
| 232 | + RowMajor layoutB{k, n, ldb}; | ||
| 233 | + RowMajor layoutC{m, n, ldc}; | ||
| 234 | + for (uint32_t loopIdx = AscendC::GetBlockIdx(); loopIdx < coreLoops; loopIdx += aicNum_) { | ||
| 235 | + const Catlass::GemmCoord blockCoord = scheduler.GetBlockCoord(loopIdx); | ||
| 236 | + const Catlass::GemmCoord actual = scheduler.GetActualBlockShape(blockCoord); | ||
| 237 | + const MatrixCoord offsetA{blockCoord.m() * tileM, blockCoord.k() * tileK}; | ||
| 238 | + const MatrixCoord offsetB{blockCoord.k() * tileK, blockCoord.n() * tileN}; | ||
| 239 | + const MatrixCoord offsetC{blockCoord.m() * tileM, blockCoord.n() * tileN}; | ||
| 240 | + blockMmad(gmA[layoutA.GetOffset(offsetA)], layoutA, gmB[layoutB.GetOffset(offsetB)], layoutB, | ||
| 241 | + gmC[layoutC.GetOffset(offsetC)], layoutC, actual); | ||
| 242 | + } | ||
| 243 | + AscendC::PipeBarrier<PIPE_ALL>(); | ||
| 244 | + AscendC::DataSyncBarrier<AscendC::MemDsbT::DDR>(); | ||
| 245 | + } | ||
| 246 | + | ||
| 247 | + __aicore__ inline void RunC1Gemm(uint32_t m, uint32_t n, uint32_t k, GM_ADDR ptrA, int64_t lda, GM_ADDR ptrB, | ||
| 248 | + int64_t ldb, GM_ADDR ptrC, int64_t ldc) | ||
| 249 | + { | ||
| 250 | + if (m >= n) { | ||
| 251 | + RunBlockLoop<C1BlockMmad, HC_GRAD_SWIZZLE_M_FIRST>(m, n, k, ptrA, lda, ptrB, ldb, ptrC, ldc); | ||
| 252 | + } else { | ||
| 253 | + RunBlockLoop<C1BlockMmad, HC_GRAD_SWIZZLE_N_FIRST>(m, n, k, ptrA, lda, ptrB, ldb, ptrC, ldc); | ||
| 254 | + } | ||
| 255 | + } | ||
| 256 | + | ||
| 257 | + __aicore__ inline void RunC1GemmColumnMajor(uint32_t m, uint32_t n, uint32_t k, GM_ADDR ptrA, int64_t lda, | ||
| 258 | + GM_ADDR ptrB, int64_t ldb, GM_ADDR ptrC, int64_t ldc) | ||
| 259 | + { | ||
| 260 | + if (m >= n) { | ||
| 261 | + RunBlockLoop<C1BlockMmadColumnMajor, HC_GRAD_SWIZZLE_M_FIRST, ColumnMajor>(m, n, k, ptrA, lda, ptrB, ldb, | ||
| 262 | + ptrC, ldc); | ||
| 263 | + } else { | ||
| 264 | + RunBlockLoop<C1BlockMmadColumnMajor, HC_GRAD_SWIZZLE_N_FIRST, ColumnMajor>(m, n, k, ptrA, lda, ptrB, ldb, | ||
| 265 | + ptrC, ldc); | ||
| 266 | + } | ||
| 267 | + } | ||
| 268 | + | ||
| 269 | + __aicore__ inline void RunDxGemm(uint32_t m, uint32_t n, uint32_t k, GM_ADDR ptrA, int64_t lda, GM_ADDR ptrB, | ||
| 270 | + int64_t ldb, GM_ADDR ptrC, int64_t ldc) | ||
| 271 | + { | ||
| 272 | + RunC1Gemm(m, n, k, ptrA, lda, ptrB, ldb, ptrC, ldc); | ||
| 273 | + } | ||
| 274 | + | ||
| 275 | + uint32_t aicNum_{1}; | ||
| 276 | +}; | ||
| 277 | + | ||
| 278 | +} // namespace TextGatedResidualGrad | ||
| 279 | + | ||
| 280 | +#endif // TEXT_GATED_RESIDUAL_GRAD_GEMM_ARCH22_H | ||
| @@ -0,0 +1,31 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file text_gated_residual_grad_kernel_arch22.h | ||
| 13 | + * \brief Atlas A2/A3 specialization of the shared TextGatedResidualGrad kernel. | ||
| 14 | + */ | ||
| 15 | +#ifndef TEXT_GATED_RESIDUAL_GRAD_KERNEL_ARCH22_H | ||
| 16 | +#define TEXT_GATED_RESIDUAL_GRAD_KERNEL_ARCH22_H | ||
| 17 | + | ||
| 18 | +#include "../arch35/text_gated_residual_grad_kernel_common.h" | ||
| 19 | +#include "text_gated_residual_grad_gemm_arch22.h" | ||
| 20 | + | ||
| 21 | +namespace TextGatedResidualGrad { | ||
| 22 | + | ||
| 23 | +// Atlas A2/A3: CATLASS AtlasA2 BlockMmad; G56 and C1b read Vector results directly as ColumnMajor, | ||
| 24 | +// as on A5. Init/Process/RunCube use the shared pipeline. | ||
| 25 | +template <typename D_T, int HAS_GAMMA> | ||
| 26 | +using KernelTextGatedResidualGradArch22 = | ||
| 27 | + KernelTextGatedResidualGrad<D_T, HAS_GAMMA, HcGradGemmArch22<D_T>, HC_GRAD_USE_DUP_T, false>; | ||
| 28 | + | ||
| 29 | +} // namespace TextGatedResidualGrad | ||
| 30 | + | ||
| 31 | +#endif // TEXT_GATED_RESIDUAL_GRAD_KERNEL_ARCH22_H | ||


C02|高|
ci-smoke|FAIL|置信度高const std::string dir = argc > DATA_DIR_ARG ? argv[DATA_DIR_ARG] : "./data";默认 example 从
./data读取meta.txt和多份.bin,但 PR 未提供这些文件或生成步骤。build.sh会从干净的构建目录无参数启动程序,因此ReadMetadata在aclInit前直接失败。该算子在tests/test_config.yaml中配置了examples: True,changed-file smoke 会选中它。**建议:**在 example 内生成确定性的输入并校验输出,或提供数据生成脚本并由构建入口显式传入数据目录。