Merged
feat(scatter_reduce): 完善 amax true/false、正确性与宽通道性能 #13812
liushucheng4Add on 15 days ago
feat(scatter_reduce): 完善 amax true/false、正确性与宽通道性能 #13812
Merged
Total: 14 files changed+3705-33
| @@ -0,0 +1,142 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License") | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#ifndef SCATTER_ELEMENTS_V2_AMAX_POLICY_H | ||
| 11 | +#define SCATTER_ELEMENTS_V2_AMAX_POLICY_H | ||
| 12 | + | ||
| 13 | +#include <algorithm> | ||
| 14 | +#include <cstdint> | ||
| 15 | + | ||
| 16 | +namespace ScatterElementsV2Amax { | ||
| 17 | +// Buffer geometry, not performance thresholds. All allocations are rounded | ||
| 18 | +// as TPipe::InitBuffer does on Ascend 910B (32 bytes). | ||
| 19 | +constexpr uint64_t kBlockBytes = 32; | ||
| 20 | +// CANN 9.1 dav-2201 Compare/Select use TMP_UB_OFFSET=184 KiB, | ||
| 21 | +// TMP_UB_SIZE=8 KiB (basic_api/utils/kernel_utils_constants.h). | ||
| 22 | +// Keep all TPipe buffers below that fixed SDK scratch region. | ||
| 23 | +constexpr uint64_t kRuntimeUbBytes = 8 * 1024; | ||
| 24 | +constexpr uint64_t kCompareElements = 64; // One FP32 vector repeat. | ||
| 25 | +constexpr uint64_t kInputTile = 2048; | ||
| 26 | +constexpr uint64_t kIndexTile = 1024; | ||
| 27 | +constexpr uint64_t kExactFloatInteger = 1ULL << 24; | ||
| 28 | + | ||
| 29 | +inline uint64_t CeilDiv(uint64_t n, uint64_t d) { return (n + d - 1) / d; } | ||
| 30 | +inline uint64_t AlignUp(uint64_t n, uint64_t a) { return CeilDiv(n, a) * a; } | ||
| 31 | +inline uint64_t BufferBytes(uint64_t bytes) { return AlignUp(bytes, kBlockBytes); } | ||
| 32 | +inline uint64_t Gcd(uint64_t a, uint64_t b) | ||
| 33 | +{ | ||
| 34 | + while (b) { | ||
| 35 | + const uint64_t next = a % b; | ||
| 36 | + a = b; | ||
| 37 | + b = next; | ||
| 38 | + } | ||
| 39 | + return a; | ||
| 40 | +} | ||
| 41 | + | ||
| 42 | +enum class Algorithm { InputPartition, OutputPartition }; | ||
| 43 | +enum class Layout { Elements, Rows }; | ||
| 44 | +enum class Reason { AtomicFits, AtomicAlignment, AtomicCapacity, RowBufferCapacity, AtomicIndexRange }; | ||
| 45 | + | ||
| 46 | +struct Plan { | ||
| 47 | + Algorithm algorithm = Algorithm::OutputPartition; | ||
| 48 | + Layout layout = Layout::Elements; | ||
| 49 | + Reason reason = Reason::AtomicCapacity; | ||
| 50 | + uint32_t cores = 0; | ||
| 51 | + uint32_t chunk = 0; // Input elements, output elements, or buffered rows. | ||
| 52 | + uint32_t tile = kInputTile; | ||
| 53 | + uint64_t ubBytes = 0; | ||
| 54 | + bool valid = false; | ||
| 55 | +}; | ||
| 56 | + | ||
| 57 | +inline uint64_t AtomicBytes(uint64_t output, uint64_t bytes) | ||
| 58 | +{ | ||
| 59 | + return BufferBytes(output * bytes) + 2 * BufferBytes(output * sizeof(int32_t)) + | ||
| 60 | + BufferBytes(kInputTile * sizeof(int64_t)) + BufferBytes(kInputTile * bytes); | ||
| 61 | +} | ||
| 62 | + | ||
| 63 | +inline uint64_t RowBytes(uint64_t rows, uint64_t columns, uint64_t bytes) | ||
| 64 | +{ | ||
| 65 | + const uint64_t stride = AlignUp(columns, kCompareElements); | ||
| 66 | + // Output, seen rows, one source row, two optional FP32 conversion rows, | ||
| 67 | + // three comparison masks; index, converted index, positions, selected, | ||
| 68 | + // and two index-selection masks. Every term matches a kernel TBuf. | ||
| 69 | + return BufferBytes(rows * stride * bytes) + BufferBytes(rows * sizeof(uint32_t)) + BufferBytes(stride * bytes) + | ||
| 70 | + (bytes == 4 ? 0 : 2 * BufferBytes(stride * 4)) + 3 * BufferBytes(stride / 8) + BufferBytes(kIndexTile * 8) + | ||
| 71 | + 3 * BufferBytes(kIndexTile * 4) + 2 * BufferBytes(kIndexTile / 8); | ||
| 72 | +} | ||
| 73 | + | ||
| 74 | +inline uint64_t ElementBytes(uint64_t chunk, uint64_t bytes) | ||
| 75 | +{ | ||
| 76 | + return BufferBytes(chunk * bytes) + BufferBytes(chunk * 4) + BufferBytes(kInputTile * 8) + | ||
| 77 | + BufferBytes(kInputTile * bytes) + 3 * BufferBytes(kInputTile * 4) + BufferBytes(kInputTile / 4); | ||
| 78 | +} | ||
| 79 | + | ||
| 80 | +inline Plan Select(uint64_t n, uint64_t e, uint64_t c, uint64_t bytes, bool compactRows, uint64_t cores, uint64_t ub) | ||
| 81 | +{ | ||
| 82 | + Plan p; | ||
| 83 | + if (!n || !e || !c || !cores || (bytes != 2 && bytes != 4) || n > UINT32_MAX || c > UINT32_MAX || e > UINT32_MAX || | ||
| 84 | + n > UINT32_MAX / c || e > UINT32_MAX / c) | ||
| 85 | + return p; | ||
| 86 | + if (ub <= kRuntimeUbBytes) | ||
| 87 | + return p; | ||
| 88 | + ub -= kRuntimeUbBytes; | ||
| 89 | + const uint64_t input = e * c, output = n * c; | ||
| 90 | + const uint64_t alignment = kBlockBytes / bytes; | ||
| 91 | + const bool aligned = output % alignment == 0; | ||
| 92 | + if (aligned && input <= uint64_t(INT32_MAX) / 2 && AtomicBytes(output, bytes) <= ub) { | ||
| 93 | + p.algorithm = Algorithm::InputPartition; | ||
| 94 | + p.reason = Reason::AtomicFits; | ||
| 95 | + p.cores = std::min(cores, CeilDiv(input, kInputTile)); | ||
| 96 | + p.chunk = CeilDiv(input, p.cores); | ||
| 97 | + p.ubBytes = AtomicBytes(output, bytes); | ||
| 98 | + p.valid = true; | ||
| 99 | + return p; | ||
| 100 | + } | ||
| 101 | + p.reason = !aligned ? Reason::AtomicAlignment : | ||
| 102 | + input > uint64_t(INT32_MAX) / 2 ? Reason::AtomicIndexRange : | ||
| 103 | + Reason::AtomicCapacity; | ||
| 104 | + if (compactRows && n <= kExactFloatInteger) { | ||
| 105 | + const uint64_t rowAlignment = kBlockBytes / Gcd(c * bytes, kBlockBytes); | ||
| 106 | + const uint64_t groups = CeilDiv(n, rowAlignment); | ||
| 107 | + const uint64_t desired = CeilDiv(groups, std::min(groups, cores)) * rowAlignment; | ||
| 108 | + // Monotonic resource fit, independent of input degree distribution. | ||
| 109 | + uint64_t low = 0, high = CeilDiv(desired, rowAlignment); | ||
| 110 | + while (low < high) { | ||
| 111 | + const uint64_t mid = (low + high + 1) / 2; | ||
| 112 | + if (RowBytes(std::min(n, mid * rowAlignment), c, bytes) <= ub) | ||
| 113 | + low = mid; | ||
| 114 | + else | ||
| 115 | + high = mid - 1; | ||
| 116 | + } | ||
| 117 | + if (low) { | ||
| 118 | + p.layout = Layout::Rows; | ||
| 119 | + p.chunk = std::min(n, low * rowAlignment); | ||
| 120 | + p.tile = kIndexTile; | ||
| 121 | + p.cores = std::min(cores, CeilDiv(n, p.chunk)); | ||
| 122 | + p.ubBytes = RowBytes(p.chunk, c, bytes); | ||
| 123 | + p.valid = true; | ||
| 124 | + return p; | ||
| 125 | + } | ||
| 126 | + p.reason = Reason::RowBufferCapacity; | ||
| 127 | + } | ||
| 128 | + const uint64_t fixed = ElementBytes(0, bytes); | ||
| 129 | + if (ub <= fixed) | ||
| 130 | + return p; | ||
| 131 | + const uint64_t capacity = (ub - fixed) / (bytes + sizeof(uint32_t)) / alignment * alignment; | ||
| 132 | + if (!capacity) | ||
| 133 | + return p; | ||
| 134 | + p.cores = std::min(cores, CeilDiv(output, alignment)); | ||
| 135 | + p.chunk = std::min(capacity, AlignUp(CeilDiv(output, p.cores), alignment)); | ||
| 136 | + p.ubBytes = ElementBytes(p.chunk, bytes); | ||
| 137 | + p.valid = p.ubBytes <= ub; | ||
| 138 | + return p; | ||
| 139 | +} | ||
| 140 | +} // namespace ScatterElementsV2Amax | ||
| 141 | + | ||
| 142 | +#endif | ||
| @@ -187,7 +187,7 @@ aclnnStatus aclnnInplaceScatterReduce( | |||
| 187 | <td>includeSelf(bool)</td> | 187 | <td>includeSelf(bool)</td> |
| 188 | <td>输入</td> | 188 | <td>输入</td> |
| 189 | <td>规约计算时是否将self中的原始值参与计算。</td> | 189 | <td>规约计算时是否将self中的原始值参与计算。</td> |
| 190 | - <td>true表示self参与规约;false表示命中位置仅累加src,未命中位置保留self;空index保留self。</td> | 190 | + <td>true表示self参与规约;false表示命中位置仅规约src,未命中位置保留self;空index保留self。</td> |
| 191 | <td>-</td> | 191 | <td>-</td> |
| 192 | <td>-</td> | 192 | <td>-</td> |
| 193 | <td>-</td> | 193 | <td>-</td> |
| @@ -398,7 +398,7 @@ aclnnStatus aclnnInplaceScatterReduce( | |||
| 398 | <td>includeSelf(bool)</td> | 398 | <td>includeSelf(bool)</td> |
| 399 | <td>输入</td> | 399 | <td>输入</td> |
| 400 | <td>规约计算时是否将selfRef中的原始值参与计算。</td> | 400 | <td>规约计算时是否将selfRef中的原始值参与计算。</td> |
| 401 | - <td>true表示self参与规约;false表示命中位置仅累加src,未命中位置保留self;空index保留self。</td> | 401 | + <td>true表示self参与规约;false表示命中位置仅规约src,未命中位置保留self;空index保留self。</td> |
| 402 | <td>-</td> | 402 | <td>-</td> |
| 403 | <td>-</td> | 403 | <td>-</td> |
| 404 | <td>-</td> | 404 | <td>-</td> |
| @@ -528,10 +528,15 @@ aclnnStatus aclnnInplaceScatterReduce( | |||
| 528 | <!-- npu="A3,910b" id8 --> | 528 | <!-- npu="A3,910b" id8 --> |
| 529 | - <term>Atlas A3系列产品</term>、<term>Atlas A2系列产品</term>: | 529 | - <term>Atlas A3系列产品</term>、<term>Atlas A2系列产品</term>: |
| 530 | - 默认确定性实现。 | 530 | - 默认确定性实现。 |
| 531 | - - `reduce` 仅支持1(add)。 | 531 | + - `reduce` 仅支持1(add)和3(max)。 |
| 532 | - `reduce=1(add)`: | 532 | - `reduce=1(add)`: |
| 533 | - `self/src/out` 的数据类型仅支持 `FLOAT32`。 | 533 | - `self/src/out` 的数据类型仅支持 `FLOAT32`。 |
| 534 | - `includeSelf=false` 且输入非空时,将 `dim` 归一化为非负轴 `d`:`d=0` 时,除首轴和末轴外,`self/index/src` 的各维大小分别相同;`d>0` 时,除第 0 轴和第 `d` 轴外,各维大小分别相同。 | 534 | - `includeSelf=false` 且输入非空时,将 `dim` 归一化为非负轴 `d`:`d=0` 时,除首轴和末轴外,`self/index/src` 的各维大小分别相同;`d>0` 时,除第 0 轴和第 `d` 轴外,各维大小分别相同。 |
| 535 | + - `reduce=3(max)`: | ||
| 536 | + - `self/src/out` 的数据类型支持 `FLOAT32`、`FLOAT16` 和 `BFLOAT16`。 | ||
| 537 | + - 支持一维向量和二维矩阵。一维时 `dim` 为 0 或 -1;二维时 `dim` 为 0 或 -2。`index` 使用 `INT64` 类型,形状与 `src` 相同;二维时 `src` 与 `self` 的列数相同。 | ||
| 538 | + - `index` 非空时,`self/src` 的各维大小及各自元素总数均不超过 4294967295。 | ||
| 539 | + - 参与比较的值中出现 NaN,结果为 NaN;正负无穷按大小正常比较;+0 与 -0 相等时,保留先参与比较的零值符号。 | ||
| 535 | <!-- end id8 --> | 540 | <!-- end id8 --> |
| 536 | 541 | ||
| 537 | ## 调用示例 | 542 | ## 调用示例 |
| @@ -189,7 +189,7 @@ aclnnStatus aclnnInplaceScatterReduce( | |||
| 189 | <td>includeSelf (bool)</td> | 189 | <td>includeSelf (bool)</td> |
| 190 | <td>Input</td> | 190 | <td>Input</td> |
| 191 | <td>Whether the original values in self participate in the reduction computation.</td> | 191 | <td>Whether the original values in self participate in the reduction computation.</td> |
| 192 | - <td>true includes self in the reduction; false sums src at updated positions and preserves self elsewhere. Empty index preserves self.</td> | 192 | + <td>true includes self in the reduction; false reduces src at updated positions and preserves self elsewhere. Empty index preserves self.</td> |
| 193 | <td>-</td> | 193 | <td>-</td> |
| 194 | <td>-</td> | 194 | <td>-</td> |
| 195 | <td>-</td> | 195 | <td>-</td> |
| @@ -400,7 +400,7 @@ aclnnStatus aclnnInplaceScatterReduce( | |||
| 400 | <td>includeSelf (bool)</td> | 400 | <td>includeSelf (bool)</td> |
| 401 | <td>Input</td> | 401 | <td>Input</td> |
| 402 | <td>Whether the original values in selfRef participate in the reduction computation.</td> | 402 | <td>Whether the original values in selfRef participate in the reduction computation.</td> |
| 403 | - <td>true includes self in the reduction; false sums src at updated positions and preserves self elsewhere. Empty index preserves self.</td> | 403 | + <td>true includes self in the reduction; false reduces src at updated positions and preserves self elsewhere. Empty index preserves self.</td> |
| 404 | <td>-</td> | 404 | <td>-</td> |
| 405 | <td>-</td> | 405 | <td>-</td> |
| 406 | <td>-</td> | 406 | <td>-</td> |
| @@ -530,10 +530,15 @@ aclnnStatus aclnnInplaceScatterReduce( | |||
| 530 | <!-- npu="A3,910b" id8 --> | 530 | <!-- npu="A3,910b" id8 --> |
| 531 | - <term>Atlas A3 products</term> and <term>Atlas A2 products</term>: | 531 | - <term>Atlas A3 products</term> and <term>Atlas A2 products</term>: |
| 532 | - The implementation is deterministic by default. | 532 | - The implementation is deterministic by default. |
| 533 | - - `reduce` supports only 1 (add). | 533 | + - `reduce` supports only 1 (add) and 3 (max). |
| 534 | - `reduce=1(add)`: | 534 | - `reduce=1(add)`: |
| 535 | - The data type of `self/src/out` must be `FLOAT32`. | 535 | - The data type of `self/src/out` must be `FLOAT32`. |
| 536 | - For nonempty inputs with `includeSelf=false`, normalize `dim` to a nonnegative axis `d`. If `d=0`, `self/index/src` sizes must match on every axis except the first and last axes. If `d>0`, their sizes must match on every axis except axis 0 and axis `d`. | 536 | - For nonempty inputs with `includeSelf=false`, normalize `dim` to a nonnegative axis `d`. If `d=0`, `self/index/src` sizes must match on every axis except the first and last axes. If `d>0`, their sizes must match on every axis except axis 0 and axis `d`. |
| 537 | + - `reduce=3(max)`: | ||
| 538 | + - The supported data types of `self/src/out` are `FLOAT32`, `FLOAT16`, and `BFLOAT16`. | ||
| 539 | + - Supports one-dimensional vectors and two-dimensional matrices. For one-dimensional inputs, `dim` is 0 or -1; for two-dimensional inputs, `dim` is 0 or -2. `index` uses the `INT64` data type and has the same shape as `src`. For two-dimensional inputs, `src` and `self` have the same number of columns. | ||
| 540 | + - For nonempty `index`, each `self/src` dimension and the element count of each tensor must be at most 4294967295. | ||
| 541 | + - If any value being compared is NaN, the result is NaN. Positive and negative infinity are compared according to their values. When +0 and -0 compare equal, the sign of the first zero being compared is retained. | ||
| 537 | <!-- end id8 --> | 542 | <!-- end id8 --> |
| 538 | 543 | ||
| 539 | ## Example | 544 | ## Example |
| @@ -119,7 +119,7 @@ int RunReduceCase(const ReduceCase& reduceCase, aclrtStream stream) | |||
| 119 | { | 119 | { |
| 120 | const std::vector<int64_t> selfShape = {4, 4}; | 120 | const std::vector<int64_t> selfShape = {4, 4}; |
| 121 | const std::vector<int64_t> indexShape = {3, 4}; | 121 | const std::vector<int64_t> indexShape = {3, 4}; |
| 122 | - const std::vector<int64_t> srcShape = {4, 4}; | 122 | + const std::vector<int64_t> srcShape = reduceCase.reduce == kReduceMax ? indexShape : selfShape; |
| 123 | const std::vector<int64_t> outShape = {4, 4}; | 123 | const std::vector<int64_t> outShape = {4, 4}; |
| 124 | const int64_t dim = 0; | 124 | const int64_t dim = 0; |
| 125 | 125 | ||
| @@ -217,8 +217,7 @@ int main() | |||
| 217 | CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("Init acl failed. ERROR: %d\n", ret); return ret); | 217 | CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("Init acl failed. ERROR: %d\n", ret); return ret); |
| 218 | 218 | ||
| 219 | const std::vector<int64_t> indexHostData = {0, 1, 2, 1, 0, 1, 2, 0, 2, 2, 1, 0}; | 219 | const std::vector<int64_t> indexHostData = {0, 1, 2, 1, 0, 1, 2, 0, 2, 2, 1, 0}; |
| 220 | - // FP32 sum supports both include_self values on Atlas A2/A3. | 220 | + // Sum and max expose both include_self values on Atlas A2/A3. |
| 221 | - // Other reductions (none/max/min/mean/mul) are intercepted at the API level. | ||
| 222 | const std::vector<ReduceCase> cases = { | 221 | const std::vector<ReduceCase> cases = { |
| 223 | {"scatter_reduce_add", | 222 | {"scatter_reduce_add", |
| 224 | kReduceAdd, | 223 | kReduceAdd, |
| @@ -234,6 +233,20 @@ int main() | |||
| 234 | indexHostData, | 233 | indexHostData, |
| 235 | {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, | 234 | {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, |
| 236 | {4, 3, 3, 18, 3, 6, 10, 3, 8, 9, 8, 3, 3, 3, 3, 3}}, | 235 | {4, 3, 3, 18, 3, 6, 10, 3, 8, 9, 8, 3, 3, 3, 3, 3}}, |
| 236 | + {"scatter_reduce_max", | ||
| 237 | + kReduceMax, | ||
| 238 | + true, | ||
| 239 | + std::vector<float>(16, 3), | ||
| 240 | + indexHostData, | ||
| 241 | + {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2}, | ||
| 242 | + std::vector<float>(16, 3)}, | ||
| 243 | + {"scatter_reduce_max_without_self", | ||
| 244 | + kReduceMax, | ||
| 245 | + false, | ||
| 246 | + std::vector<float>(16, 3), | ||
| 247 | + indexHostData, | ||
| 248 | + {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2}, | ||
| 249 | + {1, 3, 3, 2, 3, 2, 1, 0, 2, 0, 2, 3, 3, 3, 3, 3}}, | ||
| 237 | }; | 250 | }; |
| 238 | 251 | ||
| 239 | for (const auto& reduceCase : cases) { | 252 | for (const auto& reduceCase : cases) { |
| @@ -41,6 +41,7 @@ | |||
| 41 | #include "opdev/op_dfx.h" | 41 | #include "opdev/op_dfx.h" |
| 42 | #include "opdev/op_executor.h" | 42 | #include "opdev/op_executor.h" |
| 43 | #include "opdev/tensor_view_utils.h" | 43 | #include "opdev/tensor_view_utils.h" |
| 44 | +#include <limits> | ||
| 44 | #include "log/log.h" | 45 | #include "log/log.h" |
| 45 | #include "op_api/aclnn_util.h" | 46 | #include "op_api/aclnn_util.h" |
| 46 | #include "runtime/context.h" | 47 | #include "runtime/context.h" |
| @@ -574,6 +575,116 @@ static bool IsMeetScatterAddShape(const op::Shape& selfShape, const op::Shape& i | |||
| 574 | return true; | 575 | return true; |
| 575 | } | 576 | } |
| 576 | 577 | ||
| 578 | +static aclTensor* ScatterReduceRowIndex(const aclTensor* index, aclOpExecutor* executor) | ||
| 579 | +{ | ||
| 580 | + op::Shape shape; | ||
| 581 | + shape.SetDimNum(1); | ||
| 582 | + shape[0] = index->GetViewShape()[0]; | ||
| 583 | + op::Strides strides = {index->GetViewStrides()[0]}; | ||
| 584 | + auto view = executor->CreateView(index, shape, index->GetStorageShape(), strides, 0); | ||
| 585 | + CHECK_RET(view != nullptr, nullptr); | ||
| 586 | + view->SetViewOffset(index->GetViewOffset()); | ||
| 587 | + return view; | ||
| 588 | +} | ||
| 589 | + | ||
| 590 | +static aclnnStatus ExecScatterReduceMax(const aclTensor* self, const aclTensor* index, const aclTensor* src, | ||
| 591 | + aclTensor* out, bool includeSelf, aclOpExecutor* executor) | ||
| 592 | +{ | ||
| 593 | + const bool vector = self->GetViewShape().GetDimNum() == 1; | ||
| 594 | + const bool broadcast = !vector && index->GetViewStrides()[1] == 0; | ||
| 595 | + const aclTensor* indices = broadcast ? ScatterReduceRowIndex(index, executor) : index; | ||
| 596 | + CHECK_RET(indices != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 597 | + indices = l0op::Contiguous(indices, executor); | ||
| 598 | + CHECK_RET(indices != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 599 | + auto copy = executor->AllocTensor(self->GetViewShape(), self->GetDataType()); | ||
| 600 | + CHECK_RET(copy != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 601 | + const aclTensor* input = l0op::TensorMove(self, copy, executor); | ||
| 602 | + CHECK_RET(input != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 603 | + const aclTensor* updates = src; | ||
| 604 | + if (vector) { | ||
| 605 | + input = l0op::UnsqueezeNd(input, 1, executor); | ||
| 606 | + updates = l0op::UnsqueezeNd(updates, 1, executor); | ||
| 607 | + CHECK_RET(input != nullptr && updates != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 608 | + } | ||
| 609 | + auto result = l0op::ScatterElementsNoTranspose(input, indices, updates, 0, REDUCTION_MAX, executor, includeSelf); | ||
| 610 | + CHECK_RET(result != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 611 | + if (vector) { | ||
| 612 | + result = l0op::SqueezeNd(result, 1, executor); | ||
| 613 | + CHECK_RET(result != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 614 | + } | ||
| 615 | + CHECK_RET(l0op::ViewCopy(result, out, executor) != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 616 | + return ACLNN_SUCCESS; | ||
| 617 | +} | ||
| 618 | + | ||
| 619 | +static aclnnStatus CheckScatterReduceMaxSize(const aclTensor* tensor, const char* name) | ||
| 620 | +{ | ||
| 621 | + constexpr uint64_t maxElements = std::numeric_limits<uint32_t>::max(); | ||
| 622 | + const auto& shape = tensor->GetViewShape(); | ||
| 623 | + const uint64_t rows = shape[0]; | ||
| 624 | + const uint64_t columns = shape.GetDimNum() == 1 ? 1 : shape[1]; | ||
| 625 | + if (rows > maxElements || columns > maxElements) { | ||
| 626 | + OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON("ScatterReduce", name, op::ToString(shape).GetString(), | ||
| 627 | + "max dimensions must be at most 4294967295"); | ||
| 628 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 629 | + } | ||
| 630 | + if (columns != 0 && rows > maxElements / columns) { | ||
| 631 | + OP_LOGE_FOR_INVALID_VALUE_WITH_REASON("ScatterReduce", (std::string(name) + " element count").c_str(), | ||
| 632 | + std::to_string(rows * columns).c_str(), | ||
| 633 | + "max element count must be at most 4294967295"); | ||
| 634 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 635 | + } | ||
| 636 | + return ACLNN_SUCCESS; | ||
| 637 | +} | ||
| 638 | + | ||
| 639 | +static aclnnStatus CheckScatterReduceMaxParams(const aclTensor* self, const aclTensor* index, const aclTensor* src, | ||
| 640 | + const aclTensor* out, int64_t dim) | ||
| 641 | +{ | ||
| 642 | + const auto rank = self->GetViewShape().GetDimNum(); | ||
| 643 | + const auto dtype = self->GetDataType(); | ||
| 644 | + if (dtype != op::DataType::DT_FLOAT && dtype != op::DataType::DT_FLOAT16 && dtype != op::DataType::DT_BF16) { | ||
| 645 | + OP_LOGE_FOR_INVALID_DTYPE("ScatterReduce", "self", op::ToString(dtype).GetString(), "FLOAT32/FLOAT16/BFLOAT16"); | ||
| 646 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 647 | + } | ||
| 648 | + if (rank != 1 && rank != 2) { | ||
| 649 | + OP_LOGE_FOR_INVALID_VALUE_WITH_REASON("ScatterReduce", "self rank", std::to_string(rank).c_str(), | ||
| 650 | + "max supports rank 1 or 2"); | ||
| 651 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 652 | + } | ||
| 653 | + if (dim != 0 && dim != -static_cast<int64_t>(rank)) { | ||
| 654 | + const auto reason = "max requires dim=0 or " + std::to_string(-static_cast<int64_t>(rank)) + " for rank " + | ||
| 655 | + std::to_string(rank); | ||
| 656 | + OP_LOGE_FOR_INVALID_VALUE_WITH_REASON("ScatterReduce", "dim", std::to_string(dim).c_str(), reason.c_str()); | ||
| 657 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 658 | + } | ||
| 659 | + if (index->GetDataType() != op::DataType::DT_INT64) { | ||
| 660 | + OP_LOGE_FOR_INVALID_DTYPE("ScatterReduce", "index", op::ToString(index->GetDataType()).GetString(), "INT64"); | ||
| 661 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 662 | + } | ||
| 663 | + if (index->GetViewShape() != src->GetViewShape() || index->GetViewShape().GetDimNum() != rank || | ||
| 664 | + (rank == 2 && self->GetViewShape()[1] != src->GetViewShape()[1])) { | ||
| 665 | + const auto shapes = std::string(op::ToString(self->GetViewShape()).GetString()) + ", " + | ||
| 666 | + op::ToString(index->GetViewShape()).GetString() + ", " + | ||
| 667 | + op::ToString(src->GetViewShape()).GetString(); | ||
| 668 | + OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON( | ||
| 669 | + "ScatterReduce", "self, index, src", shapes.c_str(), | ||
| 670 | + "max requires equal ranks, matching index/src shapes and matching self/src columns"); | ||
| 671 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 672 | + } | ||
| 673 | + if (self->GetViewShape() != out->GetViewShape()) { | ||
| 674 | + const auto shapes = std::string(op::ToString(self->GetViewShape()).GetString()) + ", " + | ||
| 675 | + op::ToString(out->GetViewShape()).GetString(); | ||
| 676 | + OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON("ScatterReduce", "self, out", shapes.c_str(), | ||
| 677 | + "max requires matching self/out shapes"); | ||
| 678 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 679 | + } | ||
| 680 | + if (!index->IsEmpty()) { | ||
| 681 | + const auto status = CheckScatterReduceMaxSize(self, "self"); | ||
| 682 | + CHECK_RET(status == ACLNN_SUCCESS, status); | ||
| 683 | + return CheckScatterReduceMaxSize(src, "src"); | ||
| 684 | + } | ||
| 685 | + return ACLNN_SUCCESS; | ||
| 686 | +} | ||
| 687 | + | ||
| 577 | static aclnnStatus ExecScatterBase(const aclTensor* self, int64_t dim, const aclTensor* index, const aclTensor* src, | 688 | static aclnnStatus ExecScatterBase(const aclTensor* self, int64_t dim, const aclTensor* index, const aclTensor* src, |
| 578 | int64_t reduce, aclTensor* out, aclOpExecutor* executor, bool includeSelf = true) | 689 | int64_t reduce, aclTensor* out, aclOpExecutor* executor, bool includeSelf = true) |
| 579 | { | 690 | { |
| @@ -604,6 +715,10 @@ static aclnnStatus ExecScatterBase(const aclTensor* self, int64_t dim, const acl | |||
| 604 | return ACLNN_SUCCESS; | 715 | return ACLNN_SUCCESS; |
| 605 | } | 716 | } |
| 606 | 717 | ||
| 718 | + if (GetCurrentPlatformInfo().GetCurNpuArch() == NpuArch::DAV_2201 && reduce == REDUCE_MAX) { | ||
| 719 | + return ExecScatterReduceMax(selfContiguous, index, srcContiguous, out, includeSelf, executor); | ||
| 720 | + } | ||
| 721 | + | ||
| 607 | auto curArch = GetCurrentPlatformInfo().GetCurNpuArch(); | 722 | auto curArch = GetCurrentPlatformInfo().GetCurNpuArch(); |
| 608 | bool aicoreSupport = true; | 723 | bool aicoreSupport = true; |
| 609 | bool scatterAddRegbaseSupport = false; | 724 | bool scatterAddRegbaseSupport = false; |
| @@ -847,7 +962,9 @@ static aclnnStatus ExecScatterGetWorkspaceSize(const aclTensor* self, int64_t di | |||
| 847 | OP_CHECK(uniqueExecutor.get() != nullptr, | 962 | OP_CHECK(uniqueExecutor.get() != nullptr, |
| 848 | OP_LOGE_WITHOUT_REPORT(ACLNN_ERR_INNER_CREATE_EXECUTOR, "CREATE_EXECUTOR failed!"), | 963 | OP_LOGE_WITHOUT_REPORT(ACLNN_ERR_INNER_CREATE_EXECUTOR, "CREATE_EXECUTOR failed!"), |
| 849 | return ACLNN_ERR_INNER_CREATE_EXECUTOR); | 964 | return ACLNN_ERR_INNER_CREATE_EXECUTOR); |
| 850 | - bool supportNoTranspose = includeSelf && l0op::SupportNoTranspose(self, index, src, dim, GetReduceStr(reduce)); | 965 | + bool supportNoTranspose = !(GetCurrentPlatformInfo().GetCurNpuArch() == NpuArch::DAV_2201 && |
| 966 | + reduce == REDUCE_MAX) && | ||
| 967 | + includeSelf && l0op::SupportNoTranspose(self, index, src, dim, GetReduceStr(reduce)); | ||
| 851 | aclnnStatus ret; | 968 | aclnnStatus ret; |
| 852 | if (supportNoTranspose) { | 969 | if (supportNoTranspose) { |
| 853 | ret = ExecScatterNoTranspose(self, dim, index, src, reduce, out, uniqueExecutor.get(), includeSelf); | 970 | ret = ExecScatterNoTranspose(self, dim, index, src, reduce, out, uniqueExecutor.get(), includeSelf); |
| @@ -1056,16 +1173,23 @@ aclnnStatus aclnnScatterReduceGetWorkspaceSize(const aclTensor* self, int64_t di | |||
| 1056 | 1173 | ||
| 1057 | L2_DFX_PHASE_1(aclnnScatterReduce, DFX_IN(self, dim, index, src, reduce, includeSelf), DFX_OUT(out)); | 1174 | L2_DFX_PHASE_1(aclnnScatterReduce, DFX_IN(self, dim, index, src, reduce, includeSelf), DFX_OUT(out)); |
| 1058 | 1175 | ||
| 1059 | - // On 910B/910_93, only FP32 sum is supported. ExecScatterBase checks the | 1176 | + // A2/A3 support FP32 sum and first-axis FP32/FP16/BF16 max. |
| 1060 | - // AI Core shape constraints before dispatching include_self=false. | 1177 | + // Each path validates its AI Core shape constraints before dispatch. |
| 1061 | // 310P (NpuArch::DAV_2002) is not affected. | 1178 | // 310P (NpuArch::DAV_2002) is not affected. |
| 1179 | + if (reduce == REDUCE_MAX && GetCurrentPlatformInfo().GetCurNpuArch() == NpuArch::DAV_2201) { | ||
| 1180 | + auto status = CheckParams(self, index, src, out); | ||
| 1181 | + CHECK_RET(status == ACLNN_SUCCESS, status); | ||
| 1182 | + status = CheckScatterReduceMaxParams(self, index, src, out, dim); | ||
| 1183 | + CHECK_RET(status == ACLNN_SUCCESS, status); | ||
| 1184 | + } | ||
| 1062 | if (self != nullptr) { | 1185 | if (self != nullptr) { |
| 1063 | auto curArch = GetCurrentPlatformInfo().GetCurNpuArch(); | 1186 | auto curArch = GetCurrentPlatformInfo().GetCurNpuArch(); |
| 1064 | if (curArch == NpuArch::DAV_2201) { | 1187 | if (curArch == NpuArch::DAV_2201) { |
| 1065 | const std::string& reduction = GetReduceStr(reduce); | 1188 | const std::string& reduction = GetReduceStr(reduce); |
| 1066 | - if (!(self->GetDataType() == op::DataType::DT_FLOAT && reduction == REDUCTION_ADD)) { | 1189 | + if (!(self->GetDataType() == op::DataType::DT_FLOAT && reduction == REDUCTION_ADD) && |
| 1190 | + reduce != REDUCE_MAX) { | ||
| 1067 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, | 1191 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, |
| 1068 | - "scatter_reduce only supports dtype=float32, reduce=sum. " | 1192 | + "scatter_reduce supports FP32 sum and first-axis FP32/FP16/BF16 max with INT64 index. " |
| 1069 | "Got dtype=%s, reduce=%s, include_self=%d.", | 1193 | "Got dtype=%s, reduce=%s, include_self=%d.", |
| 1070 | op::ToString(self->GetDataType()).GetString(), reduction.c_str(), includeSelf ? 1 : 0); | 1194 | op::ToString(self->GetDataType()).GetString(), reduction.c_str(), includeSelf ? 1 : 0); |
| 1071 | return ACLNN_ERR_PARAM_INVALID; | 1195 | return ACLNN_ERR_PARAM_INVALID; |
| @@ -1135,16 +1259,23 @@ aclnnStatus aclnnInplaceScatterReduceGetWorkspaceSize(aclTensor* selfRef, int64_ | |||
| 1135 | 1259 | ||
| 1136 | L2_DFX_PHASE_1(aclnnInplaceScatterReduce, DFX_IN(selfRef, dim, index, src, reduce, includeSelf), DFX_OUT(selfRef)); | 1260 | L2_DFX_PHASE_1(aclnnInplaceScatterReduce, DFX_IN(selfRef, dim, index, src, reduce, includeSelf), DFX_OUT(selfRef)); |
| 1137 | 1261 | ||
| 1138 | - // On 910B/910_93, only FP32 sum is supported. ExecScatterBase checks the | 1262 | + // A2/A3 support FP32 sum and first-axis FP32/FP16/BF16 max. |
| 1139 | - // AI Core shape constraints before dispatching include_self=false. | 1263 | + // Each path validates its AI Core shape constraints before dispatch. |
| 1140 | // 310P (NpuArch::DAV_2002) is not affected. | 1264 | // 310P (NpuArch::DAV_2002) is not affected. |
| 1265 | + if (reduce == REDUCE_MAX && GetCurrentPlatformInfo().GetCurNpuArch() == NpuArch::DAV_2201) { | ||
| 1266 | + auto status = CheckParams(selfRef, index, src, selfRef); | ||
| 1267 | + CHECK_RET(status == ACLNN_SUCCESS, status); | ||
| 1268 | + status = CheckScatterReduceMaxParams(selfRef, index, src, selfRef, dim); | ||
| 1269 | + CHECK_RET(status == ACLNN_SUCCESS, status); | ||
| 1270 | + } | ||
| 1141 | if (selfRef != nullptr) { | 1271 | if (selfRef != nullptr) { |
| 1142 | auto curArch = GetCurrentPlatformInfo().GetCurNpuArch(); | 1272 | auto curArch = GetCurrentPlatformInfo().GetCurNpuArch(); |
| 1143 | if (curArch == NpuArch::DAV_2201) { | 1273 | if (curArch == NpuArch::DAV_2201) { |
| 1144 | const std::string& reduction = GetReduceStr(reduce); | 1274 | const std::string& reduction = GetReduceStr(reduce); |
| 1145 | - if (!(selfRef->GetDataType() == op::DataType::DT_FLOAT && reduction == REDUCTION_ADD)) { | 1275 | + if (!(selfRef->GetDataType() == op::DataType::DT_FLOAT && reduction == REDUCTION_ADD) && |
| 1276 | + reduce != REDUCE_MAX) { | ||
| 1146 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, | 1277 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, |
| 1147 | - "scatter_reduce only supports dtype=float32, reduce=sum. " | 1278 | + "scatter_reduce supports FP32 sum and first-axis FP32/FP16/BF16 max with INT64 index. " |
| 1148 | "Got dtype=%s, reduce=%s, include_self=%d.", | 1279 | "Got dtype=%s, reduce=%s, include_self=%d.", |
| 1149 | op::ToString(selfRef->GetDataType()).GetString(), reduction.c_str(), includeSelf ? 1 : 0); | 1280 | op::ToString(selfRef->GetDataType()).GetString(), reduction.c_str(), includeSelf ? 1 : 0); |
| 1150 | return ACLNN_ERR_PARAM_INVALID; | 1281 | return ACLNN_ERR_PARAM_INVALID; |
| @@ -22,6 +22,7 @@ | |||
| 22 | #include "scatter_elements_v2_tiling.h" | 22 | #include "scatter_elements_v2_tiling.h" |
| 23 | #include "../../common/scatter_elements_v2_low_memory_policy.h" | 23 | #include "../../common/scatter_elements_v2_low_memory_policy.h" |
| 24 | #include <vector> | 24 | #include <vector> |
| 25 | +#include "../../common/scatter_elements_v2_amax_policy.h" | ||
| 25 | 26 | ||
| 26 | using namespace std; | 27 | using namespace std; |
| 27 | using Ops::NN::Optiling::TilingRegistry; | 28 | using Ops::NN::Optiling::TilingRegistry; |
| @@ -1118,6 +1119,80 @@ ge::graphStatus ScatterElementsV2Tiling::RunCacheOpTiling() | |||
| 1118 | return SetCacheOpTiling(); | 1119 | return SetCacheOpTiling(); |
| 1119 | } | 1120 | } |
| 1120 | 1121 | ||
| 1122 | +static bool IsRowMax(const gert::TilingContext* context) | ||
| 1123 | +{ | ||
| 1124 | + const auto* attrs = context->GetAttrs(); | ||
| 1125 | + const auto* self = context->GetInputShape(0); | ||
| 1126 | + const auto* index = context->GetInputShape(1); | ||
| 1127 | + const auto* src = context->GetInputShape(2); | ||
| 1128 | + const auto* desc = context->GetInputDesc(0); | ||
| 1129 | + const auto* indexDesc = context->GetInputDesc(1); | ||
| 1130 | + if (attrs == nullptr || self == nullptr || index == nullptr || src == nullptr || desc == nullptr || | ||
| 1131 | + indexDesc == nullptr || indexDesc->GetDataType() != ge::DT_INT64) { | ||
| 1132 | + return false; | ||
| 1133 | + } | ||
| 1134 | + const auto* dim = attrs->GetAttrPointer<int64_t>(0); | ||
| 1135 | + const auto* reduce = attrs->GetAttrPointer<char>(1); | ||
| 1136 | + const auto& x = self->GetStorageShape(); | ||
| 1137 | + const auto& i = index->GetStorageShape(); | ||
| 1138 | + const auto& u = src->GetStorageShape(); | ||
| 1139 | + const auto dtype = desc->GetDataType(); | ||
| 1140 | + return dim != nullptr && (*dim == 0 || *dim == -2) && reduce != nullptr && strcmp(reduce, "max") == 0 && | ||
| 1141 | + (dtype == ge::DT_FLOAT || dtype == ge::DT_FLOAT16 || dtype == ge::DT_BF16) && x.GetDimNum() == 2 && | ||
| 1142 | + u.GetDimNum() == 2 && (i.GetDimNum() == 1 || i.GetDimNum() == 2) && x.GetDim(0) > 0 && x.GetDim(1) > 0 && | ||
| 1143 | + u.GetDim(0) > 0 && u.GetDim(1) == x.GetDim(1) && i.GetDim(0) == u.GetDim(0) && | ||
| 1144 | + (i.GetDimNum() == 1 || i.GetDim(1) == x.GetDim(1)); | ||
| 1145 | +} | ||
| 1146 | + | ||
| 1147 | +static ge::graphStatus TilingRowMax(gert::TilingContext* context, const ScatterElementsV2CompileInfo* info) | ||
| 1148 | +{ | ||
| 1149 | + const auto& x = context->GetInputShape(0)->GetStorageShape(); | ||
| 1150 | + const auto& i = context->GetInputShape(1)->GetStorageShape(); | ||
| 1151 | + const auto& u = context->GetInputShape(2)->GetStorageShape(); | ||
| 1152 | + const auto dtype = context->GetInputDesc(0)->GetDataType(); | ||
| 1153 | + const uint64_t bytes = dtype == ge::DT_FLOAT ? 4 : 2; | ||
| 1154 | + const auto plan = ScatterElementsV2Amax::Select(x.GetDim(0), u.GetDim(0), x.GetDim(1), bytes, i.GetDimNum() == 1, | ||
| 1155 | + info->totalCoreNum, info->ubSizePlatForm); | ||
| 1156 | + if (!plan.valid) { | ||
| 1157 | + const bool exceedsLimit = static_cast<uint64_t>(x.GetDim(0)) > | ||
| 1158 | + UINT32_MAX / static_cast<uint64_t>(x.GetDim(1)) || | ||
| 1159 | + static_cast<uint64_t>(u.GetDim(0)) > UINT32_MAX / static_cast<uint64_t>(x.GetDim(1)); | ||
| 1160 | + const auto reason = std::string(exceedsLimit ? | ||
| 1161 | + "max dimensions and element counts must be at most 4294967295" : | ||
| 1162 | + (info->totalCoreNum == 0 ? "max requires a positive core count" : | ||
| 1163 | + "max buffers exceed the available UB")) + | ||
| 1164 | + "; N=" + std::to_string(x.GetDim(0)) + ", E=" + std::to_string(u.GetDim(0)) + | ||
| 1165 | + ", C=" + std::to_string(x.GetDim(1)) + ", cores=" + std::to_string(info->totalCoreNum) + | ||
| 1166 | + ", UB=" + std::to_string(info->ubSizePlatForm); | ||
| 1167 | + ReportScatterElementsV2TilingError(context->GetNodeName(), reason.c_str()); | ||
| 1168 | + return ge::GRAPH_FAILED; | ||
| 1169 | + } | ||
| 1170 | + if (context->GetRawTilingData() == nullptr) { | ||
| 1171 | + ReportScatterElementsV2TilingError(context->GetNodeName(), "max tiling data is null"); | ||
| 1172 | + return ge::GRAPH_FAILED; | ||
| 1173 | + } | ||
| 1174 | + ScatterElementsV2TilingData data; | ||
| 1175 | + data.set_amaxInputRows(u.GetDim(0)); | ||
| 1176 | + data.set_amaxOutputRows(x.GetDim(0)); | ||
| 1177 | + data.set_amaxColumns(x.GetDim(1)); | ||
| 1178 | + data.set_amaxChunkLength(plan.chunk); | ||
| 1179 | + data.set_amaxTileLength(plan.tile); | ||
| 1180 | + data.set_amaxCompactRows(i.GetDimNum() == 1 ? 1 : 0); | ||
| 1181 | + const auto* includeSelf = context->GetAttrs()->GetAttrPointer<bool>(2); | ||
| 1182 | + data.set_includeSelf(includeSelf == nullptr || *includeSelf ? 1 : 0); | ||
| 1183 | + data.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); | ||
| 1184 | + context->GetRawTilingData()->SetDataSize(data.GetDataSize()); | ||
| 1185 | + context->SetBlockDim(plan.cores); | ||
| 1186 | + // Keys distinguish the three compute layouts and floating-point dtypes. | ||
| 1187 | + const uint64_t type = dtype == ge::DT_FLOAT ? 100 : (dtype == ge::DT_FLOAT16 ? 200 : 300); | ||
| 1188 | + const bool atomic = plan.algorithm == ScatterElementsV2Amax::Algorithm::InputPartition; | ||
| 1189 | + context->SetTilingKey((atomic ? 2000 : (plan.layout == ScatterElementsV2Amax::Layout::Rows ? 3000 : 1000)) + type); | ||
| 1190 | + auto workspace = context->GetWorkspaceSizes(1); | ||
| 1191 | + OP_CHECK_NULL_WITH_CONTEXT(context, workspace); | ||
| 1192 | + workspace[0] = info->workspaceSize + (atomic ? x.GetShapeSize() * (bytes + 2 * sizeof(int32_t)) : 0); | ||
| 1193 | + return ge::GRAPH_SUCCESS; | ||
| 1194 | +} | ||
| 1195 | + | ||
| 1121 | ge::graphStatus TilingScatterElementsV2(gert::TilingContext* context) | 1196 | ge::graphStatus TilingScatterElementsV2(gert::TilingContext* context) |
| 1122 | { | 1197 | { |
| 1123 | if (context == nullptr) { | 1198 | if (context == nullptr) { |
| @@ -1136,6 +1211,9 @@ ge::graphStatus TilingScatterElementsV2(gert::TilingContext* context) | |||
| 1136 | auto platformInfo = context->GetPlatformInfo(); | 1211 | auto platformInfo = context->GetPlatformInfo(); |
| 1137 | auto ascendcPlatform = platform_ascendc::PlatformAscendC(platformInfo); | 1212 | auto ascendcPlatform = platform_ascendc::PlatformAscendC(platformInfo); |
| 1138 | auto socVersion = ascendcPlatform.GetSocVersion(); | 1213 | auto socVersion = ascendcPlatform.GetSocVersion(); |
| 1214 | + if (ScatterElementsV2LowMemory::IsSupportedSoc(socVersion) && IsRowMax(context)) { | ||
| 1215 | + return TilingRowMax(context, compile_info); | ||
| 1216 | + } | ||
| 1139 | if (socVersion == platform_ascendc::SocVersion::ASCEND310P) { | 1217 | if (socVersion == platform_ascendc::SocVersion::ASCEND310P) { |
| 1140 | ScatterElementsV2Tiling310P tilingObject(context); | 1218 | ScatterElementsV2Tiling310P tilingObject(context); |
| 1141 | return tilingObject.Init(); | 1219 | return tilingObject.Init(); |
| @@ -68,6 +68,12 @@ TILING_DATA_FIELD_DEF(uint64_t, updatesDim0); | |||
| 68 | TILING_DATA_FIELD_DEF(uint64_t, updatesDim1); | 68 | TILING_DATA_FIELD_DEF(uint64_t, updatesDim1); |
| 69 | TILING_DATA_FIELD_DEF(uint64_t, batchSize); | 69 | TILING_DATA_FIELD_DEF(uint64_t, batchSize); |
| 70 | TILING_DATA_FIELD_DEF(uint64_t, realDim); | 70 | TILING_DATA_FIELD_DEF(uint64_t, realDim); |
| 71 | +TILING_DATA_FIELD_DEF(uint32_t, amaxInputRows); | ||
| 72 | +TILING_DATA_FIELD_DEF(uint32_t, amaxOutputRows); | ||
| 73 | +TILING_DATA_FIELD_DEF(uint32_t, amaxColumns); | ||
| 74 | +TILING_DATA_FIELD_DEF(uint32_t, amaxChunkLength); | ||
| 75 | +TILING_DATA_FIELD_DEF(uint32_t, amaxTileLength); | ||
| 76 | +TILING_DATA_FIELD_DEF(uint32_t, amaxCompactRows); | ||
| 71 | END_TILING_DATA_DEF; | 77 | END_TILING_DATA_DEF; |
| 72 | 78 | ||
| 73 | REGISTER_TILING_DATA_CLASS(ScatterElementsV2, ScatterElementsV2TilingData) | 79 | REGISTER_TILING_DATA_CLASS(ScatterElementsV2, ScatterElementsV2TilingData) |
| @@ -16,6 +16,7 @@ | |||
| 16 | #include "scatter_elements_v2_310p.h" | 16 | #include "scatter_elements_v2_310p.h" |
| 17 | #else | 17 | #else |
| 18 | #include "scatter_elements_v2.h" | 18 | #include "scatter_elements_v2.h" |
| 19 | +#include "scatter_elements_v2_amax.h" | ||
| 19 | #include "scatter_elements_v2_low_memory/exec_transpose_and_scatter_elements.h" | 20 | #include "scatter_elements_v2_low_memory/exec_transpose_and_scatter_elements.h" |
| 20 | 21 | ||
| 21 | __aicore__ inline bool IsCacheOpTiling(const ScatterElementsV2TilingData* tiling_data) | 22 | __aicore__ inline bool IsCacheOpTiling(const ScatterElementsV2TilingData* tiling_data) |
| @@ -97,6 +98,52 @@ extern "C" __global__ __aicore__ void scatter_elements_v2(GM_ADDR var, GM_ADDR i | |||
| 97 | } | 98 | } |
| 98 | #endif | 99 | #endif |
| 99 | #else | 100 | #else |
| 101 | + KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_MIX_AIV_1_0); | ||
| 102 | + if (TILING_KEY_IS(1100)) { | ||
| 103 | + KERNEL_TASK_TYPE(1100, KERNEL_TYPE_AIV_ONLY); | ||
| 104 | + ScatterElementsV2Amax::Run<float, 0>(var, indices, updates, workspace, tilingDevice); | ||
| 105 | + return; | ||
| 106 | + } | ||
| 107 | + if (TILING_KEY_IS(1200)) { | ||
| 108 | + KERNEL_TASK_TYPE(1200, KERNEL_TYPE_AIV_ONLY); | ||
| 109 | + ScatterElementsV2Amax::Run<half, 0>(var, indices, updates, workspace, tilingDevice); | ||
| 110 | + return; | ||
| 111 | + } | ||
| 112 | + if (TILING_KEY_IS(1300)) { | ||
| 113 | + KERNEL_TASK_TYPE(1300, KERNEL_TYPE_AIV_ONLY); | ||
| 114 | + ScatterElementsV2Amax::Run<bfloat16_t, 0>(var, indices, updates, workspace, tilingDevice); | ||
| 115 | + return; | ||
| 116 | + } | ||
| 117 | + if (TILING_KEY_IS(2100)) { | ||
| 118 | + KERNEL_TASK_TYPE(2100, KERNEL_TYPE_MIX_AIV_1_0); | ||
| 119 | + ScatterElementsV2Amax::Run<float, 2>(var, indices, updates, workspace, tilingDevice); | ||
| 120 | + return; | ||
| 121 | + } | ||
| 122 | + if (TILING_KEY_IS(2200)) { | ||
| 123 | + KERNEL_TASK_TYPE(2200, KERNEL_TYPE_MIX_AIV_1_0); | ||
| 124 | + ScatterElementsV2Amax::Run<half, 2>(var, indices, updates, workspace, tilingDevice); | ||
| 125 | + return; | ||
| 126 | + } | ||
| 127 | + if (TILING_KEY_IS(2300)) { | ||
| 128 | + KERNEL_TASK_TYPE(2300, KERNEL_TYPE_MIX_AIV_1_0); | ||
| 129 | + ScatterElementsV2Amax::Run<bfloat16_t, 2>(var, indices, updates, workspace, tilingDevice); | ||
| 130 | + return; | ||
| 131 | + } | ||
| 132 | + if (TILING_KEY_IS(3100)) { | ||
| 133 | + KERNEL_TASK_TYPE(3100, KERNEL_TYPE_AIV_ONLY); | ||
| 134 | + ScatterElementsV2Amax::Run<float, 1>(var, indices, updates, workspace, tilingDevice); | ||
| 135 | + return; | ||
| 136 | + } | ||
| 137 | + if (TILING_KEY_IS(3200)) { | ||
| 138 | + KERNEL_TASK_TYPE(3200, KERNEL_TYPE_AIV_ONLY); | ||
| 139 | + ScatterElementsV2Amax::Run<half, 1>(var, indices, updates, workspace, tilingDevice); | ||
| 140 | + return; | ||
| 141 | + } | ||
| 142 | + if (TILING_KEY_IS(3300)) { | ||
| 143 | + KERNEL_TASK_TYPE(3300, KERNEL_TYPE_AIV_ONLY); | ||
| 144 | + ScatterElementsV2Amax::Run<bfloat16_t, 1>(var, indices, updates, workspace, tilingDevice); | ||
| 145 | + return; | ||
| 146 | + } | ||
| 100 | if (TILING_KEY_IS(110)) { | 147 | if (TILING_KEY_IS(110)) { |
| 101 | CALL_OP_IMPL(float, int); | 148 | CALL_OP_IMPL(float, int); |
| 102 | } else if (TILING_KEY_IS(120)) { | 149 | } else if (TILING_KEY_IS(120)) { |
| @@ -570,10 +570,10 @@ private: | |||
| 570 | return static_cast<DataType>(inputValue * updateValue); | 570 | return static_cast<DataType>(inputValue * updateValue); |
| 571 | } | 571 | } |
| 572 | if (mode == ScatterElementsV2NS::SCATTER_MODE_MIN) { | 572 | if (mode == ScatterElementsV2NS::SCATTER_MODE_MIN) { |
| 573 | - return inputValue < updateValue ? inputValue : updateValue; | 573 | + return inputValue != inputValue || inputValue <= updateValue ? inputValue : updateValue; |
| 574 | } | 574 | } |
| 575 | if (mode == ScatterElementsV2NS::SCATTER_MODE_MAX) { | 575 | if (mode == ScatterElementsV2NS::SCATTER_MODE_MAX) { |
| 576 | - return inputValue > updateValue ? inputValue : updateValue; | 576 | + return inputValue != inputValue || inputValue >= updateValue ? inputValue : updateValue; |
| 577 | } | 577 | } |
| 578 | return updateValue; | 578 | return updateValue; |
| 579 | } | 579 | } |
| @@ -9,6 +9,7 @@ | |||
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | #include <vector> | 11 | #include <vector> |
| 12 | +#include <memory> | ||
| 12 | #include "gtest/gtest.h" | 13 | #include "gtest/gtest.h" |
| 13 | #include <dlfcn.h> | 14 | #include <dlfcn.h> |
| 14 | 15 | ||
| @@ -212,7 +213,8 @@ TEST_F(l2_scatter_reduce_test, arch22_rejects_other_reductions_and_dtypes) | |||
| 212 | op::NpuArchManager platform(NpuArch::DAV_2201); | 213 | op::NpuArchManager platform(NpuArch::DAV_2201); |
| 213 | for (auto dtype : {ACL_FLOAT, ACL_FLOAT16, ACL_BF16, ACL_INT32}) { | 214 | for (auto dtype : {ACL_FLOAT, ACL_FLOAT16, ACL_BF16, ACL_INT32}) { |
| 214 | for (int64_t reduction : {kReduceNone, kReduceAdd, kReduceMul, kReduceMax, kReduceMin, kReduceMean}) { | 215 | for (int64_t reduction : {kReduceNone, kReduceAdd, kReduceMul, kReduceMax, kReduceMin, kReduceMean}) { |
| 215 | - if (dtype == ACL_FLOAT && reduction == kReduceAdd) { | 216 | + if ((dtype == ACL_FLOAT && reduction == kReduceAdd) || |
| 217 | + ((dtype == ACL_FLOAT || dtype == ACL_FLOAT16 || dtype == ACL_BF16) && reduction == kReduceMax)) { | ||
| 216 | continue; | 218 | continue; |
| 217 | } | 219 | } |
| 218 | for (bool includeSelf : {false, true}) { | 220 | for (bool includeSelf : {false, true}) { |
| @@ -359,3 +361,124 @@ TEST_F(l2_scatter_reduce_test, ascend960_add_mul_include_self) | |||
| 359 | } | 361 | } |
| 360 | } | 362 | } |
| 361 | } | 363 | } |
| 364 | + | ||
| 365 | +TEST_F(l2_scatter_reduce_test, arch22_max_include_self) | ||
| 366 | +{ | ||
| 367 | + op::SocVersionManager soc(op::SocVersion::ASCEND910B); | ||
| 368 | + op::NpuArchManager platform(NpuArch::DAV_2201); | ||
| 369 | + for (auto dtype : {ACL_FLOAT, ACL_FLOAT16, ACL_BF16}) { | ||
| 370 | + for (const auto& shape : {vector<int64_t>{7}, vector<int64_t>{7, 33}}) { | ||
| 371 | + for (bool includeSelf : {false, true}) { | ||
| 372 | + auto self = TensorDesc(shape, dtype, ACL_FORMAT_ND); | ||
| 373 | + auto index = TensorDesc(shape, ACL_INT64, ACL_FORMAT_ND).ValueRange(0, 6); | ||
| 374 | + auto src = TensorDesc(shape, dtype, ACL_FORMAT_ND); | ||
| 375 | + auto out = TensorDesc(shape, dtype, ACL_FORMAT_ND); | ||
| 376 | + uint64_t ws = 0; | ||
| 377 | + auto ut = OP_API_UT(aclnnScatterReduce, INPUT(self, int64_t{0}, index, src, kReduceMax, includeSelf), | ||
| 378 | + OUTPUT(out)); | ||
| 379 | + EXPECT_EQ(ut.TestGetWorkspaceSize(&ws), ACLNN_SUCCESS); | ||
| 380 | + auto ip = OP_API_UT( | ||
| 381 | + aclnnInplaceScatterReduce, | ||
| 382 | + INPUT(self, -static_cast<int64_t>(shape.size()), index, src, kReduceMax, includeSelf), OUTPUT()); | ||
| 383 | + EXPECT_EQ(ip.TestGetWorkspaceSize(&ws), ACLNN_SUCCESS); | ||
| 384 | + } | ||
| 385 | + } | ||
| 386 | + } | ||
| 387 | +} | ||
| 388 | + | ||
| 389 | +TEST_F(l2_scatter_reduce_test, arch22_max_shape_limits) | ||
| 390 | +{ | ||
| 391 | + op::SocVersionManager soc(op::SocVersion::ASCEND910B); | ||
| 392 | + op::NpuArchManager platform(NpuArch::DAV_2201); | ||
| 393 | + auto self = TensorDesc({7, 33}, ACL_FLOAT, ACL_FORMAT_ND); | ||
| 394 | + auto src = TensorDesc({7, 33}, ACL_FLOAT, ACL_FORMAT_ND); | ||
| 395 | + auto index = TensorDesc({7, 33}, ACL_INT64, ACL_FORMAT_ND).ValueRange(0, 6); | ||
| 396 | + auto out = TensorDesc({7, 33}, ACL_FLOAT, ACL_FORMAT_ND); | ||
| 397 | + uint64_t ws = 0; | ||
| 398 | + auto reject = [&](TensorDesc input, TensorDesc indices, TensorDesc updates, TensorDesc output, int64_t dim, | ||
| 399 | + std::initializer_list<const char*> expected) { | ||
| 400 | + for (bool includeSelf : {false, true}) { | ||
| 401 | + auto ut = OP_API_UT(aclnnScatterReduce, INPUT(input, dim, indices, updates, kReduceMax, includeSelf), | ||
| 402 | + OUTPUT(output)); | ||
| 403 | + EXPECT_EQ(ut.TestGetWorkspaceSize(&ws), ACLNN_ERR_PARAM_INVALID); | ||
| 404 | + CheckErrorMessage(expected); | ||
| 405 | + auto ip = OP_API_UT(aclnnInplaceScatterReduce, INPUT(input, dim, indices, updates, kReduceMax, includeSelf), | ||
| 406 | + OUTPUT()); | ||
| 407 | + EXPECT_EQ(ip.TestGetWorkspaceSize(&ws), ACLNN_ERR_PARAM_INVALID); | ||
| 408 | + CheckErrorMessage(expected); | ||
| 409 | + } | ||
| 410 | + }; | ||
| 411 | + reject(self, index, src, out, 1, {"dim", "1", "dim=0 or -2", "rank 2"}); | ||
| 412 | + auto i32 = TensorDesc({7, 33}, ACL_INT32, ACL_FORMAT_ND).ValueRange(0, 6); | ||
| 413 | + reject(self, i32, src, out, 0, {"index", "INT32", "INT64"}); | ||
| 414 | + auto rank3 = TensorDesc({2, 7, 33}, ACL_FLOAT, ACL_FORMAT_ND); | ||
| 415 | + auto index3 = TensorDesc({2, 7, 33}, ACL_INT64, ACL_FORMAT_ND).ValueRange(0, 1); | ||
| 416 | + reject(rank3, index3, rank3, rank3, 0, {"self rank", "3", "rank 1 or 2"}); | ||
| 417 | + auto shortSrc = TensorDesc({6, 33}, ACL_FLOAT, ACL_FORMAT_ND); | ||
| 418 | + reject(self, index, shortSrc, out, 0, {"self, index, src", "7, 33", "6, 33", "matching index/src shapes"}); | ||
| 419 | + auto narrow = TensorDesc({7, 32}, ACL_FLOAT, ACL_FORMAT_ND); | ||
| 420 | + auto narrowIndex = TensorDesc({7, 32}, ACL_INT64, ACL_FORMAT_ND).ValueRange(0, 6); | ||
| 421 | + reject(self, narrowIndex, narrow, out, 0, {"self, index, src", "7, 33", "7, 32", "matching self/src columns"}); | ||
| 422 | + auto emptyIndex = TensorDesc({0, 33}, ACL_INT64, ACL_FORMAT_ND); | ||
| 423 | + auto emptySrc = TensorDesc({0, 33}, ACL_FLOAT, ACL_FORMAT_ND); | ||
| 424 | + auto empty = OP_API_UT(aclnnScatterReduce, INPUT(self, int64_t{0}, emptyIndex, emptySrc, kReduceMax, false), | ||
| 425 | + OUTPUT(out)); | ||
| 426 | + EXPECT_EQ(empty.TestGetWorkspaceSize(&ws), ACLNN_SUCCESS); | ||
| 427 | +} | ||
| 428 | + | ||
| 429 | +TEST_F(l2_scatter_reduce_test, arch22_max_element_count_limits) | ||
| 430 | +{ | ||
| 431 | + op::SocVersionManager soc(op::SocVersion::ASCEND910B); | ||
| 432 | + op::NpuArchManager platform(NpuArch::DAV_2201); | ||
| 433 | + auto tensor = [](const vector<int64_t>& shape, aclDataType dtype) { | ||
| 434 | + const vector<int64_t> strides = {shape[1], 1}; | ||
| 435 | + return std::unique_ptr<aclTensor, decltype(&aclDestroyTensor)>( | ||
| 436 | + aclCreateTensor(shape.data(), shape.size(), dtype, strides.data(), 0, ACL_FORMAT_ND, shape.data(), | ||
| 437 | + shape.size(), nullptr), | ||
| 438 | + aclDestroyTensor); | ||
| 439 | + }; | ||
| 440 | + for (auto dtype : {ACL_FLOAT, ACL_FLOAT16, ACL_BF16}) { | ||
| 441 | + for (bool selfLimit : {false, true}) { | ||
| 442 | + for (int64_t rows : {65536LL, 4294967296LL}) { | ||
| 443 | + const int64_t columns = rows == 65536 ? 65536 : 1; | ||
| 444 | + const vector<int64_t> large = {rows, columns}, small = {1, columns}; | ||
| 445 | + auto self = tensor(selfLimit ? large : small, dtype); | ||
| 446 | + auto src = tensor(selfLimit ? small : large, dtype); | ||
| 447 | + auto index = tensor(selfLimit ? small : large, ACL_INT64); | ||
| 448 | + ASSERT_NE(self.get(), nullptr); | ||
| 449 | + ASSERT_NE(src.get(), nullptr); | ||
| 450 | + ASSERT_NE(index.get(), nullptr); | ||
| 451 | + for (bool includeSelf : {false, true}) { | ||
| 452 | + uint64_t ws = 0; | ||
| 453 | + aclOpExecutor* executor = nullptr; | ||
| 454 | + EXPECT_EQ(aclnnScatterReduceGetWorkspaceSize(self.get(), 0, index.get(), src.get(), kReduceMax, | ||
| 455 | + includeSelf, self.get(), &ws, &executor), | ||
| 456 | + ACLNN_ERR_PARAM_INVALID); | ||
| 457 | + EXPECT_EQ(executor, nullptr); | ||
| 458 | + CheckErrorMessage({selfLimit ? "self" : "src", "4294967296", "4294967295"}); | ||
| 459 | + EXPECT_EQ(aclnnInplaceScatterReduceGetWorkspaceSize(self.get(), 0, index.get(), src.get(), | ||
| 460 | + kReduceMax, includeSelf, &ws, &executor), | ||
| 461 | + ACLNN_ERR_PARAM_INVALID); | ||
| 462 | + EXPECT_EQ(executor, nullptr); | ||
| 463 | + CheckErrorMessage({selfLimit ? "self" : "src", "4294967296", "4294967295"}); | ||
| 464 | + } | ||
| 465 | + } | ||
| 466 | + } | ||
| 467 | + } | ||
| 468 | + auto largeSelf = tensor({65536, 65536}, ACL_FLOAT); | ||
| 469 | + auto emptySrc = tensor({0, 65536}, ACL_FLOAT); | ||
| 470 | + auto emptyIndex = tensor({0, 65536}, ACL_INT64); | ||
| 471 | + for (bool includeSelf : {false, true}) { | ||
| 472 | + uint64_t ws = 0; | ||
| 473 | + aclOpExecutor* executor = nullptr; | ||
| 474 | + EXPECT_EQ(aclnnScatterReduceGetWorkspaceSize(largeSelf.get(), 0, emptyIndex.get(), emptySrc.get(), kReduceMax, | ||
| 475 | + includeSelf, largeSelf.get(), &ws, &executor), | ||
| 476 | + ACLNN_SUCCESS); | ||
| 477 | + delete executor; | ||
| 478 | + executor = nullptr; | ||
| 479 | + EXPECT_EQ(aclnnInplaceScatterReduceGetWorkspaceSize(largeSelf.get(), 0, emptyIndex.get(), emptySrc.get(), | ||
| 480 | + kReduceMax, includeSelf, &ws, &executor), | ||
| 481 | + ACLNN_SUCCESS); | ||
| 482 | + delete executor; | ||
| 483 | + } | ||
| 484 | +} | ||
| @@ -14,6 +14,7 @@ | |||
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | #include <iostream> | 16 | #include <iostream> |
| 17 | +#include "../../../../common/scatter_elements_v2_amax_policy.h" | ||
| 17 | #include <vector> | 18 | #include <vector> |
| 18 | #include <thread> | 19 | #include <thread> |
| 19 | #include <nlohmann/json.hpp> | 20 | #include <nlohmann/json.hpp> |
| @@ -74,11 +75,12 @@ struct Arch22TilingResult { | |||
| 74 | uint32_t blockDim = 0; | 75 | uint32_t blockDim = 0; |
| 75 | }; | 76 | }; |
| 76 | 77 | ||
| 77 | -static Arch22TilingResult ExecuteArch22DeterministicCase(ge::DataType inputDtype, ge::DataType indicesDtype, | 78 | +enum class Arch22Workspace { Ready, Missing, Empty }; |
| 78 | - ge::DataType updatesDtype, gert::StorageShape& inputShape, | 79 | + |
| 79 | - gert::StorageShape& indicesShape, | 80 | +static Arch22TilingResult ExecuteArch22DeterministicCase( |
| 80 | - gert::StorageShape& updatesShape, int64_t axis, | 81 | + ge::DataType inputDtype, ge::DataType indicesDtype, ge::DataType updatesDtype, gert::StorageShape& inputShape, |
| 81 | - const std::string& reduction, int32_t deterministic) | 82 | + gert::StorageShape& indicesShape, gert::StorageShape& updatesShape, int64_t axis, const std::string& reduction, |
| 83 | + int32_t deterministic, Arch22Workspace workspaceState = Arch22Workspace::Ready, bool includeSelf = true) | ||
| 82 | { | 84 | { |
| 83 | std::string op_type("ScatterElementsV2"); | 85 | std::string op_type("ScatterElementsV2"); |
| 84 | EXPECT_NE(gert::OpImplRegistry::GetInstance().GetOpImpl(op_type.c_str()), nullptr); | 86 | EXPECT_NE(gert::OpImplRegistry::GetInstance().GetOpImpl(op_type.c_str()), nullptr); |
| @@ -121,7 +123,8 @@ static Arch22TilingResult ExecuteArch22DeterministicCase(ge::DataType inputDtype | |||
| 121 | 123 | ||
| 122 | auto param = gert::TilingData::CreateCap(4096); | 124 | auto param = gert::TilingData::CreateCap(4096); |
| 123 | EXPECT_NE(param, nullptr); | 125 | EXPECT_NE(param, nullptr); |
| 124 | - auto workspace_size_holer = gert::ContinuousVector::Create<size_t>(4096); | 126 | + auto workspace_size_holer = gert::ContinuousVector::Create<size_t>(workspaceState == Arch22Workspace::Empty ? 0 : |
| 127 | + 4096); | ||
| 125 | auto ws_size = reinterpret_cast<gert::ContinuousVector*>(workspace_size_holer.get()); | 128 | auto ws_size = reinterpret_cast<gert::ContinuousVector*>(workspace_size_holer.get()); |
| 126 | auto outputShape = inputShape; | 129 | auto outputShape = inputShape; |
| 127 | 130 | ||
| @@ -133,14 +136,15 @@ static Arch22TilingResult ExecuteArch22DeterministicCase(ge::DataType inputDtype | |||
| 133 | .CompileInfo(&compile_info) | 136 | .CompileInfo(&compile_info) |
| 134 | .DeterministicInfo(reinterpret_cast<int32_t*>(deterministic)) | 137 | .DeterministicInfo(reinterpret_cast<int32_t*>(deterministic)) |
| 135 | .NodeAttrs({{"axis", Ops::NN::AnyValue::CreateFrom<int64_t>(axis)}, | 138 | .NodeAttrs({{"axis", Ops::NN::AnyValue::CreateFrom<int64_t>(axis)}, |
| 136 | - {"reduction", Ops::NN::AnyValue::CreateFrom<std::string>(reduction)}}) | 139 | + {"reduction", Ops::NN::AnyValue::CreateFrom<std::string>(reduction)}, |
| 140 | + {"include_self", Ops::NN::AnyValue::CreateFrom<bool>(includeSelf)}}) | ||
| 137 | .PlatformInfo(reinterpret_cast<char*>(&platform_info)) | 141 | .PlatformInfo(reinterpret_cast<char*>(&platform_info)) |
| 138 | .NodeInputTd(0, inputDtype, ge::FORMAT_ND, ge::FORMAT_ND) | 142 | .NodeInputTd(0, inputDtype, ge::FORMAT_ND, ge::FORMAT_ND) |
| 139 | .NodeInputTd(1, indicesDtype, ge::FORMAT_ND, ge::FORMAT_ND) | 143 | .NodeInputTd(1, indicesDtype, ge::FORMAT_ND, ge::FORMAT_ND) |
| 140 | .NodeInputTd(2, updatesDtype, ge::FORMAT_ND, ge::FORMAT_ND) | 144 | .NodeInputTd(2, updatesDtype, ge::FORMAT_ND, ge::FORMAT_ND) |
| 141 | .NodeOutputTd(0, inputDtype, ge::FORMAT_ND, ge::FORMAT_ND) | 145 | .NodeOutputTd(0, inputDtype, ge::FORMAT_ND, ge::FORMAT_ND) |
| 142 | .TilingData(param.get()) | 146 | .TilingData(param.get()) |
| 143 | - .Workspace(ws_size) | 147 | + .Workspace(workspaceState == Arch22Workspace::Missing ? nullptr : ws_size) |
| 144 | .Build(); | 148 | .Build(); |
| 145 | 149 | ||
| 146 | gert::TilingContext* tiling_context = holder.GetContext<gert::TilingContext>(); | 150 | gert::TilingContext* tiling_context = holder.GetContext<gert::TilingContext>(); |
| @@ -957,3 +961,91 @@ TEST_F(ScatterElementsV2Tiling, test_scatter_elements_v2_other_error) | |||
| 957 | // workspaces nullptr return failed | 961 | // workspaces nullptr return failed |
| 958 | EXPECT_NE(tiling_func(tiling_context), ge::GRAPH_SUCCESS); | 962 | EXPECT_NE(tiling_func(tiling_context), ge::GRAPH_SUCCESS); |
| 959 | } | 963 | } |
| 964 | + | ||
| 965 | +TEST_F(ScatterElementsV2Tiling, MaxResourcePlan) | ||
| 966 | +{ | ||
| 967 | + for (uint64_t bytes : {2U, 4U}) { | ||
| 968 | + for (uint64_t n : {3U, 63U, 64U, 1888U, 65536U}) { | ||
| 969 | + for (uint64_t c : {1U, 8U, 33U, 64U, 256U, 4097U}) { | ||
| 970 | + for (bool compact : {false, true}) { | ||
| 971 | + const auto plan = ScatterElementsV2Amax::Select(n, 4096, c, bytes, compact, 40, 196608); | ||
| 972 | + ASSERT_TRUE(plan.valid); | ||
| 973 | + EXPECT_LE(plan.ubBytes + ScatterElementsV2Amax::kRuntimeUbBytes, 196608U); | ||
| 974 | + EXPECT_GT(plan.chunk, 0U); | ||
| 975 | + EXPECT_GT(plan.cores, 0U); | ||
| 976 | + EXPECT_LE(plan.cores, 40U); | ||
| 977 | + if (plan.layout == ScatterElementsV2Amax::Layout::Rows) { | ||
| 978 | + EXPECT_TRUE(compact); | ||
| 979 | + } | ||
| 980 | + } | ||
| 981 | + } | ||
| 982 | + } | ||
| 983 | + } | ||
| 984 | + EXPECT_FALSE(ScatterElementsV2Amax::Select(3, 7, 8, 4, true, 0, 196608).valid); | ||
| 985 | + EXPECT_FALSE(ScatterElementsV2Amax::Select(3, 7, 8, 4, true, 40, 8192).valid); | ||
| 986 | +} | ||
| 987 | + | ||
| 988 | +TEST_F(ScatterElementsV2Tiling, MaxLayoutsDeterministic) | ||
| 989 | +{ | ||
| 990 | + struct MaxCase { | ||
| 991 | + int64_t n, e, c; | ||
| 992 | + bool compact; | ||
| 993 | + uint64_t key; | ||
| 994 | + }; | ||
| 995 | + for (auto dtype : {ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_BF16}) { | ||
| 996 | + for (const auto& shape : {MaxCase{1888, 4864, 256, true, 3100}, MaxCase{7, 4097, 32, false, 2100}, | ||
| 997 | + MaxCase{7, 131, 32, false, 2100}, MaxCase{3, 67, 33, false, 1100}}) { | ||
| 998 | + gert::StorageShape input({shape.n, shape.c}, {shape.n, shape.c}); | ||
| 999 | + gert::StorageShape index = shape.compact ? gert::StorageShape({shape.e}, {shape.e}) : | ||
| 1000 | + gert::StorageShape({shape.e, shape.c}, {shape.e, shape.c}); | ||
| 1001 | + gert::StorageShape updates({shape.e, shape.c}, {shape.e, shape.c}); | ||
| 1002 | + for (bool includeSelf : {false, true}) { | ||
| 1003 | + SCOPED_TRACE(includeSelf); | ||
| 1004 | + const auto normal = ExecuteArch22DeterministicCase(dtype, ge::DT_INT64, dtype, input, index, updates, 0, | ||
| 1005 | + "max", 0, Arch22Workspace::Ready, includeSelf); | ||
| 1006 | + const auto deterministic = ExecuteArch22DeterministicCase(dtype, ge::DT_INT64, dtype, input, index, | ||
| 1007 | + updates, 0, "max", 1, Arch22Workspace::Ready, | ||
| 1008 | + includeSelf); | ||
| 1009 | + EXPECT_EQ(normal.status, ge::GRAPH_SUCCESS); | ||
| 1010 | + EXPECT_EQ(deterministic.status, ge::GRAPH_SUCCESS); | ||
| 1011 | + const uint64_t key = shape.key + (dtype == ge::DT_FLOAT ? 0 : (dtype == ge::DT_FLOAT16 ? 100 : 200)); | ||
| 1012 | + EXPECT_EQ(normal.tilingKey, key); | ||
| 1013 | + EXPECT_EQ(deterministic.tilingKey, key); | ||
| 1014 | + EXPECT_EQ(normal.blockDim, deterministic.blockDim); | ||
| 1015 | + if (shape.key == 2100) { | ||
| 1016 | + EXPECT_GT(deterministic.blockDim, 1U); | ||
| 1017 | + } | ||
| 1018 | + } | ||
| 1019 | + } | ||
| 1020 | + } | ||
| 1021 | +} | ||
| 1022 | + | ||
| 1023 | +TEST_F(ScatterElementsV2Tiling, MaxElementCountLimit) | ||
| 1024 | +{ | ||
| 1025 | + for (uint64_t bytes : {2U, 4U}) { | ||
| 1026 | + EXPECT_TRUE(ScatterElementsV2Amax::Select(65535, 1, 65537, bytes, false, 48, 196608).valid); | ||
| 1027 | + EXPECT_FALSE(ScatterElementsV2Amax::Select(65536, 1, 65536, bytes, false, 48, 196608).valid); | ||
| 1028 | + EXPECT_FALSE(ScatterElementsV2Amax::Select(1, 65536, 65536, bytes, false, 48, 196608).valid); | ||
| 1029 | + EXPECT_FALSE(ScatterElementsV2Amax::Select(1ULL << 32, 1, 1, bytes, false, 48, 196608).valid); | ||
| 1030 | + } | ||
| 1031 | + gert::StorageShape input({65536, 65536}, {65536, 65536}); | ||
| 1032 | + gert::StorageShape index({1, 65536}, {1, 65536}); | ||
| 1033 | + gert::StorageShape updates({1, 65536}, {1, 65536}); | ||
| 1034 | + const auto r = ExecuteArch22DeterministicCase(ge::DT_FLOAT, ge::DT_INT64, ge::DT_FLOAT, input, index, updates, 0, | ||
| 1035 | + "max", 0); | ||
| 1036 | + EXPECT_EQ(r.status, ge::GRAPH_FAILED); | ||
| 1037 | +} | ||
| 1038 | + | ||
| 1039 | +TEST_F(ScatterElementsV2Tiling, MaxWorkspaceUnavailable) | ||
| 1040 | +{ | ||
| 1041 | + gert::StorageShape input({7, 32}, {7, 32}); | ||
| 1042 | + gert::StorageShape index({67, 32}, {67, 32}); | ||
| 1043 | + gert::StorageShape updates({67, 32}, {67, 32}); | ||
| 1044 | + for (auto dtype : {ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_BF16}) { | ||
| 1045 | + for (auto state : {Arch22Workspace::Missing, Arch22Workspace::Empty}) { | ||
| 1046 | + const auto r = ExecuteArch22DeterministicCase(dtype, ge::DT_INT64, dtype, input, index, updates, 0, "max", | ||
| 1047 | + 0, state); | ||
| 1048 | + EXPECT_EQ(r.status, ge::GRAPH_FAILED); | ||
| 1049 | + } | ||
| 1050 | + } | ||
| 1051 | +} | ||
| @@ -12,9 +12,12 @@ | |||
| 12 | * \brief | 12 | * \brief |
| 13 | */ | 13 | */ |
| 14 | #include <array> | 14 | #include <array> |
| 15 | +#include <cmath> | ||
| 16 | +#include <limits> | ||
| 15 | #include <vector> | 17 | #include <vector> |
| 16 | #include "gtest/gtest.h" | 18 | #include "gtest/gtest.h" |
| 17 | #include "../../../op_host/arch22/scatter_elements_v2_tiling.h" | 19 | #include "../../../op_host/arch22/scatter_elements_v2_tiling.h" |
| 20 | +#include "../../../common/scatter_elements_v2_amax_policy.h" | ||
| 18 | 21 | ||
| 19 | #ifdef __CCE_KT_TEST__ | 22 | #ifdef __CCE_KT_TEST__ |
| 20 | #include "tikicpulib.h" | 23 | #include "tikicpulib.h" |
| @@ -99,7 +102,7 @@ TEST_F(scatter_elements_v2_test, test_case_fp32) | |||
| 99 | } | 102 | } |
| 100 | 103 | ||
| 101 | template <typename Index> | 104 | template <typename Index> |
| 102 | -static void RunSumIncludeSelfCase(bool includeSelf, bool splitIndices) | 105 | +static void RunSumIncludeSelfCase(bool includeSelf, bool splitIndices, uint64_t mode = 2) |
| 103 | { | 106 | { |
| 104 | constexpr uint32_t rows = 2; | 107 | constexpr uint32_t rows = 2; |
| 105 | constexpr uint32_t columns = 16; | 108 | constexpr uint32_t columns = 16; |
| @@ -108,6 +111,7 @@ static void RunSumIncludeSelfCase(bool includeSelf, bool splitIndices) | |||
| 108 | vector<Index> indices(rows * updatesPerRow); | 111 | vector<Index> indices(rows * updatesPerRow); |
| 109 | vector<float> updates(rows * updatesPerRow); | 112 | vector<float> updates(rows * updatesPerRow); |
| 110 | vector<float> expected(rows * columns); | 113 | vector<float> expected(rows * columns); |
| 114 | + vector<bool> seen(rows * columns, includeSelf); | ||
| 111 | for (uint32_t row = 0; row < rows; ++row) { | 115 | for (uint32_t row = 0; row < rows; ++row) { |
| 112 | for (uint32_t column = 0; column < columns; ++column) { | 116 | for (uint32_t column = 0; column < columns; ++column) { |
| 113 | initial[row * columns + column] = 100.0f + row * columns + column; | 117 | initial[row * columns + column] = 100.0f + row * columns + column; |
| @@ -122,7 +126,24 @@ static void RunSumIncludeSelfCase(bool includeSelf, bool splitIndices) | |||
| 122 | uint32_t column = update % 3 == 0 ? 0 : (update % 3 == 1 ? 9 : 15); | 126 | uint32_t column = update % 3 == 0 ? 0 : (update % 3 == 1 ? 9 : 15); |
| 123 | indices[row * updatesPerRow + update] = static_cast<Index>(column); | 127 | indices[row * updatesPerRow + update] = static_cast<Index>(column); |
| 124 | updates[row * updatesPerRow + update] = static_cast<float>(update + row + 1); | 128 | updates[row * updatesPerRow + update] = static_cast<float>(update + row + 1); |
| 125 | - expected[row * columns + column] += updates[row * updatesPerRow + update]; | 129 | + const auto target = row * columns + column; |
| 130 | + if (mode == 4 || mode == 5) { | ||
| 131 | + auto& value = updates[row * updatesPerRow + update]; | ||
| 132 | + if (update == 0 || update == 10) { | ||
| 133 | + value = numeric_limits<float>::quiet_NaN(); | ||
| 134 | + } | ||
| 135 | + if (!seen[target]) { | ||
| 136 | + expected[target] = value; | ||
| 137 | + seen[target] = true; | ||
| 138 | + } else if (std::isnan(expected[target]) || std::isnan(value)) { | ||
| 139 | + expected[target] = numeric_limits<float>::quiet_NaN(); | ||
| 140 | + } else { | ||
| 141 | + expected[target] = mode == 4 ? std::min(expected[target], value) : | ||
| 142 | + std::max(expected[target], value); | ||
| 143 | + } | ||
| 144 | + } else { | ||
| 145 | + expected[target] += updates[row * updatesPerRow + update]; | ||
| 146 | + } | ||
| 126 | } | 147 | } |
| 127 | } | 148 | } |
| 128 | 149 | ||
| @@ -161,7 +182,7 @@ static void RunSumIncludeSelfCase(bool includeSelf, bool splitIndices) | |||
| 161 | tiling->lastOneTime = 1; | 182 | tiling->lastOneTime = 1; |
| 162 | tiling->modeFlag = splitIndices ? 0 : 1; | 183 | tiling->modeFlag = splitIndices ? 0 : 1; |
| 163 | tiling->includeSelf = includeSelf ? 1 : 0; | 184 | tiling->includeSelf = includeSelf ? 1 : 0; |
| 164 | - tiling->mode = 2; // ADD | 185 | + tiling->mode = mode; |
| 165 | 186 | ||
| 166 | ICPU_SET_TILING_KEY(sizeof(Index) == sizeof(int64_t) ? 120 : 110); | 187 | ICPU_SET_TILING_KEY(sizeof(Index) == sizeof(int64_t) ? 120 : 110); |
| 167 | AscendC::SetKernelMode(KernelMode::AIV_MODE); | 188 | AscendC::SetKernelMode(KernelMode::AIV_MODE); |
| @@ -169,7 +190,11 @@ static void RunSumIncludeSelfCase(bool includeSelf, bool splitIndices) | |||
| 169 | reinterpret_cast<uint8_t*>(indicesGm), reinterpret_cast<uint8_t*>(updatesGm), | 190 | reinterpret_cast<uint8_t*>(indicesGm), reinterpret_cast<uint8_t*>(updatesGm), |
| 170 | reinterpret_cast<uint8_t*>(inputGm), nullptr, tilingGm); | 191 | reinterpret_cast<uint8_t*>(inputGm), nullptr, tilingGm); |
| 171 | for (size_t index = 0; index < expected.size(); ++index) { | 192 | for (size_t index = 0; index < expected.size(); ++index) { |
| 172 | - EXPECT_FLOAT_EQ(inputGm[index], expected[index]) << "output index=" << index; | 193 | + if (std::isnan(expected[index])) { |
| 194 | + EXPECT_TRUE(std::isnan(inputGm[index])) << "output index=" << index; | ||
| 195 | + } else { | ||
| 196 | + EXPECT_FLOAT_EQ(inputGm[index], expected[index]) << "output index=" << index; | ||
| 197 | + } | ||
| 173 | } | 198 | } |
| 174 | AscendC::GmFree(inputGm); | 199 | AscendC::GmFree(inputGm); |
| 175 | AscendC::GmFree(indicesGm); | 200 | AscendC::GmFree(indicesGm); |
| @@ -194,3 +219,121 @@ TEST_F(scatter_elements_v2_test, sum_include_self_true_regression) | |||
| 194 | RunSumIncludeSelfCase<int32_t>(true, false); | 219 | RunSumIncludeSelfCase<int32_t>(true, false); |
| 195 | RunSumIncludeSelfCase<int64_t>(true, true); | 220 | RunSumIncludeSelfCase<int64_t>(true, true); |
| 196 | } | 221 | } |
| 222 | + | ||
| 223 | +TEST_F(scatter_elements_v2_test, max_layouts_repeatability) | ||
| 224 | +{ | ||
| 225 | + for (uint64_t key : {1100U, 2100U, 3100U}) { | ||
| 226 | + SCOPED_TRACE(key); | ||
| 227 | + const bool atomic = key == 2100; | ||
| 228 | + const uint32_t n = atomic ? 7 : 3; | ||
| 229 | + const uint32_t c = atomic ? 32 : 33; | ||
| 230 | + const uint32_t e = atomic ? 131 : 3; | ||
| 231 | + const auto plan = ScatterElementsV2Amax::Select(n, e, c, sizeof(float), key == 3100, 4, 196608); | ||
| 232 | + if (atomic) { | ||
| 233 | + ASSERT_TRUE(plan.valid); | ||
| 234 | + ASSERT_EQ(plan.algorithm, ScatterElementsV2Amax::Algorithm::InputPartition); | ||
| 235 | + ASSERT_GT(plan.cores, 1U); | ||
| 236 | + } | ||
| 237 | + for (bool includeSelf : {false, true}) { | ||
| 238 | + SCOPED_TRACE(includeSelf); | ||
| 239 | + vector<float> initial(n * c, 9.0f), source(e * c, 2.0f); | ||
| 240 | + vector<int64_t> index(key == 3100 ? e : e * c, 0); | ||
| 241 | + for (uint32_t row = 0; row < e; ++row) { | ||
| 242 | + for (uint32_t col = 0; col < c; ++col) { | ||
| 243 | + source[row * c + col] = row + col; | ||
| 244 | + } | ||
| 245 | + source[row * c + 2] = -numeric_limits<float>::infinity(); | ||
| 246 | + source[row * c + 3] = row == 0 ? -0.0f : 0.0f; | ||
| 247 | + source[row * c + 4] = row == 0 ? 0.0f : -0.0f; | ||
| 248 | + } | ||
| 249 | + source[0] = numeric_limits<float>::quiet_NaN(); | ||
| 250 | + source[(e - 1) * c] = numeric_limits<float>::quiet_NaN(); | ||
| 251 | + source[(e / 2) * c + 1] = numeric_limits<float>::infinity(); | ||
| 252 | + initial[3] = 0.0f; | ||
| 253 | + initial[4] = -0.0f; | ||
| 254 | + initial[5] = numeric_limits<float>::quiet_NaN(); | ||
| 255 | + initial[c] = -0.0f; | ||
| 256 | + initial[c + 1] = numeric_limits<float>::quiet_NaN(); | ||
| 257 | + initial[c + 2] = numeric_limits<float>::infinity(); | ||
| 258 | + initial[c + 3] = -numeric_limits<float>::infinity(); | ||
| 259 | + auto expected = initial; | ||
| 260 | + expected[0] = numeric_limits<float>::quiet_NaN(); | ||
| 261 | + expected[1] = numeric_limits<float>::infinity(); | ||
| 262 | + expected[2] = includeSelf ? initial[2] : -numeric_limits<float>::infinity(); | ||
| 263 | + expected[3] = includeSelf ? initial[3] : -0.0f; | ||
| 264 | + expected[4] = includeSelf ? initial[4] : 0.0f; | ||
| 265 | + expected[5] = includeSelf ? initial[5] : static_cast<float>(e - 1 + 5); | ||
| 266 | + for (uint32_t col = 6; col < c; ++col) { | ||
| 267 | + const float updated = static_cast<float>(e - 1 + col); | ||
| 268 | + expected[col] = includeSelf ? std::max(initial[col], updated) : updated; | ||
| 269 | + } | ||
| 270 | + auto* var = static_cast<float*>(AscendC::GmAlloc(initial.size() * sizeof(float))); | ||
| 271 | + auto* idx = static_cast<int64_t*>(AscendC::GmAlloc(index.size() * sizeof(int64_t))); | ||
| 272 | + auto* src = static_cast<float*>(AscendC::GmAlloc(source.size() * sizeof(float))); | ||
| 273 | + auto* td = static_cast<ScatterElementsV2TilingData*>(AscendC::GmAlloc(sizeof(ScatterElementsV2TilingData))); | ||
| 274 | + const size_t workspaceBytes = n * c * (sizeof(float) + 2 * sizeof(int32_t)); | ||
| 275 | + auto* workspace = atomic ? static_cast<uint8_t*>(AscendC::GmAlloc(workspaceBytes)) : nullptr; | ||
| 276 | + ASSERT_NE(var, nullptr); | ||
| 277 | + ASSERT_NE(idx, nullptr); | ||
| 278 | + ASSERT_NE(src, nullptr); | ||
| 279 | + ASSERT_NE(td, nullptr); | ||
| 280 | + if (atomic) { | ||
| 281 | + ASSERT_NE(workspace, nullptr); | ||
| 282 | + } | ||
| 283 | + memcpy(idx, index.data(), index.size() * sizeof(int64_t)); | ||
| 284 | + memcpy(src, source.data(), source.size() * sizeof(float)); | ||
| 285 | + memset(td, 0, sizeof(*td)); | ||
| 286 | + td->includeSelf = includeSelf; | ||
| 287 | + td->amaxInputRows = e; | ||
| 288 | + td->amaxOutputRows = n; | ||
| 289 | + td->amaxColumns = c; | ||
| 290 | + td->amaxChunkLength = atomic ? plan.chunk : (key == 3100 ? n : 104); | ||
| 291 | + td->amaxTileLength = atomic ? plan.tile : (key == 3100 ? 1024 : 2048); | ||
| 292 | + td->amaxCompactRows = key == 3100; | ||
| 293 | + vector<float> first(initial.size()); | ||
| 294 | + for (int repeat = 0; repeat < 3; ++repeat) { | ||
| 295 | + SCOPED_TRACE(repeat); | ||
| 296 | + memcpy(var, initial.data(), initial.size() * sizeof(float)); | ||
| 297 | + if (atomic) { | ||
| 298 | + memset(workspace, 0, workspaceBytes); | ||
| 299 | + } | ||
| 300 | + ICPU_SET_TILING_KEY(key); | ||
| 301 | + AscendC::SetKernelMode(KernelMode::AIV_MODE); | ||
| 302 | + ICPU_RUN_KF(scatter_elements_v2, atomic ? plan.cores : 1, reinterpret_cast<uint8_t*>(var), | ||
| 303 | + reinterpret_cast<uint8_t*>(idx), reinterpret_cast<uint8_t*>(src), | ||
| 304 | + reinterpret_cast<uint8_t*>(var), workspace, reinterpret_cast<uint8_t*>(td)); | ||
| 305 | + for (size_t element = 0; element < expected.size(); ++element) { | ||
| 306 | + uint32_t expectedBits = 0; | ||
| 307 | + uint32_t actualBits = 0; | ||
| 308 | + memcpy(&expectedBits, &expected[element], sizeof(expectedBits)); | ||
| 309 | + memcpy(&actualBits, &var[element], sizeof(actualBits)); | ||
| 310 | + EXPECT_EQ(actualBits, expectedBits) << "output index=" << element; | ||
| 311 | + } | ||
| 312 | + if (repeat == 0) { | ||
| 313 | + memcpy(first.data(), var, first.size() * sizeof(float)); | ||
| 314 | + } else { | ||
| 315 | + EXPECT_EQ(memcmp(first.data(), var, first.size() * sizeof(float)), 0); | ||
| 316 | + } | ||
| 317 | + } | ||
| 318 | + AscendC::GmFree(var); | ||
| 319 | + AscendC::GmFree(idx); | ||
| 320 | + AscendC::GmFree(src); | ||
| 321 | + AscendC::GmFree(td); | ||
| 322 | + if (atomic) { | ||
| 323 | + AscendC::GmFree(workspace); | ||
| 324 | + } | ||
| 325 | + } | ||
| 326 | + } | ||
| 327 | +} | ||
| 328 | + | ||
| 329 | +TEST_F(scatter_elements_v2_test, legacy_max_min_nan) | ||
| 330 | +{ | ||
| 331 | + for (uint64_t mode : {4U, 5U}) { | ||
| 332 | + for (bool includeSelf : {false, true}) { | ||
| 333 | + for (bool split : {false, true}) { | ||
| 334 | + RunSumIncludeSelfCase<int32_t>(includeSelf, split, mode); | ||
| 335 | + RunSumIncludeSelfCase<int64_t>(includeSelf, split, mode); | ||
| 336 | + } | ||
| 337 | + } | ||
| 338 | + } | ||
| 339 | +} | ||
TilingRowMax 里一串指针解引用前没有空检查。