Merged
[feat] 新增SelectTopk算子对齐DeepSelect 接口功能 #7557
nextyaleAdd on 5 days ago
[feat] 新增SelectTopk算子对齐DeepSelect 接口功能 #7557
Merged
Merged fromnextyale/ops-mathselecttopk_pr_4intocann/ops-mathmaster
Total: 58 files changed+10253-0
| @@ -0,0 +1,14 @@ | |||
| 1 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 2 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 3 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 4 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 5 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 6 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 7 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 8 | +# A2/A3 共用 arch22 实现,DMA、向量/标量访问与跨核阶段由内核显式同步。 | ||
| 9 | +set(SUPPORT_COMPUTE_UNIT "ascend910b" "ascend910_93") | ||
| 10 | +set(SUPPORT_TILING_DIR "arch22" "arch22") | ||
| 11 | +add_kernel_sources(COMPUTE_UNITS ${SUPPORT_COMPUTE_UNIT} AUTO_SYNC false) | ||
| 12 | +add_all_modules_sources(OPTYPE select_top_k ACLNNTYPE aclnn_exclude | ||
| 13 | + COMPUTE_UNIT ${SUPPORT_COMPUTE_UNIT} TILING_DIR ${SUPPORT_TILING_DIR} | ||
| 14 | + DISABLE_IN_OPP FALSE DEPENDENCIES radix_top_k sort_with_index) | ||
| @@ -0,0 +1,83 @@ | |||
| 1 | +# SelectTopK | ||
| 2 | + | ||
| 3 | +## 产品支持情况 | ||
| 4 | + | ||
| 5 | +| 产品 | 是否支持 | | ||
| 6 | +| :--- | :---: | | ||
| 7 | +| Ascend 950PR/Ascend 950DT | × | | ||
| 8 | +| Atlas A3 训练系列产品 / Atlas A3 推理系列产品 | √ | | ||
| 9 | +| Atlas A2 训练系列产品 / Atlas A2 推理系列产品 | √ | | ||
| 10 | +| Atlas 200I/500 A2 推理产品 | × | | ||
| 11 | +| Atlas 推理系列产品 | × | | ||
| 12 | +| Atlas 训练系列产品 | × | | ||
| 13 | + | ||
交付件不够完整,这个内容是 ![]() ![]() | |||
| 14 | +A2(`ascend910b`)与 A3(`ascend910_93`)共用 arch22 Host 和 Kernel 实现,资源大小由运行平台提供。 | ||
| 15 | +Radix 分流后的有序输出要求 SortWithIndex 在 A2/A3 上支持 FP16/BF16 值与原轴索引的同步排序。 | ||
| 16 | + | ||
| 17 | +## 功能说明 | ||
| 18 | + | ||
| 19 | +- 沿指定维度,从每行的有效区间选取最大或最小的 K 个元素,支持同时输出值和索引、仅值或仅索引。 | ||
| 20 | +- 支持逐行或广播区间、原轴索引偏移、INT32/INT64 索引、非连续输入与输出,以及有效元素不足 K 时的尾部填充。 | ||
| 21 | +- `order=0` 不保证有效结果顺序;`order=1` 按值排序,`largest=true` 时降序,否则升序。`order=2` 为预留值,当前不支持。 | ||
| 22 | + | ||
| 23 | +对第 r 行,将区间端点裁剪到排序轴范围 `[0,N]`,记为 b、e,有效输出数量为: | ||
| 24 | + | ||
| 25 | +$$ | ||
| 26 | +valid_r = \min(K, \max(e_r-b_r, 0)) | ||
| 27 | +$$ | ||
| 28 | + | ||
| 29 | +前 `valid_r` 项保存选中值及其原轴索引,后续项使用填充值。有效索引加 `indices_offset`, | ||
| 30 | +`index_fill` 原样写入。L2 输出形状为输入形状将排序维替换为 K。 | ||
| 31 | + | ||
| 32 | +例如输入 `[9,2,7,1,5]`,区间 `[1,4)`,K=4,选最大值并排序,偏移为10,值/索引填充为-99/-1, | ||
| 33 | +输出 values=`[7,2,1,-99]`、indices=`[12,11,13,-1]`。 | ||
| 34 | + | ||
| 35 | +## 参数说明 | ||
| 36 | + | ||
| 37 | +以下为算子原型参数。ACLNN 层另提供 `dim`、输出张量和工作区参数,完整说明见 | ||
| 38 | +[aclnnSelectTopK](docs/aclnnSelectTopK.md)。L0 输入视图为 `[outer,N,inner]`,输出为 `[outer,K,inner]`。 | ||
| 39 | + | ||
| 40 | +| 参数名 | 输入/输出/属性 | 说明 | 数据类型 | | ||
| 41 | +| :--- | :--- | :--- | :--- | | ||
| 42 | +| x | 输入 | 待选择的值,ND 格式 | FLOAT16、BFLOAT16、FLOAT | | ||
| 43 | +| begin | 可选输入 | 包含的原轴起点;一个元素广播或每行一个元素,缺省为0 | INT32 | | ||
| 44 | +| end | 可选输入 | 不包含的原轴终点;广播规则同 begin,缺省为N | INT32 | | ||
| 45 | +| values | 输出 | 选中值,类型与 x 一致 | FLOAT16、BFLOAT16、FLOAT | | ||
| 46 | +| indices | 输出 | 原轴索引或填充值 | INT32、INT64 | | ||
| 47 | +| k | 属性 | 每行输出数量,允许超过有效区间长度 | Int | | ||
| 48 | +| largest | 属性 | true 取最大值,false 取最小值;默认true | Bool | | ||
| 49 | +| order | 属性 | 0不排序,1按值排序;默认1 | Int | | ||
| 50 | +| indices_dtype | 属性 | 索引输出类型;默认INT64 | Int | | ||
| 51 | +| indices_offset | 属性 | 只作用于有效原轴索引;默认0 | Int | | ||
| 52 | +| output_mode | 属性 | 0仅值、1仅索引、2双输出;默认2 | Int | | ||
| 53 | +| value_fill | 属性 | 有效元素不足K时的值填充;默认0 | Float | | ||
| 54 | +| index_fill | 属性 | 有效元素不足K时的索引填充;默认-1 | Int | | ||
| 55 | + | ||
| 56 | +## 约束说明 | ||
| 57 | + | ||
| 58 | +- ACLNN 支持1~8维输入,排序轴长度及K不超过INT32_MAX;输出索引及其偏移不得超出指定索引类型范围。 | ||
| 59 | +- 相等值的索引选择不保证稳定;同时输出值与索引时,两者必须逐项对应。 | ||
| 60 | +- `K=0` 或行数为0时返回空输出;排序轴为空且输出非空时,全部结果使用填充值。 | ||
| 61 | +- 全轴、双输出、offset=0 的 FP16/BF16 请求满足 Radix 门控时,L2 调用 Topk/RadixTopK; | ||
| 62 | + `order=1` 再由 SortWithIndex 同时重排值及原轴索引,`order=0` 直接使用无序结果。 | ||
| 63 | +- Radix 分流接受 NaN 规范化为 `0x7fff`,非NaN值及 SelectTopK 主路径仍保持原值位模式。 | ||
| 64 | + 完整门控和配套 Radix 依赖见接口文档。 | ||
| 65 | + | ||
| 66 | +## 调用说明 | ||
| 67 | + | ||
| 68 | +| 调用方式 | 接口或样例 | 说明 | | ||
| 69 | +| :--- | :--- | :--- | | ||
| 70 | +| ACLNN | [aclnnSelectTopK](docs/aclnnSelectTopK.md#调用示例) | 两阶段接口,查询工作区后在指定流上执行 | | ||
| 71 | +| 可执行样例 | [test_aclnn_select_top_k.cpp](examples/test_aclnn_select_top_k.cpp) | 创建设备张量,执行标准模式并校验值和索引 | | ||
| 72 | +| L0 | [SelectTopK](op_api/select_top_k.h) | 构建 SelectTopK 主路径计算任务 | | ||
| 73 | + | ||
| 74 | +构建及 CPU 单元测试入口: | ||
| 75 | + | ||
| 76 | +```bash | ||
| 77 | +# 在 ops-math 仓库根目录执行。 | ||
| 78 | +bash build.sh --pkg --soc=ascend910b --ops=select_top_k --vendor_name=select_topk -j4 | ||
| 79 | +# A3 使用 --soc=ascend910_93。 | ||
| 80 | +bash build.sh -u --ophost --soc=ascend910b --ops=select_top_k --gtest_filter='SelectTopK*' -j4 | ||
| 81 | +``` | ||
| 82 | + | ||
| 83 | +测试范围见 [tests/README.md](tests/README.md)。CPU 测试不替代设备功能、位精度和完整 L2 性能验证。 | ||
| @@ -0,0 +1,83 @@ | |||
| 1 | +# SelectTopK | ||
| 2 | + | ||
| 3 | +## Supported Products | ||
| 4 | + | ||
| 5 | +| Product | Supported | | ||
| 6 | +| :--- | :---: | | ||
| 7 | +| Ascend 950PR/Ascend 950DT | × | | ||
| 8 | +| Atlas A3 training series / Atlas A3 inference series | √ | | ||
| 9 | +| Atlas A2 training series / Atlas A2 inference series | √ | | ||
| 10 | +| Atlas 200I/500 A2 inference products | × | | ||
| 11 | +| Atlas inference series | × | | ||
| 12 | +| Atlas training series | × | | ||
| 13 | + | ||
产品名称不对,没有term标记对 ![]() ![]() | |||
| 14 | +A2 (`ascend910b`) and A3 (`ascend910_93`) share the arch22 Host and Kernel implementation. Resource capacities are obtained from the runtime platform. | ||
| 15 | +Sorted output on the Radix route requires SortWithIndex to sort FP16/BF16 values together with their original-axis indices on A2/A3. | ||
| 16 | + | ||
| 17 | +## Function | ||
| 18 | + | ||
| 19 | +- Selects the K largest or smallest elements within each row's valid range along the specified dimension. Supports values and indices, values only, or indices only. | ||
| 20 | +- Supports per-row or broadcast ranges, offsets applied to original-axis indices, INT32/INT64 indices, noncontiguous inputs and outputs, and padding when fewer than K elements are valid. | ||
| 21 | +- `order=0` does not guarantee the order of valid results. `order=1` sorts by value, descending for `largest=true` and ascending otherwise. `order=2` is reserved and currently unsupported. | ||
| 22 | + | ||
| 23 | +For row r, clamp the range endpoints to `[0,N]`, obtaining b and e. The number of valid output elements is: | ||
| 24 | + | ||
| 25 | +$$ | ||
| 26 | +valid_r = \min(K, \max(e_r-b_r, 0)) | ||
| 27 | +$$ | ||
| 28 | + | ||
| 29 | +The first `valid_r` entries contain selected values and their original-axis indices. Remaining entries contain fill values. | ||
| 30 | +`indices_offset` is added only to valid indices; `index_fill` is written unchanged. The L2 output shape replaces the selected input dimension with K. | ||
| 31 | + | ||
| 32 | +For input `[9,2,7,1,5]`, range `[1,4)`, K=4, largest values sorted first, offset 10, and value/index fills -99/-1, | ||
| 33 | +the outputs are values=`[7,2,1,-99]` and indices=`[12,11,13,-1]`. | ||
| 34 | + | ||
| 35 | +## Parameters | ||
| 36 | + | ||
| 37 | +The following parameters belong to the operator prototype. The ACLNN interface additionally takes `dim`, output tensors, and workspace parameters; see | ||
| 38 | +[aclnnSelectTopK](docs/aclnnSelectTopK.md). The L0 input view is `[outer,N,inner]`, and the output view is `[outer,K,inner]`. | ||
| 39 | + | ||
| 40 | +| Name | Input/Output/Attribute | Description | Data type | | ||
| 41 | +| :--- | :--- | :--- | :--- | | ||
| 42 | +| x | Input | Values to select from, in ND format | FLOAT16, BFLOAT16, FLOAT | | ||
| 43 | +| begin | Optional input | Inclusive original-axis start; one element broadcasts to all rows, otherwise one element per row; defaults to 0 | INT32 | | ||
| 44 | +| end | Optional input | Exclusive original-axis end; same broadcasting rules as begin; defaults to N | INT32 | | ||
| 45 | +| values | Output | Selected values, with the same type as x | FLOAT16, BFLOAT16, FLOAT | | ||
| 46 | +| indices | Output | Original-axis indices or fill values | INT32, INT64 | | ||
| 47 | +| k | Attribute | Output count per row; may exceed the valid range length | Int | | ||
| 48 | +| largest | Attribute | true selects largest values, false selects smallest values; defaults to true | Bool | | ||
| 49 | +| order | Attribute | 0: no ordering guarantee; 1: sort by value; defaults to 1 | Int | | ||
| 50 | +| indices_dtype | Attribute | Output index type; defaults to INT64 | Int | | ||
| 51 | +| indices_offset | Attribute | Added only to valid original-axis indices; defaults to 0 | Int | | ||
| 52 | +| output_mode | Attribute | 0: values only; 1: indices only; 2: both; defaults to 2 | Int | | ||
| 53 | +| value_fill | Attribute | Value padding when fewer than K elements are valid; defaults to 0 | Float | | ||
| 54 | +| index_fill | Attribute | Index padding when fewer than K elements are valid; defaults to -1 | Int | | ||
| 55 | + | ||
| 56 | +## Constraints | ||
| 57 | + | ||
| 58 | +- ACLNN accepts inputs with 1 to 8 dimensions. The selected axis length and K must not exceed INT32_MAX. Indices, including offsets, must fit the requested index type. | ||
| 59 | +- Index selection among equal values is not guaranteed to be stable. When both values and indices are returned, each pair must correspond to the same input element. | ||
| 60 | +- `K=0` or zero rows produces empty outputs. An empty selected axis with nonempty outputs produces only fill values. | ||
| 61 | +- Full-axis FP16/BF16 requests with both outputs and offset=0 use Topk/RadixTopK when the Radix gate is satisfied. | ||
| 62 | + For `order=1`, SortWithIndex reorders values and original-axis indices together; `order=0` uses the unordered results directly. | ||
| 63 | +- The Radix route permits NaN normalization to `0x7fff`. Non-NaN values and the SelectTopK main path retain their original bit patterns. | ||
| 64 | + See the interface documentation for the full gate and companion Radix dependencies. | ||
| 65 | + | ||
| 66 | +## Usage | ||
| 67 | + | ||
| 68 | +| Method | Interface or example | Description | | ||
| 69 | +| :--- | :--- | :--- | | ||
| 70 | +| ACLNN | [aclnnSelectTopK](docs/aclnnSelectTopK.md#调用示例) | Two-stage API: query the workspace, then execute on the specified stream | | ||
| 71 | +| Executable example | [test_aclnn_select_top_k.cpp](examples/test_aclnn_select_top_k.cpp) | Creates device tensors, runs standard mode, and checks values and indices | | ||
| 72 | +| L0 | [SelectTopK](op_api/select_top_k.h) | Builds tasks for the SelectTopK main path | | ||
| 73 | + | ||
| 74 | +Build and CPU unit test commands: | ||
| 75 | + | ||
| 76 | +```bash | ||
| 77 | +# Run from the ops-math repository root. | ||
| 78 | +bash build.sh --pkg --soc=ascend910b --ops=select_top_k --vendor_name=select_topk -j4 | ||
| 79 | +# Use --soc=ascend910_93 for A3. | ||
| 80 | +bash build.sh -u --ophost --soc=ascend910b --ops=select_top_k --gtest_filter='SelectTopK*' -j4 | ||
| 81 | +``` | ||
| 82 | + | ||
| 83 | +See [tests/README_en.md](tests/README_en.md) for test coverage. CPU tests do not replace device functionality, bit-accuracy, or complete L2 performance validation. | ||
| @@ -0,0 +1,298 @@ | |||
| 1 | +# aclnnSelectTopK | ||
| 2 | + | ||
| 3 | +## 产品支持情况 | ||
| 4 | + | ||
| 5 | +| 产品 | 是否支持 | | ||
| 6 | +| :------------------------------------------------------- | :------: | | ||
| 7 | +| <term>Ascend 950PR/Ascend 950DT</term> | × | | ||
| 8 | +| <term>Atlas A3 训练系列产品/Atlas A3 推理系列产品</term> | √ | | ||
| 9 | +| <term>Atlas A2 训练系列产品/Atlas A2 推理系列产品</term> | √ | | ||
| 10 | +| <term>Atlas 200I/500 A2 推理产品</term> | × | | ||
| 11 | +| <term>Atlas 推理系列产品</term> | × | | ||
| 12 | +| <term>Atlas 训练系列产品</term> | × | | ||
| 13 | + | ||
| 14 | +## 功能说明 | ||
| 15 | + | ||
| 16 | +- 接口功能:沿指定维度,在每行给定区间内选取最大或最小的k个元素,支持仅输出值、仅输出索引或同时输出两者。 | ||
| 17 | +- 扩展能力:支持逐行区间、索引偏移、INT32/INT64索引和尾部填充;k可以大于排序轴或区间长度,输入和输出支持非连续Tensor。 | ||
| 18 | +- 计算规则: | ||
| 19 | + | ||
| 20 | + 记输入维数为rank,归一化后的排序轴为d,轴长度为N。去掉d维后,其余各维按原顺序展开为rows行。第r行的有效元素数为: | ||
| 21 | + | ||
| 22 | + $$ | ||
| 23 | + valid_r = \min(k, \max(e_r - b_r, 0)) | ||
| 24 | + $$ | ||
| 25 | + | ||
| 26 | + 其中,$b_r$、$e_r$为beginOptional、endOptional指定的起止位置,分别默认为0、N,并截断到[0, N]。每行从左闭右开区间$[b_r, e_r)$中选取元素,结果写入前$valid_r$项,其余位置使用valueFill、indexFill填充。 | ||
| 27 | + | ||
| 28 | + 有效索引为原排序轴坐标加indicesOffset;填充索引直接使用indexFill,不叠加偏移。输出shape为self的shape将d维替换为k。 | ||
| 29 | +- 示例: | ||
| 30 | + | ||
| 31 | + 输入`self = [9, 2, 7, 1, 5]`,`begin=1`,`end=4`,`k=4`,`largest=true`,`order=ACLNN_SELECT_TOP_K_ORDER_VALUE`,`indicesOffset=10`,`valueFill=-99`,`indexFill=-1`: | ||
| 32 | + | ||
| 33 | + - values:`[7, 2, 1, -99]` | ||
| 34 | + - indices:`[12, 11, 13, -1]` | ||
| 35 | + | ||
| 36 | +## 函数原型 | ||
| 37 | + | ||
| 38 | +本算子采用[两段式接口](../../../docs/zh/context/two_phase_api.md),先调用`aclnnSelectTopKGetWorkspaceSize`获取计算所需workspace大小及包含算子计算流程的执行器,再调用`aclnnSelectTopK`执行计算。 | ||
| 39 | + | ||
| 40 | +```cpp | ||
| 41 | +aclnnStatus aclnnSelectTopKGetWorkspaceSize( | ||
| 42 | + const aclTensor* self, | ||
| 43 | + int64_t k, | ||
| 44 | + int64_t dim, | ||
| 45 | + bool largest, | ||
| 46 | + aclDataType indicesType, | ||
| 47 | + int64_t order, | ||
| 48 | + int64_t indicesOffset, | ||
| 49 | + int64_t outputMode, | ||
| 50 | + const aclTensor* beginOptional, | ||
| 51 | + const aclTensor* endOptional, | ||
| 52 | + const aclScalar* valueFill, | ||
| 53 | + int64_t indexFill, | ||
| 54 | + aclTensor* valuesOutOptional, | ||
| 55 | + aclTensor* indicesOutOptional, | ||
| 56 | + uint64_t* workspaceSize, | ||
| 57 | + aclOpExecutor** executor); | ||
| 58 | +``` | ||
| 59 | + | ||
| 60 | +```cpp | ||
| 61 | +aclnnStatus aclnnSelectTopK( | ||
| 62 | + void* workspace, | ||
| 63 | + uint64_t workspaceSize, | ||
| 64 | + aclOpExecutor* executor, | ||
| 65 | + aclrtStream stream); | ||
| 66 | +``` | ||
| 67 | + | ||
| 68 | +### 枚举说明 | ||
| 69 | + | ||
| 70 | +| 枚举 | 值 | 含义 | | ||
| 71 | +| ------------------------------------- | -- | ------------------------------------------------- | | ||
| 72 | +| `ACLNN_SELECT_TOP_K_ORDER_NONE` | 0 | 不排序,不保证有效结果顺序 | | ||
| 73 | +| `ACLNN_SELECT_TOP_K_ORDER_VALUE` | 1 | 有效结果按值排序:`largest=true` 降序,反之升序 | | ||
| 74 | +| `ACLNN_SELECT_TOP_K_ORDER_INDEX` | 2 | 按索引排序的预留值,当前传入会报错 | | ||
| 75 | +| `ACLNN_SELECT_TOP_K_OUTPUT_VALUES` | 0 | 仅输出值 | | ||
| 76 | +| `ACLNN_SELECT_TOP_K_OUTPUT_INDICES` | 1 | 仅输出索引 | | ||
| 77 | +| `ACLNN_SELECT_TOP_K_OUTPUT_BOTH` | 2 | 同时输出值和索引 | | ||
| 78 | + | ||
| 79 | +## aclnnSelectTopKGetWorkspaceSize | ||
| 80 | + | ||
| 81 | +- **参数说明** | ||
| 82 | + | ||
| 83 | + <table style="table-layout: fixed; width: 1500px"><colgroup> | ||
| 84 | + <col style="width: 220px"> | ||
| 85 | + <col style="width: 100px"> | ||
| 86 | + <col style="width: 240px"> | ||
| 87 | + <col style="width: 440px"> | ||
| 88 | + <col style="width: 170px"> | ||
| 89 | + <col style="width: 130px"> | ||
| 90 | + <col style="width: 100px"> | ||
| 91 | + <col style="width: 100px"> | ||
| 92 | + </colgroup> | ||
| 93 | + <thead> | ||
| 94 | + <tr> | ||
| 95 | + <th>参数名</th><th>输入/输出</th><th>描述</th><th>使用说明</th> | ||
| 96 | + <th>数据类型</th><th>数据格式</th><th>维度(shape)</th><th>非连续Tensor</th> | ||
| 97 | + </tr> | ||
| 98 | + </thead> | ||
| 99 | + <tbody> | ||
| 100 | + <tr> | ||
| 101 | + <td>self(const aclTensor*)</td><td>输入</td><td>输入张量。</td> | ||
| 102 | + <td><ul><li>不可为nullptr,支持空Tensor。</li><li>排序轴长度N不超过INT32_MAX。</li></ul></td> | ||
| 103 | + <td>FLOAT、FLOAT16、BFLOAT16</td><td>ND等公开格式</td><td>1-8</td><td>√</td> | ||
| 104 | + </tr> | ||
| 105 | + <tr> | ||
| 106 | + <td>k(int64_t)</td><td>输入</td><td>每行输出的元素个数。</td> | ||
| 107 | + <td>范围为[0, INT32_MAX]。允许大于N或区间长度,不足部分填充;k=0时输出为空。</td> | ||
| 108 | + <td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 109 | + </tr> | ||
| 110 | + <tr> | ||
| 111 | + <td>dim(int64_t)</td><td>输入</td><td>排序维度。</td> | ||
| 112 | + <td>范围为[-rank, rank-1],负值加rank归一化。</td> | ||
| 113 | + <td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 114 | + </tr> | ||
| 115 | + <tr> | ||
| 116 | + <td>largest(bool)</td><td>输入</td><td>true取最大值,false取最小值。</td> | ||
| 117 | + <td>-</td><td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 118 | + </tr> | ||
| 119 | + <tr> | ||
| 120 | + <td>indicesType(aclDataType)</td><td>输入</td><td>输出索引的数据类型。</td> | ||
| 121 | + <td>仅接受ACL_INT32或ACL_INT64。仅输出值时也须传入合法类型。</td> | ||
| 122 | + <td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 123 | + </tr> | ||
| 124 | + <tr> | ||
| 125 | + <td>order(int64_t)</td><td>输入</td><td>有效结果的排列方式。</td> | ||
| 126 | + <td>支持ACLNN_SELECT_TOP_K_ORDER_VALUE和ACLNN_SELECT_TOP_K_ORDER_NONE,详见枚举说明。当order=0时,输出不保证有序或无序。</td> | ||
| 127 | + <td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 128 | + </tr> | ||
| 129 | + <tr> | ||
| 130 | + <td>indicesOffset(int64_t)</td><td>输入</td><td>有效原轴索引的标量偏移。</td> | ||
| 131 | + <td>偏移后的完整原轴索引范围须能由indicesType表示;不叠加到indexFill。</td> | ||
| 132 | + <td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 133 | + </tr> | ||
| 134 | + <tr> | ||
| 135 | + <td>outputMode(int64_t)</td><td>输入</td><td>选择输出值、索引或两者。</td> | ||
| 136 | + <td>取值见枚举说明。未启用的输出指针必须为nullptr。</td> | ||
| 137 | + <td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 138 | + </tr> | ||
| 139 | + <tr> | ||
| 140 | + <td>beginOptional(const aclTensor*)</td><td>输入</td><td>每行区间的起点,包含该位置。</td> | ||
| 141 | + <td><ul><li>可为nullptr,表示从0开始。</li><li>元素数为rows或1,单元素广播到全部行。</li><li>端点截断到[0, N]。</li></ul></td> | ||
| 142 | + <td>INT32</td><td>ND等公开格式</td><td>元素数为rows或1</td><td>√</td> | ||
| 143 | + </tr> | ||
| 144 | + <tr> | ||
| 145 | + <td>endOptional(const aclTensor*)</td><td>输入</td><td>每行区间的终点,不包含该位置。</td> | ||
| 146 | + <td><ul><li>可为nullptr,表示到N结束。</li><li>其他要求同beginOptional。</li><li>截断后end≤begin表示空区间。</li></ul></td> | ||
| 147 | + <td>INT32</td><td>ND等公开格式</td><td>元素数为rows或1</td><td>√</td> | ||
| 148 | + </tr> | ||
| 149 | + <tr> | ||
| 150 | + <td>valueFill(const aclScalar*)</td><td>输入</td><td>尾部值填充标量。</td> | ||
| 151 | + <td>nullptr表示填0;非空时须可转换为ACL_FLOAT,先转为float,再转为self类型。</td> | ||
| 152 | + <td>可转换为FLOAT的标量类型</td><td>-</td><td>-</td><td>-</td> | ||
| 153 | + </tr> | ||
| 154 | + <tr> | ||
| 155 | + <td>indexFill(int64_t)</td><td>输入</td><td>尾部索引填充值。</td> | ||
| 156 | + <td>须能由indicesType表示;原样写入,不叠加indicesOffset。</td> | ||
| 157 | + <td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 158 | + </tr> | ||
| 159 | + <tr> | ||
| 160 | + <td>valuesOutOptional(aclTensor*)</td><td>输出</td><td>TopK的值。</td> | ||
| 161 | + <td><ul><li>调用者预分配,shape为self的shape将dim维替换为k。</li><li>输出值时不可为nullptr;仅输出索引时必须为nullptr。</li></ul></td> | ||
| 162 | + <td>与self一致</td><td>ND等公开格式</td><td>与self维数一致</td><td>√</td> | ||
| 163 | + </tr> | ||
| 164 | + <tr> | ||
| 165 | + <td>indicesOutOptional(aclTensor*)</td><td>输出</td><td>TopK的索引。</td> | ||
| 166 | + <td><ul><li>调用者预分配,shape同值输出。</li><li>输出索引时不可为nullptr;仅输出值时必须为nullptr。</li></ul></td> | ||
| 167 | + <td>由indicesType指定</td><td>ND等公开格式</td><td>与self维数一致</td><td>√</td> | ||
| 168 | + </tr> | ||
| 169 | + <tr> | ||
| 170 | + <td>workspaceSize(uint64_t*)</td><td>输出</td><td>返回Device侧所需workspace大小,单位为字节。</td> | ||
| 171 | + <td>不可为nullptr。</td><td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 172 | + </tr> | ||
| 173 | + <tr> | ||
| 174 | + <td>executor(aclOpExecutor**)</td><td>输出</td><td>返回包含完整计算流程的执行器。</td> | ||
| 175 | + <td>不可为nullptr,用于第二段接口。</td><td>-</td><td>-</td><td>-</td><td>-</td> | ||
| 176 | + </tr> | ||
| 177 | + </tbody></table> | ||
| 178 | + | ||
| 179 | + 公开格式包括ND、NC、NCL、NCHW等普通步长格式,不支持私有存储格式。区间张量按除dim外各维的原顺序逐行对应,shape无需与行shape一致;负端点截断为0,不按负下标解释。 | ||
| 180 | +- **返回值** | ||
| 181 | + | ||
| 182 | + aclnnStatus:成功返回`ACLNN_SUCCESS`,其他状态码参见[aclnn返回码](../../../docs/zh/context/aclnn_return_code.md)。主要异常如下: | ||
| 183 | + | ||
| 184 | + <table style="table-layout: fixed; width: 1000px"><colgroup> | ||
| 185 | + <col style="width: 300px"><col style="width: 150px"><col style="width: 550px"> | ||
| 186 | + </colgroup> | ||
| 187 | + <thead> | ||
| 188 | + <tr><th>返回值</th><th>错误码</th><th>描述</th></tr> | ||
| 189 | + </thead> | ||
| 190 | + <tbody> | ||
| 191 | + <tr><td>ACLNN_ERR_PARAM_NULLPTR</td><td>161001</td><td>self、workspaceSize或executor为空指针。</td></tr> | ||
| 192 | + <tr><td>ACLNN_ERR_PARAM_INVALID</td><td>161002</td><td>参数范围、数据类型、shape、格式、输出指针组合或产品架构不符合要求,包括启用的输出为空或未启用的输出非空。</td></tr> | ||
| 193 | + <tr><td>ACLNN_ERR_INNER_NULLPTR</td><td>561103</td><td>缺少配套的ops-math实现,或构建内部张量、算子失败。</td></tr> | ||
| 194 | + </tbody></table> | ||
| 195 | + | ||
| 196 | + 执行器创建或框架资源分配失败时,还可能返回其他框架错误码。 | ||
| 197 | + | ||
| 198 | +## aclnnSelectTopK | ||
| 199 | + | ||
| 200 | +- **参数说明** | ||
| 201 | + | ||
| 202 | + <table style="table-layout: fixed; width: 1000px"><colgroup> | ||
| 203 | + <col style="width: 220px"><col style="width: 100px"><col style="width: 680px"> | ||
| 204 | + </colgroup> | ||
| 205 | + <thead> | ||
| 206 | + <tr><th>参数名</th><th>输入/输出</th><th>描述</th></tr> | ||
| 207 | + </thead> | ||
| 208 | + <tbody> | ||
| 209 | + <tr><td>workspace(void*)</td><td>输入</td><td>按第一段接口查询结果分配的Device侧内存;大小为0时可为nullptr,须保持有效直到流执行完成。</td></tr> | ||
| 210 | + <tr><td>workspaceSize(uint64_t)</td><td>输入</td><td>第一段接口返回的workspace大小,单位为字节。</td></tr> | ||
| 211 | + <tr><td>executor(aclOpExecutor*)</td><td>输入</td><td>第一段接口返回的执行器,包含本次输入、输出及计算流程。</td></tr> | ||
| 212 | + <tr><td>stream(aclrtStream)</td><td>输入</td><td>执行计算的Stream。读取输出或释放相关内存前须等待流执行完成。</td></tr> | ||
| 213 | + </tbody></table> | ||
| 214 | +- **返回值** | ||
| 215 | + | ||
| 216 | + aclnnStatus:成功返回`ACLNN_SUCCESS`,其他状态码参见[aclnn返回码](../../../docs/zh/context/aclnn_return_code.md)。接口异步执行,设备计算结果还须检查流同步状态。 | ||
| 217 | + | ||
| 218 | +## 约束说明 | ||
| 219 | + | ||
| 220 | +- `ORDER_VALUE`仅保证有效结果按值排序,填充部分不参与排序;`ORDER_NONE`不保证顺序。相等值不保证稳定排序,同时输出值和索引时两者逐项对应。 | ||
| 221 | +- indicesOffset须满足`type_min ≤ indicesOffset ≤ type_max - max(N-1, 0)`,indexFill须在同一类型范围内,其中type_min、type_max为indicesType的取值边界。仅输出值时也校验这两个参数;indexFill可与有效索引相同,不能用其数值判断有效项数。 | ||
| 222 | +- 维度乘积不得溢出INT64,`rows×N`和`rows×k`均不得超过`INT64_MAX/sizeof(int64_t)`。 | ||
| 223 | +- `rows=0`或`k=0`时,参数和产品检查通过后返回workspaceSize为0的空执行器。 | ||
| 224 | +- `N=0、rows>0、k>0`时仍执行填充,全部输出使用valueFill、indexFill。 | ||
| 225 | +- 不支持按索引排序,也不提供遇到NaN中止或按行输出哨兵值的语义。 | ||
| 226 | + | ||
| 227 | +## RadixTopK 调用路径 | ||
| 228 | + | ||
| 229 | +在当前支持的轴长范围 `N≤INT32_MAX` 内,全轴(begin/end 均为空)、双输出、offset=0、 | ||
| 230 | +`K≤N` 的 FP16/BF16 请求采用以下门控:`K≥8`,且 `12000000≤N<100000000` 时 | ||
| 231 | +`N>100K`,或 `N≥100000000` 时 `N>50K`。 | ||
| 232 | +区间、偏移、单输出、需要填充及不满足门控的请求继续走 SelectTopK。 | ||
| 233 | + | ||
| 234 | +### 门控后:RadixTopK 与 SortWithIndex | ||
| 235 | + | ||
| 236 | +```mermaid | ||
| 237 | +flowchart LR | ||
| 238 | + X[输入] --> P[Contiguous / 非末轴交换排序轴与末轴] | ||
| 239 | + P --> T[l0op::Topk:sorted 对应 order] | ||
| 240 | + T --> R[配套 math 门控选择 RadixTopK] | ||
| 241 | + R --> D{order=1?} | ||
| 242 | + D -->|是| S[SortWithIndex:同时排序值和原轴索引] | ||
| 243 | + S --> U[非末轴反向 Transpose / 索引按需 Cast / ViewCopy] | ||
| 244 | + D -->|否:直接使用值和索引| U | ||
| 245 | +``` | ||
| 246 | + | ||
| 247 | +该分流保持输入原维数,不额外 Reshape,也不从原输入回读值。按值排序时调用 | ||
| 248 | +`SortWithIndex(values, indices, lastDim, largest, true)`,同时重排值和原轴索引; | ||
| 249 | +只有 order=1 执行此排序, | ||
| 250 | +order=0 直接使用 Radix 的值和原轴索引,不保证输出顺序,相同值的索引选择不保证稳定。 | ||
| 251 | +INT64 中间索引直接透传;当前接口轴长上限内实际使用 INT32 中间索引。 | ||
| 252 | + | ||
| 253 | +该路径要求配套的 Topk/RadixTopK 实现支持上述 K≥8 门控;有序分流还要求 | ||
| 254 | +SortWithIndex 在 A2/A3 上支持 FP16/BF16 值和原轴索引的同步排序。 | ||
| 255 | +NaN 输出沿用 aclnnTopk 的规范化行为(FP16/BF16 为 0x7fff),不保留其原始载荷和符号; | ||
| 256 | +非 NaN 值和未命中此分流的 SelectTopK 路径保持原值位模式。 | ||
| 257 | +所有准备、选择、SortWithIndex、Cast 和输出恢复均计入完整 L2 成本。 | ||
| 258 | + | ||
| 259 | +## 调用示例 | ||
| 260 | + | ||
| 261 | +以下示例展示两段式调用。调用前须初始化ACL、选择A2或A3并创建stream,准备self为`[2,5]`的FLOAT设备张量、values为`[2,3]`的FLOAT张量、indices为`[2,3]`的INT64张量。张量及底层设备内存由调用方创建和释放;具体编译和执行过程参见[编译与运行样例](../../../docs/zh/context/compile_and_run_sample.md)。 | ||
| 262 | + | ||
| 263 | +```cpp | ||
| 264 | +#include "acl/acl.h" | ||
| 265 | +#include "aclnn_select_top_k.h" | ||
| 266 | + | ||
| 267 | +// 查询并执行标准模式;调用方在返回后检查状态并读取输出。 | ||
| 268 | +int RunStandardTopk(const aclTensor* self, aclTensor* values, aclTensor* indices, aclrtStream stream) | ||
| 269 | +{ | ||
| 270 | + constexpr int64_t OUTPUT_COUNT = 3; | ||
| 271 | + constexpr int64_t LAST_AXIS = -1; | ||
| 272 | + constexpr int64_t UNUSED_INDEX_FILL = -1; | ||
| 273 | + uint64_t workspaceSize = 0; | ||
| 274 | + aclOpExecutor* executor = nullptr; | ||
| 275 | + auto status = aclnnSelectTopKGetWorkspaceSize( | ||
| 276 | + self, OUTPUT_COUNT, LAST_AXIS, true, ACL_INT64, ACLNN_SELECT_TOP_K_ORDER_VALUE, | ||
| 277 | + 0, ACLNN_SELECT_TOP_K_OUTPUT_BOTH, nullptr, nullptr, nullptr, UNUSED_INDEX_FILL, | ||
| 278 | + values, indices, &workspaceSize, &executor); | ||
| 279 | + if (status != ACL_SUCCESS) { | ||
| 280 | + return status; | ||
| 281 | + } | ||
| 282 | + void* workspace = nullptr; | ||
| 283 | + if (workspaceSize != 0) { | ||
| 284 | + const auto allocStatus = aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST); | ||
| 285 | + if (allocStatus != ACL_SUCCESS) { | ||
| 286 | + return allocStatus; | ||
| 287 | + } | ||
| 288 | + } | ||
| 289 | + status = aclnnSelectTopK(workspace, workspaceSize, executor, stream); | ||
| 290 | + const auto syncStatus = aclrtSynchronizeStream(stream); | ||
| 291 | + if (workspace != nullptr) { | ||
| 292 | + aclrtFree(workspace); | ||
| 293 | + } | ||
| 294 | + return status == ACL_SUCCESS ? syncStatus : status; | ||
| 295 | +} | ||
| 296 | +``` | ||
| 297 | + | ||
| 298 | +输入`self = [[4,1,7,2,5],[0,8,3,6,9]]`时,输出values为`[[7,5,4],[9,8,6]]`,indices为`[[2,4,0],[4,1,3]]`。 | ||
| @@ -0,0 +1,155 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +#include <cstdio> | ||
| 12 | +#include <memory> | ||
| 13 | +#include <type_traits> | ||
| 14 | +#include <vector> | ||
| 15 | +#include "acl/acl.h" | ||
| 16 | +#include "aclnnop/aclnn_select_top_k.h" | ||
| 17 | + | ||
| 18 | +namespace { | ||
| 19 | +constexpr int64_t ROW_COUNT = 2; | ||
| 20 | +constexpr int64_t AXIS_LENGTH = 5; | ||
| 21 | +constexpr int64_t OUTPUT_COUNT = 3; | ||
| 22 | +constexpr int64_t LAST_AXIS = -1; | ||
| 23 | +constexpr int64_t INDEX_FILL = -1; | ||
| 24 | +using StreamPtr = std::unique_ptr<std::remove_pointer<aclrtStream>::type, decltype(&aclrtDestroyStream)>; | ||
| 25 | +using DeviceMemPtr = std::unique_ptr<void, decltype(&aclrtFree)>; | ||
| 26 | +using TensorPtr = std::unique_ptr<aclTensor, decltype(&aclDestroyTensor)>; | ||
| 27 | + | ||
| 28 | +// 创建本示例的连续二维张量,并把主机数据复制到设备。 | ||
| 29 | +template <typename T> | ||
| 30 | +int CreateTensor(const std::vector<T>& data, int64_t columns, aclDataType type, DeviceMemPtr& memory, TensorPtr& tensor) | ||
| 31 | +{ | ||
| 32 | + const size_t bytes = data.size() * sizeof(T); | ||
| 33 | + void* address = nullptr; | ||
| 34 | + auto status = aclrtMalloc(&address, bytes, ACL_MEM_MALLOC_HUGE_FIRST); | ||
| 35 | + if (status != ACL_SUCCESS) { | ||
| 36 | + return status; | ||
| 37 | + } | ||
| 38 | + memory.reset(address); | ||
| 39 | + status = aclrtMemcpy(address, bytes, data.data(), bytes, ACL_MEMCPY_HOST_TO_DEVICE); | ||
| 40 | + if (status != ACL_SUCCESS) { | ||
| 41 | + return status; | ||
| 42 | + } | ||
| 43 | + const std::vector<int64_t> shape = {ROW_COUNT, columns}; | ||
| 44 | + const std::vector<int64_t> strides = {columns, 1}; | ||
| 45 | + tensor.reset(aclCreateTensor(shape.data(), shape.size(), type, strides.data(), 0, ACL_FORMAT_ND, shape.data(), | ||
| 46 | + shape.size(), address)); | ||
| 47 | + return tensor ? ACL_SUCCESS : ACL_ERROR_FAILURE; | ||
| 48 | +} | ||
| 49 | + | ||
| 50 | +// 将输出读回主机,与无重复值输入的确定结果逐项比较。 | ||
| 51 | +template <typename T> | ||
| 52 | +int CheckOutput(void* memory, const std::vector<T>& expected) | ||
| 53 | +{ | ||
| 54 | + std::vector<T> actual(expected.size()); | ||
| 55 | + const size_t bytes = actual.size() * sizeof(T); | ||
| 56 | + const auto status = aclrtMemcpy(actual.data(), bytes, memory, bytes, ACL_MEMCPY_DEVICE_TO_HOST); | ||
| 57 | + if (status != ACL_SUCCESS) { | ||
| 58 | + return status; | ||
| 59 | + } | ||
| 60 | + if (actual != expected) { | ||
| 61 | + std::printf("SelectTopK output mismatch.\n"); | ||
| 62 | + return ACL_ERROR_FAILURE; | ||
| 63 | + } | ||
| 64 | + return ACL_SUCCESS; | ||
| 65 | +} | ||
| 66 | + | ||
| 67 | +// 演示全轴、双输出、INT64索引、零偏移的两段式调用。 | ||
| 68 | +int RunExample(aclrtStream stream) | ||
| 69 | +{ | ||
| 70 | + DeviceMemPtr inputMemory(nullptr, &aclrtFree); | ||
| 71 | + DeviceMemPtr valuesMemory(nullptr, &aclrtFree); | ||
| 72 | + DeviceMemPtr indicesMemory(nullptr, &aclrtFree); | ||
| 73 | + DeviceMemPtr workspace(nullptr, &aclrtFree); | ||
| 74 | + TensorPtr input(nullptr, &aclDestroyTensor); | ||
| 75 | + TensorPtr values(nullptr, &aclDestroyTensor); | ||
| 76 | + TensorPtr indices(nullptr, &aclDestroyTensor); | ||
| 77 | + const std::vector<float> data = {9, 2, 7, 1, 5, -3, 8, 4, 6, 0}; | ||
| 78 | + auto status = CreateTensor(data, AXIS_LENGTH, ACL_FLOAT, inputMemory, input); | ||
| 79 | + if (status != ACL_SUCCESS) { | ||
| 80 | + return status; | ||
| 81 | + } | ||
| 82 | + status = CreateTensor(std::vector<float>(ROW_COUNT * OUTPUT_COUNT), OUTPUT_COUNT, ACL_FLOAT, valuesMemory, values); | ||
| 83 | + if (status != ACL_SUCCESS) { | ||
| 84 | + return status; | ||
| 85 | + } | ||
| 86 | + status = CreateTensor(std::vector<int64_t>(ROW_COUNT * OUTPUT_COUNT), OUTPUT_COUNT, ACL_INT64, indicesMemory, | ||
| 87 | + indices); | ||
| 88 | + if (status != ACL_SUCCESS) { | ||
| 89 | + return status; | ||
| 90 | + } | ||
| 91 | + | ||
| 92 | + uint64_t workspaceSize = 0; | ||
| 93 | + aclOpExecutor* executor = nullptr; | ||
| 94 | + status = aclnnSelectTopKGetWorkspaceSize(input.get(), OUTPUT_COUNT, LAST_AXIS, true, ACL_INT64, | ||
| 95 | + ACLNN_SELECT_TOP_K_ORDER_VALUE, 0, ACLNN_SELECT_TOP_K_OUTPUT_BOTH, nullptr, | ||
| 96 | + nullptr, nullptr, INDEX_FILL, values.get(), indices.get(), &workspaceSize, | ||
| 97 | + &executor); | ||
| 98 | + if (status != ACL_SUCCESS) { | ||
| 99 | + return status; | ||
| 100 | + } | ||
| 101 | + if (workspaceSize != 0) { | ||
| 102 | + void* address = nullptr; | ||
| 103 | + status = aclrtMalloc(&address, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST); | ||
| 104 | + if (status != ACL_SUCCESS) { | ||
| 105 | + return status; | ||
| 106 | + } | ||
| 107 | + workspace.reset(address); | ||
| 108 | + } | ||
| 109 | + status = aclnnSelectTopK(workspace.get(), workspaceSize, executor, stream); | ||
| 110 | + // 即使任务提交返回错误,也先等待已提交任务结束,再析构张量及设备内存。 | ||
| 111 | + const auto syncStatus = aclrtSynchronizeStream(stream); | ||
| 112 | + if (status != ACL_SUCCESS) { | ||
| 113 | + return status; | ||
| 114 | + } | ||
| 115 | + if (syncStatus != ACL_SUCCESS) { | ||
| 116 | + return syncStatus; | ||
| 117 | + } | ||
| 118 | + status = CheckOutput<float>(valuesMemory.get(), {9, 7, 5, 8, 6, 4}); | ||
| 119 | + if (status != ACL_SUCCESS) { | ||
| 120 | + return status; | ||
| 121 | + } | ||
| 122 | + return CheckOutput<int64_t>(indicesMemory.get(), {0, 2, 4, 1, 3, 2}); | ||
| 123 | +} | ||
| 124 | +} // namespace | ||
| 125 | + | ||
| 126 | +// 初始化设备和流,执行示例后按相反顺序释放资源。 | ||
| 127 | +int main() | ||
| 128 | +{ | ||
| 129 | + constexpr int32_t DEVICE_ID = 0; | ||
| 130 | + auto status = aclInit(nullptr); | ||
| 131 | + if (status != ACL_SUCCESS) { | ||
| 132 | + return status; | ||
| 133 | + } | ||
| 134 | + bool deviceSet = false; | ||
| 135 | + std::shared_ptr<void> aclGuard(nullptr, [&](void*) { | ||
| 136 | + if (deviceSet) { | ||
| 137 | + aclrtResetDevice(DEVICE_ID); | ||
| 138 | + } | ||
| 139 | + aclFinalize(); | ||
| 140 | + }); | ||
| 141 | + status = aclrtSetDevice(DEVICE_ID); | ||
| 142 | + if (status != ACL_SUCCESS) { | ||
| 143 | + return status; | ||
| 144 | + } | ||
| 145 | + deviceSet = true; | ||
| 146 | + aclrtStream rawStream = nullptr; | ||
| 147 | + status = aclrtCreateStream(&rawStream); | ||
| 148 | + if (status != ACL_SUCCESS) { | ||
| 149 | + return status; | ||
| 150 | + } | ||
| 151 | + StreamPtr stream(rawStream, &aclrtDestroyStream); | ||
| 152 | + status = RunExample(stream.get()); | ||
| 153 | + std::printf("SelectTopK example: %s (status=%d).\n", status == ACL_SUCCESS ? "PASS" : "FAIL", status); | ||
| 154 | + return status; | ||
| 155 | +} | ||
| @@ -0,0 +1,360 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#include "aclnn_select_top_k.h" | ||
| 11 | +#include "select_top_k.h" | ||
| 12 | +#include "../../topk/op_api/topk.h" | ||
| 13 | +#include "../../sort_with_index/op_api/sort_with_index.h" | ||
| 14 | +#include "aclnn_kernels/cast.h" | ||
| 15 | +#include "aclnn_kernels/contiguous.h" | ||
| 16 | +#include "aclnn_kernels/reshape.h" | ||
| 17 | +#include "aclnn_kernels/transpose.h" | ||
| 18 | +#include "aclnn_kernels/transdata.h" | ||
| 19 | +#include "aclnn_kernels/common/op_error_check.h" | ||
| 20 | +#include "opdev/data_type_utils.h" | ||
| 21 | +#include "opdev/format_utils.h" | ||
| 22 | +#include "opdev/make_op_executor.h" | ||
| 23 | +#include "opdev/op_dfx.h" | ||
| 24 | +#include "opdev/platform.h" | ||
| 25 | +#include "opdev/shape_utils.h" | ||
| 26 | +#include "opdev/tensor_view_utils.h" | ||
| 27 | +#include <algorithm> | ||
| 28 | +#include <limits> | ||
| 29 | +#include <vector> | ||
| 30 | + | ||
| 31 | +namespace SelectTopKPolicy { | ||
| 32 | +constexpr int64_t RADIX_LARGE_AXIS_MIN = 12000000; // 第一档长轴门槛。 | ||
| 33 | +constexpr int64_t RADIX_LARGE_AXIS_K_RATIO = 100; // 第一档要求 N > 100K。 | ||
| 34 | +constexpr int64_t RADIX_VERY_LARGE_AXIS_MIN = 100000000; // 第二档长轴门槛。 | ||
| 35 | +constexpr int64_t RADIX_VERY_LARGE_AXIS_K_RATIO = 50; // 第二档要求 N > 50K。 | ||
| 36 | +constexpr int64_t RADIX_MIN_K = 8; // 与 aclnnTopk PR 14011 的包含式下限一致。 | ||
| 37 | + | ||
| 38 | +// 判断当前接口轴长范围内的 Radix 收益条件;芯片、类型及扩展语义由 L2 单独校验。 | ||
| 39 | +bool IsRadixShapeSupported(int64_t axis, int64_t k) | ||
| 40 | +{ | ||
| 41 | + if (k < RADIX_MIN_K || k > axis || axis > std::numeric_limits<int32_t>::max()) { | ||
| 42 | + return false; | ||
| 43 | + } | ||
| 44 | + if (axis >= RADIX_VERY_LARGE_AXIS_MIN) { | ||
| 45 | + return axis > RADIX_VERY_LARGE_AXIS_K_RATIO * k; | ||
| 46 | + } | ||
| 47 | + return axis >= RADIX_LARGE_AXIS_MIN && axis > RADIX_LARGE_AXIS_K_RATIO * k; | ||
| 48 | +} | ||
| 49 | +} // namespace SelectTopKPolicy | ||
| 50 | + | ||
| 51 | +namespace { | ||
| 52 | +constexpr int64_t MAX_INPUT_RANK = 8; // 接口支持的最大输入维数。 | ||
| 53 | +constexpr int64_t MAX_FUSED_INNER = 256; // A2 融合布局的最大轴后元素数,与Host资源规划上限一致。 | ||
| 54 | +constexpr int64_t INNER_LAYOUT_DIM = 2; // 三维布局 [outer, axis, inner] 的末维位置。 | ||
| 55 | +constexpr size_t AXIS_LAYOUT_RANK = 3; // 非末轴输入与输出使用的中间布局维数。 | ||
| 56 | +constexpr int64_t MAX_ELEMENT_BYTES = sizeof(int64_t); // 地址溢出检查采用最宽的索引类型。 | ||
| 57 | + | ||
| 58 | +// 将任意输入维度拆成排序轴前、轴内和轴后三部分。 | ||
| 59 | +struct Layout { | ||
| 60 | + int64_t outer = 1; | ||
| 61 | + int64_t axis = 0; | ||
| 62 | + int64_t inner = 1; | ||
| 63 | + int64_t rows = 0; | ||
| 64 | + op::Shape outputShape; | ||
| 65 | +}; | ||
| 66 | + | ||
| 67 | +// 接受具有普通步长的公开格式,拒绝需要特殊存储解释的私有格式。 | ||
| 68 | +bool IsPlainFormat(const aclTensor* tensor) | ||
| 69 | +{ | ||
| 70 | + // NC/NCL/NCHW 与 ND 一样按普通步长解释;分块私有格式不满足该条件。 | ||
| 71 | + return !op::IsPrivateFormat(tensor->GetViewFormat()) && !op::IsPrivateFormat(tensor->GetStorageFormat()); | ||
| 72 | +} | ||
| 73 | + | ||
| 74 | +// 计算非负维度乘积,在有符号地址范围内检查溢出。 | ||
| 75 | +bool Multiply(int64_t a, int64_t b, int64_t& result) | ||
| 76 | +{ | ||
| 77 | + if (a < 0 || b < 0 || (b && a > INT64_MAX / b)) { | ||
| 78 | + return false; | ||
| 79 | + } | ||
| 80 | + result = a * b; | ||
| 81 | + return true; | ||
| 82 | +} | ||
| 83 | + | ||
| 84 | +// 检查可选区间张量:INT32、公开格式,且为逐行数据或单值广播。 | ||
| 85 | +bool CheckBounds(const aclTensor* bounds, int64_t rows) | ||
| 86 | +{ | ||
| 87 | + if (bounds == nullptr) { | ||
| 88 | + return true; | ||
| 89 | + } | ||
| 90 | + return bounds->GetDataType() == op::DataType::DT_INT32 && IsPlainFormat(bounds) && | ||
| 91 | + (bounds->GetViewShape().GetShapeSize() == rows || bounds->GetViewShape().GetShapeSize() == 1); | ||
| 92 | +} | ||
| 93 | + | ||
| 94 | +// 按输出模式验证指针、数据类型及完整输出形状。 | ||
| 95 | +bool CheckOutput(const aclTensor* output, bool enabled, op::DataType dtype, const op::Shape& shape) | ||
| 96 | +{ | ||
| 97 | + if (!enabled) { | ||
| 98 | + return output == nullptr; | ||
| 99 | + } | ||
| 100 | + return output != nullptr && output->GetDataType() == dtype && IsPlainFormat(output) && | ||
| 101 | + output->GetViewShape() == shape; | ||
| 102 | +} | ||
| 103 | + | ||
| 104 | +// 校验接口参数并计算轴布局,同时保证偏移、填充值和地址乘积均可表示。 | ||
| 105 | +aclnnStatus CheckParams(const aclTensor* self, int64_t k, int64_t dim, aclDataType indicesType, int64_t order, | ||
| 106 | + int64_t offset, int64_t mode, const aclTensor* begin, const aclTensor* end, int64_t indexFill, | ||
| 107 | + const aclTensor* values, const aclTensor* indices, Layout& layout) | ||
| 108 | +{ | ||
| 109 | + if (self == nullptr) { | ||
| 110 | + return ACLNN_ERR_PARAM_NULLPTR; | ||
| 111 | + } | ||
| 112 | + const auto& shape = self->GetViewShape(); | ||
| 113 | + const int64_t rank = shape.GetDimNum(); | ||
| 114 | + const auto dtype = self->GetDataType(); | ||
| 115 | + if (rank < 1 || rank > MAX_INPUT_RANK || dim < -rank || dim >= rank || k < 0 || k > INT32_MAX || | ||
| 116 | + (indicesType != ACL_INT32 && indicesType != ACL_INT64) || | ||
| 117 | + (order != ACLNN_SELECT_TOP_K_ORDER_VALUE && order != ACLNN_SELECT_TOP_K_ORDER_NONE) || | ||
| 118 | + mode < ACLNN_SELECT_TOP_K_OUTPUT_VALUES || mode > ACLNN_SELECT_TOP_K_OUTPUT_BOTH || | ||
| 119 | + (dtype != op::DataType::DT_FLOAT && dtype != op::DataType::DT_FLOAT16 && dtype != op::DataType::DT_BF16) || | ||
| 120 | + !IsPlainFormat(self)) { | ||
| 121 | + OP_LOGE(ACLNN_ERR_PARAM_INVALID, "SelectTopK: invalid rank/dim/k, dtype, ND format, order or output mode"); | ||
| 122 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 123 | + } | ||
| 124 | + dim = dim < 0 ? dim + rank : dim; | ||
| 125 | + layout.axis = shape.GetDim(dim); | ||
| 126 | + if (layout.axis < 0 || layout.axis > INT32_MAX) { | ||
| 127 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 128 | + } | ||
| 129 | + for (int64_t i = 0; i < rank; ++i) { | ||
| 130 | + if (i < dim && !Multiply(layout.outer, shape.GetDim(i), layout.outer)) { | ||
| 131 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 132 | + } | ||
| 133 | + if (i > dim && !Multiply(layout.inner, shape.GetDim(i), layout.inner)) { | ||
| 134 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 135 | + } | ||
| 136 | + } | ||
| 137 | + int64_t inputItems = 0, outputItems = 0; | ||
| 138 | + if (!Multiply(layout.outer, layout.inner, layout.rows) || !Multiply(layout.rows, layout.axis, inputItems) || | ||
| 139 | + !Multiply(layout.rows, k, outputItems) || inputItems > INT64_MAX / MAX_ELEMENT_BYTES || | ||
| 140 | + outputItems > INT64_MAX / MAX_ELEMENT_BYTES) { | ||
| 141 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 142 | + } | ||
| 143 | + layout.outputShape = shape; | ||
| 144 | + layout.outputShape.SetDim(dim, k); | ||
| 145 | + const int64_t maxIndex = std::max<int64_t>(layout.axis - 1, 0); | ||
| 146 | + const int64_t upper = indicesType == ACL_INT32 ? INT32_MAX : INT64_MAX; | ||
| 147 | + const int64_t lower = indicesType == ACL_INT32 ? INT32_MIN : INT64_MIN; | ||
| 148 | + if (offset < lower || offset > upper - maxIndex || indexFill < lower || indexFill > upper || | ||
| 149 | + !CheckBounds(begin, layout.rows) || !CheckBounds(end, layout.rows) || | ||
| 150 | + !CheckOutput(values, mode != ACLNN_SELECT_TOP_K_OUTPUT_INDICES, dtype, layout.outputShape) || | ||
| 151 | + !CheckOutput(indices, mode != ACLNN_SELECT_TOP_K_OUTPUT_VALUES, static_cast<op::DataType>(indicesType), | ||
| 152 | + layout.outputShape)) { | ||
| 153 | + OP_LOGE(ACLNN_ERR_PARAM_INVALID, "SelectTopK: invalid bounds/output shape or index overflow"); | ||
| 154 | + return ACLNN_ERR_PARAM_INVALID; | ||
| 155 | + } | ||
| 156 | + return ACLNN_SUCCESS; | ||
| 157 | +} | ||
| 158 | + | ||
| 159 | +// 将形状参数交给执行器管理,并创建新的张量视图。 | ||
| 160 | +const aclTensor* Reshape(const aclTensor* tensor, const std::vector<int64_t>& shape, aclOpExecutor* executor) | ||
| 161 | +{ | ||
| 162 | + auto dims = executor->AllocIntArray(shape.data(), shape.size()); | ||
| 163 | + return dims == nullptr ? nullptr : l0op::Reshape(tensor, dims, executor); | ||
| 164 | +} | ||
| 165 | + | ||
| 166 | +// 准备统一三维输入;融合路径保留 [outer, axis, inner],其余路径整理为 [rows, axis, 1]。 | ||
| 167 | +const aclTensor* PrepareInput(const aclTensor* self, const Layout& layout, aclOpExecutor* executor, bool fusedInput) | ||
| 168 | +{ | ||
| 169 | + if (layout.axis == 0) { | ||
| 170 | + // Reshape 不接受空形状,直接创建空视图;内核仅写入请求的填充值和填充索引。 | ||
| 171 | + auto empty = executor->CreateView(self, op::Shape{layout.rows, 0, 1}, self->GetViewOffset()); | ||
| 172 | + return empty == nullptr ? nullptr : l0op::ReFormat(empty, op::Format::FORMAT_ND); | ||
| 173 | + } | ||
| 174 | + auto input = l0op::Contiguous(self, executor); | ||
| 175 | + if (input == nullptr) { | ||
| 176 | + return nullptr; | ||
| 177 | + } | ||
| 178 | + if (layout.inner > 1 && layout.axis > 1 && !fusedInput) { | ||
| 179 | + input = Reshape(input, {layout.outer, layout.axis, layout.inner}, executor); | ||
| 180 | + const int64_t permutation[] = {0, INNER_LAYOUT_DIM, 1}; | ||
| 181 | + auto axes = executor->AllocIntArray(permutation, AXIS_LAYOUT_RANK); | ||
| 182 | + if (input == nullptr || axes == nullptr) { | ||
| 183 | + return nullptr; | ||
| 184 | + } | ||
| 185 | + input = l0op::Transpose(input, axes, executor); | ||
| 186 | + if (input == nullptr) { | ||
| 187 | + return nullptr; | ||
| 188 | + } | ||
| 189 | + } | ||
| 190 | + return fusedInput ? Reshape(input, {layout.outer, layout.axis, layout.inner}, executor) : | ||
| 191 | + Reshape(input, {layout.rows, layout.axis, 1}, executor); | ||
| 192 | +} | ||
| 193 | + | ||
| 194 | +// 将计算结果恢复到原轴位置;融合输出已经完成搬运时只恢复形状。 | ||
| 195 | +const aclTensor* RestoreOutput(const aclTensor* tensor, const Layout& layout, int64_t k, aclOpExecutor* executor, | ||
| 196 | + bool restored) | ||
| 197 | +{ | ||
| 198 | + if (!restored && layout.inner > 1 && k > 1) { | ||
| 199 | + tensor = Reshape(tensor, {layout.outer, layout.inner, k}, executor); | ||
| 200 | + const int64_t permutation[] = {0, INNER_LAYOUT_DIM, 1}; | ||
| 201 | + auto axes = executor->AllocIntArray(permutation, AXIS_LAYOUT_RANK); | ||
| 202 | + if (tensor == nullptr || axes == nullptr) { | ||
| 203 | + return nullptr; | ||
| 204 | + } | ||
| 205 | + tensor = l0op::Transpose(tensor, axes, executor); | ||
| 206 | + if (tensor == nullptr) { | ||
| 207 | + return nullptr; | ||
| 208 | + } | ||
| 209 | + } | ||
| 210 | + const auto shape = op::ToShapeVector(layout.outputShape); | ||
| 211 | + auto dims = executor->AllocIntArray(shape.data(), shape.size()); | ||
| 212 | + return dims == nullptr ? nullptr : l0op::Reshape(tensor, dims, executor); | ||
| 213 | +} | ||
| 214 | +// 将 aclnnTopk 的 Radix 门控限制在当前接口的标准语义内,扩展请求继续使用 SelectTopK。 | ||
| 215 | +bool IsRadixTopKSupported(const aclTensor* self, const Layout& layout, int64_t k, int64_t outputMode, | ||
| 216 | + int64_t indicesOffset, const aclTensor* begin, const aclTensor* end) | ||
| 217 | +{ | ||
| 218 | + const auto version = op::GetCurrentPlatformInfo().GetSocVersion(); | ||
| 219 | + const bool socSupported = version == op::SocVersion::ASCEND910B || version == op::SocVersion::ASCEND910_93; | ||
| 220 | + const auto dtype = self->GetDataType(); | ||
| 221 | + const bool dtypeSupported = dtype == op::DataType::DT_FLOAT16 || dtype == op::DataType::DT_BF16; | ||
| 222 | + return socSupported && dtypeSupported && outputMode == ACLNN_SELECT_TOP_K_OUTPUT_BOTH && indicesOffset == 0 && | ||
| 223 | + begin == nullptr && end == nullptr && SelectTopKPolicy::IsRadixShapeSupported(layout.axis, k); | ||
| 224 | +} | ||
| 225 | + | ||
| 226 | +// Radix 分流保持原维数;非末轴先交换排序轴与末轴,排序时同步维护原轴索引。 | ||
| 227 | +aclnnStatus RunRadixTopK(const aclTensor* self, int64_t k, int64_t dim, bool largest, int64_t order, | ||
| 228 | + op::DataType indicesDType, aclTensor* valuesOut, aclTensor* indicesOut, | ||
| 229 | + aclOpExecutor* executor) | ||
| 230 | +{ | ||
| 231 | + auto input = l0op::Contiguous(self, executor); | ||
| 232 | + CHECK_RET(input != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 233 | + const int64_t rank = input->GetViewShape().GetDimNum(); | ||
| 234 | + const int64_t positiveDim = dim < 0 ? dim + rank : dim; | ||
| 235 | + const int64_t lastDim = rank - 1; | ||
| 236 | + aclIntArray* axes = nullptr; | ||
| 237 | + if (positiveDim != lastDim) { | ||
| 238 | + std::vector<int64_t> permutation(rank); | ||
| 239 | + for (int64_t i = 0; i < rank; ++i) { | ||
| 240 | + permutation[i] = i; | ||
| 241 | + } | ||
| 242 | + std::swap(permutation[positiveDim], permutation[lastDim]); | ||
| 243 | + axes = executor->AllocIntArray(permutation.data(), rank); | ||
| 244 | + CHECK_RET(axes != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 245 | + input = l0op::Transpose(input, axes, executor); | ||
| 246 | + CHECK_RET(input != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 247 | + } | ||
| 248 | + | ||
| 249 | + OP_LOGD("SelectTopK L2 route=Topk, radix_gate=true, dim=%ld, k=%ld, order=%ld", lastDim, k, order); | ||
| 250 | + const auto selected = l0op::Topk(input, k, lastDim, largest, order == ACLNN_SELECT_TOP_K_ORDER_VALUE, indicesDType, | ||
| 251 | + executor); | ||
| 252 | + const aclTensor* values = std::get<0>(selected); | ||
| 253 | + const aclTensor* indices = std::get<1>(selected); | ||
| 254 | + CHECK_RET(values != nullptr && indices != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 255 | + // 按值排序时同步重排 Radix 返回的原轴索引;无序输出直接透传。 | ||
| 256 | + if (order == ACLNN_SELECT_TOP_K_ORDER_VALUE && indices->GetDataType() != op::DataType::DT_INT64) { | ||
| 257 | + const auto ordered = l0op::SortWithIndex(values, indices, lastDim, largest, true, executor); | ||
| 258 | + values = std::get<0>(ordered); | ||
| 259 | + indices = std::get<1>(ordered); | ||
| 260 | + CHECK_RET(values != nullptr && indices != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 261 | + } | ||
| 262 | + if (axes != nullptr) { | ||
| 263 | + values = l0op::Transpose(values, axes, executor); | ||
| 264 | + indices = l0op::Transpose(indices, axes, executor); | ||
| 265 | + CHECK_RET(values != nullptr && indices != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 266 | + } | ||
| 267 | + CHECK_RET(l0op::ViewCopy(values, valuesOut, executor) != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 268 | + if (indices->GetDataType() != indicesDType) { | ||
| 269 | + indices = l0op::Cast(indices, indicesDType, executor); | ||
| 270 | + CHECK_RET(indices != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 271 | + } | ||
| 272 | + CHECK_RET(l0op::ViewCopy(indices, indicesOut, executor) != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 273 | + return ACLNN_SUCCESS; | ||
| 274 | +} | ||
| 275 | +} // namespace | ||
| 276 | + | ||
| 277 | +// 构建完整 L2 执行计划,包含输入准备、TopK、索引转换及输出恢复。 | ||
| 278 | +aclnnStatus aclnnSelectTopKGetWorkspaceSize(const aclTensor* self, int64_t k, int64_t dim, bool largest, | ||
| 279 | + aclDataType indicesType, int64_t order, int64_t indicesOffset, | ||
| 280 | + int64_t outputMode, const aclTensor* beginOptional, | ||
| 281 | + const aclTensor* endOptional, const aclScalar* valueFill, int64_t indexFill, | ||
| 282 | + aclTensor* valuesOutOptional, aclTensor* indicesOutOptional, | ||
| 283 | + uint64_t* workspaceSize, aclOpExecutor** executor) | ||
| 284 | +{ | ||
| 285 | + L2_DFX_PHASE_1(aclnnSelectTopK, | ||
| 286 | + DFX_IN(self, k, dim, largest, indicesType, order, indicesOffset, outputMode, beginOptional, | ||
| 287 | + endOptional, valueFill, indexFill), | ||
| 288 | + DFX_OUT(valuesOutOptional, indicesOutOptional)); | ||
| 289 | + CHECK_RET(workspaceSize != nullptr && executor != nullptr, ACLNN_ERR_PARAM_NULLPTR); | ||
| 290 | + CHECK_RET(valueFill == nullptr || op::CanCast(valueFill->GetDataType(), op::DataType::DT_FLOAT), | ||
| 291 | + ACLNN_ERR_PARAM_INVALID); | ||
| 292 | + Layout layout; | ||
| 293 | + auto status = CheckParams(self, k, dim, indicesType, order, indicesOffset, outputMode, beginOptional, endOptional, | ||
| 294 | + indexFill, valuesOutOptional, indicesOutOptional, layout); | ||
| 295 | + CHECK_RET(status == ACLNN_SUCCESS, status); | ||
| 296 | + CHECK_RET(op::GetCurrentPlatformInfo().GetCurNpuArch() == NpuArch::DAV_2201, ACLNN_ERR_PARAM_INVALID); | ||
| 297 | + auto uniqueExecutor = CREATE_EXECUTOR(); | ||
| 298 | + CHECK_RET(uniqueExecutor.get() != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR); | ||
| 299 | + if (layout.rows == 0 || k == 0) { | ||
| 300 | + *workspaceSize = 0; | ||
| 301 | + uniqueExecutor.ReleaseTo(executor); | ||
| 302 | + return ACLNN_SUCCESS; | ||
| 303 | + } | ||
| 304 | + if (IsRadixTopKSupported(self, layout, k, outputMode, indicesOffset, beginOptional, endOptional)) { | ||
| 305 | + status = RunRadixTopK(self, k, dim, largest, order, static_cast<op::DataType>(indicesType), valuesOutOptional, | ||
| 306 | + indicesOutOptional, uniqueExecutor.get()); | ||
| 307 | + CHECK_RET(status == ACLNN_SUCCESS, status); | ||
| 308 | + *workspaceSize = uniqueExecutor->GetWorkspaceSize(); | ||
| 309 | + uniqueExecutor.ReleaseTo(executor); | ||
| 310 | + return ACLNN_SUCCESS; | ||
| 311 | + } | ||
| 312 | + const bool fusedOutput = outputMode == ACLNN_SELECT_TOP_K_OUTPUT_BOTH && beginOptional == nullptr && | ||
| 313 | + endOptional == nullptr && indicesOffset == 0 && layout.inner > 1 && | ||
| 314 | + layout.inner <= MAX_FUSED_INNER && k > 1 && k <= layout.axis; | ||
| 315 | + const bool fusedInput = fusedOutput && layout.axis > 1; | ||
| 316 | + auto input = PrepareInput(self, layout, uniqueExecutor.get(), fusedInput); | ||
| 317 | + CHECK_RET(input != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 318 | + const aclTensor* begin = beginOptional == nullptr ? nullptr : l0op::Contiguous(beginOptional, uniqueExecutor.get()); | ||
| 319 | + const aclTensor* end = endOptional == nullptr ? nullptr : l0op::Contiguous(endOptional, uniqueExecutor.get()); | ||
| 320 | + CHECK_RET((beginOptional == nullptr || begin != nullptr) && (endOptional == nullptr || end != nullptr), | ||
| 321 | + ACLNN_ERR_INNER_NULLPTR); | ||
| 322 | + // A2 的 INT64 转置开销较高;索引范围可表示时先用 INT32 恢复轴布局,再扩宽。 | ||
| 323 | + // 无法用 INT32 表示的偏移和填充索引始终保留 INT64 路径。 | ||
| 324 | + const bool widenAfterTranspose = !fusedOutput && outputMode != ACLNN_SELECT_TOP_K_OUTPUT_VALUES && | ||
| 325 | + indicesType == ACL_INT64 && layout.inner > 1 && k > 1 && | ||
| 326 | + indicesOffset >= INT32_MIN && | ||
| 327 | + indicesOffset <= INT32_MAX - std::max<int64_t>(0, layout.axis - 1) && | ||
| 328 | + indexFill >= INT32_MIN && indexFill <= INT32_MAX; | ||
| 329 | + const auto kernelIndicesType = widenAfterTranspose ? op::DataType::DT_INT32 : | ||
| 330 | + static_cast<op::DataType>(indicesType); | ||
| 331 | + auto result = l0op::SelectTopK(input, begin, end, k, largest, order, kernelIndicesType, indicesOffset, outputMode, | ||
| 332 | + valueFill == nullptr ? 0.0f : valueFill->ToFloat(), indexFill, uniqueExecutor.get()); | ||
| 333 | + if (outputMode != ACLNN_SELECT_TOP_K_OUTPUT_INDICES) { | ||
| 334 | + CHECK_RET(std::get<0>(result) != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 335 | + auto values = RestoreOutput(std::get<0>(result), layout, k, uniqueExecutor.get(), fusedOutput); | ||
| 336 | + CHECK_RET(values != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 337 | + CHECK_RET(l0op::ViewCopy(values, valuesOutOptional, uniqueExecutor.get()) != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 338 | + } | ||
| 339 | + if (outputMode != ACLNN_SELECT_TOP_K_OUTPUT_VALUES) { | ||
| 340 | + CHECK_RET(std::get<1>(result) != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 341 | + auto indices = RestoreOutput(std::get<1>(result), layout, k, uniqueExecutor.get(), fusedOutput); | ||
| 342 | + CHECK_RET(indices != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 343 | + if (widenAfterTranspose) { | ||
| 344 | + indices = l0op::Cast(indices, op::DataType::DT_INT64, uniqueExecutor.get()); | ||
| 345 | + CHECK_RET(indices != nullptr, ACLNN_ERR_INNER_NULLPTR); | ||
| 346 | + } | ||
| 347 | + CHECK_RET(l0op::ViewCopy(indices, indicesOutOptional, uniqueExecutor.get()) != nullptr, | ||
| 348 | + ACLNN_ERR_INNER_NULLPTR); | ||
| 349 | + } | ||
| 350 | + *workspaceSize = uniqueExecutor->GetWorkspaceSize(); | ||
| 351 | + uniqueExecutor.ReleaseTo(executor); | ||
| 352 | + return ACLNN_SUCCESS; | ||
| 353 | +} | ||
| 354 | + | ||
| 355 | +// 在指定流上执行第一阶段生成的完整计算计划。 | ||
| 356 | +aclnnStatus aclnnSelectTopK(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, aclrtStream stream) | ||
| 357 | +{ | ||
| 358 | + L2_DFX_PHASE_2(aclnnSelectTopK); | ||
| 359 | + return CommonOpExecutorRun(workspace, workspaceSize, executor, stream); | ||
| 360 | +} | ||
| @@ -0,0 +1,92 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#ifndef ACLNN_SELECT_TOP_K_H_ | ||
| 11 | +#define ACLNN_SELECT_TOP_K_H_ | ||
| 12 | + | ||
| 13 | +#include "aclnn/aclnn_base.h" | ||
| 14 | +#include "aclnn_util.h" | ||
| 15 | + | ||
| 16 | +#ifdef __cplusplus | ||
| 17 | +extern "C" { | ||
| 18 | +#endif | ||
| 19 | + | ||
| 20 | +typedef enum { | ||
| 21 | + ACLNN_SELECT_TOP_K_ORDER_NONE = 0, /* 不保证有效结果的排列顺序。 */ | ||
| 22 | + ACLNN_SELECT_TOP_K_ORDER_VALUE = 1, /* 有效结果按值排序。 */ | ||
| 23 | + ACLNN_SELECT_TOP_K_ORDER_INDEX = 2 /* 索引排序预留值,当前不支持。 */ | ||
| 24 | +} aclnnSelectTopKOrder; | ||
| 25 | + | ||
| 26 | +typedef enum { | ||
| 27 | + ACLNN_SELECT_TOP_K_OUTPUT_VALUES = 0, | ||
| 28 | + ACLNN_SELECT_TOP_K_OUTPUT_INDICES = 1, | ||
| 29 | + ACLNN_SELECT_TOP_K_OUTPUT_BOTH = 2 | ||
| 30 | +} aclnnSelectTopKOutputMode; | ||
| 31 | + | ||
| 32 | +/** | ||
| 33 | + * @brief aclnnSelectTopK 的第一阶段接口,构建 A2/A3 TopK 执行计划并查询工作区大小。 | ||
| 34 | + * | ||
| 35 | + * 记输入维数为 rank,排序轴长度为 N,其余各维长度的乘积为 rows。 | ||
| 36 | + * 每行在裁剪到 [0, N] 后的 [begin, end) 区间内选取极值,end <= begin 时该行区间为空。 | ||
| 37 | + * 输出形状为 self 的形状将 dim 维长度替换为 k;不足 k 个有效元素时,在尾部填充。 | ||
| 38 | + * 本阶段记录调用者预分配的输出张量,实际数据由第二阶段写入。 | ||
| 39 | + * | ||
| 40 | + * @param[in] self 输入张量,不可为 nullptr。支持公开格式和非连续张量,类型为 | ||
| 41 | + * ACL_FLOAT16、ACL_BF16 或 ACL_FLOAT;rank 为 1~8,N 不超过 INT32_MAX。 | ||
| 42 | + * @param[in] k 每行请求的输出元素数,范围为 [0, INT32_MAX];允许大于有效区间长度或 N。 | ||
| 43 | + * @param[in] dim 排序轴,范围为 [-rank, rank - 1];负数表示从末维向前计数。 | ||
| 44 | + * @param[in] largest 为 true 时选取最大值,为 false 时选取最小值。 | ||
| 45 | + * @param[in] indicesType 输出索引类型,仅接受 ACL_INT32 或 ACL_INT64;仅输出值时也须传入合法类型。 | ||
| 46 | + * @param[in] order 有效结果的排列方式。0(ACLNN_SELECT_TOP_K_ORDER_NONE)不排序; | ||
| 47 | + * 1(ACLNN_SELECT_TOP_K_ORDER_VALUE)按值排序,largest 为 true 时降序,否则升序; | ||
| 48 | + * 2(ACLNN_SELECT_TOP_K_ORDER_INDEX)为索引排序预留值,当前不支持。 | ||
| 49 | + * @param[in] indicesOffset 仅加到有效元素的原排序轴索引上的偏移量;不使用区间内的相对索引, | ||
| 50 | + * 也不加到 indexFill 上。偏移后完整原轴的索引范围须能用 indicesType 表示。 | ||
| 51 | + * @param[in] outputMode 输出选择:ACLNN_SELECT_TOP_K_OUTPUT_VALUES 仅输出值, | ||
| 52 | + * ACLNN_SELECT_TOP_K_OUTPUT_INDICES 仅输出索引,ACLNN_SELECT_TOP_K_OUTPUT_BOTH 同时输出两者。 | ||
| 53 | + * @param[in] beginOptional 每行区间的起点(包含),为公开格式的 ACL_INT32 设备张量,允许非连续。 | ||
| 54 | + * 元素数为 rows 时按除 dim 外各维的原顺序逐行对应,元素数为 1 时广播到全部行; | ||
| 55 | + * nullptr 表示所有行从 0 开始。端点按原轴坐标解释,负值截断为 0。 | ||
| 56 | + * @param[in] endOptional 每行区间的终点(不包含),类型、格式及逐行对应或广播规则与 beginOptional 相同; | ||
| 57 | + * nullptr 表示所有行到 N 结束。端点按原轴坐标解释,负值截断为 0。 | ||
| 58 | + * @param[in] valueFill 有效元素不足 k 时的值填充标量;nullptr 表示填零。 | ||
| 59 | + * 标量须可转换为 ACL_FLOAT,实现先转为 float,再转换为 self 的类型写入。 | ||
| 60 | + * @param[in] indexFill 有效元素不足 k 时的索引填充值,须能用 indicesType 表示;原样写入,不叠加偏移。 | ||
| 61 | + * @param[out] valuesOutOptional 调用者预分配的值输出张量,须为公开格式、类型与 self 相同且符合上述输出形状。 | ||
| 62 | + * 输出值时不可为 nullptr;仅输出索引时必须为 nullptr。 | ||
| 63 | + * @param[out] indicesOutOptional 调用者预分配的索引输出张量,须为公开格式、类型为 indicesType 且符合上述输出形状。 | ||
| 64 | + * 输出索引时不可为 nullptr;仅输出值时必须为 nullptr。 | ||
| 65 | + * @param[out] workspaceSize 返回第二阶段所需的设备工作区字节数,指针不可为 nullptr。 | ||
| 66 | + * @param[out] executor 返回包含完整计算流程的执行器,指针不可为 nullptr;交给 aclnnSelectTopK 执行。 | ||
| 67 | + * @return 成功返回 ACLNN_SUCCESS,否则返回对应的错误码。 | ||
| 68 | + */ | ||
| 69 | +ACLNN_API aclnnStatus aclnnSelectTopKGetWorkspaceSize(const aclTensor* self, int64_t k, int64_t dim, bool largest, | ||
| 70 | + aclDataType indicesType, int64_t order, int64_t indicesOffset, | ||
| 71 | + int64_t outputMode, const aclTensor* beginOptional, | ||
| 72 | + const aclTensor* endOptional, const aclScalar* valueFill, | ||
| 73 | + int64_t indexFill, aclTensor* valuesOutOptional, | ||
| 74 | + aclTensor* indicesOutOptional, uint64_t* workspaceSize, | ||
| 75 | + aclOpExecutor** executor); | ||
| 76 | + | ||
| 77 | +/** | ||
| 78 | + * @brief aclnnSelectTopK 的第二阶段接口,在指定流上执行完整 TopK 计算。 | ||
| 79 | + * | ||
| 80 | + * @param[in] workspace 调用者按第一阶段查询结果分配的设备工作区;相关设备内存须保持有效直到流执行完成。 | ||
| 81 | + * @param[in] workspaceSize 工作区大小,单位为字节,使用 aclnnSelectTopKGetWorkspaceSize 返回的大小。 | ||
| 82 | + * @param[in] executor aclnnSelectTopKGetWorkspaceSize 返回的执行器,包含输入、输出和计算流程信息。 | ||
| 83 | + * @param[in] stream 执行计算的设备流;调用者在读取输出数据前须等待该流上的计算完成。 | ||
| 84 | + * @return 成功返回 ACLNN_SUCCESS,否则返回对应的错误码。 | ||
| 85 | + */ | ||
| 86 | +ACLNN_API aclnnStatus aclnnSelectTopK(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor, | ||
| 87 | + aclrtStream stream); | ||
| 88 | + | ||
| 89 | +#ifdef __cplusplus | ||
| 90 | +} | ||
| 91 | +#endif | ||
| 92 | +#endif | ||
| @@ -0,0 +1,43 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#include "select_top_k.h" | ||
| 11 | +#include "../op_kernel/select_top_k_config.h" | ||
| 12 | +#include "opdev/make_op_executor.h" | ||
| 13 | +#include "opdev/op_def.h" | ||
| 14 | +#include "opdev/op_dfx.h" | ||
| 15 | + | ||
| 16 | +namespace l0op { | ||
| 17 | +OP_TYPE_REGISTER(SelectTopK); | ||
| 18 | + | ||
| 19 | +// 分配输出张量并注册统一的 A2 计算任务,具体算法由 Host 的 tiling key 区分。 | ||
| 20 | +std::tuple<aclTensor*, aclTensor*> SelectTopK(const aclTensor* self, const aclTensor* begin, const aclTensor* end, | ||
| 21 | + int64_t k, bool largest, int64_t order, op::DataType indicesType, | ||
| 22 | + int64_t indicesOffset, int64_t outputMode, float valueFill, | ||
| 23 | + int64_t indexFill, aclOpExecutor* executor) | ||
| 24 | +{ | ||
| 25 | + L0_DFX(SelectTopK, self, begin, end, k, largest, order, indicesType, indicesOffset, outputMode, valueFill, | ||
| 26 | + indexFill); | ||
| 27 | + auto shape = self->GetViewShape(); | ||
| 28 | + if (shape.GetDimNum() != ::SelectTopK::INPUT_RANK) { | ||
| 29 | + return {nullptr, nullptr}; | ||
| 30 | + } | ||
| 31 | + // 排序轴固定在中间维,输入、输出的 outer 和 inner 保持一致。 | ||
| 32 | + shape.SetDim(1, k); | ||
| 33 | + auto values = executor->AllocTensor(shape, self->GetDataType(), op::Format::FORMAT_ND); | ||
| 34 | + auto indices = executor->AllocTensor(shape, indicesType, op::Format::FORMAT_ND); | ||
| 35 | + if (values == nullptr || indices == nullptr) { | ||
| 36 | + return {nullptr, nullptr}; | ||
| 37 | + } | ||
| 38 | + ADD_TO_LAUNCHER_LIST_AICORE( | ||
| 39 | + SelectTopK, OP_INPUT(self, begin, end), OP_OUTPUT(values, indices), | ||
| 40 | + OP_ATTR(k, largest, order, indicesType, indicesOffset, outputMode, valueFill, indexFill)); | ||
| 41 | + return {values, indices}; | ||
| 42 | +} | ||
| 43 | +} // namespace l0op | ||
| @@ -0,0 +1,21 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#ifndef MATH_SELECT_TOP_K_H_ | ||
| 11 | +#define MATH_SELECT_TOP_K_H_ | ||
| 12 | +#include "opdev/op_executor.h" | ||
| 13 | +#include <tuple> | ||
| 14 | +namespace l0op { | ||
| 15 | +// 构建 SelectTopK 计算任务;输入为 [outer, axis, inner],inner 为 1 表示排序轴连续。 | ||
| 16 | +std::tuple<aclTensor*, aclTensor*> SelectTopK(const aclTensor* self, const aclTensor* begin, const aclTensor* end, | ||
| 17 | + int64_t k, bool largest, int64_t order, op::DataType indicesType, | ||
| 18 | + int64_t indicesOffset, int64_t outputMode, float valueFill, | ||
| 19 | + int64_t indexFill, aclOpExecutor* executor); | ||
| 20 | +} // namespace l0op | ||
| 21 | +#endif | ||
| @@ -0,0 +1,426 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +#ifndef SELECT_TOP_K_BASE_PLAN_H | ||
| 12 | +#define SELECT_TOP_K_BASE_PLAN_H | ||
| 13 | + | ||
| 14 | +#include <algorithm> | ||
| 15 | +#include <cstdint> | ||
| 16 | +#include <limits> | ||
| 17 | + | ||
| 18 | +#include "../../op_kernel/select_top_k_base_tiling_data.h" | ||
| 19 | +#include "../../op_kernel/select_top_k_base_select_layout.h" | ||
| 20 | + | ||
| 21 | +namespace SelectTopKBase { | ||
| 22 | +using SelectTopK::CandidateMerge; | ||
| 23 | +using SelectTopK::KernelTemplate; | ||
| 24 | +using SelectTopK::SelectPartition; | ||
| 25 | +// 核数及 UB 总容量由本次平台资源查询获得;调优常量见 select_top_k_base_config.h。 | ||
| 26 | +inline uint64_t CeilDiv(uint64_t n, uint64_t d) | ||
| 27 | +{ | ||
| 28 | + // 避免使用 (n + d - 1) / d 时,加法先发生溢出。 | ||
| 29 | + return n / d + (n % d != 0); | ||
| 30 | +} | ||
| 31 | + | ||
| 32 | +// 将元素数或字节数向上对齐到指定粒度。 | ||
| 33 | +inline uint64_t Align(uint64_t n, uint64_t alignment) { return CeilDiv(n, alignment) * alignment; } | ||
| 34 | + | ||
| 35 | +// 检查无符号乘法溢出,成功时写出乘积。 | ||
| 36 | +inline bool Mul(uint64_t a, uint64_t b, uint64_t& out) | ||
| 37 | +{ | ||
| 38 | + if (b != 0 && a > std::numeric_limits<uint64_t>::max() / b) { | ||
| 39 | + return false; | ||
| 40 | + } | ||
| 41 | + out = a * b; | ||
| 42 | + return true; | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +// 选择与页数互质的扫描步长,使各页恰好访问一次。 | ||
| 46 | +inline uint32_t SelectPageStep(uint32_t pages) | ||
| 47 | +{ | ||
| 48 | + // 与页数互质的步长构成全排列;不依赖输入数值,也不分配 N 规模的页索引表。 | ||
| 49 | + uint32_t step = std::max<uint32_t>( | ||
| 50 | + 1, (static_cast<uint64_t>(pages) * Tuning::PAGE_STEP_MULTIPLIER) >> MASK_WORD32_BITS); | ||
| 51 | + while (true) { | ||
| 52 | + uint32_t a = pages; | ||
| 53 | + uint32_t b = step; | ||
| 54 | + while (b != 0) { | ||
| 55 | + const uint32_t remainder = a % b; | ||
| 56 | + a = b; | ||
| 57 | + b = remainder; | ||
| 58 | + } | ||
| 59 | + if (a == 1) { | ||
| 60 | + return step; | ||
| 61 | + } | ||
| 62 | + ++step; | ||
| 63 | + } | ||
| 64 | +} | ||
| 65 | + | ||
| 66 | +// 根据通用归并布局返回完整 UB 占用字节数。 | ||
| 67 | +inline uint64_t UbBytes(uint32_t tile, uint32_t window, uint64_t k, uint32_t dtypeBytes, uint32_t route) | ||
| 68 | +{ | ||
| 69 | + return GeneralMergeLayout(tile, window, k, dtypeBytes, route).bytes; | ||
| 70 | +} | ||
| 71 | + | ||
| 72 | +// 从大到小选择满足 UB 预算的排序块与归并窗口。 | ||
| 73 | +inline bool SelectTile(uint64_t usableUbBytes, uint32_t dtypeBytes, uint64_t k, uint32_t route, uint32_t& tile, | ||
| 74 | + uint32_t& window, uint64_t& usedUbBytes) | ||
| 75 | +{ | ||
| 76 | + // Sort32 每次处理 32 个元素,repeat 最大 255;从大到小选择能容纳全部缓冲的 tile。 | ||
| 77 | + for (tile = SORT32_MAX_ELEMENTS; tile >= SORT32_ELEMENTS; tile -= SORT32_ELEMENTS) { | ||
| 78 | + // 窗口取 tile 的 1/8,再向下对齐到 4 个 pair,保证每路窗口的 32B 对齐。 | ||
| 79 | + window = std::min<uint32_t>(MERGE_MAX_ALIGNED_ITEMS, | ||
| 80 | + std::max<uint32_t>(PAIR_ALIGNMENT, tile / Tuning::GENERAL_WINDOW_DIVISOR / | ||
| 81 | + PAIR_ALIGNMENT * PAIR_ALIGNMENT)); | ||
| 82 | + if (route == ROUTE_GM_MERGE) { | ||
| 83 | + // 流式归并与叶子排序分时复用 UB;大 K 使用较宽窗口以摊薄 DMA 和归并准备开销。 | ||
| 84 | + window = std::min<uint64_t>(std::min<uint32_t>(tile, MERGE_MAX_ALIGNED_ITEMS), | ||
| 85 | + usableUbBytes / MERGE_WORK_BYTES / PAIR_ALIGNMENT * PAIR_ALIGNMENT); | ||
| 86 | + if (window < PAIR_ALIGNMENT) { | ||
| 87 | + return false; | ||
| 88 | + } | ||
| 89 | + } | ||
| 90 | + usedUbBytes = UbBytes(tile, window, k, dtypeBytes, route); | ||
| 91 | + if (usedUbBytes <= usableUbBytes && usedUbBytes <= UINT32_MAX) { | ||
| 92 | + return true; | ||
| 93 | + } | ||
| 94 | + } | ||
| 95 | + return false; | ||
| 96 | +} | ||
| 97 | + | ||
| 98 | +// 资源预算与算法参数分离;usableUbBytes 已在 BuildPlan 中扣过一次系统预留。 | ||
| 99 | +struct PlanResources { | ||
| 100 | + uint64_t usableUbBytes; | ||
| 101 | + uint32_t cores; | ||
| 102 | + uint64_t smallKMinAxis = Tuning::FILTER_MIN_AXIS; | ||
| 103 | + uint32_t maxLeafTile = Tuning::MAX_LEAF_BLOCK; | ||
| 104 | + bool allowWideRowBatch = false; | ||
| 105 | +}; | ||
| 106 | + | ||
| 107 | +enum class PlanAttempt { NOT_APPLICABLE, SUCCESS, INVALID }; | ||
| 108 | + | ||
| 109 | +// 判断 N/K 是否落在过滤选择允许的范围。 | ||
| 110 | +inline bool IsFilterRange(const SelectTopKBaseTilingData& plan, const PlanResources& resources) | ||
| 111 | +{ | ||
| 112 | + const uint64_t minAxis = plan.k <= Tuning::SMALL_K ? resources.smallKMinAxis : Tuning::FILTER_MIN_AXIS; | ||
| 113 | + return plan.axis >= minAxis && plan.k >= Tuning::FILTER_MIN_K && plan.k <= Tuning::FILTER_MAX_K; | ||
| 114 | +} | ||
| 115 | + | ||
| 116 | +// 短轴每行补到 32 项,用一个 Sort32 repeat 独立处理一行。 | ||
| 117 | +inline PlanAttempt TryShortRowBatch(SelectTopKBaseTilingData& plan, const PlanResources& resources) | ||
| 118 | +{ | ||
| 119 | + if (!(plan.axis > 1 && plan.axis <= SORT32_ELEMENTS && (plan.sorted || plan.k != plan.axis))) { | ||
| 120 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 121 | + } | ||
| 122 | + | ||
| 123 | + // R1 短轴子路径:每行补至 32 项,Sort32 的每个 repeat 排一行,无需归并。 | ||
| 124 | + // raw、FP32 key、UINT32 index 和交错 pair 同时存活;所有行天然按 32B 对齐。 | ||
| 125 | + const uint64_t bytesPerRow = SORT32_ELEMENTS_64 * (plan.dtypeBytes + VALUE32_BYTES + INDEX32_BYTES + PAIR_BYTES); | ||
| 126 | + const uint64_t maxRowsPerBatch = std::min<uint64_t>(VECTOR_MAX_REPEATS, resources.usableUbBytes / bytesPerRow); | ||
| 127 | + if (maxRowsPerBatch == 0) { | ||
| 128 | + return PlanAttempt::INVALID; | ||
| 129 | + } | ||
| 130 | + // 短轴优先让每核一次处理至少 16 行,避免少量数据分散到大量核后的小块 DMA 争用。 | ||
| 131 | + // 大输入仍按实际可用核数均分;小 UB 和 repeat 上限始终优先约束批量。 | ||
| 132 | + const uint64_t preferredRows = std::max<uint64_t>(Tuning::SHORT_ROWS_MIN_BATCH, | ||
| 133 | + CeilDiv(plan.rows, resources.cores)); | ||
| 134 | + const uint64_t rowsPerBatch = std::min(maxRowsPerBatch, std::min(plan.rows, preferredRows)); | ||
| 135 | + plan.route = ROUTE_ROW_SORT; | ||
| 136 | + plan.tile = static_cast<uint32_t>(rowsPerBatch * SORT32_ELEMENTS); | ||
| 137 | + plan.batchRows = static_cast<uint32_t>(rowsPerBatch); | ||
| 138 | + plan.kernel = KernelTemplate::SHORT_ROWS; | ||
| 139 | + plan.outputTile = SORT32_ELEMENTS; | ||
| 140 | + plan.tiles = 1; | ||
| 141 | + plan.ubBytes = static_cast<uint32_t>(rowsPerBatch * bytesPerRow); | ||
| 142 | + plan.cores = static_cast<uint32_t>(std::min<uint64_t>(resources.cores, CeilDiv(plan.rows, rowsPerBatch))); | ||
| 143 | + return PlanAttempt::SUCCESS; | ||
| 144 | +} | ||
| 145 | + | ||
| 146 | +// 行并行充分时,每核独立维护本行 TopK 阈值和候选。 | ||
| 147 | +inline PlanAttempt TryRowFilter(SelectTopKBaseTilingData& plan, const PlanResources& resources) | ||
| 148 | +{ | ||
| 149 | + if (!(IsFilterRange(plan, resources) && plan.rows >= resources.cores)) { | ||
| 150 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 151 | + } | ||
| 152 | + | ||
| 153 | + // 分阶段复用候选/排序临时区后,大 K 也尝试 2048;实际放不下时继续逐级减半。 | ||
| 154 | + const uint32_t preferredBlock = Tuning::FILTER_PREFERRED_BLOCK; | ||
| 155 | + for (uint32_t block = preferredBlock; block >= Tuning::FILTER_MIN_BLOCK; block /= HALF_DIVISOR) { | ||
| 156 | + const SelectLayout layout(block, static_cast<uint32_t>(plan.k), plan.dtypeBytes); | ||
| 157 | + if (layout.bytes <= resources.usableUbBytes) { | ||
| 158 | + plan.route = ROUTE_FILTER_MERGE; | ||
| 159 | + plan.kernel = KernelTemplate::SELECT; | ||
| 160 | + plan.partition = SelectPartition::ROWS; | ||
| 161 | + plan.cores = resources.cores; | ||
| 162 | + plan.tile = block; | ||
| 163 | + plan.pendingLimit = block; | ||
| 164 | + plan.outputTile = std::min<uint32_t>(block, static_cast<uint32_t>(plan.k)); | ||
| 165 | + plan.tiles = CeilDiv(plan.axis, block); | ||
| 166 | + plan.pageStep = SelectPageStep(static_cast<uint32_t>(plan.tiles)); | ||
| 167 | + plan.ubBytes = layout.bytes; | ||
| 168 | + return PlanAttempt::SUCCESS; | ||
| 169 | + } | ||
| 170 | + } | ||
| 171 | + // UB 无法容纳最小扫描块及完整 Acc 时,沿用原有 R1/R2/R3 规划与错误处理。 | ||
| 172 | + | ||
| 173 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 174 | +} | ||
| 175 | + | ||
| 176 | +// 少行小 K 采用宽块过滤;每行按实际核数分片。 | ||
| 177 | +inline PlanAttempt TryWideFilter(SelectTopKBaseTilingData& plan, const PlanResources& resources) | ||
| 178 | +{ | ||
| 179 | + if (!(IsFilterRange(plan, resources) && (plan.rows < resources.cores || resources.allowWideRowBatch) && | ||
| 180 | + plan.k <= Tuning::SMALL_K && plan.rows > 1)) { | ||
| 181 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 182 | + } | ||
| 183 | + | ||
| 184 | + // 小 K 的候选很少,优先扩大一次扫描量;B 由实际 UB 决定,不能固定为某个芯片容量。 | ||
| 185 | + // 每项三份 4B 数组、1bit mask,另留完整 Acc;精确布局检查负责对齐后的边界。 | ||
| 186 | + const uint64_t accumulatorBytes = Align(plan.k, SORT32_ELEMENTS) * PAIR_BYTES; | ||
| 187 | + uint32_t block = resources.usableUbBytes > accumulatorBytes ? | ||
| 188 | + static_cast<uint32_t>(std::min<uint64_t>((resources.usableUbBytes - accumulatorBytes) * | ||
| 189 | + BITS_PER_BYTE / WIDE_SCAN_BITS_PER_ITEM / | ||
| 190 | + FLOAT_VECTOR_ELEMENTS * FLOAT_VECTOR_ELEMENTS, | ||
| 191 | + FLOAT_VECTOR_ELEMENTS * VECTOR_MAX_REPEATS)) : | ||
| 192 | + 0; | ||
| 193 | + for (; block >= Tuning::WIDE_MIN_BLOCK; block -= FLOAT_VECTOR_ELEMENTS) { | ||
| 194 | + const WideSelectLayout layout(block, static_cast<uint32_t>(plan.k)); | ||
| 195 | + if (layout.bytes > resources.usableUbBytes) { | ||
| 196 | + continue; | ||
| 197 | + } | ||
| 198 | + const uint32_t seed = plan.k <= Tuning::TINY_K ? Tuning::WIDE_TINY_SEED : Tuning::WIDE_SMALL_SEED; | ||
| 199 | + // 仅当每行在建立初始候选后只需扫描一次时,才将多行合并为一个批次。 | ||
| 200 | + if (plan.rows >= resources.cores && plan.axis > block + seed) { | ||
| 201 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 202 | + } | ||
| 203 | + const uint64_t shards = std::min<uint64_t>(std::max<uint64_t>(1, resources.cores / plan.rows), | ||
| 204 | + CeilDiv(plan.axis, block + seed)); | ||
| 205 | + plan.route = ROUTE_WIDE_FILTER; | ||
| 206 | + plan.kernel = KernelTemplate::WIDE_SELECT; | ||
| 207 | + plan.cores = static_cast<uint32_t>(std::min<uint64_t>(resources.cores, plan.rows * shards)); | ||
| 208 | + plan.tile = block; | ||
| 209 | + plan.seedItems = seed; | ||
| 210 | + plan.candidateSortTile = Tuning::WIDE_SORT_CHUNK; | ||
| 211 | + while (plan.candidateSortTile * DOUBLE_PAIR_BYTES > block * VALUE32_BYTES) { | ||
| 212 | + plan.candidateSortTile /= HALF_DIVISOR; | ||
| 213 | + } | ||
| 214 | + plan.tiles = CeilDiv(plan.axis, block); | ||
| 215 | + plan.shardCount = static_cast<uint32_t>(shards); | ||
| 216 | + plan.ubBytes = layout.bytes; | ||
| 217 | + if (shards > 1) { | ||
| 218 | + plan.rowWorkspacePairs = shards * Align(plan.k, UB_BLOCK_BYTES / PAIR_BYTES); | ||
| 219 | + if (!Mul(plan.rows, plan.rowWorkspacePairs * PAIR_BYTES, plan.workspaceBytes)) { | ||
| 220 | + return PlanAttempt::INVALID; | ||
| 221 | + } | ||
| 222 | + } | ||
| 223 | + return PlanAttempt::SUCCESS; | ||
| 224 | + } | ||
| 225 | + | ||
| 226 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 227 | +} | ||
| 228 | + | ||
| 229 | +// 汇总各叶子的对齐候选数,保留调用方的最低容量;上限及 UB 校验仍由各规划规则决定。 | ||
| 230 | +inline uint64_t LeafCandidateCapacity(uint64_t shards, uint64_t k, uint32_t minimum) | ||
| 231 | +{ | ||
| 232 | + return std::max<uint64_t>(minimum, Align(shards * Align(k, PAIR_ALIGNMENT), SORT32_ELEMENTS)); | ||
| 233 | +} | ||
| 234 | + | ||
| 235 | +// 每片能一次排序且独占一核时,直接局部排序再归并候选。 | ||
| 236 | +inline PlanAttempt TryLeafSort(SelectTopKBaseTilingData& plan, const PlanResources& resources) | ||
| 237 | +{ | ||
| 238 | + if (!(IsFilterRange(plan, resources) && plan.rows < resources.cores)) { | ||
| 239 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 240 | + } | ||
| 241 | + // 所有片均能在 UB 内一次排序,且每片有独立 AIV 时,直接局部排序并保留 TopK。 | ||
| 242 | + // block 仍受实际 UB/核数约束;条件不满足继续采用下方多页过滤。 | ||
| 243 | + // 单行小 K 用更多叶子并行;多行宽块筛选优先,资源不足时仍能落到本分支。 | ||
| 244 | + const uint32_t firstLeafBlock = plan.k <= Tuning::TINY_K ? | ||
| 245 | + Tuning::TINY_LEAF_BLOCK : | ||
| 246 | + (plan.k <= Tuning::SMALL_K ? Tuning::SMALL_LEAF_BLOCK : Tuning::MAX_LEAF_BLOCK); | ||
| 247 | + const uint32_t maxLeafBlock = std::min<uint32_t>(resources.maxLeafTile, Tuning::MAX_EXTENDED_LEAF_BLOCK); | ||
| 248 | + // 大 K 优先减少候选列表数;小 K 保留最小可行叶子,仅在核数不足时尝试扩展叶子。 | ||
| 249 | + const bool preferLarge = maxLeafBlock > Tuning::MAX_LEAF_BLOCK && plan.k >= Tuning::LARGE_LEAF_MIN_K; | ||
| 250 | + const uint32_t candidates[Tuning::LEAF_CANDIDATE_COUNT] = { | ||
| 251 | + preferLarge ? maxLeafBlock : 0, firstLeafBlock, firstLeafBlock * HALF_DIVISOR, firstLeafBlock * MERGE_WAYS, | ||
| 252 | + preferLarge ? 0 : maxLeafBlock}; | ||
| 253 | + for (uint32_t attempt = 0; attempt < Tuning::LEAF_CANDIDATE_COUNT; ++attempt) { | ||
| 254 | + const uint32_t block = candidates[attempt]; | ||
| 255 | + if (block < firstLeafBlock || block > maxLeafBlock || | ||
| 256 | + (attempt > 0 && attempt < Tuning::LAST_REGULAR_LEAF_CANDIDATE && block > Tuning::MAX_LEAF_BLOCK)) { | ||
| 257 | + continue; | ||
| 258 | + } | ||
| 259 | + const uint64_t shards = CeilDiv(plan.axis, block); | ||
| 260 | + // 优先一次读入全部片候选;受 4096 项归并容量与实际 UB 约束。 | ||
| 261 | + uint32_t finalCapacity = static_cast<uint32_t>( | ||
| 262 | + std::min<uint64_t>(block > Tuning::MAX_LEAF_BLOCK ? maxLeafBlock : Tuning::MAX_LEAF_BLOCK, | ||
| 263 | + LeafCandidateCapacity(shards, plan.k, block))); | ||
| 264 | + LeafSelectLayout layout(block, static_cast<uint32_t>(plan.k), plan.dtypeBytes, finalCapacity); | ||
| 265 | + if (layout.bytes > resources.usableUbBytes) { | ||
| 266 | + finalCapacity = block; | ||
| 267 | + layout = LeafSelectLayout(block, static_cast<uint32_t>(plan.k), plan.dtypeBytes, finalCapacity); | ||
| 268 | + } | ||
| 269 | + if (layout.bytes > resources.usableUbBytes || plan.k > block || shards > resources.cores / plan.rows) { | ||
| 270 | + continue; | ||
| 271 | + } | ||
| 272 | + plan.route = ROUTE_SHARDED_SELECT; | ||
| 273 | + plan.cores = static_cast<uint32_t>(plan.rows * shards); | ||
| 274 | + plan.tile = block; | ||
| 275 | + plan.kernel = KernelTemplate::LEAF_SELECT; | ||
| 276 | + plan.candidateMerge = plan.k >= Tuning::PREFIX_MERGE_MIN_K ? CandidateMerge::HALF_PREFIX : | ||
| 277 | + CandidateMerge::FULL_LISTS; | ||
| 278 | + plan.finalCapacity = finalCapacity; | ||
| 279 | + plan.tiles = shards; | ||
| 280 | + plan.shardCount = static_cast<uint32_t>(shards); | ||
| 281 | + plan.ubBytes = layout.bytes; | ||
| 282 | + plan.rowWorkspacePairs = shards * Align(plan.k, UB_BLOCK_BYTES / PAIR_BYTES); | ||
| 283 | + if (!Mul(plan.rows, plan.rowWorkspacePairs * PAIR_BYTES, plan.workspaceBytes)) { | ||
| 284 | + return PlanAttempt::INVALID; | ||
| 285 | + } | ||
| 286 | + return PlanAttempt::SUCCESS; | ||
| 287 | + } | ||
| 288 | + | ||
| 289 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 290 | +} | ||
| 291 | + | ||
| 292 | +// 其余少行特化按多页分片过滤,最终合并各片有序 TopK。 | ||
| 293 | +inline PlanAttempt TryShardedFilter(SelectTopKBaseTilingData& plan, const PlanResources& resources) | ||
| 294 | +{ | ||
| 295 | + if (!(IsFilterRange(plan, resources) && plan.rows < resources.cores)) { | ||
| 296 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 297 | + } | ||
| 298 | + // 少行长轴按页分片,每片独立过滤并保留 TopK;最后只合并各片候选。 | ||
| 299 | + // 小 K 允许单块分片缩短扫描;较大 K 保留至少约 4096 项,限制候选归并量。 | ||
| 300 | + for (uint32_t block = Tuning::FILTER_PREFERRED_BLOCK; block >= Tuning::FILTER_MIN_BLOCK; block /= HALF_DIVISOR) { | ||
| 301 | + const SelectLayout layout(block, static_cast<uint32_t>(plan.k), plan.dtypeBytes); | ||
| 302 | + // 最终归并需同时容纳两份 K;ping 至少能读入一份完整局部候选。 | ||
| 303 | + if (layout.bytes > resources.usableUbBytes || layout.capacity < plan.k) { | ||
| 304 | + continue; | ||
| 305 | + } | ||
| 306 | + const uint64_t pages = CeilDiv(plan.axis, block); | ||
| 307 | + const uint64_t minShardItems = plan.k <= Tuning::SMALL_K ? Tuning::SMALL_K_MIN_SHARD_ITEMS : | ||
| 308 | + Tuning::LARGE_K_MIN_SHARD_ITEMS; | ||
| 309 | + const uint64_t maxShards = std::max<uint64_t>(1, plan.axis / minShardItems); | ||
| 310 | + const uint64_t shards = std::min<uint64_t>(resources.cores / plan.rows, maxShards); | ||
| 311 | + plan.route = ROUTE_SHARDED_SELECT; | ||
| 312 | + plan.cores = static_cast<uint32_t>(std::min<uint64_t>(resources.cores, plan.rows * shards)); | ||
| 313 | + plan.tile = block; | ||
| 314 | + plan.pendingLimit = block; | ||
| 315 | + plan.kernel = KernelTemplate::SELECT; | ||
| 316 | + plan.partition = SelectPartition::SHARDS; | ||
| 317 | + plan.outputTile = std::min<uint32_t>(block, static_cast<uint32_t>(plan.k)); | ||
| 318 | + plan.tiles = pages; | ||
| 319 | + plan.shardCount = static_cast<uint32_t>(shards); | ||
| 320 | + plan.ubBytes = layout.bytes; | ||
| 321 | + // 扫描结束后复用 arena;四路前缀的输入/输出均须容纳,复制前 K 项不能碰到活跃 Acc。 | ||
| 322 | + const uint64_t prefix = (plan.k + (HALF_PREFIX_ALIGNMENT - 1)) / HALF_PREFIX_ALIGNMENT * PAIR_ALIGNMENT; | ||
| 323 | + const uint64_t stride = Align(plan.k, PAIR_ALIGNMENT); | ||
| 324 | + const bool prefixFits = prefix * MERGE_WORK_BYTES <= layout.bytes && | ||
| 325 | + prefix * MERGE_INPUT_BYTES + stride * PAIR_BYTES <= layout.accumulator; | ||
| 326 | + if (plan.k >= Tuning::PREFIX_MERGE_MIN_K && shards >= Tuning::PREFIX_MIN_SHARDS && prefixFits) { | ||
| 327 | + plan.candidateMerge = CandidateMerge::HALF_PREFIX; | ||
| 328 | + } | ||
| 329 | + if (shards > 1) { | ||
| 330 | + // 每片槽位按 32B 对齐;仅一份 GM 候选平面,最终归并直接写输出。 | ||
| 331 | + plan.rowWorkspacePairs = shards * Align(plan.k, UB_BLOCK_BYTES / PAIR_BYTES); | ||
| 332 | + if (!Mul(plan.rows, plan.rowWorkspacePairs * PAIR_BYTES, plan.workspaceBytes)) { | ||
| 333 | + return PlanAttempt::INVALID; | ||
| 334 | + } | ||
| 335 | + } | ||
| 336 | + return PlanAttempt::SUCCESS; | ||
| 337 | + } | ||
| 338 | + | ||
| 339 | + return PlanAttempt::NOT_APPLICABLE; | ||
| 340 | +} | ||
| 341 | + | ||
| 342 | +// 通用路径:整行排序 -> UB 累积;容量或行并行度不足时回退到 GM 分层归并。 | ||
| 343 | +inline bool PlanGeneralMerge(SelectTopKBaseTilingData& plan, const PlanResources& resources, uint64_t elements) | ||
| 344 | +{ | ||
| 345 | + uint64_t usedUbBytes = 0; | ||
| 346 | + uint32_t tile = 0; | ||
| 347 | + uint32_t window = 0; | ||
| 348 | + // R0:全量且不要求排序,或轴长为 1;其余场景先按 R1 的缓冲布局选择 tile。 | ||
| 349 | + const bool copyOnly = (!plan.sorted && plan.k == plan.axis) || plan.axis == 1; | ||
| 350 | + plan.route = copyOnly ? ROUTE_COPY : ROUTE_ROW_SORT; | ||
| 351 | + if (!SelectTile(resources.usableUbBytes, plan.dtypeBytes, plan.k, plan.route, tile, window, usedUbBytes)) { | ||
| 352 | + return false; | ||
| 353 | + } | ||
| 354 | + | ||
| 355 | + if (!copyOnly && plan.axis > tile) { | ||
| 356 | + uint32_t accumulateTile = 0; | ||
| 357 | + uint32_t accumulateWindow = 0; | ||
| 358 | + uint64_t accumulateUbBytes = 0; | ||
| 359 | + // R2:行数足以覆盖可用 AIV,且两份 Acc 连同全部临时区都能放入 UB。 | ||
| 360 | + // 加入 Acc 后必须重新选择 tile,不能沿用 R1 的容量计算。 | ||
| 361 | + if (plan.rows >= resources.cores && | ||
| 362 | + // 从大到小选择满足 UB 预算的排序块与归并窗口。 | ||
| 363 | + SelectTile(resources.usableUbBytes, plan.dtypeBytes, plan.k, ROUTE_UB_ACCUMULATE, accumulateTile, | ||
| 364 | + accumulateWindow, accumulateUbBytes) && | ||
| 365 | + accumulateTile >= plan.k) { | ||
| 366 | + plan.route = ROUTE_UB_ACCUMULATE; | ||
| 367 | + tile = accumulateTile; | ||
| 368 | + window = accumulateWindow; | ||
| 369 | + usedUbBytes = accumulateUbBytes; | ||
| 370 | + } else { | ||
| 371 | + // R3:行并行度不足或 UB 放不下 Acc,通过 GM 候选段进行分层归并。 | ||
| 372 | + plan.route = ROUTE_GM_MERGE; | ||
| 373 | + if (!SelectTile(resources.usableUbBytes, plan.dtypeBytes, plan.k, plan.route, tile, window, usedUbBytes)) { | ||
| 374 | + return false; | ||
| 375 | + } | ||
| 376 | + } | ||
| 377 | + } | ||
| 378 | + | ||
| 379 | + plan.tile = tile; | ||
| 380 | + plan.mergeWindow = window; | ||
| 381 | + plan.outputTile = std::min(tile, window); | ||
| 382 | + plan.ubBytes = static_cast<uint32_t>(usedUbBytes); | ||
| 383 | + plan.tiles = CeilDiv(plan.axis, tile); | ||
| 384 | + | ||
| 385 | + // R1/R2 按行分配任务;R0 可以跨行按连续片段分配任务。 | ||
| 386 | + uint64_t tasks = plan.rows; | ||
| 387 | + if (plan.route == ROUTE_COPY) { | ||
| 388 | + tasks = CeilDiv(elements, tile); | ||
| 389 | + } | ||
| 390 | + if (plan.route == ROUTE_GM_MERGE) { | ||
| 391 | + // 每个叶子段最多保留 min(plan.k, tile) 个 pair;每行的固定槽位覆盖全部叶子段。 | ||
| 392 | + const uint64_t leafCandidatePairs = std::min<uint64_t>(plan.k, tile); | ||
| 393 | + uint64_t rowCandidatePairs = 0; | ||
| 394 | + if (!Mul(plan.tiles, leafCandidatePairs, rowCandidatePairs) || | ||
| 395 | + rowCandidatePairs > INT64_MAX / PAIR_BYTES - PAIR_ALIGNMENT) { | ||
| 396 | + return false; | ||
| 397 | + } | ||
| 398 | + // 每行按 4 个 pair 对齐,使两份 GM 平面中的各行起点都满足 32B 对齐。 | ||
| 399 | + plan.rowWorkspacePairs = Align(rowCandidatePairs, UB_BLOCK_BYTES / PAIR_BYTES); | ||
| 400 | + uint64_t planePairs = 0; | ||
| 401 | + if (!Mul(plan.rows, plan.rowWorkspacePairs, planePairs)) { | ||
| 402 | + return false; | ||
| 403 | + } | ||
| 404 | + // GM ping/pong 各一份:plan.rows * rowWorkspacePairs * 8B * 2;不含框架系统 workspace。 | ||
| 405 | + if (!Mul(planePairs, DOUBLE_BUFFER_COUNT * PAIR_BYTES, plan.workspaceBytes) || | ||
| 406 | + plan.workspaceBytes > INT64_MAX) { | ||
| 407 | + return false; | ||
| 408 | + } | ||
| 409 | + // R3 的叶子排序按“行 × tile”切任务,短行数场景也可以使用多个 AIV。 | ||
| 410 | + if (!Mul(plan.rows, plan.tiles, tasks)) { | ||
| 411 | + return false; | ||
| 412 | + } | ||
| 413 | + } | ||
| 414 | + | ||
| 415 | + // 核数由实际任务量和平台可用 AIV 共同决定,不固定核数。 | ||
| 416 | + plan.cores = static_cast<uint32_t>(std::min<uint64_t>(resources.cores, tasks)); | ||
| 417 | + if (plan.route == ROUTE_GM_MERGE) { | ||
| 418 | + // 输出阶段可复用完整叶子缓冲;同时保留足够输出块,使工作分布到所有活动核。 | ||
| 419 | + plan.outputTile = std::min<uint64_t>( | ||
| 420 | + tile, std::max<uint64_t>(window, Align(CeilDiv(plan.rows * plan.k, plan.cores), SORT32_ELEMENTS))); | ||
| 421 | + } | ||
| 422 | + return true; | ||
| 423 | +} | ||
| 424 | + | ||
| 425 | +} // namespace SelectTopKBase | ||
| 426 | +#endif | ||
| @@ -0,0 +1,217 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#ifndef SELECT_TOP_K_PLAN_H_ | ||
| 11 | +#define SELECT_TOP_K_PLAN_H_ | ||
| 12 | +#include "../../op_kernel/select_top_k_tiling_data.h" | ||
| 13 | +#include "../../op_kernel/select_top_k_output_layout.h" | ||
| 14 | +#include "../../op_kernel/select_top_k_output_reuse_layout.h" | ||
| 15 | +#include "select_top_k_targeted_plan.h" | ||
| 16 | +#include <type_traits> | ||
| 17 | + | ||
| 18 | +namespace SelectTopK { | ||
| 19 | +using namespace SelectTopKBase; | ||
| 20 | +// 为一个工作区片段计算 32B 对齐后的偏移,并提前检查地址范围溢出。 | ||
| 21 | +inline bool AddWorkspace(uint64_t count, uint32_t bytes, uint64_t& total, uint64_t& offset) | ||
| 22 | +{ | ||
| 23 | + uint64_t size = 0; | ||
| 24 | + if (!SelectTopKBase::Mul(count, bytes, size) || size > INT64_MAX - (UB_BLOCK_BYTES - 1) || | ||
| 25 | + total > INT64_MAX - size - (UB_BLOCK_BYTES - 1)) { | ||
| 26 | + return false; | ||
| 27 | + } | ||
| 28 | + offset = total; | ||
| 29 | + total += SelectTopKBase::Align(size, UB_BLOCK_BYTES); | ||
| 30 | + return true; | ||
| 31 | +} | ||
| 32 | + | ||
| 33 | +// 按原有增长序列搜索输出列块,统一检查并行度、Gather 容量与 UB 预算。 | ||
| 34 | +template <bool Reuse> | ||
| 35 | +inline bool PlanOutputLayout(uint64_t ub, uint32_t indicesBytes, SelectTopKTilingData& data) | ||
| 36 | +{ | ||
| 37 | + using Layout = typename std::conditional<Reuse, OutputReuseLayout, OutputLayout>::type; | ||
| 38 | + const uint64_t inner = data.outputInner, k = data.outputK; | ||
| 39 | + const uint32_t bytes = data.base.dtypeBytes; | ||
| 40 | + const uint64_t columnsPerCore = CeilDiv((data.base.rows / inner) * k, std::max<uint32_t>(1, data.base.cores)); | ||
| 41 | + const uint32_t limit = Reuse ? MAX_REUSED_OUTPUT_BLOCK : | ||
| 42 | + (inner >= GM_MIN_INNER ? (k > LARGE_K_THRESHOLD ? WIDE_OUTPUT_BLOCK : GM_MIN_INNER) : | ||
| 43 | + MAX_OUTPUT_BLOCK); | ||
| 44 | + const uint32_t capacity = Reuse ? MAX_REUSED_CAPACITY : MAX_OUTPUT_CAPACITY; | ||
| 45 | + uint32_t block = MIN_LAYOUT_BLOCK; | ||
| 46 | + auto layout = Layout(inner, block, k, bytes, indicesBytes); | ||
| 47 | + if (layout.bytes > ub - UB_RESERVE) | ||
| 48 | + return false; | ||
| 49 | + while (block < limit && block < columnsPerCore) { | ||
| 50 | + const uint32_t nextBlock = Reuse && block == WIDE_OUTPUT_BLOCK ? | ||
| 51 | + REUSED_MID_BLOCK : | ||
| 52 | + (Reuse && block == REUSED_MID_BLOCK ? MAX_REUSED_OUTPUT_BLOCK : | ||
| 53 | + block * DOUBLE_BUFFER_COUNT); | ||
| 54 | + const auto next = Layout(inner, nextBlock, k, bytes, indicesBytes); | ||
| 55 | + if (next.capacity > capacity || next.bytes > ub - UB_RESERVE) | ||
| 56 | + break; | ||
| 57 | + block = nextBlock; | ||
| 58 | + layout = next; | ||
| 59 | + } | ||
| 60 | + data.outputBlock = block; | ||
| 61 | + data.base.ubBytes = std::max<uint32_t>(data.base.ubBytes, layout.bytes); | ||
| 62 | + return true; | ||
| 63 | +} | ||
| 64 | + | ||
| 65 | +// 校验完整接口,依次规划基础排序、过滤选择和已定位特化,再适配扩展收尾与输入输出布局。 | ||
| 66 | +inline bool BuildPlan(uint64_t rows, uint64_t axis, uint64_t k, uint32_t bytes, bool largest, bool sorted, | ||
| 67 | + uint32_t indicesBytes, int64_t offset, int64_t mode, uint32_t boundsFlags, float valueFill, | ||
| 68 | + int64_t indexFill, uint64_t ub, uint32_t cores, SelectTopKTilingData& data, | ||
| 69 | + uint64_t outputInner = 1, uint64_t inputInner = 1) | ||
| 70 | +{ | ||
| 71 | + if (cores == 0 || ub <= SelectTopKBase::UB_RESERVE || k > INT32_MAX || axis > INT32_MAX || | ||
| 72 | + (indicesBytes != INDEX32_BYTES && indicesBytes != INDEX64_BYTES) || mode < 0 || mode > OUTPUT_BOTH) { | ||
| 73 | + return false; | ||
| 74 | + } | ||
| 75 | + const int64_t upper = indicesBytes == INDEX32_BYTES ? INT32_MAX : INT64_MAX; | ||
| 76 | + const int64_t lower = indicesBytes == INDEX32_BYTES ? INT32_MIN : INT64_MIN; | ||
| 77 | + const int64_t maxIndex = axis == 0 ? 0 : static_cast<int64_t>(axis - 1); | ||
| 78 | + uint64_t inputItems = 0, outputItems = 0; | ||
| 79 | + if (offset < lower || offset > upper - maxIndex || indexFill < lower || indexFill > upper || | ||
| 80 | + !SelectTopKBase::Mul(rows, axis, inputItems) || !SelectTopKBase::Mul(rows, k, outputItems) || | ||
| 81 | + inputItems > INT64_MAX / INDEX64_BYTES || outputItems > INT64_MAX / INDEX64_BYTES) { | ||
| 82 | + return false; | ||
| 83 | + } | ||
| 84 | + if (outputInner == 0 || outputInner > MAX_LAYOUT_INNER || rows % outputInner != 0 || | ||
| 85 | + (outputInner > 1 && (k <= 1 || k > axis || boundsFlags != 0 || mode != OUTPUT_BOTH || offset != 0))) { | ||
| 86 | + return false; | ||
| 87 | + } | ||
| 88 | + data = {}; | ||
| 89 | + data.outputInner = outputInner; | ||
| 90 | + if (inputInner == 0 || inputInner > MAX_LAYOUT_INNER || | ||
| 91 | + (inputInner > 1 && (inputInner != outputInner || axis <= 1))) | ||
| 92 | + return false; | ||
| 93 | + data.inputInner = inputInner; | ||
| 94 | + data.outputK = k; | ||
| 95 | + data.indicesOffset = offset; | ||
| 96 | + data.indexFill = indexFill; | ||
| 97 | + data.valueFill = valueFill; | ||
| 98 | + data.flags = boundsFlags; | ||
| 99 | + const bool ranged = (data.flags & (SelectTopK::HAS_BEGIN | SelectTopK::HAS_END)) != 0 || data.outputK > axis; | ||
| 100 | + if (ranged) { | ||
| 101 | + data.flags |= SelectTopK::RANGE; | ||
| 102 | + } | ||
| 103 | + const uint64_t selectedK = std::min<uint64_t>(axis, data.outputK); | ||
| 104 | + if (!BuildSortPlan(rows, axis, selectedK, bytes, largest, sorted, ub, cores, data.base, data.schedule)) { | ||
| 105 | + return false; | ||
| 106 | + } | ||
| 107 | + if (ranged) | ||
| 108 | + data.base.kernel = KernelTemplate::RANGE_MERGE; | ||
| 109 | + if (!ranged) { | ||
| 110 | + TryFilterSelectPlan(ub, cores, data.base, data.schedule); | ||
| 111 | + TryLeafOptimization(ub, cores, data.base, data.schedule); | ||
| 112 | + TryBalancedLargePrefix(ub, cores, data.base); | ||
| 113 | + TryDenseLeafPrefix(ub, cores, data.base); | ||
| 114 | + TryTop1Optimization(ub, cores, data.base, data.schedule); | ||
| 115 | + TryShortRowOptimization(ub, cores, indicesBytes, offset, mode, outputInner, data.base, data.schedule); | ||
| 116 | + TryCompactRowOptimization(ub, cores, data.base, data.schedule); | ||
| 117 | + TryCopyAllOptimization(ub, cores, indicesBytes, offset, mode, outputInner, data.base, data.schedule); | ||
| 118 | + TryResidentMergeOptimization(ub, cores, data.base, data.schedule); | ||
| 119 | + TryGroupedResidentOptimization(ub, cores, data.base, data.schedule); | ||
| 120 | + TryTwoLeafLocalOptimization(ub, cores, data.base, data.schedule); | ||
| 121 | + TryTinyRowOptimization(ub, indicesBytes, offset, mode, outputInner, data.base, data.schedule); | ||
| 122 | + TryScalarRowOptimization(ub, data.base); | ||
| 123 | + TryPrefixRowOptimization(ub, cores, indicesBytes, offset, mode, outputInner, inputInner, data.base, | ||
| 124 | + data.schedule); | ||
| 125 | + } | ||
| 126 | + if (ranged && rows != 0 && selectedK != 0) { | ||
| 127 | + // 区间长度来自设备输入,按每行真实有效长度做受限归并,避免用数值哨兵表示被排除的元素。 | ||
| 128 | + data.base = {}; | ||
| 129 | + data.base.rows = rows; | ||
| 130 | + data.base.axis = axis; | ||
| 131 | + data.base.k = selectedK; | ||
| 132 | + data.base.dtypeBytes = bytes; | ||
| 133 | + data.base.largest = largest; | ||
| 134 | + data.base.sorted = true; | ||
| 135 | + if (!SelectTopKBase::PlanGeneralMerge(data.base, {ub - SelectTopKBase::UB_RESERVE, cores}, rows * axis)) { | ||
| 136 | + return false; | ||
| 137 | + } | ||
| 138 | + if (data.base.route == SelectTopKBase::ROUTE_COPY) { | ||
| 139 | + data.base.route = SelectTopKBase::ROUTE_ROW_SORT; | ||
| 140 | + if (!SelectTopKBase::SelectTile(ub - SelectTopKBase::UB_RESERVE, bytes, selectedK, data.base.route, | ||
| 141 | + data.base.tile, data.base.mergeWindow, data.workspaceBytes)) { | ||
| 142 | + return false; | ||
| 143 | + } | ||
| 144 | + data.base.ubBytes = static_cast<uint32_t>(data.workspaceBytes); | ||
| 145 | + data.base.outputTile = std::min(data.base.tile, data.base.mergeWindow); | ||
| 146 | + data.base.cores = std::min<uint64_t>(rows, cores); | ||
| 147 | + } | ||
| 148 | + } | ||
| 149 | + const bool reuseOutput = !ranged && outputInner >= GM_MIN_INNER && | ||
| 150 | + selectedK > SelectTopKBase::Tuning::MAX_LEAF_BLOCK && indicesBytes == INDEX64_BYTES; | ||
| 151 | + if (outputInner > 1) { | ||
| 152 | + const bool planned = reuseOutput ? PlanOutputLayout<true>(ub, indicesBytes, data) : | ||
| 153 | + PlanOutputLayout<false>(ub, indicesBytes, data); | ||
| 154 | + if (!planned) | ||
| 155 | + return false; | ||
| 156 | + } | ||
| 157 | + data.workspaceBytes = SelectTopKBase::Align(data.base.workspaceBytes, UB_BLOCK_BYTES); | ||
| 158 | + const bool values = mode != 1, indices = mode != 0; | ||
| 159 | + if (values) { | ||
| 160 | + data.flags |= SelectTopK::WRITE_VALUES; | ||
| 161 | + } | ||
| 162 | + if (indices) { | ||
| 163 | + data.flags |= SelectTopK::WRITE_INDICES; | ||
| 164 | + } | ||
| 165 | + const bool directIndices64 = outputInner == 1 && !ranged && values && indices && indicesBytes == INDEX64_BYTES && | ||
| 166 | + offset == 0; | ||
| 167 | + if (directIndices64) { | ||
| 168 | + data.flags |= SelectTopK::DIRECT_INDICES64; | ||
| 169 | + } | ||
| 170 | + const bool tempValues = ranged || !values || outputInner > 1; | ||
| 171 | + const bool tempIndices = !directIndices64 && | ||
| 172 | + (ranged || !indices || indicesBytes == INDEX64_BYTES || offset != 0 || outputInner > 1); | ||
| 173 | + uint64_t items = 0; | ||
| 174 | + if (!SelectTopKBase::Mul(rows, selectedK, items)) { | ||
| 175 | + return false; | ||
| 176 | + } | ||
| 177 | + if (tempValues) { | ||
| 178 | + data.flags |= SelectTopK::TEMP_VALUES; | ||
| 179 | + if (!AddWorkspace(items, bytes, data.workspaceBytes, data.valuesWorkspaceOffset)) { | ||
| 180 | + return false; | ||
| 181 | + } | ||
| 182 | + } | ||
| 183 | + if (tempIndices) { | ||
| 184 | + data.flags |= SelectTopK::TEMP_INDICES; | ||
| 185 | + if (!AddWorkspace(items, INDEX32_BYTES, data.workspaceBytes, data.indicesWorkspaceOffset)) { | ||
| 186 | + return false; | ||
| 187 | + } | ||
| 188 | + } | ||
| 189 | + if (ranged || (indices && tempIndices) || outputInner > 1) { | ||
| 190 | + data.flags |= SelectTopK::FINALIZE; | ||
| 191 | + } | ||
| 192 | + if (data.base.cores == 0) { | ||
| 193 | + data.base.cores = std::max<uint32_t>(1, std::min<uint64_t>(rows, cores)); | ||
| 194 | + } | ||
| 195 | + if (inputInner > 1) { | ||
| 196 | + uint32_t block = MIN_LAYOUT_BLOCK; | ||
| 197 | + const uint64_t perColumn = inputInner * (DOUBLE_BUFFER_COUNT * bytes + INDEX32_BYTES); | ||
| 198 | + if (block * perColumn > ub - SelectTopKBase::UB_RESERVE) | ||
| 199 | + return false; | ||
| 200 | + while (block < MAX_INPUT_BLOCK && block < axis && | ||
| 201 | + block * DOUBLE_BUFFER_COUNT * inputInner <= MAX_REUSED_CAPACITY && | ||
| 202 | + block * DOUBLE_BUFFER_COUNT * perColumn <= ub - SelectTopKBase::UB_RESERVE) { | ||
| 203 | + block *= DOUBLE_BUFFER_COUNT; | ||
| 204 | + } | ||
| 205 | + data.inputBlock = block; | ||
| 206 | + data.base.ubBytes = std::max<uint32_t>(data.base.ubBytes, block * perColumn); | ||
| 207 | + if (!AddWorkspace(inputItems, bytes, data.workspaceBytes, data.inputWorkspaceOffset)) | ||
| 208 | + return false; | ||
| 209 | + } | ||
| 210 | + // 输出布局属于基础路径的适配阶段,不引入性能特化 key。 | ||
| 211 | + if (reuseOutput) { | ||
| 212 | + data.flags |= SelectTopK::REUSE_LAYOUT; | ||
| 213 | + } | ||
| 214 | + return true; | ||
| 215 | +} | ||
| 216 | +} // namespace SelectTopK | ||
| 217 | +#endif | ||
| @@ -0,0 +1,50 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#ifndef SELECT_TOP_K_SELECT_PLAN_H_ | ||
| 11 | +#define SELECT_TOP_K_SELECT_PLAN_H_ | ||
| 12 | +#include "select_top_k_sort_plan.h" | ||
| 13 | + | ||
| 14 | +namespace SelectTopK { | ||
| 15 | +// 仅复制语义字段,为每种选择方案独立计算资源,避免残留基础归并工作区。 | ||
| 16 | +inline SelectTopKBaseTilingData SelectionInput(const SelectTopKBaseTilingData& source) | ||
| 17 | +{ | ||
| 18 | + SelectTopKBaseTilingData result{}; | ||
| 19 | + result.rows = source.rows; | ||
| 20 | + result.axis = source.axis; | ||
| 21 | + result.k = source.k; | ||
| 22 | + result.dtypeBytes = source.dtypeBytes; | ||
| 23 | + result.largest = source.largest; | ||
| 24 | + result.sorted = source.sorted; | ||
| 25 | + return result; | ||
| 26 | +} | ||
| 27 | + | ||
| 28 | +// 在长轴小 K 上尝试扫描、过滤、压缩;资源不适用时保留已建立的 基础计划。 | ||
| 29 | +inline void TryFilterSelectPlan(uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan, SortSchedule& schedule) | ||
| 30 | +{ | ||
| 31 | + using namespace SelectTopKBase; | ||
| 32 | + if (plan.rows == 0 || plan.axis < SelectTopKBase::Tuning::FILTER_MIN_AXIS || | ||
| 33 | + plan.k < SelectTopKBase::Tuning::FILTER_MIN_K || plan.k > SelectTopKBase::Tuning::FILTER_MAX_K) | ||
| 34 | + return; | ||
| 35 | + const PlanResources resources{ub - UB_RESERVE, cores, SelectTopKBase::Tuning::FILTER_MIN_AXIS, | ||
| 36 | + SelectTopKBase::Tuning::MAX_LEAF_BLOCK, true}; | ||
| 37 | + // 各候选由资源规划器直接给出模板/分片/候选策略,成功后整体提交。 | ||
| 38 | + using Planner = PlanAttempt (*)(SelectTopKBaseTilingData&, const PlanResources&); | ||
| 39 | + const Planner planners[] = {TryWideFilter, TryRowFilter, TryShardedFilter}; | ||
| 40 | + for (auto planner : planners) { | ||
| 41 | + auto candidate = SelectionInput(plan); | ||
| 42 | + if (planner(candidate, resources) == PlanAttempt::SUCCESS) { | ||
| 43 | + plan = candidate; | ||
| 44 | + schedule = {}; | ||
| 45 | + return; | ||
| 46 | + } | ||
| 47 | + } | ||
| 48 | +} | ||
| 49 | +} // namespace SelectTopK | ||
| 50 | +#endif | ||
| @@ -0,0 +1,104 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#ifndef SELECT_TOP_K_SORT_PLAN_H_ | ||
| 11 | +#define SELECT_TOP_K_SORT_PLAN_H_ | ||
| 12 | +#include "../../op_kernel/select_top_k_tiling_data.h" | ||
| 13 | +#include "select_top_k_base_plan.h" | ||
| 14 | + | ||
| 15 | +namespace SelectTopK { | ||
| 16 | +// 按连续行块分核;AscendC DataCopyPad 仅写有效字节,无需沿用原生 DMA 的短输出单核限制。 | ||
| 17 | +inline void PlanRowBlocks(uint64_t rows, uint32_t cores, SortSchedule& schedule, SelectTopKBaseTilingData& plan) | ||
| 18 | +{ | ||
| 19 | + schedule.rowsPerCore = SelectTopKBase::CeilDiv(rows, cores); | ||
| 20 | + plan.cores = SelectTopKBase::CeilDiv(rows, schedule.rowsPerCore); | ||
| 21 | +} | ||
| 22 | + | ||
| 23 | +// 按 四种模式规划分核和分层归并;容量来自 AscendC 实际缓冲,不使用场景特化阈值。 | ||
| 24 | +inline bool BuildSortPlan(uint64_t rows, uint64_t axis, uint64_t k, uint32_t bytes, bool largest, bool sorted, | ||
| 25 | + uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan, SortSchedule& schedule) | ||
| 26 | +{ | ||
| 27 | + using namespace SelectTopKBase; | ||
| 28 | + plan = {}; | ||
| 29 | + schedule = {}; | ||
| 30 | + plan.rows = rows; | ||
| 31 | + plan.axis = axis; | ||
| 32 | + plan.k = k; | ||
| 33 | + plan.dtypeBytes = bytes; | ||
| 34 | + plan.largest = largest; | ||
| 35 | + plan.sorted = sorted; | ||
| 36 | + plan.kernel = KernelTemplate::ROW_SORT; | ||
| 37 | + if (cores == 0 || ub <= UB_RESERVE || (bytes != VALUE16_BYTES && bytes != VALUE32_BYTES)) | ||
| 38 | + return false; | ||
| 39 | + if (rows == 0 || axis == 0 || k == 0) { | ||
| 40 | + plan.cores = 1; | ||
| 41 | + plan.route = ROUTE_EMPTY; | ||
| 42 | + return true; | ||
| 43 | + } | ||
| 44 | + const uint64_t available = ub - UB_RESERVE; | ||
| 45 | + const uint32_t rowBytes = ROW_SORT_ARRAY_BYTES + ROW_INPUT_BUFFER_COUNT * bytes; | ||
| 46 | + const uint32_t rowCapacity = std::min<uint64_t>(SORT32_MAX_ELEMENTS, | ||
| 47 | + available / rowBytes / SORT32_ELEMENTS * SORT32_ELEMENTS); | ||
| 48 | + const uint32_t leafCapacity = std::min<uint64_t>( | ||
| 49 | + MAX_LEAF_SORT_ITEMS, available / COMPACT_SORT_BYTES / SHARD_ALIGNMENT * SHARD_ALIGNMENT); | ||
| 50 | + if (rowCapacity < SORT32_ELEMENTS || leafCapacity == 0) | ||
| 51 | + return false; | ||
| 52 | + | ||
| 53 | + if (Align(axis, SORT32_ELEMENTS) <= rowCapacity) { | ||
| 54 | + schedule.pattern = SortPattern::ROWS; | ||
| 55 | + schedule.rowWidth = Align(axis, SORT32_ELEMENTS); | ||
| 56 | + PlanRowBlocks(rows, cores, schedule, plan); | ||
| 57 | + plan.batchRows = std::min<uint64_t>(schedule.rowsPerCore, rowCapacity / schedule.rowWidth); | ||
| 58 | + plan.tile = plan.batchRows * schedule.rowWidth; | ||
| 59 | + plan.tiles = 1; | ||
| 60 | + plan.shardCount = 1; | ||
| 61 | + plan.outputTile = schedule.rowWidth; | ||
| 62 | + plan.route = ROUTE_COMPACT_ROW_SORT; | ||
| 63 | + plan.ubBytes = plan.tile * rowBytes; | ||
| 64 | + return true; | ||
| 65 | + } | ||
| 66 | + | ||
| 67 | + // 空闲核按整行数均分,保持每个轴片段连续;不是所有叶子在全核间轮转。 | ||
| 68 | + const uint64_t shardsAvailable = cores / rows; | ||
| 69 | + schedule.pattern = SortPattern::LOCAL_MERGE; | ||
| 70 | + schedule.shardElements = axis; | ||
| 71 | + plan.shardCount = 1; | ||
| 72 | + if (shardsAvailable > 1) { | ||
| 73 | + schedule.shardElements = Align(CeilDiv(axis, shardsAvailable), SHARD_ALIGNMENT); | ||
| 74 | + schedule.pattern = schedule.shardElements <= leafCapacity ? SortPattern::CROSS_MERGE : | ||
| 75 | + SortPattern::LOCAL_CROSS_MERGE; | ||
| 76 | + // 局部前缀工作区仍按 pair 块对齐;行首核的最终输出使用非对齐字节拷贝。 | ||
| 77 | + // 保留真实 K,不额外计算 KFirst32,也不引入裁剪、Gather 或全核同步。 | ||
| 78 | + plan.shardCount = CeilDiv(axis, schedule.shardElements); | ||
| 79 | + plan.cores = rows * plan.shardCount; | ||
| 80 | + schedule.rowsPerCore = 1; | ||
| 81 | + if (plan.shardCount == 1) | ||
| 82 | + schedule.pattern = SortPattern::LOCAL_MERGE; | ||
| 83 | + } else { | ||
| 84 | + PlanRowBlocks(rows, cores, schedule, plan); | ||
| 85 | + } | ||
| 86 | + plan.kernel = KernelTemplate::MERGE_SORT; | ||
| 87 | + plan.tile = std::min<uint64_t>(leafCapacity, Align(schedule.shardElements, SORT32_ELEMENTS)); | ||
| 88 | + schedule.leavesPerShard = CeilDiv(schedule.shardElements, plan.tile); | ||
| 89 | + schedule.leafStride = Align(std::min<uint64_t>(k, plan.tile), PAIR_ALIGNMENT); | ||
| 90 | + plan.tiles = plan.shardCount * schedule.leavesPerShard; | ||
| 91 | + plan.route = ROUTE_GM_MERGE; | ||
| 92 | + plan.mergeWindow = std::min<uint64_t>(plan.tile, available / MERGE_WORK_BYTES / SORT32_ELEMENTS * SORT32_ELEMENTS); | ||
| 93 | + if (plan.mergeWindow == 0) | ||
| 94 | + return false; | ||
| 95 | + plan.outputTile = plan.mergeWindow; | ||
| 96 | + const GeneralMergeLayout layout(plan.tile, plan.mergeWindow, k, bytes, plan.route); | ||
| 97 | + plan.ubBytes = layout.bytes; | ||
| 98 | + plan.rowWorkspacePairs = plan.tiles * schedule.leafStride; | ||
| 99 | + uint64_t pairs = 0; | ||
| 100 | + return layout.bytes <= available && Mul(rows, plan.rowWorkspacePairs, pairs) && | ||
| 101 | + Mul(pairs, DOUBLE_PAIR_BYTES, plan.workspaceBytes) && plan.workspaceBytes <= INT64_MAX; | ||
| 102 | +} | ||
| 103 | +} // namespace SelectTopK | ||
| 104 | +#endif | ||
| @@ -0,0 +1,459 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#ifndef SELECT_TOP_K_TARGETED_PLAN_H_ | ||
| 11 | +#define SELECT_TOP_K_TARGETED_PLAN_H_ | ||
| 12 | +#include "select_top_k_select_plan.h" | ||
| 13 | +#include "../../op_kernel/select_top_k_two_leaf_layout.h" | ||
| 14 | +#include "../../op_kernel/select_top_k_scalar_layout.h" | ||
| 15 | +namespace SelectTopK { | ||
| 16 | +// 计算完整候选列表所需的四路归并层数,用于权衡局部排序与最终归并。 | ||
| 17 | +inline uint32_t LeafMergeRounds(uint64_t lists) | ||
| 18 | +{ | ||
| 19 | + uint32_t rounds = 0; | ||
| 20 | + while (lists > 1) { | ||
| 21 | + lists = SelectTopKBase::CeilDiv(lists, SelectTopKBase::MERGE_WAYS); | ||
| 22 | + ++rounds; | ||
| 23 | + } | ||
| 24 | + return rounds; | ||
| 25 | +} | ||
| 26 | + | ||
| 27 | +// 在核数和 UB 允许时重设叶子块长,并同步更新候选工作区与核数。 | ||
| 28 | +inline bool RetileLeaf(SelectTopKBaseTilingData& plan, uint32_t tile, uint32_t capacity, uint64_t ub, uint32_t cores) | ||
| 29 | +{ | ||
| 30 | + using namespace SelectTopKBase; | ||
| 31 | + const uint64_t shards = CeilDiv(plan.axis, tile); | ||
| 32 | + const LeafSelectLayout layout(tile, plan.k, plan.dtypeBytes, capacity); | ||
| 33 | + if (plan.k > tile || shards > cores / plan.rows || layout.bytes > ub - UB_RESERVE) | ||
| 34 | + return false; | ||
| 35 | + plan.tile = tile; | ||
| 36 | + plan.tiles = plan.shardCount = shards; | ||
| 37 | + plan.cores = plan.rows * shards; | ||
| 38 | + plan.finalCapacity = capacity; | ||
| 39 | + plan.ubBytes = layout.bytes; | ||
| 40 | + plan.rowWorkspacePairs = shards * Align(plan.k, PAIR_ALIGNMENT); | ||
| 41 | + plan.workspaceBytes = plan.rows * plan.rowWorkspacePairs * PAIR_BYTES; | ||
| 42 | + return true; | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +// 按已验证的UB容量和候选段数均衡叶子,不改变基础排序或逐页过滤的规划。 | ||
| 46 | +inline void TuneLeafPlan(SelectTopKBaseTilingData& plan, uint64_t ub, uint32_t cores) | ||
| 47 | +{ | ||
| 48 | + using namespace SelectTopKBase; | ||
| 49 | + const uint64_t rows = plan.rows, axis = plan.axis, selectedK = plan.k; | ||
| 50 | + const uint32_t bytes = plan.dtypeBytes; | ||
| 51 | + // 四个完整前缀加一个可选短尾只需一次全核同步。 | ||
| 52 | + if (plan.candidateMerge == CandidateMerge::FULL_LISTS && selectedK > SelectTopKBase::Tuning::SMALL_K && | ||
| 53 | + selectedK <= Tuning::PREFIX_SMALL_K && plan.tile <= SelectTopKBase::Tuning::MAX_LEAF_BLOCK && | ||
| 54 | + (plan.shardCount == MERGE_WAYS || plan.shardCount == MERGE_WAYS + 1)) { | ||
| 55 | + plan.candidateMerge = CandidateMerge::HALF_PREFIX; | ||
| 56 | + } | ||
| 57 | + if (plan.candidateMerge == CandidateMerge::FULL_LISTS && plan.shardCount > Tuning::BALANCED_LEAVES && | ||
| 58 | + plan.shardCount * SelectTopKBase::Align(selectedK, PAIR_ALIGNMENT) > plan.finalCapacity) { | ||
| 59 | + const SelectTopKBase::LeafSelectLayout layout(plan.tile, selectedK, bytes, plan.finalCapacity); | ||
| 60 | + const uint64_t prefix = (selectedK + HALF_PREFIX_ALIGNMENT - 1) / HALF_PREFIX_ALIGNMENT * PAIR_ALIGNMENT; | ||
| 61 | + if (prefix * MERGE_WORK_BYTES <= layout.accumulator) { | ||
| 62 | + plan.candidateMerge = CandidateMerge::HALF_PREFIX; | ||
| 63 | + } | ||
| 64 | + } | ||
| 65 | + | ||
| 66 | + // 保持独立叶子数量不变,以 Sort32 的粒度均衡每个叶子的工作量。 | ||
| 67 | + if (plan.kernel == KernelTemplate::LEAF_SELECT && plan.tile > SelectTopKBase::Tuning::MAX_LEAF_BLOCK) { | ||
| 68 | + const uint32_t balanced = SelectTopKBase::Align(SelectTopKBase::CeilDiv(axis, plan.shardCount), | ||
| 69 | + SORT32_ELEMENTS); | ||
| 70 | + if (balanced > SelectTopKBase::Tuning::MAX_LEAF_BLOCK && balanced < plan.tile) { | ||
| 71 | + const SelectTopKBase::LeafSelectLayout layout(balanced, selectedK, bytes, plan.finalCapacity); | ||
| 72 | + if (layout.bytes <= ub - SelectTopKBase::UB_RESERVE) { | ||
| 73 | + plan.tile = balanced; | ||
| 74 | + plan.ubBytes = layout.bytes; | ||
| 75 | + } | ||
| 76 | + } | ||
| 77 | + } | ||
| 78 | + // 八个均衡叶子可利用空闲核,同时保持批量归并层数。 | ||
| 79 | + // 常规叶子使用固定长度模板;扩展叶子按 Sort32 粒度归并,可细分工作量。 | ||
| 80 | + if (plan.kernel == KernelTemplate::LEAF_SELECT && plan.tile > SelectTopKBase::Tuning::MAX_LEAF_BLOCK) { | ||
| 81 | + if (selectedK <= SelectTopKBase::Tuning::TINY_LEAF_BLOCK && plan.shardCount < Tuning::BALANCED_LEAVES && | ||
| 82 | + cores / rows >= Tuning::BALANCED_LEAVES) { | ||
| 83 | + const uint32_t tile = SelectTopKBase::Align(SelectTopKBase::CeilDiv(axis, Tuning::BALANCED_LEAVES), | ||
| 84 | + SORT32_ELEMENTS); | ||
| 85 | + if (tile > SelectTopKBase::Tuning::MAX_LEAF_BLOCK && tile < plan.tile) | ||
| 86 | + RetileLeaf(plan, tile, plan.finalCapacity, ub, cores); | ||
| 87 | + } else if (selectedK >= SelectTopKBase::Tuning::SMALL_LEAF_BLOCK && plan.shardCount > Tuning::BALANCED_LEAVES && | ||
| 88 | + (selectedK <= SelectTopKBase::Tuning::SMALL_LEAF_BLOCK || | ||
| 89 | + SelectTopKBase::CeilDiv(axis, SelectTopKBase::Tuning::MAX_LEAF_BLOCK) <= | ||
| 90 | + Tuning::MAX_FIXED_LEAVES)) { | ||
| 91 | + // 第九个扩展叶子会增加一层前缀归并;常规叶子模板可在相同层数下完成排序。 | ||
| 92 | + RetileLeaf(plan, SelectTopKBase::Tuning::MAX_LEAF_BLOCK, SelectTopKBase::Tuning::MAX_LEAF_BLOCK, ub, cores); | ||
| 93 | + } | ||
| 94 | + } | ||
| 95 | + // 将小叶子容量加倍可保持归并层数,并将小 K 的候选列表和并发 DMA 写入数减半。 | ||
| 96 | + if (plan.candidateMerge == CandidateMerge::FULL_LISTS && plan.tile == SelectTopKBase::Tuning::TINY_LEAF_BLOCK && | ||
| 97 | + selectedK <= Tuning::SMALL_LEAF_MAX_K && axis >= SelectTopKBase::Tuning::FILTER_MIN_AXIS) { | ||
| 98 | + const uint64_t shards = SelectTopKBase::CeilDiv(axis, SelectTopKBase::Tuning::SMALL_LEAF_BLOCK); | ||
| 99 | + const uint32_t capacity = LeafCandidateCapacity(shards, selectedK, SelectTopKBase::Tuning::SMALL_LEAF_BLOCK); | ||
| 100 | + RetileLeaf(plan, SelectTopKBase::Tuning::SMALL_LEAF_BLOCK, capacity, ub, cores); | ||
| 101 | + } | ||
| 102 | + // 单行小 K 用更多空闲核缩短局部排序,但不因此增加最终候选的归并层数。 | ||
| 103 | + // 核数、候选容量或 UB 不足时,仍保留原计划。 | ||
| 104 | + if (rows == 1 && plan.candidateMerge == CandidateMerge::FULL_LISTS && | ||
| 105 | + plan.tile == SelectTopKBase::Tuning::SMALL_LEAF_BLOCK && selectedK <= Tuning::SMALL_LEAF_MAX_K && | ||
| 106 | + axis >= SelectTopKBase::Tuning::FILTER_MIN_AXIS) { | ||
| 107 | + const uint64_t shards = SelectTopKBase::CeilDiv(axis, SelectTopKBase::Tuning::TINY_LEAF_BLOCK); | ||
| 108 | + const uint32_t capacity = LeafCandidateCapacity(shards, selectedK, SelectTopKBase::Tuning::SMALL_LEAF_BLOCK); | ||
| 109 | + if (LeafMergeRounds(shards) <= LeafMergeRounds(plan.shardCount)) { | ||
| 110 | + RetileLeaf(plan, SelectTopKBase::Tuning::TINY_LEAF_BLOCK, capacity, ub, cores); | ||
| 111 | + } | ||
| 112 | + } | ||
| 113 | + // 中小K的候选段较多时,用固定4096叶子减少段数和末级归并工作量。 | ||
| 114 | + // 保留K<=64的小叶子方案;容量、核数或UB不满足时仍使用原计划。 | ||
| 115 | + if (plan.candidateMerge == CandidateMerge::FULL_LISTS && plan.tile == SelectTopKBase::Tuning::SMALL_LEAF_BLOCK && | ||
| 116 | + selectedK > Tuning::SMALL_LEAF_MAX_K && selectedK <= SelectTopKBase::Tuning::SMALL_K && | ||
| 117 | + plan.shardCount > Tuning::BALANCED_LEAVES) { | ||
| 118 | + const uint32_t tile = SelectTopKBase::Tuning::MAX_LEAF_BLOCK; | ||
| 119 | + const uint64_t shards = CeilDiv(axis, tile); | ||
| 120 | + const uint32_t capacity = LeafCandidateCapacity(shards, selectedK, tile); | ||
| 121 | + RetileLeaf(plan, tile, capacity, ub, cores); | ||
| 122 | + } | ||
| 123 | +} | ||
| 124 | + | ||
| 125 | +// 轴可拆成每核一个叶子时,候选在UB内完成归并;容不下仍保留扫描过滤或基础排序。 | ||
| 126 | +inline void TryLeafOptimization(uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan, SortSchedule& schedule) | ||
| 127 | +{ | ||
| 128 | + using namespace SelectTopKBase; | ||
| 129 | + if (plan.rows == 0 || plan.rows >= cores || plan.k < SelectTopKBase::Tuning::FILTER_MIN_K || | ||
| 130 | + plan.k > SelectTopKBase::Tuning::FILTER_MAX_K) | ||
| 131 | + return; | ||
| 132 | + uint32_t maxLeafTile = SelectTopKBase::Tuning::MAX_LEAF_BLOCK; | ||
| 133 | + // 四个常规叶子和一个短尾已能占满该行核预算时,避免扩大单核排序。 | ||
| 134 | + const bool shortAxisLargeK = plan.k >= SelectTopKBase::Tuning::SMALL_LEAF_BLOCK && | ||
| 135 | + plan.axis <= MERGE_WAYS * SelectTopKBase::Tuning::MAX_LEAF_BLOCK + SORT32_ELEMENTS && | ||
| 136 | + plan.rows <= Tuning::SHORT_AXIS_MAX_ROWS && | ||
| 137 | + plan.rows * CeilDiv(plan.axis, SelectTopKBase::Tuning::MAX_LEAF_BLOCK) <= cores; | ||
| 138 | + if (plan.k > SelectTopKBase::Tuning::SMALL_K && !shortAxisLargeK) { | ||
| 139 | + for (uint32_t block = SelectTopKBase::Tuning::MAX_EXTENDED_LEAF_BLOCK; | ||
| 140 | + block > SelectTopKBase::Tuning::MAX_LEAF_BLOCK; block -= SORT32_ELEMENTS) { | ||
| 141 | + const LeafSelectLayout layout(block, plan.k, plan.dtypeBytes, block); | ||
| 142 | + if (layout.bytes <= ub - UB_RESERVE) { | ||
| 143 | + maxLeafTile = block; | ||
| 144 | + break; | ||
| 145 | + } | ||
| 146 | + } | ||
| 147 | + } | ||
| 148 | + auto candidate = SelectionInput(plan); | ||
| 149 | + const PlanResources resources{ub - UB_RESERVE, cores, SelectTopKBase::Tuning::SMALL_LEAF_BLOCK, maxLeafTile}; | ||
| 150 | + if (TryLeafSort(candidate, resources) != PlanAttempt::SUCCESS) | ||
| 151 | + return; | ||
| 152 | + // 宽块过滤只需一次扫描时,较大的叶子并不一定更好;仅接受小叶子或原两片路径。 | ||
| 153 | + if (plan.kernel == KernelTemplate::WIDE_SELECT && candidate.tile > SelectTopKBase::Tuning::SMALL_LEAF_BLOCK && | ||
| 154 | + plan.shardCount != TWO_MERGE_RUNS) | ||
| 155 | + return; | ||
| 156 | + TuneLeafPlan(candidate, ub, cores); | ||
| 157 | + plan = candidate; | ||
| 158 | + schedule = {}; | ||
| 159 | +} | ||
| 160 | + | ||
| 161 | +// K=1只求一个极值;规划原值位恢复及每片一个候选,不执行排序或前缀归并。 | ||
| 162 | +inline void TryTop1Optimization(uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan, SortSchedule& schedule) | ||
| 163 | +{ | ||
| 164 | + using namespace SelectTopKBase; | ||
| 165 | + const uint64_t bytes = TOP1_TILE * (plan.dtypeBytes + INDEX32_BYTES_64) + TOP1_FIXED_UB; | ||
| 166 | + if (plan.rows == 0 || plan.k != 1 || plan.axis <= SORT32_ELEMENTS || bytes > ub - UB_RESERVE) | ||
| 167 | + return; | ||
| 168 | + auto candidate = SelectionInput(plan); | ||
| 169 | + candidate.kernel = KernelTemplate::TOP1; | ||
| 170 | + candidate.route = ROUTE_TOP1_REDUCE; | ||
| 171 | + candidate.tile = TOP1_TILE; | ||
| 172 | + candidate.tiles = CeilDiv(plan.axis, TOP1_TILE); | ||
| 173 | + candidate.outputTile = 1; | ||
| 174 | + candidate.ubBytes = bytes; | ||
| 175 | + candidate.shardCount = std::min<uint64_t>(CeilDiv(plan.axis, SelectTopKBase::Tuning::SMALL_LEAF_BLOCK), | ||
| 176 | + std::max<uint64_t>(1, cores / plan.rows)); | ||
| 177 | + candidate.cores = std::min<uint64_t>(cores, plan.rows * candidate.shardCount); | ||
| 178 | + candidate.rowWorkspacePairs = candidate.shardCount; | ||
| 179 | + candidate.workspaceBytes = candidate.shardCount == 1 ? | ||
| 180 | + 0 : | ||
| 181 | + Align(plan.rows * candidate.shardCount * PAIR_BYTES, UB_BLOCK_BYTES); | ||
| 182 | + plan = candidate; | ||
| 183 | + schedule = {}; | ||
| 184 | +} | ||
| 185 | + | ||
| 186 | +// 标准连续短行直接生成INT64索引,按一批多个Sort32行降低DMA及初始化成本。 | ||
| 187 | +inline void TryShortRowOptimization(uint64_t ub, uint32_t cores, uint32_t indexBytes, int64_t offset, int64_t mode, | ||
| 188 | + uint64_t outputInner, SelectTopKBaseTilingData& plan, SortSchedule& schedule) | ||
| 189 | +{ | ||
| 190 | + using namespace SelectTopKBase; | ||
| 191 | + if (plan.rows == 0 || plan.k == 0 || indexBytes != INDEX64_BYTES || offset != 0 || mode != OUTPUT_BOTH || | ||
| 192 | + outputInner != 1) | ||
| 193 | + return; | ||
| 194 | + auto candidate = SelectionInput(plan); | ||
| 195 | + if (TryShortRowBatch(candidate, {ub - UB_RESERVE, cores}) != PlanAttempt::SUCCESS) | ||
| 196 | + return; | ||
| 197 | + candidate.kernel = KernelTemplate::SHORT_ROWS; | ||
| 198 | + plan = candidate; | ||
| 199 | + schedule = {}; | ||
| 200 | +} | ||
| 201 | + | ||
| 202 | +// 完整且无序的标准输出直接复制位模式;连续行内生成原轴INT64索引。 | ||
| 203 | +inline void TryCopyAllOptimization(uint64_t ub, uint32_t cores, uint32_t indexBytes, int64_t offset, int64_t mode, | ||
| 204 | + uint64_t outputInner, SelectTopKBaseTilingData& plan, SortSchedule& schedule) | ||
| 205 | +{ | ||
| 206 | + using namespace SelectTopKBase; | ||
| 207 | + if (plan.rows == 0 || plan.axis <= SORT32_ELEMENTS || plan.k != plan.axis || plan.sorted || | ||
| 208 | + indexBytes != INDEX64_BYTES || offset != 0 || mode != OUTPUT_BOTH || outputInner != 1) | ||
| 209 | + return; | ||
| 210 | + auto candidate = SelectionInput(plan); | ||
| 211 | + candidate.tile = Align(std::min<uint64_t>(plan.axis, COPY_ALL_TILE), SORT32_ELEMENTS); | ||
| 212 | + const uint64_t bytes = uint64_t(candidate.tile) * (plan.dtypeBytes + INDEX32_BYTES + INDEX64_BYTES); | ||
| 213 | + if (bytes > ub - UB_RESERVE) | ||
| 214 | + return; | ||
| 215 | + candidate.kernel = KernelTemplate::COPY_ALL; | ||
| 216 | + candidate.route = ROUTE_COPY; | ||
| 217 | + candidate.tiles = CeilDiv(plan.axis, candidate.tile); | ||
| 218 | + candidate.cores = std::min<uint64_t>(cores, plan.rows * candidate.tiles); | ||
| 219 | + candidate.ubBytes = bytes; | ||
| 220 | + plan = candidate; | ||
| 221 | + schedule = {}; | ||
| 222 | +} | ||
| 223 | + | ||
| 224 | +// 每核最多一条短行时,复用输入/排序缓冲,省去批处理和预取管理。 | ||
| 225 | +inline void TryCompactRowOptimization(uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan, | ||
| 226 | + SortSchedule& schedule) | ||
| 227 | +{ | ||
| 228 | + using namespace SelectTopKBase; | ||
| 229 | + if (plan.rows == 0 || plan.rows > cores || plan.axis <= SORT32_ELEMENTS || | ||
| 230 | + plan.k < SelectTopKBase::Tuning::FILTER_MIN_K || plan.axis > COMPACT_ROW_MAX_AXIS || | ||
| 231 | + plan.kernel != KernelTemplate::ROW_SORT) | ||
| 232 | + return; | ||
| 233 | + auto candidate = SelectionInput(plan); | ||
| 234 | + candidate.tile = Align(plan.axis, SORT32_ELEMENTS); | ||
| 235 | + const uint64_t bytes = uint64_t(candidate.tile) * COMPACT_SORT_BYTES; | ||
| 236 | + if (bytes > ub - UB_RESERVE) | ||
| 237 | + return; | ||
| 238 | + candidate.kernel = KernelTemplate::MERGE_SORT; | ||
| 239 | + candidate.route = ROUTE_COMPACT_ROW_SORT; | ||
| 240 | + candidate.tiles = candidate.shardCount = 1; | ||
| 241 | + candidate.cores = plan.rows; | ||
| 242 | + candidate.outputTile = candidate.tile; | ||
| 243 | + candidate.ubBytes = bytes; | ||
| 244 | + plan = candidate; | ||
| 245 | + schedule = {}; | ||
| 246 | +} | ||
| 247 | + | ||
| 248 | +// 候选可同时驻留时避免窗口重载;核内两叶减少逐叶DMA,跨核仍用四叶并行。 | ||
| 249 | +inline void TryResidentMergeOptimization(uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan, | ||
| 250 | + SortSchedule& schedule) | ||
| 251 | +{ | ||
| 252 | + using namespace SelectTopKBase; | ||
| 253 | + if (plan.rows == 0 || plan.kernel != KernelTemplate::MERGE_SORT || schedule.pattern == SortPattern::ROWS || | ||
| 254 | + schedule.storage != MergeStorage::WINDOWS) | ||
| 255 | + return; | ||
| 256 | + const uint32_t twoLeafTile = Align(CeilDiv(plan.axis, TWO_MERGE_RUNS), SORT32_ELEMENTS); | ||
| 257 | + const uint32_t ways = plan.rows >= cores && plan.k >= twoLeafTile ? TWO_MERGE_RUNS : MERGE_WAYS; | ||
| 258 | + const uint32_t tile = Align(CeilDiv(plan.axis, ways), SORT32_ELEMENTS); | ||
| 259 | + const uint64_t leaves = CeilDiv(plan.axis, tile); | ||
| 260 | + const uint64_t slots = leaves * tile; | ||
| 261 | + const uint32_t tileLimit = ways == TWO_MERGE_RUNS ? TWO_MERGE_RUNS * MERGE_MAX_ALIGNED_ITEMS : | ||
| 262 | + MERGE_MAX_ALIGNED_ITEMS; | ||
| 263 | + if (tile > tileLimit || plan.k < tile || slots * COMPACT_SORT_BYTES > ub - UB_RESERVE) | ||
| 264 | + return; | ||
| 265 | + const bool parallel = plan.rows * leaves <= cores; | ||
| 266 | + // 中间核预算保留基础切分;新路径只使用完整的每叶一核或完整的每核多行。 | ||
| 267 | + if (!parallel && plan.rows < cores) | ||
| 268 | + return; | ||
| 269 | + auto candidate = SelectionInput(plan); | ||
| 270 | + SortSchedule candidateSchedule{}; | ||
| 271 | + candidateSchedule.pattern = parallel ? SortPattern::CROSS_MERGE : SortPattern::LOCAL_MERGE; | ||
| 272 | + candidateSchedule.rowsPerCore = parallel ? 1 : CeilDiv(plan.rows, cores); | ||
| 273 | + candidateSchedule.shardElements = parallel ? tile : plan.axis; | ||
| 274 | + candidateSchedule.leavesPerShard = parallel ? 1 : leaves; | ||
| 275 | + candidateSchedule.leafStride = tile; | ||
| 276 | + candidate.kernel = KernelTemplate::MERGE_SORT; | ||
| 277 | + candidateSchedule.storage = MergeStorage::ALL_CANDIDATES; | ||
| 278 | + candidate.route = ROUTE_GM_MERGE; | ||
| 279 | + candidate.tile = tile; | ||
| 280 | + candidate.tiles = leaves; | ||
| 281 | + candidate.shardCount = parallel ? leaves : 1; | ||
| 282 | + candidate.cores = parallel ? plan.rows * leaves : CeilDiv(plan.rows, candidateSchedule.rowsPerCore); | ||
| 283 | + candidate.outputTile = plan.k; | ||
| 284 | + candidate.rowWorkspacePairs = slots; | ||
| 285 | + if (!Mul(plan.rows, slots, candidate.workspaceBytes) || | ||
| 286 | + !Mul(candidate.workspaceBytes, PAIR_BYTES, candidate.workspaceBytes) || candidate.workspaceBytes > INT64_MAX) | ||
| 287 | + return; | ||
| 288 | + candidate.ubBytes = slots * COMPACT_SORT_BYTES; | ||
| 289 | + plan = candidate; | ||
| 290 | + schedule = candidateSchedule; | ||
| 291 | +} | ||
| 292 | + | ||
| 293 | +// 五到八个常规叶子分两批归并;整批结果和一份TopK同时驻留,避免中间GM回写。 | ||
| 294 | +inline void TryGroupedResidentOptimization(uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan, | ||
| 295 | + SortSchedule& schedule) | ||
| 296 | +{ | ||
| 297 | + using namespace SelectTopKBase; | ||
| 298 | + if (plan.rows == 0 || plan.kernel != KernelTemplate::MERGE_SORT || schedule.pattern == SortPattern::ROWS || | ||
| 299 | + schedule.storage != MergeStorage::WINDOWS || plan.k < SHARD_ALIGNMENT) | ||
| 300 | + return; | ||
| 301 | + const uint64_t leaves = CeilDiv(plan.axis, SHARD_ALIGNMENT); | ||
| 302 | + const uint64_t alignedK = Align(plan.k, SORT32_ELEMENTS); | ||
| 303 | + const uint64_t capacity = std::max<uint64_t>(MERGE_WAYS * SHARD_ALIGNMENT, DOUBLE_BUFFER_COUNT * alignedK); | ||
| 304 | + const uint64_t bytes = capacity * DOUBLE_PAIR_BYTES + alignedK * PAIR_BYTES; | ||
| 305 | + if (leaves <= MERGE_WAYS || leaves > DOUBLE_BUFFER_COUNT * MERGE_WAYS || plan.rows * leaves > cores || | ||
| 306 | + alignedK > DOUBLE_BUFFER_COUNT * MERGE_MAX_ALIGNED_ITEMS || bytes > ub - UB_RESERVE) | ||
| 307 | + return; | ||
| 308 | + auto candidate = SelectionInput(plan); | ||
| 309 | + candidate.kernel = KernelTemplate::MERGE_SORT; | ||
| 310 | + candidate.route = ROUTE_GM_MERGE; | ||
| 311 | + candidate.tile = SHARD_ALIGNMENT; | ||
| 312 | + candidate.tiles = candidate.shardCount = leaves; | ||
| 313 | + candidate.cores = plan.rows * leaves; | ||
| 314 | + candidate.outputTile = plan.k; | ||
| 315 | + candidate.finalCapacity = capacity; | ||
| 316 | + candidate.rowWorkspacePairs = leaves * SHARD_ALIGNMENT; | ||
| 317 | + if (!Mul(plan.rows, candidate.rowWorkspacePairs, candidate.workspaceBytes) || | ||
| 318 | + !Mul(candidate.workspaceBytes, PAIR_BYTES, candidate.workspaceBytes) || candidate.workspaceBytes > INT64_MAX) | ||
| 319 | + return; | ||
| 320 | + candidate.ubBytes = bytes; | ||
| 321 | + schedule = {}; | ||
| 322 | + schedule.pattern = SortPattern::CROSS_MERGE; | ||
| 323 | + schedule.storage = MergeStorage::GROUPED_CANDIDATES; | ||
| 324 | + schedule.rowsPerCore = 1; | ||
| 325 | + schedule.shardElements = candidate.tile; | ||
| 326 | + schedule.leavesPerShard = 1; | ||
| 327 | + schedule.leafStride = candidate.tile; | ||
| 328 | + plan = candidate; | ||
| 329 | +} | ||
| 330 | + | ||
| 331 | +// 多行大K恰需两个叶子时,以叶子任务数判断并行度,扩大两路流式窗口减少重载。 | ||
| 332 | +// 行数少于核数仍可用两个叶子填满各核;最终行归并允许部分核没有任务。 | ||
| 333 | +inline void TryTwoLeafLocalOptimization(uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan, | ||
| 334 | + SortSchedule& schedule) | ||
| 335 | +{ | ||
| 336 | + using namespace SelectTopKBase; | ||
| 337 | + if (plan.kernel != KernelTemplate::MERGE_SORT || schedule.pattern != SortPattern::LOCAL_MERGE || | ||
| 338 | + schedule.storage != MergeStorage::WINDOWS || plan.rows < CeilDiv(cores, TWO_MERGE_RUNS) || | ||
| 339 | + schedule.leavesPerShard != TWO_MERGE_RUNS || plan.shardCount != 1 || plan.k <= plan.tile || | ||
| 340 | + plan.mergeWindow >= TWO_LEAF_WINDOW) | ||
| 341 | + return; | ||
| 342 | + const TwoLeafMergeLayout layout(plan.tile, TWO_LEAF_WINDOW); | ||
| 343 | + if (layout.bytes > ub - UB_RESERVE) | ||
| 344 | + return; | ||
| 345 | + schedule.storage = MergeStorage::TWO_LEAF_WINDOWS; | ||
| 346 | + plan.mergeWindow = plan.outputTile = TWO_LEAF_WINDOW; | ||
| 347 | + plan.ubBytes = layout.bytes; | ||
| 348 | + plan.cores = cores; | ||
| 349 | + schedule.rowsPerCore = CeilDiv(plan.rows, cores); | ||
| 350 | +} | ||
| 351 | + | ||
| 352 | +// 大K的五至七个扩展叶子在有空闲核时均衡为八片,由单核依次处理两个四路组。 | ||
| 353 | +inline void TryBalancedLargePrefix(uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan) | ||
| 354 | +{ | ||
| 355 | + using namespace SelectTopKBase; | ||
| 356 | + if (plan.kernel != KernelTemplate::LEAF_SELECT || plan.candidateMerge != CandidateMerge::HALF_PREFIX || | ||
| 357 | + plan.k < SelectTopKBase::Tuning::SMALL_LEAF_BLOCK || plan.tile <= SelectTopKBase::Tuning::MAX_LEAF_BLOCK || | ||
| 358 | + plan.shardCount <= MERGE_WAYS || plan.shardCount >= Tuning::BALANCED_LEAVES || | ||
| 359 | + cores / plan.rows < Tuning::BALANCED_LEAVES) | ||
| 360 | + return; | ||
| 361 | + const uint32_t tile = Align(CeilDiv(plan.axis, Tuning::BALANCED_LEAVES), SORT32_ELEMENTS); | ||
| 362 | + if (tile <= SelectTopKBase::Tuning::MAX_LEAF_BLOCK || tile >= plan.tile) | ||
| 363 | + return; | ||
| 364 | + auto candidate = plan; | ||
| 365 | + if (RetileLeaf(candidate, tile, plan.finalCapacity, ub, cores)) { | ||
| 366 | + plan = candidate; | ||
| 367 | + } | ||
| 368 | +} | ||
| 369 | + | ||
| 370 | +// 每叶一核的4096项固定排序模板;四路短前缀逐组累积,省去中间GM写回与额外同步。 | ||
| 371 | +inline void TryDenseLeafPrefix(uint64_t ub, uint32_t cores, SelectTopKBaseTilingData& plan) | ||
| 372 | +{ | ||
| 373 | + using namespace SelectTopKBase; | ||
| 374 | + if (plan.kernel != KernelTemplate::LEAF_SELECT || plan.candidateMerge != CandidateMerge::HALF_PREFIX || | ||
| 375 | + plan.k <= SelectTopKBase::Tuning::SMALL_LEAF_BLOCK || plan.k > SelectTopKBase::Tuning::MAX_LEAF_BLOCK) | ||
| 376 | + return; | ||
| 377 | + const uint64_t leaves = CeilDiv(plan.axis, SelectTopKBase::Tuning::MAX_LEAF_BLOCK); | ||
| 378 | + if (leaves <= LOCAL_PREFIX_MAX_LISTS || leaves > DENSE_PREFIX_MAX_LEAVES) | ||
| 379 | + return; | ||
| 380 | + auto candidate = plan; | ||
| 381 | + if (RetileLeaf(candidate, SelectTopKBase::Tuning::MAX_LEAF_BLOCK, SelectTopKBase::Tuning::MAX_LEAF_BLOCK, ub, | ||
| 382 | + cores)) { | ||
| 383 | + candidate.candidateMerge = CandidateMerge::DENSE_PREFIX; | ||
| 384 | + plan = candidate; | ||
| 385 | + } | ||
| 386 | +} | ||
| 387 | + | ||
| 388 | +// 标准单行短轴使用固定小UB入口,避免批处理、输出分段和通用收尾的固定开销。 | ||
| 389 | +inline void TryTinyRowOptimization(uint64_t ub, uint32_t indexBytes, int64_t offset, int64_t mode, uint64_t outputInner, | ||
| 390 | + SelectTopKBaseTilingData& plan, SortSchedule& schedule) | ||
| 391 | +{ | ||
| 392 | + using namespace SelectTopKBase; | ||
| 393 | + if (plan.rows != 1 || plan.axis <= 1 || plan.axis > SINGLE_ROW_MAX_AXIS || plan.k == 0 || | ||
| 394 | + (plan.axis > TINY_ROW_ITEMS && plan.kernel == KernelTemplate::COPY_ALL) || | ||
| 395 | + plan.kernel == KernelTemplate::TOP1 || indexBytes != INDEX64_BYTES || offset != 0 || mode != OUTPUT_BOTH || | ||
| 396 | + outputInner != 1) | ||
| 397 | + return; | ||
| 398 | + auto candidate = SelectionInput(plan); | ||
| 399 | + // 容量作为 tiling 数据,由同一入口在内部选择固定容量实现。 | ||
| 400 | + const uint32_t slots = plan.axis <= SORT32_ELEMENTS ? | ||
| 401 | + SORT32_ELEMENTS : | ||
| 402 | + (plan.axis <= TINY_ROW_ITEMS ? TINY_ROW_ITEMS : SINGLE_ROW_MAX_AXIS); | ||
| 403 | + const uint32_t keySlots = Align(slots, FLOAT_VECTOR_ELEMENTS); | ||
| 404 | + const uint64_t bytes = uint64_t(slots) * (plan.dtypeBytes + INDEX32_BYTES + DOUBLE_PAIR_BYTES) + | ||
| 405 | + uint64_t(keySlots) * VALUE32_BYTES; | ||
| 406 | + if (bytes > ub - UB_RESERVE) | ||
| 407 | + return; | ||
| 408 | + candidate.kernel = KernelTemplate::TINY_ROW; | ||
| 409 | + candidate.route = ROUTE_COMPACT_ROW_SORT; | ||
| 410 | + candidate.tile = slots; | ||
| 411 | + candidate.tiles = candidate.shardCount = candidate.cores = 1; | ||
| 412 | + candidate.outputTile = plan.k; | ||
| 413 | + candidate.ubBytes = bytes; | ||
| 414 | + plan = candidate; | ||
| 415 | + schedule = {}; | ||
| 416 | +} | ||
| 417 | + | ||
| 418 | +// 已核验的标准单行最多16项时,使用固定标量缓冲,省去向量索引、掩码及归并初始化。 | ||
| 419 | +inline void TryScalarRowOptimization(uint64_t ub, SelectTopKBaseTilingData& plan) | ||
| 420 | +{ | ||
| 421 | + using namespace SelectTopKBase; | ||
| 422 | + if (plan.kernel != KernelTemplate::TINY_ROW || plan.tile != SORT32_ELEMENTS || plan.axis > SCALAR_ROW_ITEMS) | ||
| 423 | + return; | ||
| 424 | + // 无序且只舍弃一项继续使用线性压缩;其他较长短行避免标量插入排序的二次比较。 | ||
| 425 | + if (plan.axis > SCALAR_SORT_MAX_ITEMS && (plan.sorted || plan.k + 1 != plan.axis)) | ||
| 426 | + return; | ||
| 427 | + const ScalarRowLayout layout(plan.dtypeBytes); | ||
| 428 | + if (layout.bytes > ub - UB_RESERVE) | ||
| 429 | + return; | ||
| 430 | + plan.tile = SCALAR_ROW_ITEMS; | ||
| 431 | + plan.ubBytes = layout.bytes; | ||
| 432 | +} | ||
| 433 | +// 连续标准多行在归并层间只保留64项;容量、行数及范围由输入特征决定。 | ||
| 434 | +inline void TryPrefixRowOptimization(uint64_t ub, uint32_t cores, uint32_t indexBytes, int64_t offset, int64_t mode, | ||
| 435 | + uint64_t outputInner, uint64_t inputInner, SelectTopKBaseTilingData& plan, | ||
| 436 | + SortSchedule& schedule) | ||
| 437 | +{ | ||
| 438 | + using namespace SelectTopKBase; | ||
| 439 | + if (plan.kernel != KernelTemplate::ROW_SORT || indexBytes != INDEX64_BYTES || offset != 0 || mode != OUTPUT_BOTH || | ||
| 440 | + outputInner != 1 || inputInner != 1 || plan.axis <= PREFIX_ROW_MIN_AXIS || plan.axis > PREFIX_ROW_WIDTH || | ||
| 441 | + plan.k <= SORT32_ELEMENTS || plan.k > PREFIX_ROW_ITEMS || | ||
| 442 | + plan.rows < static_cast<uint64_t>(cores) * PREFIX_ROW_MIN_BATCHES) | ||
| 443 | + return; | ||
| 444 | + const uint32_t rowBytes = ROW_SORT_ARRAY_BYTES + ROW_INPUT_BUFFER_COUNT * plan.dtypeBytes; | ||
| 445 | + const uint32_t capacity = std::min<uint64_t>(SORT32_MAX_ELEMENTS, (ub - UB_RESERVE) / rowBytes); | ||
| 446 | + // 首级四路Sort32组已经可以独立裁剪;末级不足四路由同一模板处理。 | ||
| 447 | + const uint32_t width = Align(plan.axis, SORT32_ELEMENTS * MERGE_WAYS); | ||
| 448 | + const uint32_t batch = std::min<uint64_t>(schedule.rowsPerCore, capacity / width); | ||
| 449 | + if (batch == 0) | ||
| 450 | + return; | ||
| 451 | + plan.kernel = KernelTemplate::PREFIX_ROWS; | ||
| 452 | + plan.batchRows = batch; | ||
| 453 | + plan.tile = batch * width; | ||
| 454 | + plan.ubBytes = plan.tile * rowBytes; | ||
| 455 | + plan.outputTile = PREFIX_ROW_ITEMS; | ||
| 456 | + schedule.rowWidth = width; | ||
| 457 | +} | ||
| 458 | +} // namespace SelectTopK | ||
| 459 | +#endif | ||
| @@ -0,0 +1,18 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#include "select_top_k_tiling_common.h" | ||
| 11 | +namespace optiling { | ||
| 12 | +// 统一规划普通路径与大 K 的 GM 路径,通过 tiling key 选择设备实现。 | ||
| 13 | +static ge::graphStatus TileSelectTopK(gert::TilingContext* context) { return TileSelectTopKPlan(context); } | ||
| 14 | +struct SelectTopKCompileInfo {}; | ||
| 15 | +// 扩展模块无需额外编译期数据,返回解析成功。 | ||
| 16 | +static ge::graphStatus ParseSelectTopK(gert::TilingParseContext*) { return ge::GRAPH_SUCCESS; } | ||
| 17 | +IMPL_OP_OPTILING(SelectTopK).Tiling(TileSelectTopK).TilingParse<SelectTopKCompileInfo>(ParseSelectTopK); | ||
| 18 | +} // namespace optiling | ||
| @@ -0,0 +1,136 @@ | |||
| 1 | +#ifndef SELECT_TOP_K_TILING_COMMON_H_ | ||
| 2 | +#define SELECT_TOP_K_TILING_COMMON_H_ | ||
| 3 | +/** | ||
| 4 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | + */ | ||
| 12 | +#include "select_top_k_plan.h" | ||
| 13 | +#include "register/op_impl_registry.h" | ||
| 14 | +#include "tiling/platform/platform_ascendc.h" | ||
| 15 | +#include "log/log.h" | ||
| 16 | +#include <cstring> | ||
| 17 | + | ||
| 18 | +namespace optiling { | ||
| 19 | +namespace select_top_k_detail { | ||
| 20 | +// 读取可选边界张量的描述,校验广播长度并设置区间标记。 | ||
| 21 | +inline bool ReadBounds(gert::TilingContext* context, size_t input, uint64_t rows, uint32_t& flags) | ||
| 22 | +{ | ||
| 23 | + auto shape = context->GetOptionalInputShape(input); | ||
| 24 | + if (shape == nullptr) { | ||
| 25 | + return true; | ||
| 26 | + } | ||
| 27 | + auto desc = context->GetOptionalInputDesc(input); | ||
| 28 | + if (desc == nullptr || desc->GetDataType() != ge::DT_INT32) { | ||
| 29 | + return false; | ||
| 30 | + } | ||
| 31 | + const int64_t count = shape->GetStorageShape().GetShapeSize(); | ||
| 32 | + if (count != 1 && (count < 0 || static_cast<uint64_t>(count) != rows)) { | ||
| 33 | + return false; | ||
| 34 | + } | ||
| 35 | + flags |= input == SelectTopK::INPUT_BEGIN ? SelectTopK::HAS_BEGIN : SelectTopK::HAS_END; | ||
| 36 | + if (count == 1) { | ||
| 37 | + flags |= input == SelectTopK::INPUT_BEGIN ? SelectTopK::BROADCAST_BEGIN : SelectTopK::BROADCAST_END; | ||
| 38 | + } | ||
| 39 | + return true; | ||
| 40 | +} | ||
| 41 | +} // namespace select_top_k_detail | ||
| 42 | + | ||
| 43 | +// 读取框架输入与平台资源,生成计划并提交 tiling、核数和工作区。 | ||
| 44 | +inline ge::graphStatus TileSelectTopKPlan(gert::TilingContext* context) | ||
| 45 | +{ | ||
| 46 | + auto input = context->GetInputShape(0); | ||
| 47 | + auto desc = context->GetInputDesc(0); | ||
| 48 | + auto indexDesc = context->GetOutputDesc(1); | ||
| 49 | + auto attrs = context->GetAttrs(); | ||
| 50 | + auto platformInfo = context->GetPlatformInfo(); | ||
| 51 | + if (!input || !desc || !indexDesc || !attrs || !platformInfo) { | ||
| 52 | + return ge::GRAPH_FAILED; | ||
| 53 | + } | ||
| 54 | + const auto& shape = input->GetStorageShape(); | ||
| 55 | + const auto* k = attrs->GetInt(0); | ||
| 56 | + const auto* largest = attrs->GetBool(1); | ||
| 57 | + const auto* order = attrs->GetInt(SelectTopK::ATTR_ORDER); | ||
| 58 | + const auto* indicesType = attrs->GetInt(SelectTopK::ATTR_INDICES_TYPE); | ||
| 59 | + const auto* offset = attrs->GetInt(SelectTopK::ATTR_INDICES_OFFSET); | ||
| 60 | + const auto* mode = attrs->GetInt(SelectTopK::ATTR_OUTPUT_MODE); | ||
| 61 | + const auto* valueFill = attrs->GetFloat(SelectTopK::ATTR_VALUE_FILL); | ||
| 62 | + const auto* indexFill = attrs->GetInt(SelectTopK::ATTR_INDEX_FILL); | ||
| 63 | + const auto dtype = desc->GetDataType(); | ||
| 64 | + if (shape.GetDimNum() != SelectTopK::INPUT_RANK || shape.GetDim(0) < 0 || shape.GetDim(1) < 0 || | ||
| 65 | + shape.GetDim(1) > INT32_MAX || shape.GetDim(SelectTopK::INNER_LAYOUT_DIM) < 1 || | ||
| 66 | + shape.GetDim(SelectTopK::INNER_LAYOUT_DIM) > SelectTopK::MAX_LAYOUT_INNER || !k || !largest || !order || | ||
| 67 | + !indicesType || !offset || !mode || !valueFill || !indexFill || *k < 0 || *k > INT32_MAX || | ||
| 68 | + (*order != SelectTopK::ORDER_VALUE && *order != SelectTopK::ORDER_NONE) || *mode < 0 || | ||
| 69 | + *mode > SelectTopK::OUTPUT_BOTH || (dtype != ge::DT_FLOAT && dtype != ge::DT_FLOAT16 && dtype != ge::DT_BF16) || | ||
| 70 | + (*indicesType != ge::DT_INT32 && *indicesType != ge::DT_INT64) || indexDesc->GetDataType() != *indicesType) { | ||
| 71 | + return ge::GRAPH_FAILED; | ||
| 72 | + } | ||
| 73 | + const uint64_t inner = shape.GetDim(SelectTopK::INNER_LAYOUT_DIM); | ||
| 74 | + const uint64_t outer = shape.GetDim(0), axis = shape.GetDim(1); | ||
| 75 | + if (outer > static_cast<uint64_t>(INT64_MAX) / inner) { | ||
| 76 | + return ge::GRAPH_FAILED; | ||
| 77 | + } | ||
| 78 | + // 三维形状保留原轴后跨度;内核内部仍按 rows 行计算,不增加 GM 搬运。 | ||
| 79 | + const uint64_t rows = outer * inner; | ||
| 80 | + const int64_t maxIndex = axis == 0 ? 0 : static_cast<int64_t>(axis - 1); | ||
| 81 | + const int64_t upper = *indicesType == ge::DT_INT32 ? INT32_MAX : INT64_MAX; | ||
| 82 | + const int64_t lower = *indicesType == ge::DT_INT32 ? INT32_MIN : INT64_MIN; | ||
| 83 | + if (*offset < lower || *offset > upper - maxIndex || *indexFill < lower || *indexFill > upper) { | ||
| 84 | + return ge::GRAPH_FAILED; | ||
| 85 | + } | ||
| 86 | + SelectTopKTilingData data; | ||
| 87 | + data.outputK = *k; | ||
| 88 | + data.indicesOffset = *offset; | ||
| 89 | + data.indexFill = *indexFill; | ||
| 90 | + data.valueFill = *valueFill; | ||
| 91 | + if (!select_top_k_detail::ReadBounds(context, SelectTopK::INPUT_BEGIN, rows, data.flags) || | ||
| 92 | + !select_top_k_detail::ReadBounds(context, SelectTopK::INPUT_END, rows, data.flags)) { | ||
| 93 | + return ge::GRAPH_FAILED; | ||
| 94 | + } | ||
| 95 | + platform_ascendc::PlatformAscendC platform(platformInfo); | ||
| 96 | + uint64_t ub = 0; | ||
| 97 | + platform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ub); | ||
| 98 | + const uint32_t bytes = dtype == ge::DT_FLOAT ? SelectTopKBase::VALUE32_BYTES : SelectTopKBase::VALUE16_BYTES; | ||
| 99 | + if (!SelectTopK::BuildPlan( | ||
| 100 | + rows, axis, *k, bytes, *largest, *order == SelectTopK::ORDER_VALUE, | ||
| 101 | + *indicesType == ge::DT_INT64 ? SelectTopKBase::INDEX64_BYTES : SelectTopKBase::INDEX32_BYTES, *offset, | ||
| 102 | + *mode, data.flags, *valueFill, *indexFill, ub, platform.GetCoreNumAiv(), data, inner, inner)) { | ||
| 103 | + return ge::GRAPH_FAILED; | ||
| 104 | + } | ||
| 105 | + const uint32_t key = SelectTopK::TilingKey(data.base.kernel); | ||
| 106 | + if (key == UINT32_MAX) { | ||
| 107 | + return ge::GRAPH_FAILED; | ||
| 108 | + } | ||
| 109 | + auto raw = context->GetRawTilingData(); | ||
| 110 | + auto workspace = context->GetWorkspaceSizes(1); | ||
| 111 | + const uint64_t system = platform.GetLibApiWorkSpaceSize(); | ||
| 112 | + if (!raw || !workspace || raw->GetCapacity() < sizeof(data) || data.workspaceBytes > UINT64_MAX - system) { | ||
| 113 | + return ge::GRAPH_FAILED; | ||
| 114 | + } | ||
| 115 | + workspace[0] = data.workspaceBytes + system; | ||
| 116 | + std::memcpy(raw->GetData(), &data, sizeof(data)); | ||
| 117 | + raw->SetDataSize(sizeof(data)); | ||
| 118 | + context->SetBlockDim(data.base.cores); | ||
| 119 | + context->SetTilingKey(key); | ||
| 120 | + context->SetScheduleMode(1); | ||
| 121 | + OP_LOGI(context->GetNodeName(), | ||
| 122 | + "SelectTopK A2: rows=%lu N=%lu k=%lu coreK=%lu key=%u flags=%u offset=%ld inputInner=%u inputBlock=%u " | ||
| 123 | + "outputInner=%u outputBlock=%u tile=%u shards=%u cores=%u " | ||
| 124 | + "sortPattern=%u rowBlock=%lu rowBatch=%u shardItems=%lu localLeaves=%lu leafStride=%lu mergeWindow=%u " | ||
| 125 | + "template=%u partition=%u candidateMerge=%u mergeStorage=%u", | ||
| 126 | + rows, axis, data.outputK, data.base.k, key, data.flags, data.indicesOffset, data.inputInner, | ||
| 127 | + data.inputBlock, data.outputInner, data.outputBlock, data.base.tile, data.base.shardCount, data.base.cores, | ||
| 128 | + static_cast<uint32_t>(data.schedule.pattern), data.schedule.rowsPerCore, data.base.batchRows, | ||
| 129 | + data.schedule.shardElements, data.schedule.leavesPerShard, data.schedule.leafStride, data.base.mergeWindow, | ||
| 130 | + static_cast<uint32_t>(data.base.kernel), static_cast<uint32_t>(data.base.partition), | ||
| 131 | + static_cast<uint32_t>(data.base.candidateMerge), static_cast<uint32_t>(data.schedule.storage)); | ||
| 132 | + return ge::GRAPH_SUCCESS; | ||
| 133 | +} | ||
| 134 | + | ||
| 135 | +} // namespace optiling | ||
| 136 | +#endif | ||


产品名称不对啊