Merged
feat: RadixTopK 新增 int64 索引变体并修复 largest 场景 NaN 与 WS 同步问题 #6335
feat: RadixTopK 新增 int64 索引变体并修复 largest 场景 NaN 与 WS 同步问题 #6335
Merged
Nice tryAdd on Oct 14
Total: 14 files changed+1588-406
@@ -1,143 +1,281 @@
1{1{
2 "op_type": "RadixTopK",2 "op_type": "RadixTopK",
3 "op_list": [3 "op_list": [
4- {4+ {
5- "bin_filename": "RadixTopK_x_bfloat16_indices_int32_k_int32",5+ "bin_filename": "RadixTopK_x_bfloat16_indices_int32_k_int32",
6- "inputs": [6+ "inputs": [
7- {7+ {
8- "name": "x",8+ "name": "x",
9- "index": 0,9+ "index": 0,
10- "dtype": "bfloat16",10+ "dtype": "bfloat16",
11- "format": "ND",11+ "format": "ND",
12- "paramType": "required",12+ "paramType": "required",
13- "shape": [13+ "shape": [
14- -214+ -2
15- ]
16- },
17- {
18- "name": "k",
19- "index": 1,
20- "dtype": "int32",
21- "format": "ND",
22- "paramType": "required",
23- "shape": [
24- -2
25- ]
26- }
27- ],
28- "outputs": [
29- {
30- "name": "values",
31- "index": 0,
32- "dtype": "bfloat16",
33- "format": "ND",
34- "paramType": "required",
35- "shape": [
36- -2
37- ]
38- },
39- {
40- "name": "indices",
41- "index": 1,
42- "dtype": "int32",
43- "format": "ND",
44- "paramType": "required",
45- "shape": [
46- -2
47- ]
48- }
49- ],
50- "attrs": [
51- {
52- "name": "sorted",
53- "dtype": "bool",
54- "value": null
55- },
56- {
57- "name": "dim",
58- "dtype": "int",
59- "value": null
60- },
61- {
62- "name": "largest",
63- "dtype": "bool",
64- "value": null
65- },
66- {
67- "name": "indices_dtype",
68- "dtype": "int",
69- "value": 3
70- }
71 ]15 ]
72- },16+ },
73- {17+ {
74- "bin_filename": "RadixTopK_x_float16_indices_int32_k_int32",18+ "name": "k",
75- "inputs": [19+ "index": 1,
76- {20+ "dtype": "int32",
77- "name": "x",21+ "format": "ND",
78- "index": 0,22+ "paramType": "required",
79- "dtype": "float16",23+ "shape": [
80- "format": "ND",24+ -2
81- "paramType": "required",
82- "shape": [
83- -2
84- ]
85- },
86- {
87- "name": "k",
88- "index": 1,
89- "dtype": "int32",
90- "format": "ND",
91- "paramType": "required",
92- "shape": [
93- -2
94- ]
95- }
96- ],
97- "outputs": [
98- {
99- "name": "values",
100- "index": 0,
101- "dtype": "float16",
102- "format": "ND",
103- "paramType": "required",
104- "shape": [
105- -2
106- ]
107- },
108- {
109- "name": "indices",
110- "index": 1,
111- "dtype": "int32",
112- "format": "ND",
113- "paramType": "required",
114- "shape": [
115- -2
116- ]
117- }
118- ],
119- "attrs": [
120- {
121- "name": "sorted",
122- "dtype": "bool",
123- "value": null
124- },
125- {
126- "name": "dim",
127- "dtype": "int",
128- "value": null
129- },
130- {
131- "name": "largest",
132- "dtype": "bool",
133- "value": null
134- },
135- {
136- "name": "indices_dtype",
137- "dtype": "int",
138- "value": 3
139- }
140 ]25 ]
141- }26+ }
27+ ],
28+ "outputs": [
29+ {
30+ "name": "values",
31+ "index": 0,
32+ "dtype": "bfloat16",
33+ "format": "ND",
34+ "paramType": "required",
35+ "shape": [
36+ -2
37+ ]
38+ },
39+ {
40+ "name": "indices",
41+ "index": 1,
42+ "dtype": "int32",
43+ "format": "ND",
44+ "paramType": "required",
45+ "shape": [
46+ -2
47+ ]
48+ }
49+ ],
50+ "attrs": [
51+ {
52+ "name": "sorted",
53+ "dtype": "bool",
54+ "value": null
55+ },
56+ {
57+ "name": "dim",
58+ "dtype": "int",
59+ "value": null
60+ },
61+ {
62+ "name": "largest",
63+ "dtype": "bool",
64+ "value": null
65+ },
66+ {
67+ "name": "indices_dtype",
68+ "dtype": "int",
69+ "value": 3
70+ }
71+ ]
72+ },
73+ {
74+ "bin_filename": "RadixTopK_x_float16_indices_int32_k_int32",
75+ "inputs": [
76+ {
77+ "name": "x",
78+ "index": 0,
79+ "dtype": "float16",
80+ "format": "ND",
81+ "paramType": "required",
82+ "shape": [
83+ -2
84+ ]
85+ },
86+ {
87+ "name": "k",
88+ "index": 1,
89+ "dtype": "int32",
90+ "format": "ND",
91+ "paramType": "required",
92+ "shape": [
93+ -2
94+ ]
95+ }
96+ ],
97+ "outputs": [
98+ {
99+ "name": "values",
100+ "index": 0,
101+ "dtype": "float16",
102+ "format": "ND",
103+ "paramType": "required",
104+ "shape": [
105+ -2
106+ ]
107+ },
108+ {
109+ "name": "indices",
110+ "index": 1,
111+ "dtype": "int32",
112+ "format": "ND",
113+ "paramType": "required",
114+ "shape": [
115+ -2
116+ ]
117+ }
118+ ],
119+ "attrs": [
120+ {
121+ "name": "sorted",
122+ "dtype": "bool",
123+ "value": null
124+ },
125+ {
126+ "name": "dim",
127+ "dtype": "int",
128+ "value": null
129+ },
130+ {
131+ "name": "largest",
132+ "dtype": "bool",
133+ "value": null
134+ },
135+ {
136+ "name": "indices_dtype",
137+ "dtype": "int",
138+ "value": 3
139+ }
140+ ]
141+ },
142+ {
143+ "bin_filename": "RadixTopK_x_bfloat16_indices_int64_k_int32",
144+ "inputs": [
145+ {
146+ "name": "x",
147+ "index": 0,
148+ "dtype": "bfloat16",
149+ "format": "ND",
150+ "paramType": "required",
151+ "shape": [
152+ -2
153+ ]
154+ },
155+ {
156+ "name": "k",
157+ "index": 1,
158+ "dtype": "int32",
159+ "format": "ND",
160+ "paramType": "required",
161+ "shape": [
162+ -2
163+ ]
164+ }
165+ ],
166+ "outputs": [
167+ {
168+ "name": "values",
169+ "index": 0,
170+ "dtype": "bfloat16",
171+ "format": "ND",
172+ "paramType": "required",
173+ "shape": [
174+ -2
175+ ]
176+ },
177+ {
178+ "name": "indices",
179+ "index": 1,
180+ "dtype": "int64",
181+ "format": "ND",
182+ "paramType": "required",
183+ "shape": [
184+ -2
185+ ]
186+ }
187+ ],
188+ "attrs": [
189+ {
190+ "name": "sorted",
191+ "dtype": "bool",
192+ "value": null
193+ },
194+ {
195+ "name": "dim",
196+ "dtype": "int",
197+ "value": null
198+ },
199+ {
200+ "name": "largest",
201+ "dtype": "bool",
202+ "value": null
203+ },
204+ {
205+ "name": "indices_dtype",
206+ "dtype": "int",
207+ "value": 9
208+ }
209+ ]
210+ },
211+ {
212+ "bin_filename": "RadixTopK_x_float16_indices_int64_k_int32",
213+ "inputs": [
214+ {
215+ "name": "x",
216+ "index": 0,
217+ "dtype": "float16",
218+ "format": "ND",
219+ "paramType": "required",
220+ "shape": [
221+ -2
222+ ]
223+ },
224+ {
225+ "name": "k",
226+ "index": 1,
227+ "dtype": "int32",
228+ "format": "ND",
229+ "paramType": "required",
230+ "shape": [
231+ -2
232+ ]
233+ }
234+ ],
235+ "outputs": [
236+ {
237+ "name": "values",
238+ "index": 0,
239+ "dtype": "float16",
240+ "format": "ND",
241+ "paramType": "required",
242+ "shape": [
243+ -2
244+ ]
245+ },
246+ {
247+ "name": "indices",
248+ "index": 1,
249+ "dtype": "int64",
250+ "format": "ND",
251+ "paramType": "required",
252+ "shape": [
253+ -2
254+ ]
255+ }
256+ ],
257+ "attrs": [
258+ {
259+ "name": "sorted",
260+ "dtype": "bool",
261+ "value": null
262+ },
263+ {
264+ "name": "dim",
265+ "dtype": "int",
266+ "value": null
267+ },
268+ {
269+ "name": "largest",
270+ "dtype": "bool",
271+ "value": null
272+ },
273+ {
274+ "name": "indices_dtype",
275+ "dtype": "int",
276+ "value": 9
277+ }
278+ ]
279+ }
142 ]280 ]
143-}281+ }
@@ -1,143 +1,281 @@
1{1{
2- "op_type": "RadixTopK",2+ "op_type": "RadixTopK",
3- "op_list": [3+ "op_list": [
4+ {
5+ "bin_filename": "RadixTopK_x_bfloat16_indices_int32_k_int32",
6+ "inputs": [
4 {7 {
5- "bin_filename": "RadixTopK_x_bfloat16_indices_int32_k_int32",8+ "name": "x",
6- "inputs": [9+ "index": 0,
7- {10+ "dtype": "bfloat16",
8- "name": "x",11+ "format": "ND",
9- "index": 0,12+ "paramType": "required",
10- "dtype": "bfloat16",13+ "shape": [
11- "format": "ND",14+ -2
12- "paramType": "required",15+ ]
13- "shape": [
14- -2
15- ]
16- },
17- {
18- "name": "k",
19- "index": 1,
20- "dtype": "int32",
21- "format": "ND",
22- "paramType": "required",
23- "shape": [
24- -2
25- ]
26- }
27- ],
28- "outputs": [
29- {
30- "name": "values",
31- "index": 0,
32- "dtype": "bfloat16",
33- "format": "ND",
34- "paramType": "required",
35- "shape": [
36- -2
37- ]
38- },
39- {
40- "name": "indices",
41- "index": 1,
42- "dtype": "int32",
43- "format": "ND",
44- "paramType": "required",
45- "shape": [
46- -2
47- ]
48- }
49- ],
50- "attrs": [
51- {
52- "name": "sorted",
53- "dtype": "bool",
54- "value": null
55- },
56- {
57- "name": "dim",
58- "dtype": "int",
59- "value": null
60- },
61- {
62- "name": "largest",
63- "dtype": "bool",
64- "value": null
65- },
66- {
67- "name": "indices_dtype",
68- "dtype": "int",
69- "value": 3
70- }
71- ]
72 },16 },
73 {17 {
74- "bin_filename": "RadixTopK_x_float16_indices_int32_k_int32",18+ "name": "k",
75- "inputs": [19+ "index": 1,
76- {20+ "dtype": "int32",
77- "name": "x",21+ "format": "ND",
78- "index": 0,22+ "paramType": "required",
79- "dtype": "float16",23+ "shape": [
80- "format": "ND",24+ -2
81- "paramType": "required",25+ ]
82- "shape": [
83- -2
84- ]
85- },
86- {
87- "name": "k",
88- "index": 1,
89- "dtype": "int32",
90- "format": "ND",
91- "paramType": "required",
92- "shape": [
93- -2
94- ]
95- }
96- ],
97- "outputs": [
98- {
99- "name": "values",
100- "index": 0,
101- "dtype": "float16",
102- "format": "ND",
103- "paramType": "required",
104- "shape": [
105- -2
106- ]
107- },
108- {
109- "name": "indices",
110- "index": 1,
111- "dtype": "int32",
112- "format": "ND",
113- "paramType": "required",
114- "shape": [
115- -2
116- ]
117- }
118- ],
119- "attrs": [
120- {
121- "name": "sorted",
122- "dtype": "bool",
123- "value": null
124- },
125- {
126- "name": "dim",
127- "dtype": "int",
128- "value": null
129- },
130- {
131- "name": "largest",
132- "dtype": "bool",
133- "value": null
134- },
135- {
136- "name": "indices_dtype",
137- "dtype": "int",
138- "value": 3
139- }
140- ]
141 }26 }
142- ]27+ ],
143-}28+ "outputs": [
29+ {
30+ "name": "values",
31+ "index": 0,
32+ "dtype": "bfloat16",
33+ "format": "ND",
34+ "paramType": "required",
35+ "shape": [
36+ -2
37+ ]
38+ },
39+ {
40+ "name": "indices",
41+ "index": 1,
42+ "dtype": "int32",
43+ "format": "ND",
44+ "paramType": "required",
45+ "shape": [
46+ -2
47+ ]
48+ }
49+ ],
50+ "attrs": [
51+ {
52+ "name": "sorted",
53+ "dtype": "bool",
54+ "value": null
55+ },
56+ {
57+ "name": "dim",
58+ "dtype": "int",
59+ "value": null
60+ },
61+ {
62+ "name": "largest",
63+ "dtype": "bool",
64+ "value": null
65+ },
66+ {
67+ "name": "indices_dtype",
68+ "dtype": "int",
69+ "value": 3
70+ }
71+ ]
72+ },
73+ {
74+ "bin_filename": "RadixTopK_x_float16_indices_int32_k_int32",
75+ "inputs": [
76+ {
77+ "name": "x",
78+ "index": 0,
79+ "dtype": "float16",
80+ "format": "ND",
81+ "paramType": "required",
82+ "shape": [
83+ -2
84+ ]
85+ },
86+ {
87+ "name": "k",
88+ "index": 1,
89+ "dtype": "int32",
90+ "format": "ND",
91+ "paramType": "required",
92+ "shape": [
93+ -2
94+ ]
95+ }
96+ ],
97+ "outputs": [
98+ {
99+ "name": "values",
100+ "index": 0,
101+ "dtype": "float16",
102+ "format": "ND",
103+ "paramType": "required",
104+ "shape": [
105+ -2
106+ ]
107+ },
108+ {
109+ "name": "indices",
110+ "index": 1,
111+ "dtype": "int32",
112+ "format": "ND",
113+ "paramType": "required",
114+ "shape": [
115+ -2
116+ ]
117+ }
118+ ],
119+ "attrs": [
120+ {
121+ "name": "sorted",
122+ "dtype": "bool",
123+ "value": null
124+ },
125+ {
126+ "name": "dim",
127+ "dtype": "int",
128+ "value": null
129+ },
130+ {
131+ "name": "largest",
132+ "dtype": "bool",
133+ "value": null
134+ },
135+ {
136+ "name": "indices_dtype",
137+ "dtype": "int",
138+ "value": 3
139+ }
140+ ]
141+ },
142+ {
143+ "bin_filename": "RadixTopK_x_bfloat16_indices_int64_k_int32",
144+ "inputs": [
145+ {
146+ "name": "x",
147+ "index": 0,
148+ "dtype": "bfloat16",
149+ "format": "ND",
150+ "paramType": "required",
151+ "shape": [
152+ -2
153+ ]
154+ },
155+ {
156+ "name": "k",
157+ "index": 1,
158+ "dtype": "int32",
159+ "format": "ND",
160+ "paramType": "required",
161+ "shape": [
162+ -2
163+ ]
164+ }
165+ ],
166+ "outputs": [
167+ {
168+ "name": "values",
169+ "index": 0,
170+ "dtype": "bfloat16",
171+ "format": "ND",
172+ "paramType": "required",
173+ "shape": [
174+ -2
175+ ]
176+ },
177+ {
178+ "name": "indices",
179+ "index": 1,
180+ "dtype": "int64",
181+ "format": "ND",
182+ "paramType": "required",
183+ "shape": [
184+ -2
185+ ]
186+ }
187+ ],
188+ "attrs": [
189+ {
190+ "name": "sorted",
191+ "dtype": "bool",
192+ "value": null
193+ },
194+ {
195+ "name": "dim",
196+ "dtype": "int",
197+ "value": null
198+ },
199+ {
200+ "name": "largest",
201+ "dtype": "bool",
202+ "value": null
203+ },
204+ {
205+ "name": "indices_dtype",
206+ "dtype": "int",
207+ "value": 9
208+ }
209+ ]
210+ },
211+ {
212+ "bin_filename": "RadixTopK_x_float16_indices_int64_k_int32",
213+ "inputs": [
214+ {
215+ "name": "x",
216+ "index": 0,
217+ "dtype": "float16",
218+ "format": "ND",
219+ "paramType": "required",
220+ "shape": [
221+ -2
222+ ]
223+ },
224+ {
225+ "name": "k",
226+ "index": 1,
227+ "dtype": "int32",
228+ "format": "ND",
229+ "paramType": "required",
230+ "shape": [
231+ -2
232+ ]
233+ }
234+ ],
235+ "outputs": [
236+ {
237+ "name": "values",
238+ "index": 0,
239+ "dtype": "float16",
240+ "format": "ND",
241+ "paramType": "required",
242+ "shape": [
243+ -2
244+ ]
245+ },
246+ {
247+ "name": "indices",
248+ "index": 1,
249+ "dtype": "int64",
250+ "format": "ND",
251+ "paramType": "required",
252+ "shape": [
253+ -2
254+ ]
255+ }
256+ ],
257+ "attrs": [
258+ {
259+ "name": "sorted",
260+ "dtype": "bool",
261+ "value": null
262+ },
263+ {
264+ "name": "dim",
265+ "dtype": "int",
266+ "value": null
267+ },
268+ {
269+ "name": "largest",
270+ "dtype": "bool",
271+ "value": null
272+ },
273+ {
274+ "name": "indices_dtype",
275+ "dtype": "int",
276+ "value": 9
277+ }
278+ ]
279+ }
280+ ]
281+}
@@ -13,6 +13,8 @@
13 * \brief13 * \brief
14 */14 */
15 15 
16+#include <cstdint>
17+#include <limits>
16#include "tiling/tiling_api.h"18#include "tiling/tiling_api.h"
17#include "register/tilingdata_base.h"19#include "register/tilingdata_base.h"
18#include "register/op_def_registry.h"20#include "register/op_def_registry.h"
@@ -31,6 +33,13 @@ static constexpr uint64_t MAX_TILE_LEN = 12032;
31static constexpr uint64_t LARGE_SORT_LEN = 1440000000;33static constexpr uint64_t LARGE_SORT_LEN = 1440000000;
32static constexpr uint64_t LARGE_TILE_LEN = 10496;34static constexpr uint64_t LARGE_TILE_LEN = 10496;
33static constexpr uint64_t MAX_TILE_NUM_IN_UB = 6144;35static constexpr uint64_t MAX_TILE_NUM_IN_UB = 6144;
36+// sortLen 超过 int32 上限时切换 int64 模板(索引值域与跨核计数可达 sortLen)
37+static constexpr uint64_t INT32_SORT_LEN_MAX = static_cast<uint64_t>(std::numeric_limits<int32_t>::max());
38+// checkUb 中 UB 占用按「每 tile 元素数 × 系数」估算:
39+// int32 索引:xBuf ping/pong(fp32) + outValue(fp32) + outIndex(int32) = 16 元素/tile;
40+// int64 索引:outIndexBuf 翻倍(int64),系数 16 -> 20。
41+static constexpr uint64_t UB_COEF_PER_TILE = 16;
42+static constexpr uint64_t UB_COEF_PER_TILE_I64 = 20;
34static constexpr uint64_t INPUT_X = 0;43static constexpr uint64_t INPUT_X = 0;
35static constexpr uint64_t NUM_VALUE_2BIT = 4;44static constexpr uint64_t NUM_VALUE_2BIT = 4;
36static constexpr uint64_t BIT_PER_BYTE = 8;45static constexpr uint64_t BIT_PER_BYTE = 8;
@@ -79,6 +88,16 @@ ge::graphStatus RadixTopKTiling::GetShapeAttrsInfo()
79 tilingData_.sorted = *attrs->GetAttrPointer<bool>(ATTR_INDEX_SORTED);88 tilingData_.sorted = *attrs->GetAttrPointer<bool>(ATTR_INDEX_SORTED);
80 tilingData_.largest = *attrs->GetAttrPointer<bool>(ATTR_INDEX_LARGEST);89 tilingData_.largest = *attrs->GetAttrPointer<bool>(ATTR_INDEX_LARGEST);
81 isLargeShape_ = tilingData_.sortLen > LARGE_SORT_LEN;90 isLargeShape_ = tilingData_.sortLen > LARGE_SORT_LEN;
91+ isInt64Shape_ = tilingData_.sortLen > INT32_SORT_LEN_MAX;
92+ // int64 模板输出 int64 索引,防御性校验输出 dtype 匹配(int32 组合不允许超 int32 shape)
93+ if (isInt64Shape_) {
94+ auto indicesDesc = context_->GetOutputDesc(1);
95+ OP_CHECK_NULL_WITH_CONTEXT(context_, indicesDesc);
96+ OP_CHECK_IF((indicesDesc->GetDataType() != ge::DT_INT64),
97+ OP_LOGE(opName_, "RadixTopK sortLen [%lu] exceeds int32, indices output must be int64.",
98+ tilingData_.sortLen),
99+ return ge::GRAPH_FAILED);
100+ }
82 101 
83 xDType_ = context_->GetInputDesc(INPUT_X)->GetDataType();102 xDType_ = context_->GetInputDesc(INPUT_X)->GetDataType();
84 xDtypeSize_ = ge::GetSizeByDataType(xDType_);103 xDtypeSize_ = ge::GetSizeByDataType(xDType_);
@@ -91,21 +110,30 @@ ge::graphStatus RadixTopKTiling::DoOpTiling()
91{110{
92 OP_LOGD(opName_, "RadixTopKTiling DoOpTiling.");111 OP_LOGD(opName_, "RadixTopKTiling DoOpTiling.");
93 bool isCalcSuccess = false;112 bool isCalcSuccess = false;
94- if (!isLargeShape_) {113+ if (isInt64Shape_) {
114+ isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen, UB_COEF_PER_TILE_I64);
115+ } else if (!isLargeShape_) {
95 isCalcSuccess = CalcTilingParams(tilingData_.sortLen);116 isCalcSuccess = CalcTilingParams(tilingData_.sortLen);
96 if (!isCalcSuccess) {117 if (!isCalcSuccess) {
97 isLargeShape_ = true;118 isLargeShape_ = true;
98- isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen);119+ isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen, UB_COEF_PER_TILE);
99 }120 }
100 } else {121 } else {
101- isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen);122+ isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen, UB_COEF_PER_TILE);
102 }123 }
103 OP_CHECK_IF((!isCalcSuccess), OP_LOGE(opName_, "RadixTopK Tiling failed."), return ge::GRAPH_FAILED);124 OP_CHECK_IF((!isCalcSuccess), OP_LOGE(opName_, "RadixTopK Tiling failed."), return ge::GRAPH_FAILED);
125+ // int64 模板的 coreTopK/边界计数等 int32 通道要求 kValue 不超 int32 上限
126+ OP_CHECK_IF((isInt64Shape_ && tilingData_.kValue > INT32_SORT_LEN_MAX),
127+ OP_LOGE(opName_, "RadixTopK kValue [%lu] exceeds int32 limit for int64 shape.", tilingData_.kValue),
128+ return ge::GRAPH_FAILED);
129+ // per-core 直方图计数保持 int32:per-core 元素数须 < 2^31(S < 2^31*coreNum,物理内存内恒成立,防御性校验)
130+ OP_CHECK_IF((isInt64Shape_ && tilingData_.formerTileNum * tilingData_.formerTileLen > INT32_SORT_LEN_MAX),
131+ OP_LOGE(opName_, "RadixTopK per-core element num exceeds int32 limit."), return ge::GRAPH_FAILED);
104 132 
105 blockDim_ = tilingData_.formerCoreNum + tilingData_.tailCoreNum;133 blockDim_ = tilingData_.formerCoreNum + tilingData_.tailCoreNum;
106 tilingData_.coreNum = blockDim_;134 tilingData_.coreNum = blockDim_;
107- // 仅 UB 变体可复用 indices 内存(WS 变体的 tileTopK/tileHist 与 CopyOutResult 跨 core 冲突)135+ // 仅 UB 变体可复用 indices 内存(WS/INT64 变体的 tileTopK/tileHist 与 CopyOutResult 跨 core 冲突)
108- if (isLargeShape_) {136+ if (isLargeShape_ || isInt64Shape_) {
109 tilingData_.needWorkspace = true;137 tilingData_.needWorkspace = true;
110 } else {138 } else {
111 uint64_t wsInt32Cnt = tilingData_.coreNum * (NUM_VALUE_2BIT + 2);139 uint64_t wsInt32Cnt = tilingData_.coreNum * (NUM_VALUE_2BIT + 2);
@@ -124,10 +152,11 @@ uint64_t RadixTopKTiling::GetTilingKey() const
124{152{
125 OP_LOGD(opName_, "RadixTopKTiling GetTilingKey.");153 OP_LOGD(opName_, "RadixTopKTiling GetTilingKey.");
126 154 
127- const uint64_t tilingKey = GET_TPL_TILING_KEY(tilingData_.sorted, tilingData_.largest, isLargeShape_);155+ const uint64_t tilingKey = GET_TPL_TILING_KEY(tilingData_.sorted, tilingData_.largest, isLargeShape_,
156+ isInt64Shape_);
128 OP_LOGD(opName_, "tilingKey is: [%lu]", tilingKey);157 OP_LOGD(opName_, "tilingKey is: [%lu]", tilingKey);
129- OP_LOGD(opName_, "sorted, largest, isLargeShape is: [%d, %d, %d]", tilingData_.sorted, tilingData_.largest,158+ OP_LOGD(opName_, "sorted, largest, isLargeShape, isInt64Shape is: [%d, %d, %d, %d]", tilingData_.sorted,
130- isLargeShape_);159+ tilingData_.largest, isLargeShape_, isInt64Shape_);
131 160 
132 return tilingKey;161 return tilingKey;
133}162}
@@ -139,7 +168,7 @@ ge::graphStatus RadixTopKTiling::GetWorkspaceSize()
139 workspaceSize_ = 0;168 workspaceSize_ = 0;
140 return ge::GRAPH_SUCCESS;169 return ge::GRAPH_SUCCESS;
141 }170 }
142- if (isLargeShape_) {171+ if (isLargeShape_ || isInt64Shape_) {
143 workspaceSize_ = (NUM_VALUE_2BIT * tilingData_.coreNum + tilingData_.coreNum + tilingData_.coreNum +172 workspaceSize_ = (NUM_VALUE_2BIT * tilingData_.coreNum + tilingData_.coreNum + tilingData_.coreNum +
144 WS_INT32_PER_TILE * tilingData_.totalTileNum) *173 WS_INT32_PER_TILE * tilingData_.totalTileNum) *
145 sizeof(int32_t);174 sizeof(int32_t);
@@ -199,6 +228,7 @@ void RadixTopKTiling::PrintTilingData()
199 OP_LOGD(opName_, "kValue: %lu.", tilingData_.kValue);228 OP_LOGD(opName_, "kValue: %lu.", tilingData_.kValue);
200 OP_LOGD(opName_, "largest: %d.", tilingData_.largest);229 OP_LOGD(opName_, "largest: %d.", tilingData_.largest);
201 OP_LOGD(opName_, "sorted: %d.", tilingData_.sorted);230 OP_LOGD(opName_, "sorted: %d.", tilingData_.sorted);
231+ OP_LOGD(opName_, "isInt64Shape: %d.", isInt64Shape_);
202 OP_LOGD(opName_, "needWorkspace: %d.", tilingData_.needWorkspace);232 OP_LOGD(opName_, "needWorkspace: %d.", tilingData_.needWorkspace);
203}233}
204 234 
@@ -235,14 +265,19 @@ bool RadixTopKTiling::TryCalcTileDistribution(const uint64_t& dataNum, uint64_t
235 return false;265 return false;
236}266}
237 267 
238-bool RadixTopKTiling::CalcLargeTilingParams(const uint64_t& dataNum)268+/**
269+ * @brief 大 shape / int64 shape 的 tile 切分(两者仅 UB 系数不同,合并为一个带参函数)
270+ * @param dataNum 单 core 处理的数据量
271+ * @param ubCoefPerTile UB 占用中「每 tile 元素数」的系数(int32 索引 16、int64 索引 20)
272+ */
273+bool RadixTopKTiling::CalcLargeTilingParams(const uint64_t& dataNum, const uint64_t& ubCoefPerTile)
239{274{
240 uint64_t dataAlign = CACHE_LINE / xDtypeSize_;275 uint64_t dataAlign = CACHE_LINE / xDtypeSize_;
241 uint64_t minTileLen = dataAlign;276 uint64_t minTileLen = dataAlign;
242 277 
243- auto checkUb = [this](uint64_t tileLen) -> bool {278+ auto checkUb = [this, ubCoefPerTile](uint64_t tileLen) -> bool {
244 uint64_t cmpMaskSize = tileLen / 8;279 uint64_t cmpMaskSize = tileLen / 8;
245- uint64_t ubUsed = 16 * tileLen + cmpMaskSize + MAX_TILE_NUM_IN_UB * sizeof(int32_t) + BLOCK_SIZE;280+ uint64_t ubUsed = ubCoefPerTile * tileLen + cmpMaskSize + MAX_TILE_NUM_IN_UB * sizeof(int32_t) + BLOCK_SIZE;
246 return ubUsed <= ubSize_;281 return ubUsed <= ubSize_;
247 };282 };
248 283 
@@ -47,7 +47,7 @@ protected:
47 // 7、保存Tiling数据47 // 7、保存Tiling数据
48 ge::graphStatus PostTiling() override;48 ge::graphStatus PostTiling() override;
49 49 
50- bool CalcLargeTilingParams(const uint64_t& dataNum);50+ bool CalcLargeTilingParams(const uint64_t& dataNum, const uint64_t& ubCoefPerTile);
51 bool CalcTilingParams(const uint64_t& dataNum);51 bool CalcTilingParams(const uint64_t& dataNum);
52 void PrintTilingData();52 void PrintTilingData();
53 53 
@@ -63,6 +63,7 @@ private:
63 uint32_t xDtypeSize_{0};63 uint32_t xDtypeSize_{0};
64 uint64_t xAlign_{0};64 uint64_t xAlign_{0};
65 bool isLargeShape_{false};65 bool isLargeShape_{false};
66+ bool isInt64Shape_{false};
66 RadixTopKTilingData tilingData_{};67 RadixTopKTilingData tilingData_{};
67 68 
68 void SetTilingData();69 void SetTilingData();
@@ -10,23 +10,29 @@
10 10 
11/*!11/*!
12 * \file radix_top_k.cpp12 * \file radix_top_k.cpp
13- * \brief radix top k op kernel 13+ * \brief radix top k op kernel
14 */14 */
15#include "kernel_operator.h"15#include "kernel_operator.h"
16#include "radix_top_k_struct.h"16#include "radix_top_k_struct.h"
17#include "radix_top_k_tiling_key.h"17#include "radix_top_k_tiling_key.h"
18#include "radix_top_k_ub.h"18#include "radix_top_k_ub.h"
19#include "radix_top_k_ws.h"19#include "radix_top_k_ws.h"
20+#include "radix_top_k_ws_i64.h"
20using namespace AscendC;21using namespace AscendC;
21 22 
22-template <bool sorted, bool largest, bool isLargeShape>23+template <bool sorted, bool largest, bool isLargeShape, bool isInt64Shape>
23-__global__ __aicore__ void radix_top_k(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling)24+__global__ __aicore__ void radix_top_k(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace,
25+ GM_ADDR tiling)
24{26{
25 TPipe tpipe;27 TPipe tpipe;
26 REGISTER_TILING_DEFAULT(RadixTopKTilingData);28 REGISTER_TILING_DEFAULT(RadixTopKTilingData);
27 GET_TILING_DATA(tilingData, tiling);29 GET_TILING_DATA(tilingData, tiling);
28#if (defined(DTYPE_X))30#if (defined(DTYPE_X))
29- if constexpr (isLargeShape) {31+ if constexpr (isInt64Shape) {
32+ RadixTopK::RadixTopKWsI64<DTYPE_X, largest> op(tpipe, tilingData);
33+ op.Init(x, k, values, indices, workspace);
34+ op.Process();
35+ } else if constexpr (isLargeShape) {
30 RadixTopK::RadixTopKWs<DTYPE_X, largest> op(tpipe, tilingData);36 RadixTopK::RadixTopKWs<DTYPE_X, largest> op(tpipe, tilingData);
31 op.Init(x, k, values, indices, workspace);37 op.Init(x, k, values, indices, workspace);
32 op.Process();38 op.Process();
@@ -40,10 +46,9 @@ __global__ __aicore__ void radix_top_k(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_
40 46 
41// op_kernel UT 链接桩:仅提供 extern "C" 符号供链接,不参与实际执行逻辑。47// op_kernel UT 链接桩:仅提供 extern "C" 符号供链接,不参与实际执行逻辑。
42#ifdef __CCE_UT_TEST__48#ifdef __CCE_UT_TEST__
43-extern "C" __global__ __aicore__ void radix_top_k(49+extern "C" __global__ __aicore__ void radix_top_k(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices,
44- GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices,50+ GM_ADDR workspace, GM_ADDR tiling)
45- GM_ADDR workspace, GM_ADDR tiling)
46{51{
47- radix_top_k<true, true, false>(x, k, values, indices, workspace, tiling);52+ radix_top_k<true, true, false, false>(x, k, values, indices, workspace, tiling);
48}53}
49-#endif54+#endif
@@ -59,7 +59,7 @@ using namespace AscendC;
59#define PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileKVal, vecIndexExpr) \59#define PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileKVal, vecIndexExpr) \
60 if (curTileKVal > 0) { \60 if (curTileKVal > 0) { \
61 auto xLocal = curBuf.template Get<T>(); \61 auto xLocal = curBuf.template Get<T>(); \
62- auto cmpMaskTensor = tempBuf_.template Get<uint8_t>(); \62+ auto cmpMaskTensor = this->tempBuf_.template Get<uint8_t>(); \
63 auto vecIndex = (vecIndexExpr); \63 auto vecIndex = (vecIndexExpr); \
64 if constexpr (IsSameType<T, bfloat16_t>::value) { \64 if constexpr (IsSameType<T, bfloat16_t>::value) { \
65 auto xLocalFp32 = xLocal.template ReinterpretCast<float>(); \65 auto xLocalFp32 = xLocal.template ReinterpretCast<float>(); \
@@ -68,6 +68,9 @@ using namespace AscendC;
68 cmpMaskTensor, vecIndex, tileOffset); \68 cmpMaskTensor, vecIndex, tileOffset); \
69 } else if constexpr (IsSameType<T, half>::value) { \69 } else if constexpr (IsSameType<T, half>::value) { \
70 auto xLocalHalf = xLocal[this->tileLen_]; \70 auto xLocalHalf = xLocal[this->tileLen_]; \
71+ if constexpr (largest) { \
72+ this->CanonicalizeDataForLargest(curBuf, curTileLen); \
73+ } \
71 this->template ProcessOneTileTopK<half>(xLocalHalf, curTileLen, curTileKVal, coreTopKOffset, maxIndexLen, \74 this->template ProcessOneTileTopK<half>(xLocalHalf, curTileLen, curTileKVal, coreTopKOffset, maxIndexLen, \
72 cmpMaskTensor, vecIndex, tileOffset); \75 cmpMaskTensor, vecIndex, tileOffset); \
73 } \76 } \
@@ -101,7 +104,7 @@ protected:
101 const int32_t& tileHistBinStride, const int32_t& tileHistBinOffset,104 const int32_t& tileHistBinStride, const int32_t& tileHistBinOffset,
102 const uint64_t& curTileLen);105 const uint64_t& curTileLen);
103 106 
104- __aicore__ inline void NegateDataForLargest(TBuf<TPosition::VECIN>& curBuf, const uint64_t& curTileLen);107+ __aicore__ inline void CanonicalizeDataForLargest(TBuf<TPosition::VECIN>& curBuf, const uint64_t& curTileLen);
105 108 
106 template <bool isInit>109 template <bool isInit>
107 __aicore__ inline void CopyOut2Ws(const uint64_t& maskLen, const uint64_t& gmOffset);110 __aicore__ inline void CopyOut2Ws(const uint64_t& maskLen, const uint64_t& gmOffset);
@@ -143,6 +146,13 @@ protected:
143 template <typename CT>146 template <typename CT>
144 __aicore__ inline CT ExtractBoundaryNumber();147 __aicore__ inline CT ExtractBoundaryNumber();
145 148 
149+ /**
150+ * @brief 边界值是否为 NaN(位模式判定,不做浮点自比较以免触发浮点相等告警)
151+ * NaN ⟺ 指数位全 1 且尾数非 0(对 ±NaN、任意 payload 都成立)
152+ */
153+ template <typename CT>
154+ __aicore__ inline bool IsNaNValue(const CT& value);
155+ 
146 __aicore__ inline uint64_t CalcBlockOffset(uint64_t batchId);156 __aicore__ inline uint64_t CalcBlockOffset(uint64_t batchId);
147 157 
148protected:158protected:
@@ -271,6 +281,16 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::TwiddleInB16(TBuf<TPosit
271 Not(temp1, temp1, calcLen);281 Not(temp1, temp1, calcLen);
272 PipeBarrier<PIPE_V>();282 PipeBarrier<PIPE_V>();
273 And(xLocalInt16, xLocalInt16, temp1, calcLen);283 And(xLocalInt16, xLocalInt16, temp1, calcLen);
284+ if constexpr (!largest && (sizeof(T) == sizeof(int16_t))) {
285+ // largest=false 的降序由键域取补承担:先做升序键(上面的 twiddle),再整体取补。
286+ // 对全部 65536 个 16 位模式恒有 ~twiddle(x) == twiddle(bits(-x)),故除 NaN 外本步
287+ // 与「先取反再 twiddle」逐位一致,序键语义不变;仅 NaN 受影响:
288+ // CanonicalizeDataForLargest 已把 ±NaN 规范化为同一个正 NaN(实测 fp16 与 bf16 都是
289+ // 0x7FFF,与输入 NaN 的符号无关),其升序键为 0xFFFF,取补后为 0x0000,低于 -Inf 的
290+ // 0x007F(bf16)/0x03FF(fp16),落到序键最底端 ⇒ 在计数相不被计入 k 槽、提取相也不会被选中。
291+ PipeBarrier<PIPE_V>();
292+ Not(xLocalInt16, xLocalInt16, calcLen);
293+ }
274}294}
275 295 
276/**296/**
@@ -339,23 +359,49 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::CalcCumsumHistogram16(
339}359}
340 360 
341/**361/**
342- * @brief 当 largest=false 时对数据取反,将求最小值转为求最大值362+ * @brief ±NaN 规范化(两个方向都要做):把 ±NaN 统一变成同一个正 NaN(16 位类型的原域),
363+ * 使 TwiddleInB16 键域对 NaN 的记账与提取相的掩码口径一致;有限值与 ±Inf 逐位不变
364+ * (fp16/bf16 → fp32 → ×1 → 回 16 位是精确往返)。
365+ * - largest=false:规范化后 NaN 键为 0xFFFF,经 TwiddleInB16 末尾的键域取补落 0x0000
366+ * (低于 -Inf 的 0x007F/0x03FF)⇒ 计数相不计入 k 槽、提取相 <LT> 也不匹配
367+ * ⇒ NaN 不入选(与 torch 语义一致)。降序本身由该键域取补承担,故此处不取反。
368+ * - largest=true:规范化后 NaN 键为 0xFFFF(最大端)⇒ 计数相把 NaN 计入该 tile 的
369+ * k 配额,与提取相 NOT(LE) = (x > b) ∪ isNaN 口径一致 ⇒ NaN 必入选(与 torch 一致)。
370+ * **缺了这一步会丢 −NaN**:−NaN 原样 twiddle 后键为 ~0xFE00=0x01FF(fp16) /
371+ * ~0xFFC0=0x003F(bf16),落在 −Inf 之下 ⇒ 直方图不计入配额,但提取掩码仍匹配
372+ * ⇒ rsvdCntV 超出该 tile 配额,被 SubTopKAndCopyOut 的 min(rsvdCntV, curTileK)
373+ * 截断(GatherMask 按 lane 序保留靠前的匹配)⇒ −NaN 槽丢空、背景值顶替。
374+ * 实测(S=1.2e7+4 / k=1001 / 尾块 8 个 ±NaN / largest=true):fp16 只中 +NaN 半边、
375+ * −NaN 0/4、values 多重集错位 410/1001;bf16 同类(−NaN 0/4,错位 8/1001,量化粗
376+ * 使错位值多为等值)。全 +NaN 输入(8/8、错位 0)与 largest=false(NaN 0 入选、
377+ * 错位 0)是对照。
343 * @param curBuf 当前数据缓冲区378 * @param curBuf 当前数据缓冲区
344 * @param curTileLen 当前 tile 数据长度379 * @param curTileLen 当前 tile 数据长度
345 */380 */
346template <typename T, bool largest>381template <typename T, bool largest>
347-__aicore__ inline void RadixTopKBaseKernel<T, largest>::NegateDataForLargest(TBuf<TPosition::VECIN>& curBuf,382+__aicore__ inline void RadixTopKBaseKernel<T, largest>::CanonicalizeDataForLargest(TBuf<TPosition::VECIN>& curBuf,
348- const uint64_t& curTileLen)383+ const uint64_t& curTileLen)
349{384{
350- if constexpr (!largest) {385+ LocalTensor<T> xCanon = curBuf.Get<T>();
351- LocalTensor<T> xNgt = curBuf.Get<T>();386+ PipeBarrier<PIPE_V>();
387+ if constexpr (IsSameType<T, bfloat16_t>::value) {
388+ // bf16:Muls 不支持 bf16(编译报错),只能借 fp32 域做规范化 —— 实测该往返把 ±NaN 归成
389+ // 0x7FFF,±Inf/±0/次正规码无损。
390+ // 注意这里是乘 +1 而非 −1:降序由 TwiddleInB16 末尾的键域取补承担,本步只需把 ±NaN
391+ // 统一成同一个正 NaN(否则 −NaN 取补后会落到序键顶端而被误选,即 q7 缺陷)。
392+ // (坑位记录:当年 bf16 的取反是「把位模式 reinterpret 成 half 再乘 −1」,
393+ // |x|≥2^121 与 ±Inf/NaN 会落进 half 的非规格区退化为 NaN ⇒ D64/S2。)
394+ LocalTensor<float> xFp32 = outValueBuf_.Get<float>();
395+ Cast(xFp32, xCanon[tileLen_], RoundMode::CAST_NONE, curTileLen);
352 PipeBarrier<PIPE_V>();396 PipeBarrier<PIPE_V>();
353- if constexpr (IsSameType<T, bfloat16_t>::value) {397+ Muls(xFp32, xFp32, (float)1, curTileLen);
354- Muls(xNgt[tileLen_].template ReinterpretCast<half>(), xNgt[tileLen_].template ReinterpretCast<half>(),398+ PipeBarrier<PIPE_V>();
355- (half)-1, curTileLen);399+ Cast(xCanon[tileLen_], xFp32, RoundMode::CAST_RINT, curTileLen);
356- } else {400+ } else if constexpr (IsSameType<T, half>::value) {
357- Muls(xNgt[tileLen_], xNgt[tileLen_], (T)-1, curTileLen);401+ // fp16:原生同类型 Muls(x, 1)。实测:与「Cast→fp32→Muls(+1)→Cast 回」对
358- }402+ // ±NaN/±Inf/±0/次正规码等全部边界输入逐位一致(±NaN 都规范化成 0x7FFF)。
403+ // 省掉 2 条向量指令与 2 次 PipeBarrier,也不再需要 fp32 暂存。
404+ Muls(xCanon[tileLen_], xCanon[tileLen_], (half)1, curTileLen);
359 }405 }
360}406}
361 407 
@@ -407,6 +453,23 @@ __aicore__ inline uint32_t RadixTopKBaseKernel<T, largest>::CreateVecIndex4TopK(
407 return maxIndexLen;453 return maxIndexLen;
408}454}
409 455 
456+/**
457+ * @brief 单个 tile 内按 cmpMode 提取匹配 boundaryNumber 的元素并输出
458+ */
459+template <typename T, bool largest>
460+template <typename CT>
461+__aicore__ inline bool RadixTopKBaseKernel<T, largest>::IsNaNValue(const CT& value)
462+{
463+ float valueFp32;
464+ if constexpr (IsSameType<CT, float>::value) {
465+ valueFp32 = value;
466+ } else {
467+ valueFp32 = static_cast<float>(value); // half -> fp32,NaN 语义保持
468+ }
469+ uint32_t bits = *reinterpret_cast<uint32_t*>(&valueFp32);
470+ return (bits & 0x7FFFFFFFU) > 0x7F800000U; // 指数位全 1 且尾数非 0 ⇒ NaN
471+}
472+ 
410/**473/**
411 * @brief 单个 tile 内按 cmpMode 提取匹配 boundaryNumber 的元素并输出474 * @brief 单个 tile 内按 cmpMode 提取匹配 boundaryNumber 的元素并输出
412 * 先 Compares → GatherMask values/indices → CopyOutResult475 * 先 Compares → GatherMask values/indices → CopyOutResult
@@ -438,7 +501,37 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::SubTopKAndCopyOut(
438 uint64_t curTileLenAlign = Ops::Base::CeilAlign(curTileLen, REPEAT_SIZE / sizeof(CT));501 uint64_t curTileLenAlign = Ops::Base::CeilAlign(curTileLen, REPEAT_SIZE / sizeof(CT));
439 502 
440 PipeBarrier<PIPE_V>();503 PipeBarrier<PIPE_V>();
441- Compares<CT, uint8_t>(cmpMaskTensor, xLocal, boundaryNumber, cmpMode, curTileLenAlign);504+ if constexpr (largest && cmpMode != AscendC::CMPMODE::EQ &&
505+ (IsSameType<CT, float>::value || IsSameType<CT, half>::value)) {
506+ if (IsNaNValue(boundaryNumber)) {
507+ // 边界值本身就是 NaN:候选入选集合只有 NaN(key 域 0xFFFF 组,计数相已把 ±NaN 归入同一
508+ // bin 并按 NaN 个数分配配额)。此时上式的 NOT(LE) 会退化成「全真」——x <= NaN 对**有限值
509+ // 同样为 false**,取反后掩码全 1,GatherMask 于是把 tile 前 curTileK 个(有限的)元素当成
510+ // 入选者拷出;实测混合数据 + largest=true 时表现为 top-k 里 NaN 只剩零星几个(1015 个 NaN
511+ // 只选中 3 个,其余槽被背景元素顶替)。
512+ // 改取 isNaN 掩码:NaN == NaN 恒为 false,Compare(x, x, EQ) 逐位取反即 isNaN,
513+ // 对 ±NaN 与任意 payload 都成立,与计数相的配额口径完全一致。
514+ Compare<CT, uint8_t>(cmpMaskTensor, xLocal, xLocal, AscendC::CMPMODE::EQ, curTileLenAlign);
515+ PipeBarrier<PIPE_V>();
516+ Not(cmpMaskTensor.template ReinterpretCast<int16_t>(), cmpMaskTensor.template ReinterpretCast<int16_t>(),
517+ static_cast<uint32_t>((curTileLenAlign >> 3) / sizeof(int16_t)));
518+ } else {
519+ // largest=true 的「必然入选」通过:NaN 的序键在 twiddle 域位于最大端(key ∈ [0xFF81, 0xFFFF]),
520+ // 直方图按 key 统计时已把 NaN 计入该 tile 的配额,但 x > b 对 NaN 恒为 false,
521+ // 于是 NaN 不被拷出、配额反被背景元素顶替(S1:被顶替槽数精确等于注入的 NaN 数)。
522+ // 这里不直接 Compares(GT),而取 NOT(LE):LE 对 NaN 恒为 false,逐位取反即为 true,
523+ // 掩码 = (x > b) ∪ isNaN,与 key 域的配额统计口径一致(逐位运算,全值域有效)。
524+ // largest=false 不加:key 域中 +NaN 不计入入选,且 torch 语义要求 NaN 不入选;
525+ // 整型 CT 无 NaN,走下面的直通分支。
526+ Compares<CT, uint8_t>(cmpMaskTensor, xLocal, boundaryNumber, AscendC::CMPMODE::LE, curTileLenAlign);
527+ PipeBarrier<PIPE_V>();
528+ Not(cmpMaskTensor.ReinterpretCast<int16_t>(), cmpMaskTensor.ReinterpretCast<int16_t>(),
529+ static_cast<uint32_t>((curTileLenAlign >> 3) / sizeof(int16_t)));
530+ }
531+ } else {
532+ PipeBarrier<PIPE_V>();
533+ Compares<CT, uint8_t>(cmpMaskTensor, xLocal, boundaryNumber, cmpMode, curTileLenAlign);
534+ }
442 uint64_t rsvdCntV = 0, rsvdCntI = 0;535 uint64_t rsvdCntV = 0, rsvdCntI = 0;
443 WaitFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForValue_);536 WaitFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForValue_);
444 PipeBarrier<PIPE_V>();537 PipeBarrier<PIPE_V>();
@@ -467,7 +560,11 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::SubTopKAndCopyOut(
467 {1, 1, 0, 0}, rsvdCntI);560 {1, 1, 0, 0}, rsvdCntI);
468 }561 }
469 VToSSync();562 VToSSync();
470- uint32_t count = (cmpMode != AscendC::CMPMODE::EQ) ? rsvdCntV : curTileK;563+ // 非 EQ 通过按 tile 配额封顶:NaN 掩码并入后 rsvdCntV 可能略多于直方图统计出的配额
564+ // (例如混入 key 域未计入的 -NaN),封顶可避免拷出到 k 槽之外的越界写
565+ uint64_t passRsvdCnt = (cmpMode != AscendC::CMPMODE::EQ) ? rsvdCntV : static_cast<uint64_t>(curTileK);
566+ uint32_t count = static_cast<uint32_t>(
567+ passRsvdCnt < static_cast<uint64_t>(curTileK) ? passRsvdCnt : static_cast<uint64_t>(curTileK));
471 SToVSync();568 SToVSync();
472 if constexpr (IsSameType<T, bfloat16_t>::value) {569 if constexpr (IsSameType<T, bfloat16_t>::value) {
473 PipeBarrier<PIPE_V>();570 PipeBarrier<PIPE_V>();
@@ -493,12 +590,16 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::CopyOutResult(const int3
493{590{
494 LocalTensor<T> outValueLocal = outValueBuf_.Get<T>();591 LocalTensor<T> outValueLocal = outValueBuf_.Get<T>();
495 LocalTensor<int32_t> outIndexLocal = outIndexBuf_.Get<int32_t>();592 LocalTensor<int32_t> outIndexLocal = outIndexBuf_.Get<int32_t>();
496- 593+ if (count > 0) {
497- DataCopyPad(valueGm_[coreTopKOffset], outValueLocal,594+ DataCopyPad(valueGm_[coreTopKOffset], outValueLocal,
498- DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(T) * count), 0, 0, 0});595+ DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(T) * count), 0, 0, 0});
596+ }
499 SetFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForValue_);597 SetFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForValue_);
500- DataCopyPad(indexGm_[coreTopKOffset], outIndexLocal,598+ if (count > 0) {
501- DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * count), 0, 0, 0});599+ DataCopyPad(
600+ indexGm_[coreTopKOffset], outIndexLocal,
601+ DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * count), 0, 0, 0});
602+ }
502 SetFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForIndex_);603 SetFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForIndex_);
503}604}
504 605 
@@ -22,23 +22,23 @@
22 * 包含数据切分、核分配、tile 参数等编译期确定的常量信息22 * 包含数据切分、核分配、tile 参数等编译期确定的常量信息
23 */23 */
24struct RadixTopKTilingData {24struct RadixTopKTilingData {
25- uint64_t coreNum; /**< 总 core 数 */25+ uint64_t coreNum; /**< 总 core 数 */
26- uint64_t formerCoreNum; /**< 主核数量(处理 formerTileNum 个 tile) */26+ uint64_t formerCoreNum; /**< 主核数量(处理 formerTileNum 个 tile) */
27- uint64_t tailCoreNum; /**< 尾核数量(处理 tailTileNum 个 tile) */27+ uint64_t tailCoreNum; /**< 尾核数量(处理 tailTileNum 个 tile) */
28- uint64_t totalTileNum; /**< 总 tile 数 */28+ uint64_t totalTileNum; /**< 总 tile 数 */
29- uint64_t formerTileNum; /**< 主核 tile 数 */29+ uint64_t formerTileNum; /**< 主核 tile 数 */
30- uint64_t tailTileNum; /**< 尾核 tile 数 */30+ uint64_t tailTileNum; /**< 尾核 tile 数 */
31- uint64_t formerTileLen; /**< 每个 tile 的数据长度(主核/尾核常规tile) */31+ uint64_t formerTileLen; /**< 每个 tile 的数据长度(主核/尾核常规tile) */
32- uint64_t tailTileLen; /**< 最后一个 tail tile 的数据长度 */32+ uint64_t tailTileLen; /**< 最后一个 tail tile 的数据长度 */
33- uint64_t batch; /**< batch 数(除最后一维外各维度乘积) */33+ uint64_t batch; /**< batch 数(除最后一维外各维度乘积) */
34 uint64_t sortLen; /**< 排序轴长度(输入最后一维长度) */34 uint64_t sortLen; /**< 排序轴长度(输入最后一维长度) */
35- uint64_t kValue; /**< topK 值 */35+ uint64_t kValue; /**< topK 值 */
36 36 
37- bool largest; /**< 是否求最大 k 值(true: largest, false: smallest) */37+ bool largest; /**< 是否求最大 k 值(true: largest, false: smallest) */
38- bool sorted; /**< 输出结果是否排序(当前仅支持 false) */38+ bool sorted; /**< 输出结果是否排序(当前仅支持 false) */
39- bool needWorkspace; /**< 是否使用 workspace(false: 复用 indices 内存做暂存) */39+ bool needWorkspace; /**< 是否使用 workspace(false: 复用 indices 内存做暂存) */
40};40};
41 41 
42#pragma pack(pop)42#pragma pack(pop)
43 43 
44-#endif44+#endif
@@ -24,23 +24,18 @@
24 * sorted: 输出是否排序(0/1)24 * sorted: 输出是否排序(0/1)
25 * largest: 是否求最大 k 值(0/1)25 * largest: 是否求最大 k 值(0/1)
26 * isLargeShape: 是否大 shape 场景(0/1),用于选择 UB/WS 变体26 * isLargeShape: 是否大 shape 场景(0/1),用于选择 UB/WS 变体
27+ * isInt64Shape: sortLen 超过 int32 上限(0/1),选择 int64 变体(追加在最高位,
28+ * 现有 3 参数 key 编码值不变)
27 */29 */
28-ASCENDC_TPL_ARGS_DECL(RadixTopK,30+ASCENDC_TPL_ARGS_DECL(RadixTopK, ASCENDC_TPL_BOOL_DECL(sorted, 0, 1), ASCENDC_TPL_BOOL_DECL(largest, 0, 1),
29- ASCENDC_TPL_BOOL_DECL(sorted, 0, 1),31+ ASCENDC_TPL_BOOL_DECL(isLargeShape, 0, 1), ASCENDC_TPL_BOOL_DECL(isInt64Shape, 0, 1));
30- ASCENDC_TPL_BOOL_DECL(largest, 0, 1),
31- ASCENDC_TPL_BOOL_DECL(isLargeShape, 0, 1)
32-);
33 32 
34/**33/**
35 * @brief 模板参数组合选择34 * @brief 模板参数组合选择
36 * 用于 GET_TPL_TILING_KEY 获取 TilingKey 时校验 TilingKey 合法性35 * 用于 GET_TPL_TILING_KEY 获取 TilingKey 时校验 TilingKey 合法性
37 */36 */
38-ASCENDC_TPL_SEL(37+ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_BOOL_SEL(sorted, 0, 1), ASCENDC_TPL_BOOL_SEL(largest, 0, 1),
39- ASCENDC_TPL_ARGS_SEL(38+ ASCENDC_TPL_BOOL_SEL(isLargeShape, 0, 1),
40- ASCENDC_TPL_BOOL_SEL(sorted, 0, 1),39+ ASCENDC_TPL_BOOL_SEL(isInt64Shape, 0, 1)), );
41- ASCENDC_TPL_BOOL_SEL(largest, 0, 1),
42- ASCENDC_TPL_BOOL_SEL(isLargeShape, 0, 1)
43- ),
44-);
45 40 
46#endif // RADIX_TOP_K_TILING_KEY_H41#endif // RADIX_TOP_K_TILING_KEY_H
@@ -73,7 +73,7 @@ private:
73template <typename T, bool largest>73template <typename T, bool largest>
74__aicore__ inline void RadixTopKUb<T, largest>::Process()74__aicore__ inline void RadixTopKUb<T, largest>::Process()
75{75{
76- for (int32_t batchId = 0; batchId < this->batch_; batchId++) {76+ for (uint64_t batchId = 0; batchId < this->batch_; batchId++) {
77 SubProcess(batchId);77 SubProcess(batchId);
78 SyncAll();78 SyncAll();
79 }79 }
@@ -183,7 +183,7 @@ __aicore__ inline void RadixTopKUb<T, largest>::SubProcess(uint64_t batchId)
183 for (int32_t tileId = 0; tileId < this->tileNum_; tileId++) {183 for (int32_t tileId = 0; tileId < this->tileNum_; tileId++) {
184 PINGPONG_TILE_BEGIN(tileId, blockOffset);184 PINGPONG_TILE_BEGIN(tileId, blockOffset);
185 185 
186- this->NegateDataForLargest(curBuf, curTileLen);186+ this->CanonicalizeDataForLargest(curBuf, curTileLen);
187 this->TwiddleInB16(curBuf, curTileLen);187 this->TwiddleInB16(curBuf, curTileLen);
188 this->DoAndMask(curBuf, curTileLen);188 this->DoAndMask(curBuf, curTileLen);
189 LocalTensor<int32_t> tileHist = tempBuf_.Get<int32_t>();189 LocalTensor<int32_t> tileHist = tempBuf_.Get<int32_t>();
@@ -294,6 +294,8 @@ __aicore__ inline void RadixTopKUb<T, largest>::HandleLastRoundBoundary(LocalTen
294 LocalTensor<float>& tileHistFp32)294 LocalTensor<float>& tileHistFp32)
295{295{
296 PipeBarrier<PIPE_V>();296 PipeBarrier<PIPE_V>();
297+ // 覆盖两个分支:ReduceGlobalHist 末尾的 Scalar SetValue 写过 resTensor,
298+ // 后续 V pipe 的 Sub(写 resTensor) 与 else 分支 Cast(写 tileHistFp32=resTensor) 均需 S->V 可见性
297 SToVSync();299 SToVSync();
298 if (this->boundaryBinPrev < this->numValue_) {300 if (this->boundaryBinPrev < this->numValue_) {
299 Sub(resTensor, tileHist[this->boundaryBin * this->tileNumAlign_],301 Sub(resTensor, tileHist[this->boundaryBin * this->tileNumAlign_],
@@ -387,6 +389,13 @@ __aicore__ inline void RadixTopKUb<T, largest>::TileTopK(const uint64_t& batchId
387 uint64_t firstLen = (firstId == this->tileNum_ - 1) ? this->tailTileLen_ : this->tileLen_;389 uint64_t firstLen = (firstId == this->tileNum_ - 1) ? this->tailTileLen_ : this->tileLen_;
388 this->CopyIn(this->xBufPing_, firstLen, blockOffset + firstId * this->tileLen_);390 this->CopyIn(this->xBufPing_, firstLen, blockOffset + firstId * this->tileLen_);
389 uint32_t maxIndexLen = CreateVecIndex4TopK();391 uint32_t maxIndexLen = CreateVecIndex4TopK();
392+ // maxIndexLen == 0 表示 tempBuf_ 剩余空间放不下 512 个预生成索引,此时
393+ // SubTopKAndCopyOut 会走「现算索引」兜底分支,vecIndex 不会被使用。
394+ // 所以必须判空后再取句柄,0 时用 buffer 首地址占位。
395+ LocalTensor<int32_t> vecIndexTopK = maxIndexLen > 0 ?
396+ tempBuf_.GetWithOffset<int32_t>(
397+ maxIndexLen, this->tileLen_ * sizeof(uint8_t) / BYTE_SIZE) :
398+ tempBuf_.Get<int32_t>(); // 占位句柄,0 时不会被使用
390 399 
391 this->InitTopKEvent();400 this->InitTopKEvent();
392 401 
@@ -398,8 +407,7 @@ __aicore__ inline void RadixTopKUb<T, largest>::TileTopK(const uint64_t& batchId
398 407 
399 int32_t curTileK = tileTopK.GetValue(tileId);408 int32_t curTileK = tileTopK.GetValue(tileId);
400 PipeBarrier<PIPE_V>();409 PipeBarrier<PIPE_V>();
401- PROCESS_ONE_TILE_TOPK_OR_SKIP(410+ PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileK, vecIndexTopK);
402- curTileK, tempBuf_.GetWithOffset<int32_t>(maxIndexLen, this->tileLen_ * sizeof(uint8_t) / BYTE_SIZE));
403 411 
404 PINGPONG_TILE_END(tileId);412 PINGPONG_TILE_END(tileId);
405 }413 }
@@ -30,36 +30,20 @@ public:
30 30 
31 __aicore__ inline void Init(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace)31 __aicore__ inline void Init(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace)
32 {32 {
33- InitParams();33+ InitWsGm(x, k, values, indices, workspace);
34- this->xGm_.SetGlobalBuffer((__gm__ T*)x);34+ InitBuffers(sizeof(int32_t)); // 索引输出 int32;int64 大 shape 子类改传 sizeof(int64_t)
35- this->valueGm_.SetGlobalBuffer((__gm__ T*)values);
36- this->kGm_.SetGlobalBuffer((__gm__ int32_t*)k);
37- this->indexGm_.SetGlobalBuffer((__gm__ int32_t*)indices);
38- this->globalHistGm_.SetGlobalBuffer((__gm__ int32_t*)workspace);
39- this->boundaryBinCumSumGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_);
40- this->coreTopKGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ +
41- this->coreNum_);
42- uint32_t tileTopKOffset = 0;
43- if (this->blockIdx_ < this->formerCoreNum_) {
44- tileTopKOffset = this->blockIdx_ * this->formerTileNum_;
45- } else {
46- tileTopKOffset = this->formerCoreNum_ * this->formerTileNum_ +
47- (this->blockIdx_ - this->formerCoreNum_) * this->tailTileNum_;
48- }
49- uint32_t tileHistOffset = this->numValue_ * tileTopKOffset;
50- tileTopKGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_ * 2 +
51- tileTopKOffset);
52- tileHistGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_ * 2 +
53- this->totalTileNum_ + tileHistOffset);
54- InitBuffers();
55 }35 }
56 36 
57 __aicore__ inline void SubProcess(uint64_t batchId);37 __aicore__ inline void SubProcess(uint64_t batchId);
58 __aicore__ inline void Process();38 __aicore__ inline void Process();
59 39 
60-private:40+ // int64 大 shape 子类 RadixTopKWsI64 复用以下成员与 WS 搬运工具(仅访问级别,无逻辑变化)
41+protected:
42+ /** @brief 初始化 GM/WS 布局(含 indexGm_ 的 int32 视图);int64 子类复用后再补设 indexGm64_ */
43+ __aicore__ inline void InitWsGm(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace);
61 __aicore__ inline void InitParams();44 __aicore__ inline void InitParams();
62- __aicore__ inline void InitBuffers();45+ /** @brief 形参为索引输出元素字节数:int32 变体 sizeof(int32_t)、int64 变体 sizeof(int64_t) */
46+ __aicore__ inline void InitBuffers(const uint32_t& indexElemSize);
63 __aicore__ inline void ClearTileTopKInWs();47 __aicore__ inline void ClearTileTopKInWs();
64 __aicore__ inline void ClearHistInWs(const int32_t& roundId);48 __aicore__ inline void ClearHistInWs(const int32_t& roundId);
65 __aicore__ inline void ClearTileHistInWs(const uint32_t& dstOffset, const uint32_t& dataNum);49 __aicore__ inline void ClearTileHistInWs(const uint32_t& dstOffset, const uint32_t& dataNum);
@@ -86,6 +70,8 @@ private:
86 __aicore__ inline void AddTileHistToTileTopK(LocalTensor<int32_t>& globalHist);70 __aicore__ inline void AddTileHistToTileTopK(LocalTensor<int32_t>& globalHist);
87 __aicore__ inline void HandleLastRoundBoundary(LocalTensor<int32_t>& resTensor, LocalTensor<float>& tileHistFp32);71 __aicore__ inline void HandleLastRoundBoundary(LocalTensor<int32_t>& resTensor, LocalTensor<float>& tileHistFp32);
88 __aicore__ inline void WriteCoreTopKFromWs(LocalTensor<int32_t>& resTensor, LocalTensor<float>& tileHistFp32);72 __aicore__ inline void WriteCoreTopKFromWs(LocalTensor<int32_t>& resTensor, LocalTensor<float>& tileHistFp32);
73+ /** @brief 前驱 core 边界 bin 累计数(int64 累加,总量可达 sortLen > 2^31;int32 形状下同样安全) */
74+ __aicore__ inline void ComputeCumSumPrevI64(LocalTensor<int32_t>& resTensor, int64_t& cumSumValuePrev);
89 75 
90 GlobalTensor<int32_t> tileTopKGm_; /**< WS 中 tileTopK 段的 Global Tensor */76 GlobalTensor<int32_t> tileTopKGm_; /**< WS 中 tileTopK 段的 Global Tensor */
91 GlobalTensor<int32_t> tileHistGm_; /**< WS 中 tileHist 段的 Global Tensor */77 GlobalTensor<int32_t> tileHistGm_; /**< WS 中 tileHist 段的 Global Tensor */
@@ -107,13 +93,43 @@ private:
107template <typename T, bool largest>93template <typename T, bool largest>
108__aicore__ inline void RadixTopKWs<T, largest>::Process()94__aicore__ inline void RadixTopKWs<T, largest>::Process()
109{95{
110- for (int32_t batchId = 0; batchId < this->batch_; batchId++) {96+ for (uint64_t batchId = 0; batchId < this->batch_; batchId++) {
111 SubProcess(batchId);97 SubProcess(batchId);
112 SyncAll();98 SyncAll();
113 }99 }
114 GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(this->eventIDMTE2ToVForX_);100 GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(this->eventIDMTE2ToVForX_);
115}101}
116 102 
103+/**
104+ * @brief 初始化 GM/WS 布局(含 indexGm_ 的 int32 视图)
105+ * int64 大 shape 子类 RadixTopKWsI64 复用本函数后再补设 indexGm64_,不重复整段布局代码
106+ */
107+template <typename T, bool largest>
108+__aicore__ inline void RadixTopKWs<T, largest>::InitWsGm(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices,
109+ GM_ADDR workspace)
110+{
111+ InitParams();
112+ this->xGm_.SetGlobalBuffer((__gm__ T*)x);
113+ this->valueGm_.SetGlobalBuffer((__gm__ T*)values);
114+ this->kGm_.SetGlobalBuffer((__gm__ int32_t*)k);
115+ this->indexGm_.SetGlobalBuffer((__gm__ int32_t*)indices);
116+ this->globalHistGm_.SetGlobalBuffer((__gm__ int32_t*)workspace);
117+ this->boundaryBinCumSumGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_);
118+ this->coreTopKGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_);
119+ uint32_t tileTopKOffset = 0;
120+ if (this->blockIdx_ < this->formerCoreNum_) {
121+ tileTopKOffset = this->blockIdx_ * this->formerTileNum_;
122+ } else {
123+ tileTopKOffset = this->formerCoreNum_ * this->formerTileNum_ +
124+ (this->blockIdx_ - this->formerCoreNum_) * this->tailTileNum_;
125+ }
126+ uint32_t tileHistOffset = this->numValue_ * tileTopKOffset;
127+ tileTopKGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_ * 2 +
128+ tileTopKOffset);
129+ tileHistGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_ * 2 +
130+ this->totalTileNum_ + tileHistOffset);
131+}
132+ 
117/**133/**
118 * @brief 初始化基本参数(调用基类)+ WS 变体特有参数(tileNum 分段次数)134 * @brief 初始化基本参数(调用基类)+ WS 变体特有参数(tileNum 分段次数)
119 */135 */
@@ -132,9 +148,10 @@ __aicore__ inline void RadixTopKWs<T, largest>::InitParams()
132/**148/**
133 * @brief 初始化 UB 缓冲区:xBuf Ping-Pong、tileHistAndTopKBuf、tempBuf、outBuf、globalHistBuf149 * @brief 初始化 UB 缓冲区:xBuf Ping-Pong、tileHistAndTopKBuf、tempBuf、outBuf、globalHistBuf
134 * WS 变体中 tileHist/tileTopK 存 GM,UB 只用作分段读写缓冲150 * WS 变体中 tileHist/tileTopK 存 GM,UB 只用作分段读写缓冲
151+ * @param indexElemSize 索引输出元素字节数(int32 变体 4、int64 变体 8)
135 */152 */
136template <typename T, bool largest>153template <typename T, bool largest>
137-__aicore__ inline void RadixTopKWs<T, largest>::InitBuffers()154+__aicore__ inline void RadixTopKWs<T, largest>::InitBuffers(const uint32_t& indexElemSize)
138{155{
139 uint32_t cmpMaskSize = this->tileLen_ * sizeof(uint8_t) / BYTE_SIZE;156 uint32_t cmpMaskSize = this->tileLen_ * sizeof(uint8_t) / BYTE_SIZE;
140 this->tpipe_.InitBuffer(this->xBufPing_, this->tileLen_ * sizeof(float));157 this->tpipe_.InitBuffer(this->xBufPing_, this->tileLen_ * sizeof(float));
@@ -142,7 +159,7 @@ __aicore__ inline void RadixTopKWs<T, largest>::InitBuffers()
142 this->tpipe_.InitBuffer(tileHistAndTopKBuf_, MAX_TILE_NUM_IN_UB * sizeof(int32_t));159 this->tpipe_.InitBuffer(tileHistAndTopKBuf_, MAX_TILE_NUM_IN_UB * sizeof(int32_t));
143 this->tpipe_.InitBuffer(tempBuf_, cmpMaskSize);160 this->tpipe_.InitBuffer(tempBuf_, cmpMaskSize);
144 this->tpipe_.InitBuffer(this->outValueBuf_, this->tileLen_ * sizeof(float));161 this->tpipe_.InitBuffer(this->outValueBuf_, this->tileLen_ * sizeof(float));
145- this->tpipe_.InitBuffer(this->outIndexBuf_, this->tileLen_ * sizeof(int32_t));162+ this->tpipe_.InitBuffer(this->outIndexBuf_, this->tileLen_ * indexElemSize);
146 this->tpipe_.InitBuffer(this->globalHistBuf_, this->numValue_ * sizeof(int32_t));163 this->tpipe_.InitBuffer(this->globalHistBuf_, this->numValue_ * sizeof(int32_t));
147}164}
148 165 
@@ -181,7 +198,7 @@ __aicore__ inline void RadixTopKWs<T, largest>::SubProcess(uint64_t batchId)
181 for (int32_t tileId = startTileId; tileId < endTileId; tileId++) {198 for (int32_t tileId = startTileId; tileId < endTileId; tileId++) {
182 PINGPONG_TILE_BEGIN(tileId, blockOffset);199 PINGPONG_TILE_BEGIN(tileId, blockOffset);
183 200 
184- this->NegateDataForLargest(curBuf, curTileLen);201+ this->CanonicalizeDataForLargest(curBuf, curTileLen);
185 this->TwiddleInB16(curBuf, curTileLen);202 this->TwiddleInB16(curBuf, curTileLen);
186 this->DoAndMask(curBuf, curTileLen);203 this->DoAndMask(curBuf, curTileLen);
187 LocalTensor<int32_t> tileHist = tileHistAndTopKBuf_.Get<int32_t>();204 LocalTensor<int32_t> tileHist = tileHistAndTopKBuf_.Get<int32_t>();
@@ -206,6 +223,8 @@ __aicore__ inline void RadixTopKWs<T, largest>::SubProcess(uint64_t batchId)
206 this->andMask16_ >>= BITS_PER_ROUND;223 this->andMask16_ >>= BITS_PER_ROUND;
207 if (Update(roundId))224 if (Update(roundId))
208 break;225 break;
226+ // 与 ub 模板对齐:确保所有核完成 Update 内 CopyInGlobalHist 对 globalHistGm_ 的读取,
227+ // 再允许快核进入下一轮 CopyOut2Ws<true> 覆盖
209 SyncAll();228 SyncAll();
210 }229 }
211 SyncAll();230 SyncAll();
@@ -261,8 +280,13 @@ __aicore__ inline void RadixTopKWs<T, largest>::AddTileHistToTileTopK(LocalTenso
261}280}
262 281 
263/**282/**
264- * @brief WS 变体:处理最后一轮边界 bin 的元素分配283+ * @brief 处理最后一轮边界 bin 的元素分配:从 WS 读取各 tile 的边界 bin / 前驱 bin 计数,
265- * 从 WS 读取各 tile 边界 bin 数据,计算每个 core 的元素分配284+ * 按 (core, tile) 顺序把剩余配额 remainK_ 分配到各 tile 的 tileTopK
285+ *
286+ * 计数与配额统一用 int64 标量(跨核总量可达 sortLen > 2^31;int32 形状下同样安全),
287+ * 分配按 MAX_TILE_NUM_IN_UB_BY3 分段:段内先求差分总和 segSum,若剩余配额能吞下整段
288+ * 则按元素向量 Add 给段内各 tile 加各自差分、跳过逐 tile 标量扫描,只在配额耗尽的那个
289+ * 段内退化为逐 tile 标量扫描。两条变体共用本实现(int64 变体不再单独 override)。
266 * @param resTensor 临时 Tensor290 * @param resTensor 临时 Tensor
267 * @param tileHistFp32 tileHist 的 fp32 视图291 * @param tileHistFp32 tileHist 的 fp32 视图
268 */292 */
@@ -270,7 +294,9 @@ template <typename T, bool largest>
270__aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTensor<int32_t>& resTensor,294__aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTensor<int32_t>& resTensor,
271 LocalTensor<float>& tileHistFp32)295 LocalTensor<float>& tileHistFp32)
272{296{
273- uint32_t reduceSumValue = 0;297+ int64_t reduceSumValue = 0;
298+ // 首个迭代前,ReduceGlobalHist 末尾用 Scalar SetValue 写过 resTensor[0..3],
299+ // 需保证 S 写对后续 MTE2 覆写 resTensor 可见(与 ub 模板 HandleLastRoundBoundary 的 SToVSync 对齐)
274 SToMTE2Sync();300 SToMTE2Sync();
275 for (int32_t i = 0; i <= tileNumBy2RepeatTimes_; i++) {301 for (int32_t i = 0; i <= tileNumBy2RepeatTimes_; i++) {
276 uint32_t dataNum = i == tileNumBy2RepeatTimes_ ? tileNumBy2Remain_ : MAX_TILE_NUM_IN_UB_BY2;302 uint32_t dataNum = i == tileNumBy2RepeatTimes_ ? tileNumBy2Remain_ : MAX_TILE_NUM_IN_UB_BY2;
@@ -295,17 +321,19 @@ __aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTen
295 SToVSync();321 SToVSync();
296 ReduceSum(tileHistFp32, tileHistFp32[segOff], tileHistFp32[segOff], curSegLen);322 ReduceSum(tileHistFp32, tileHistFp32[segOff], tileHistFp32[segOff], curSegLen);
297 VToSSync();323 VToSSync();
298- reduceSumValue += static_cast<int32_t>(tileHistFp32.GetValue(0));324+ // 分段和 <= 2^24 精确;累加器 int64(跨段总量 = per-core 边界 bin 元素数)
325+ reduceSumValue += static_cast<int64_t>(tileHistFp32.GetValue(0));
299 }326 }
300 }327 }
301- resTensor.SetValue(0, reduceSumValue);328+ // per-core 边界 bin 计数 < 2^31(per-core 元素数上限),int32 写回安全
329+ resTensor.SetValue(0, static_cast<int32_t>(reduceSumValue));
302 SToMTE3Sync();330 SToMTE3Sync();
303 this->CopyOutBoundaryBinCumSum(resTensor);331 this->CopyOutBoundaryBinCumSum(resTensor);
304 SyncAll();332 SyncAll();
305 333 
306- int32_t cumSumValuePrev = 0;334+ int64_t cumSumValuePrev = 0;
307- this->ComputeCumSumPrev(resTensor, cumSumValuePrev);335+ ComputeCumSumPrevI64(resTensor, cumSumValuePrev);
308- int32_t remainCoreBoundaryNum = this->remainK_ - cumSumValuePrev;336+ int64_t remainCoreBoundaryNum = static_cast<int64_t>(this->remainK_) - cumSumValuePrev;
309 if (remainCoreBoundaryNum > 0) {337 if (remainCoreBoundaryNum > 0) {
310 LocalTensor<int32_t> tileTopK = tileHistAndTopKBuf_.Get<int32_t>();338 LocalTensor<int32_t> tileTopK = tileHistAndTopKBuf_.Get<int32_t>();
311 LocalTensor<int32_t> tileHistBoundary = tileTopK[MAX_TILE_NUM_IN_UB_BY3];339 LocalTensor<int32_t> tileHistBoundary = tileTopK[MAX_TILE_NUM_IN_UB_BY3];
@@ -320,19 +348,61 @@ __aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTen
320 CopyTileHistWs2Ub(tileHistBoundary, this->boundaryBin * this->tileNum_ + startTileId, dataNum);348 CopyTileHistWs2Ub(tileHistBoundary, this->boundaryBin * this->tileNum_ + startTileId, dataNum);
321 CopyTileHistWs2Ub(tileHistBoundaryPrev, this->boundaryBinPrev * this->tileNum_ + startTileId, dataNum);349 CopyTileHistWs2Ub(tileHistBoundaryPrev, this->boundaryBinPrev * this->tileNum_ + startTileId, dataNum);
322 CopyTileTopKWs2Ub(tileTopK, startTileId, dataNum);350 CopyTileTopKWs2Ub(tileTopK, startTileId, dataNum);
323- MTE2ToSSync();351+ MTE2ToVSync();
324- for (int32_t tileId = startTileId; tileId < endTileId; tileId++) {352+ // 段级定位:先算本段边界 bin 差分总和 segSum(向量),若剩余配额能吞下整段,
325- int32_t curTileBoundaryNum = tileHistBoundary.GetValue(tileId - startTileId);353+ // 则按元素向量 Add 给段内各 tile 加各自差分(tileTopK[i] += diff[i]),
326- if (this->boundaryBinPrev < this->numValue_) {354+ // 跳过逐 tile 标量扫描——标量扫描只在配额耗尽的那个段内进行(≤2048 次后必然 break)。
327- curTileBoundaryNum -= tileHistBoundaryPrev.GetValue(tileId - startTileId);355+ // 注意:整段吞下不是加常量 segSum(那会给每个 tile 超发全段总和的配额,
356+ // 导致 curTileK 超过 tile 实际 EQ 元素数、拷出陈旧 lane——NaN/-1 输出的根源)
357+ if (this->boundaryBinPrev < this->numValue_) {
358+ Sub(tileHistBoundary, tileHistBoundary, tileHistBoundaryPrev, dataNum);
359+ PipeBarrier<PIPE_V>();
360+ }
361+ int32_t int32Align = BLOCK_SIZE / sizeof(int32_t);
362+ int32_t safeSumTileNum = FLOAT32_SAFE_INT / this->tileLen_ / int32Align * int32Align;
363+ // fp32 视图借用已被 Sub 消费的 prev 区域,保持 tileHistBoundary 的 int32 差分
364+ // 不被 Cast 覆写(整段 Add 与段内标量扫描都还要复用它)
365+ LocalTensor<float> segFp32 = tileHistBoundaryPrev.template ReinterpretCast<float>();
366+ PipeBarrier<PIPE_V>();
367+ Cast(segFp32, tileHistBoundary, RoundMode::CAST_NONE, dataNum);
368+ PipeBarrier<PIPE_V>();
369+ int64_t segSum = 0;
370+ int32_t segSize = dataNum < static_cast<uint32_t>(safeSumTileNum) ? static_cast<int32_t>(dataNum) :
371+ safeSumTileNum;
372+ for (int32_t segOff = 0; segOff < static_cast<int32_t>(dataNum); segOff += segSize) {
373+ int32_t curSegLen = AscendC::Std::min(segSize, static_cast<int32_t>(dataNum - segOff));
374+ SToVSync();
375+ ReduceSum(segFp32, segFp32[segOff], segFp32[segOff], curSegLen);
376+ VToSSync();
377+ segSum += static_cast<int64_t>(segFp32.GetValue(0));
378+ }
379+ if (remainCoreBoundaryNum >= segSum) {
380+ // 整段吞下:tileTopK += diff 向量按元素加(int32 Add)
381+ SToVSync();
382+ PipeBarrier<PIPE_V>();
383+ Add(tileTopK, tileTopK, tileHistBoundary, dataNum);
384+ PipeBarrier<PIPE_V>();
385+ remainCoreBoundaryNum -= segSum;
386+ SToMTE3Sync();
387+ CopyTileTopKUb2Ws(tileTopK, startTileId, dataNum);
388+ MTE3ToMTE2Sync();
389+ if (remainCoreBoundaryNum == 0) {
390+ break;
328 }391 }
392+ continue;
393+ }
394+ // 段内部分配:逐 tile 标量扫描(本段必是配额耗尽段;差分已在 tileHistBoundary 内,
395+ // 标量侧无需再减 prev——prev 区域已被 segFp32 借用覆写)
396+ MTE2ToSSync();
397+ for (int32_t tileId = startTileId; tileId < static_cast<int32_t>(endTileId); tileId++) {
398+ int64_t curTileBoundaryNum = tileHistBoundary.GetValue(tileId - startTileId);
329 if (curTileBoundaryNum < remainCoreBoundaryNum) {399 if (curTileBoundaryNum < remainCoreBoundaryNum) {
330 remainCoreBoundaryNum -= curTileBoundaryNum;400 remainCoreBoundaryNum -= curTileBoundaryNum;
331- tileTopK.SetValue(tileId - startTileId,401+ tileTopK.SetValue(tileId - startTileId, tileTopK.GetValue(tileId - startTileId) +
332- tileTopK.GetValue(tileId - startTileId) + curTileBoundaryNum);402+ static_cast<int32_t>(curTileBoundaryNum));
333 } else {403 } else {
334- tileTopK.SetValue(tileId - startTileId,404+ tileTopK.SetValue(tileId - startTileId, tileTopK.GetValue(tileId - startTileId) +
335- tileTopK.GetValue(tileId - startTileId) + remainCoreBoundaryNum);405+ static_cast<int32_t>(remainCoreBoundaryNum));
336 remainCoreBoundaryNum = 0;406 remainCoreBoundaryNum = 0;
337 break;407 break;
338 }408 }
@@ -345,7 +415,8 @@ __aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTen
345}415}
346 416 
347/**417/**
348- * @brief WS 变体:从 WS 分段读取 tileTopK 累加得到本 core 总 TopK 计数并写回 WS418+ * @brief 从 WS 分段读取 tileTopK 累加得到本 core 总 TopK 计数并写回 WS
419+ * 累加器 int64(int32 形状下同样安全),两条变体共用
349 * @param resTensor 临时 Tensor420 * @param resTensor 临时 Tensor
350 * @param tileHistFp32 tileHist 的 fp32 视图421 * @param tileHistFp32 tileHist 的 fp32 视图
351 */422 */
@@ -354,7 +425,7 @@ __aicore__ inline void RadixTopKWs<T, largest>::WriteCoreTopKFromWs(LocalTensor<
354 LocalTensor<float>& tileHistFp32)425 LocalTensor<float>& tileHistFp32)
355{426{
356 LocalTensor<int32_t> tileTopK = tileHistAndTopKBuf_.Get<int32_t>();427 LocalTensor<int32_t> tileTopK = tileHistAndTopKBuf_.Get<int32_t>();
357- int32_t totalTileTopKInCore = 0;428+ int64_t totalTileTopKInCore = 0;
358 int32_t int32Align = BLOCK_SIZE / sizeof(int32_t);429 int32_t int32Align = BLOCK_SIZE / sizeof(int32_t);
359 int32_t safeSumTileNum = FLOAT32_SAFE_INT / this->tileLen_ / int32Align * int32Align;430 int32_t safeSumTileNum = FLOAT32_SAFE_INT / this->tileLen_ / int32Align * int32Align;
360 for (int32_t i = 0; i <= tileNumRepeatTimes_; i++) {431 for (int32_t i = 0; i <= tileNumRepeatTimes_; i++) {
@@ -371,10 +442,29 @@ __aicore__ inline void RadixTopKWs<T, largest>::WriteCoreTopKFromWs(LocalTensor<
371 PipeBarrier<PIPE_V>();442 PipeBarrier<PIPE_V>();
372 ReduceSum(tileHistFp32, tileHistFp32, tileHistFp32, curSegLen);443 ReduceSum(tileHistFp32, tileHistFp32, tileHistFp32, curSegLen);
373 VToSSync();444 VToSSync();
374- totalTileTopKInCore += static_cast<int32_t>(tileHistFp32.GetValue(0));445+ totalTileTopKInCore += static_cast<int64_t>(tileHistFp32.GetValue(0));
375 }446 }
376 }447 }
377- this->CopyOutCoreTopK(resTensor, totalTileTopKInCore);448+ // per-core 计数 < 2^31(per-core 元素数上限),coreTopK 通道保持 int32
449+ this->CopyOutCoreTopK(resTensor, static_cast<int32_t>(totalTileTopKInCore));
450+}
451+ 
452+/**
453+ * @brief 计算排在当前 core 之前所有 core 的边界 bin 累计元素数(int64 累加,总量可达 sortLen)
454+ */
455+template <typename T, bool largest>
456+__aicore__ inline void RadixTopKWs<T, largest>::ComputeCumSumPrevI64(LocalTensor<int32_t>& resTensor,
457+ int64_t& cumSumValuePrev)
458+{
459+ DataCopyPad(
460+ resTensor, this->boundaryBinCumSumGm_,
461+ DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * this->coreNum_), 0, 0, 0},
462+ DataCopyPadExtParams<int32_t>{true, 0, 0, 0});
463+ MTE2ToSSync();
464+ cumSumValuePrev = 0;
465+ for (int32_t coreId = 0; coreId < this->blockIdx_; coreId++) {
466+ cumSumValuePrev += resTensor.GetValue(coreId);
467+ }
378}468}
379 469 
380/**470/**
@@ -446,6 +536,12 @@ __aicore__ inline void RadixTopKWs<T, largest>::TileTopK(const uint64_t& batchId
446 uint64_t firstLen = (firstId == this->tileNum_ - 1) ? this->tailTileLen_ : this->tileLen_;536 uint64_t firstLen = (firstId == this->tileNum_ - 1) ? this->tailTileLen_ : this->tileLen_;
447 this->CopyIn(this->xBufPing_, firstLen, blockOffset + firstId * this->tileLen_);537 this->CopyIn(this->xBufPing_, firstLen, blockOffset + firstId * this->tileLen_);
448 uint32_t maxIndexLen = CreateVecIndex4TopK();538 uint32_t maxIndexLen = CreateVecIndex4TopK();
539+ // 同 UB 变体:maxIndexLen == 0 时 SubTopKAndCopyOut 走现算索引兜底分支、
540+ // 不使用 vecIndex;因此 0 时改用 buffer 首地址占位,不做零长度取句柄。
541+ LocalTensor<int32_t> vecIndexTopK = maxIndexLen > 0 ?
542+ tileHistAndTopKBuf_.GetWithOffset<int32_t>(
543+ maxIndexLen, this->tileNumAlign_ * sizeof(int32_t)) :
544+ tileHistAndTopKBuf_.Get<int32_t>(); // 占位句柄,0 时不会被使用
449 545 
450 this->InitTopKEvent();546 this->InitTopKEvent();
451 uint64_t startRepeatId = firstId / MAX_TILE_NUM_IN_UB;547 uint64_t startRepeatId = firstId / MAX_TILE_NUM_IN_UB;
@@ -467,8 +563,7 @@ __aicore__ inline void RadixTopKWs<T, largest>::TileTopK(const uint64_t& batchId
467 563 
468 int32_t curTileK = tileTopK.GetValue(tileId - repeatId * MAX_TILE_NUM_IN_UB);564 int32_t curTileK = tileTopK.GetValue(tileId - repeatId * MAX_TILE_NUM_IN_UB);
469 PipeBarrier<PIPE_V>();565 PipeBarrier<PIPE_V>();
470- PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileK, tileHistAndTopKBuf_.GetWithOffset<int32_t>(566+ PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileK, vecIndexTopK);
471- maxIndexLen, this->tileNumAlign_ * sizeof(int32_t)));
472 567 
473 PINGPONG_TILE_END(tileId);568 PINGPONG_TILE_END(tileId);
474 }569 }
@@ -0,0 +1,646 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file radix_top_k_ws_i64.h
13+ * \brief Radix TopK kernel — int64 大 shape 变体(sortLen > INT32_MAX)
14+ *
15+ * 计算逻辑与 RadixTopKWs 完全一致(继承复用其全部 WS 搬运工具与同步结构),
16+ * 仅以下位置 int64 化。910B 向量指令(Add/Adds/Duplicate/GatherMask)不支持
17+ * int64,唯一可用的 int64 向量能力是 Cast 的 s32<->s64 转换通道,且不能承担
18+ * 加法/合并,因此 int64 算术一律走标量:
19+ * 1) 跨核 globalHist 归约与边界查找(总量可达 sortLen > 2^31):标量 int64 累加,
20+ * 每轮 coreNum*numValue_(<=200)次,代价可忽略;
21+ * 2) 边界计数跨核前缀和(ComputeCumSumPrevI64)与 remainCoreBoundaryNum:int64 标量;
22+ * 3) 索引输出:GatherMask 仍在 int32 域压缩 tile 内索引,逐元素标量加 int64
23+ * tileOffset 后就地后向扩展写 int64,成本 O(kValue)。
24+ *
25+ * 与基类 RadixTopKWs 的复用边界(避免两处同步修改):
26+ * - 复用(本类不重写):InitWsGm(GM/WS 布局)、InitBuffers(差一个索引元素字节数)、
27+ * HandleLastRoundBoundary(int64 计数 + 段级快速路径)、WriteCoreTopKFromWs、
28+ * ComputeCumSumPrevI64、全部 WS 搬运/清除/同步工具;
29+ * - 必须重写:Process/SubProcess(非虚成员 + 静态派发,基类体内会静态绑定到基类 Update)、
30+ * Update 及其 int64 归约/边界查找/neighbor 累加、TileTopK、索引输出相关函数。
31+ * 若要进一步合并 Process/SubProcess,需要引入 CRTP 或把基类模板化为 <T, largest, IndexT>,
32+ * 属结构性重构,未在本 PR 处理。
33+ *
34+ * int32 安全域(保持不变):per-core globalHist/tileHist/tileTopK 计数与 coreTopK
35+ * (per-core 元素数 = sortLen/coreNum < 2^31,由 tiling 保证,物理内存内恒成立;
36+ * kValue <= INT32_MAX 由 tiling 校验)、vecIndex(< tileLen)、x/value 数据通路。
37+ */
38+ 
39+#ifndef RADIX_TOP_K_WS_I64_H
40+#define RADIX_TOP_K_WS_I64_H
41+ 
42+#include "radix_top_k_ws.h"
43+ 
44+namespace RadixTopK {
45+using namespace AscendC;
46+ 
47+/**
48+ * @brief i64 变体专用的 tile 循环前导:与 PROCESS_ONE_TILE_TOPK_OR_SKIP 的唯一区别是
49+ * 不传 vecIndex/maxIndexLen——i64 路径的索引一律在 SubTopKAndCopyOutI64 内现算
50+ * (VectorMergeI64),预生成索引链路整条不适用。
51+ * 前置条件:curBuf, coreTopKOffset, tileOffset, curTileLen 在作用域内
52+ */
53+#define PROCESS_ONE_TILE_TOPK_OR_SKIP_I64(curTileKVal) \
54+ if (curTileKVal > 0) { \
55+ auto xLocal = curBuf.template Get<T>(); \
56+ auto cmpMaskTensor = this->tempBuf_.template Get<uint8_t>(); \
57+ if constexpr (IsSameType<T, bfloat16_t>::value) { \
58+ auto xLocalFp32 = xLocal.template ReinterpretCast<float>(); \
59+ Cast(xLocalFp32, xLocal[this->tileLen_], RoundMode::CAST_NONE, curTileLen); \
60+ this->template ProcessOneTileTopK<float>(xLocalFp32, curTileLen, curTileKVal, coreTopKOffset, \
61+ cmpMaskTensor, tileOffset); \
62+ } else if constexpr (IsSameType<T, half>::value) { \
63+ auto xLocalHalf = xLocal[this->tileLen_]; \
64+ if constexpr (largest) { \
65+ this->CanonicalizeDataForLargest(curBuf, curTileLen); \
66+ } \
67+ this->template ProcessOneTileTopK<half>(xLocalHalf, curTileLen, curTileKVal, coreTopKOffset, \
68+ cmpMaskTensor, tileOffset); \
69+ } \
70+ } else { \
71+ PipeBarrier<PIPE_MTE2>(); \
72+ }
73+ 
74+template <typename T, bool largest>
75+class RadixTopKWsI64 : public RadixTopKWs<T, largest> {
76+ using Base = RadixTopKWs<T, largest>;
77+ 
78+public:
79+ __aicore__ inline RadixTopKWsI64(TPipe& tpipe, const RadixTopKTilingData& tilingData) : Base(tpipe, tilingData) {}
80+ 
81+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace);
82+ __aicore__ inline void SubProcess(uint64_t batchId);
83+ __aicore__ inline void Process();
84+ 
85+private:
86+ __aicore__ inline void ReduceGlobalHistI64(LocalTensor<int32_t>& globalHist, LocalTensor<int64_t>& reduced);
87+ __aicore__ inline void FindBoundaryBinI64(const LocalTensor<int64_t>& reduced, int32_t roundId);
88+ __aicore__ inline void AddTileHistToTileTopKI64(const LocalTensor<int64_t>& reduced);
89+ __aicore__ inline bool Update(const int32_t& roundId);
90+ __aicore__ inline void CopyOutResultI64(const int32_t& count, const uint64_t& coreTopKOffset);
91+ 
92+ template <AscendC::CMPMODE cmpMode, typename CT>
93+ __aicore__ inline void SubTopKAndCopyOutI64(LocalTensor<CT>& xLocal, const CT& boundaryNumber,
94+ const uint64_t& curTileLen, int32_t& curTileK, uint64_t& coreTopKOffset,
95+ LocalTensor<uint8_t>& cmpMaskTensor, const int64_t& tileOffset);
96+ 
97+ template <typename CT>
98+ __aicore__ inline void ProcessOneTileTopK(LocalTensor<CT>& xLocalTyped, const uint64_t& curTileLen,
99+ int32_t& curTileK, uint64_t& coreTopKOffset,
100+ LocalTensor<uint8_t>& cmpMaskTensor, const int64_t& tileOffset);
@@ -131,6 +131,9 @@ __aicore__ inline void RadixTopKWs<T, largest>::CopyTileHistInWs(const uint32_t&
131 DataCopyExtParams tileHistOutParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * dataNum),131 DataCopyExtParams tileHistOutParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * dataNum),
132 0, 0, 0};132 0, 0, 0};
133 DataCopyPad(tileHistGm_[dstOffset + i * MAX_TILE_NUM_IN_UB], tileHist, tileHistOutParams);133 DataCopyPad(tileHistGm_[dstOffset + i * MAX_TILE_NUM_IN_UB], tileHist, tileHistOutParams);
134+ // 迭代间必须等 MTE3 搬完 UB 再发起下一块 MTE2(复用同一 UB 段),
135+ // 否则下一迭代的 GM→UB 覆盖尚未搬走的头部数据,GM 目标段头部被后块源污染
136+ // (i64 场景 tileNum>6144 进入多块循环才触发,实测 K=240000 出现 tile 级 ±1 配额错位)
134 MTE3ToMTE2Sync();137 MTE3ToMTE2Sync();
135 }138 }
136}139}
@@ -1,10 +1,10 @@
1# ---------------------------------------------------------------------------------------------------------1# ---------------------------------------------------------------------------------------------------------
2# Copyright (c) 2026 Huawei Technologies Co., Ltd.2# Copyright (c) 2026 Huawei Technologies Co., Ltd.
3-# This program is free software, you can redistribute it and/or modify it under the terms and conditions of 3+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4# CANN Open Software License Agreement Version 2.0 (the "License").4# CANN Open Software License Agreement Version 2.0 (the "License").
5# Please refer to the License for details. You may not use this file except in compliance with the License.5# Please refer to the License for details. You may not use this file except in compliance with the License.
6-# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, 6+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. 7+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8# See LICENSE in the root of the software repository for the full text of the License.8# See LICENSE in the root of the software repository for the full text of the License.
9# ---------------------------------------------------------------------------------------------------------9# ---------------------------------------------------------------------------------------------------------
10 10 
@@ -15,4 +15,14 @@ if(UT_TEST_ALL OR OP_KERNEL_UT)
15 AddOpTestCase(radix_top_k "ascend910b"15 AddOpTestCase(radix_top_k "ascend910b"
16 "-DDTYPE_X=half -D__CCE_UT_TEST__ -DTestUtDefaultTilingStruct=RadixTopKTilingData"16 "-DDTYPE_X=half -D__CCE_UT_TEST__ -DTestUtDefaultTilingStruct=RadixTopKTilingData"
17 "${radix_top_k_tiling_files}")17 "${radix_top_k_tiling_files}")
18+ 
19+ if(TARGET ${OP_KERNEL_MODULE_NAME}_common_obj)
20+ target_compile_definitions(${OP_KERNEL_MODULE_NAME}_common_obj
21+ PRIVATE BUILD_SOC_VERSION=ascend910b)
22+ message(STATUS "[radix_top_k UT] BUILD_SOC_VERSION=ascend910b -> ${OP_KERNEL_MODULE_NAME}_common_obj")
23+ else()
24+ cmake_language(DEFER DIRECTORY ${CMAKE_SOURCE_DIR} CALL target_compile_definitions
25+ ${OP_KERNEL_MODULE_NAME}_common_obj PRIVATE BUILD_SOC_VERSION=ascend910b)
26+ message(STATUS "[radix_top_k UT] BUILD_SOC_VERSION=ascend910b -> ${OP_KERNEL_MODULE_NAME}_common_obj (deferred)")
27+ endif()
18endif()28endif()
@@ -119,7 +119,7 @@ static bool IsAiCoreSupport(const aclTensor* self, int64_t k)
119 return self->GetDataType() == op::DataType::DT_FLOAT16;119 return self->GetDataType() == op::DataType::DT_FLOAT16;
120}120}
121 121 
122-static bool IsRadixTopKSupport(const aclTensor* self, int64_t k)122+static bool IsRadixTopKSupport(const aclTensor* self, int64_t k, op::DataType indicesDType)
123{123{
124 SocVersion version = GetCurrentPlatformInfo().GetSocVersion();124 SocVersion version = GetCurrentPlatformInfo().GetSocVersion();
125 auto inputShape = self->GetViewShape();125 auto inputShape = self->GetViewShape();
@@ -127,10 +127,13 @@ static bool IsRadixTopKSupport(const aclTensor* self, int64_t k)
127 int64_t sortLen = inputShape.GetDim(dimNum - 1);127 int64_t sortLen = inputShape.GetDim(dimNum - 1);
128 bool socCheck = version == SocVersion::ASCEND910B || version == SocVersion::ASCEND910_93;128 bool socCheck = version == SocVersion::ASCEND910B || version == SocVersion::ASCEND910_93;
129 bool dtypeCheck = CheckType(self->GetDataType(), {op::DataType::DT_FLOAT16, op::DataType::DT_BF16});129 bool dtypeCheck = CheckType(self->GetDataType(), {op::DataType::DT_FLOAT16, op::DataType::DT_BF16});
130- bool shapeCheck = false;130+ // sortLen 超 int32 上限 → int64 大 shape 模板:只要求 indices 为 int64 且 k > 0,
131+ // 不套用下面的 k 比例门限,故此处直接判定
131 if (sortLen > MAX_INT32_INPUTSIZE) {132 if (sortLen > MAX_INT32_INPUTSIZE) {
132- shapeCheck = false;133+ return socCheck && dtypeCheck && (indicesDType == op::DataType::DT_INT64) && k > 0;
133- } else if (sortLen >= RADIX_TOP_K_S_THRESHOLD_2) {134+ }
135+ bool shapeCheck = false;
136+ if (sortLen >= RADIX_TOP_K_S_THRESHOLD_2) {
134 shapeCheck = sortLen > RADIX_TOP_K_S_K_RATIO_2 * k;137 shapeCheck = sortLen > RADIX_TOP_K_S_K_RATIO_2 * k;
135 } else if (sortLen >= RADIX_TOP_K_S_THRESHOLD_1) {138 } else if (sortLen >= RADIX_TOP_K_S_THRESHOLD_1) {
136 shapeCheck = sortLen > RADIX_TOP_K_S_K_RATIO_1 * k;139 shapeCheck = sortLen > RADIX_TOP_K_S_K_RATIO_1 * k;
@@ -230,9 +233,13 @@ std::tuple<aclTensor*, aclTensor*> TopkCommon(const aclTensor* self, int64_t k,
230 233 
231 // AICore执行路径234 // AICore执行路径
232 if (IsAiCoreSupport(self, k)) {235 if (IsAiCoreSupport(self, k)) {
236+ int64_t topkDimNum = static_cast<int64_t>(self->GetViewShape().GetDimNum());
237+ int64_t sortLen = self->GetViewShape().GetDim(topkDimNum - 1);
233 aclTensor* indicesOut = nullptr;238 aclTensor* indicesOut = nullptr;
234 if (IsRegBase() && !IsSortWithIndex(self, k, sorted)) {239 if (IsRegBase() && !IsSortWithIndex(self, k, sorted)) {
235 indicesOut = executor->AllocTensor(outShape, indicesDType, self->GetStorageFormat());240 indicesOut = executor->AllocTensor(outShape, indicesDType, self->GetStorageFormat());
241+ } else if (sortLen > MAX_INT32_INPUTSIZE && indicesDType == op::DataType::DT_INT64) {
242+ indicesOut = executor->AllocTensor(outShape, op::DataType::DT_INT64, self->GetStorageFormat());
236 } else {243 } else {
237 indicesOut = executor->AllocTensor(outShape, op::DataType::DT_INT32, self->GetStorageFormat());244 indicesOut = executor->AllocTensor(outShape, op::DataType::DT_INT32, self->GetStorageFormat());
238 }245 }
@@ -243,7 +250,7 @@ std::tuple<aclTensor*, aclTensor*> TopkCommon(const aclTensor* self, int64_t k,
243 if (IsRegBase()) {250 if (IsRegBase()) {
244 return TopkV2AiCoreForDavid(self, kTensor, dim, largest, sorted, valuesOut, indicesOut, indicesDType,251 return TopkV2AiCoreForDavid(self, kTensor, dim, largest, sorted, valuesOut, indicesOut, indicesDType,
245 sortPolicy, executor);252 sortPolicy, executor);
246- } else if (IsRadixTopKSupport(self, k)) {253+ } else if (IsRadixTopKSupport(self, k, indicesDType)) {
247 return RadixTopK(self, kTensor, dim, largest, sorted, valuesOut, indicesOut, indicesDType, executor);254 return RadixTopK(self, kTensor, dim, largest, sorted, valuesOut, indicesOut, indicesDType, executor);
248 } else {255 } else {
249 return TopkV2AiCore(self, kTensor, dim, largest, sorted, ge::DT_INT32, sortPolicy, valuesOut,256 return TopkV2AiCore(self, kTensor, dim, largest, sorted, ge::DT_INT32, sortPolicy, valuesOut,