Merged
feat: RadixTopK 新增 int64 索引变体并修复 largest 场景 NaN 与 WS 同步问题 #6335
Nice tryAdd on Oct 14
feat: RadixTopK 新增 int64 索引变体并修复 largest 场景 NaN 与 WS 同步问题 #6335
Merged
Merged fromNice_try/ops-math_workRadixTopKInt64Nanintocann/ops-mathmaster
Total: 14 files changed+1588-406
| @@ -1,143 +1,281 @@ | |||
| 1 | { | 1 | { |
| 2 | "op_type": "RadixTopK", | 2 | "op_type": "RadixTopK", |
| 3 | "op_list": [ | 3 | "op_list": [ |
| 4 | - { | 4 | + { |
| 5 | - "bin_filename": "RadixTopK_x_bfloat16_indices_int32_k_int32", | 5 | + "bin_filename": "RadixTopK_x_bfloat16_indices_int32_k_int32", |
| 6 | - "inputs": [ | 6 | + "inputs": [ |
| 7 | - { | 7 | + { |
| 8 | - "name": "x", | 8 | + "name": "x", |
| 9 | - "index": 0, | 9 | + "index": 0, |
| 10 | - "dtype": "bfloat16", | 10 | + "dtype": "bfloat16", |
| 11 | - "format": "ND", | 11 | + "format": "ND", |
| 12 | - "paramType": "required", | 12 | + "paramType": "required", |
| 13 | - "shape": [ | 13 | + "shape": [ |
| 14 | - -2 | 14 | + -2 |
| 15 | - ] | ||
| 16 | - }, | ||
| 17 | - { | ||
| 18 | - "name": "k", | ||
| 19 | - "index": 1, | ||
| 20 | - "dtype": "int32", | ||
| 21 | - "format": "ND", | ||
| 22 | - "paramType": "required", | ||
| 23 | - "shape": [ | ||
| 24 | - -2 | ||
| 25 | - ] | ||
| 26 | - } | ||
| 27 | - ], | ||
| 28 | - "outputs": [ | ||
| 29 | - { | ||
| 30 | - "name": "values", | ||
| 31 | - "index": 0, | ||
| 32 | - "dtype": "bfloat16", | ||
| 33 | - "format": "ND", | ||
| 34 | - "paramType": "required", | ||
| 35 | - "shape": [ | ||
| 36 | - -2 | ||
| 37 | - ] | ||
| 38 | - }, | ||
| 39 | - { | ||
| 40 | - "name": "indices", | ||
| 41 | - "index": 1, | ||
| 42 | - "dtype": "int32", | ||
| 43 | - "format": "ND", | ||
| 44 | - "paramType": "required", | ||
| 45 | - "shape": [ | ||
| 46 | - -2 | ||
| 47 | - ] | ||
| 48 | - } | ||
| 49 | - ], | ||
| 50 | - "attrs": [ | ||
| 51 | - { | ||
| 52 | - "name": "sorted", | ||
| 53 | - "dtype": "bool", | ||
| 54 | - "value": null | ||
| 55 | - }, | ||
| 56 | - { | ||
| 57 | - "name": "dim", | ||
| 58 | - "dtype": "int", | ||
| 59 | - "value": null | ||
| 60 | - }, | ||
| 61 | - { | ||
| 62 | - "name": "largest", | ||
| 63 | - "dtype": "bool", | ||
| 64 | - "value": null | ||
| 65 | - }, | ||
| 66 | - { | ||
| 67 | - "name": "indices_dtype", | ||
| 68 | - "dtype": "int", | ||
| 69 | - "value": 3 | ||
| 70 | - } | ||
| 71 | ] | 15 | ] |
| 72 | - }, | 16 | + }, |
| 73 | - { | 17 | + { |
| 74 | - "bin_filename": "RadixTopK_x_float16_indices_int32_k_int32", | 18 | + "name": "k", |
| 75 | - "inputs": [ | 19 | + "index": 1, |
| 76 | - { | 20 | + "dtype": "int32", |
| 77 | - "name": "x", | 21 | + "format": "ND", |
| 78 | - "index": 0, | 22 | + "paramType": "required", |
| 79 | - "dtype": "float16", | 23 | + "shape": [ |
| 80 | - "format": "ND", | 24 | + -2 |
| 81 | - "paramType": "required", | ||
| 82 | - "shape": [ | ||
| 83 | - -2 | ||
| 84 | - ] | ||
| 85 | - }, | ||
| 86 | - { | ||
| 87 | - "name": "k", | ||
| 88 | - "index": 1, | ||
| 89 | - "dtype": "int32", | ||
| 90 | - "format": "ND", | ||
| 91 | - "paramType": "required", | ||
| 92 | - "shape": [ | ||
| 93 | - -2 | ||
| 94 | - ] | ||
| 95 | - } | ||
| 96 | - ], | ||
| 97 | - "outputs": [ | ||
| 98 | - { | ||
| 99 | - "name": "values", | ||
| 100 | - "index": 0, | ||
| 101 | - "dtype": "float16", | ||
| 102 | - "format": "ND", | ||
| 103 | - "paramType": "required", | ||
| 104 | - "shape": [ | ||
| 105 | - -2 | ||
| 106 | - ] | ||
| 107 | - }, | ||
| 108 | - { | ||
| 109 | - "name": "indices", | ||
| 110 | - "index": 1, | ||
| 111 | - "dtype": "int32", | ||
| 112 | - "format": "ND", | ||
| 113 | - "paramType": "required", | ||
| 114 | - "shape": [ | ||
| 115 | - -2 | ||
| 116 | - ] | ||
| 117 | - } | ||
| 118 | - ], | ||
| 119 | - "attrs": [ | ||
| 120 | - { | ||
| 121 | - "name": "sorted", | ||
| 122 | - "dtype": "bool", | ||
| 123 | - "value": null | ||
| 124 | - }, | ||
| 125 | - { | ||
| 126 | - "name": "dim", | ||
| 127 | - "dtype": "int", | ||
| 128 | - "value": null | ||
| 129 | - }, | ||
| 130 | - { | ||
| 131 | - "name": "largest", | ||
| 132 | - "dtype": "bool", | ||
| 133 | - "value": null | ||
| 134 | - }, | ||
| 135 | - { | ||
| 136 | - "name": "indices_dtype", | ||
| 137 | - "dtype": "int", | ||
| 138 | - "value": 3 | ||
| 139 | - } | ||
| 140 | ] | 25 | ] |
| 141 | - } | 26 | + } |
| 27 | + ], | ||
| 28 | + "outputs": [ | ||
| 29 | + { | ||
| 30 | + "name": "values", | ||
| 31 | + "index": 0, | ||
| 32 | + "dtype": "bfloat16", | ||
| 33 | + "format": "ND", | ||
| 34 | + "paramType": "required", | ||
| 35 | + "shape": [ | ||
| 36 | + -2 | ||
| 37 | + ] | ||
| 38 | + }, | ||
| 39 | + { | ||
| 40 | + "name": "indices", | ||
| 41 | + "index": 1, | ||
| 42 | + "dtype": "int32", | ||
| 43 | + "format": "ND", | ||
| 44 | + "paramType": "required", | ||
| 45 | + "shape": [ | ||
| 46 | + -2 | ||
| 47 | + ] | ||
| 48 | + } | ||
| 49 | + ], | ||
| 50 | + "attrs": [ | ||
| 51 | + { | ||
| 52 | + "name": "sorted", | ||
| 53 | + "dtype": "bool", | ||
| 54 | + "value": null | ||
| 55 | + }, | ||
| 56 | + { | ||
| 57 | + "name": "dim", | ||
| 58 | + "dtype": "int", | ||
| 59 | + "value": null | ||
| 60 | + }, | ||
| 61 | + { | ||
| 62 | + "name": "largest", | ||
| 63 | + "dtype": "bool", | ||
| 64 | + "value": null | ||
| 65 | + }, | ||
| 66 | + { | ||
| 67 | + "name": "indices_dtype", | ||
| 68 | + "dtype": "int", | ||
| 69 | + "value": 3 | ||
| 70 | + } | ||
| 71 | + ] | ||
| 72 | + }, | ||
| 73 | + { | ||
| 74 | + "bin_filename": "RadixTopK_x_float16_indices_int32_k_int32", | ||
| 75 | + "inputs": [ | ||
| 76 | + { | ||
| 77 | + "name": "x", | ||
| 78 | + "index": 0, | ||
| 79 | + "dtype": "float16", | ||
| 80 | + "format": "ND", | ||
| 81 | + "paramType": "required", | ||
| 82 | + "shape": [ | ||
| 83 | + -2 | ||
| 84 | + ] | ||
| 85 | + }, | ||
| 86 | + { | ||
| 87 | + "name": "k", | ||
| 88 | + "index": 1, | ||
| 89 | + "dtype": "int32", | ||
| 90 | + "format": "ND", | ||
| 91 | + "paramType": "required", | ||
| 92 | + "shape": [ | ||
| 93 | + -2 | ||
| 94 | + ] | ||
| 95 | + } | ||
| 96 | + ], | ||
| 97 | + "outputs": [ | ||
| 98 | + { | ||
| 99 | + "name": "values", | ||
| 100 | + "index": 0, | ||
| 101 | + "dtype": "float16", | ||
| 102 | + "format": "ND", | ||
| 103 | + "paramType": "required", | ||
| 104 | + "shape": [ | ||
| 105 | + -2 | ||
| 106 | + ] | ||
| 107 | + }, | ||
| 108 | + { | ||
| 109 | + "name": "indices", | ||
| 110 | + "index": 1, | ||
| 111 | + "dtype": "int32", | ||
| 112 | + "format": "ND", | ||
| 113 | + "paramType": "required", | ||
| 114 | + "shape": [ | ||
| 115 | + -2 | ||
| 116 | + ] | ||
| 117 | + } | ||
| 118 | + ], | ||
| 119 | + "attrs": [ | ||
| 120 | + { | ||
| 121 | + "name": "sorted", | ||
| 122 | + "dtype": "bool", | ||
| 123 | + "value": null | ||
| 124 | + }, | ||
| 125 | + { | ||
| 126 | + "name": "dim", | ||
| 127 | + "dtype": "int", | ||
| 128 | + "value": null | ||
| 129 | + }, | ||
| 130 | + { | ||
| 131 | + "name": "largest", | ||
| 132 | + "dtype": "bool", | ||
| 133 | + "value": null | ||
| 134 | + }, | ||
| 135 | + { | ||
| 136 | + "name": "indices_dtype", | ||
| 137 | + "dtype": "int", | ||
| 138 | + "value": 3 | ||
| 139 | + } | ||
| 140 | + ] | ||
| 141 | + }, | ||
| 142 | + { | ||
| 143 | + "bin_filename": "RadixTopK_x_bfloat16_indices_int64_k_int32", | ||
| 144 | + "inputs": [ | ||
| 145 | + { | ||
| 146 | + "name": "x", | ||
| 147 | + "index": 0, | ||
| 148 | + "dtype": "bfloat16", | ||
| 149 | + "format": "ND", | ||
| 150 | + "paramType": "required", | ||
| 151 | + "shape": [ | ||
| 152 | + -2 | ||
| 153 | + ] | ||
| 154 | + }, | ||
| 155 | + { | ||
| 156 | + "name": "k", | ||
| 157 | + "index": 1, | ||
| 158 | + "dtype": "int32", | ||
| 159 | + "format": "ND", | ||
| 160 | + "paramType": "required", | ||
| 161 | + "shape": [ | ||
| 162 | + -2 | ||
| 163 | + ] | ||
| 164 | + } | ||
| 165 | + ], | ||
| 166 | + "outputs": [ | ||
| 167 | + { | ||
| 168 | + "name": "values", | ||
| 169 | + "index": 0, | ||
| 170 | + "dtype": "bfloat16", | ||
| 171 | + "format": "ND", | ||
| 172 | + "paramType": "required", | ||
| 173 | + "shape": [ | ||
| 174 | + -2 | ||
| 175 | + ] | ||
| 176 | + }, | ||
| 177 | + { | ||
| 178 | + "name": "indices", | ||
| 179 | + "index": 1, | ||
| 180 | + "dtype": "int64", | ||
| 181 | + "format": "ND", | ||
| 182 | + "paramType": "required", | ||
| 183 | + "shape": [ | ||
| 184 | + -2 | ||
| 185 | + ] | ||
| 186 | + } | ||
| 187 | + ], | ||
| 188 | + "attrs": [ | ||
| 189 | + { | ||
| 190 | + "name": "sorted", | ||
| 191 | + "dtype": "bool", | ||
| 192 | + "value": null | ||
| 193 | + }, | ||
| 194 | + { | ||
| 195 | + "name": "dim", | ||
| 196 | + "dtype": "int", | ||
| 197 | + "value": null | ||
| 198 | + }, | ||
| 199 | + { | ||
| 200 | + "name": "largest", | ||
| 201 | + "dtype": "bool", | ||
| 202 | + "value": null | ||
| 203 | + }, | ||
| 204 | + { | ||
| 205 | + "name": "indices_dtype", | ||
| 206 | + "dtype": "int", | ||
| 207 | + "value": 9 | ||
| 208 | + } | ||
| 209 | + ] | ||
| 210 | + }, | ||
| 211 | + { | ||
| 212 | + "bin_filename": "RadixTopK_x_float16_indices_int64_k_int32", | ||
| 213 | + "inputs": [ | ||
| 214 | + { | ||
| 215 | + "name": "x", | ||
| 216 | + "index": 0, | ||
| 217 | + "dtype": "float16", | ||
| 218 | + "format": "ND", | ||
| 219 | + "paramType": "required", | ||
| 220 | + "shape": [ | ||
| 221 | + -2 | ||
| 222 | + ] | ||
| 223 | + }, | ||
| 224 | + { | ||
| 225 | + "name": "k", | ||
| 226 | + "index": 1, | ||
| 227 | + "dtype": "int32", | ||
| 228 | + "format": "ND", | ||
| 229 | + "paramType": "required", | ||
| 230 | + "shape": [ | ||
| 231 | + -2 | ||
| 232 | + ] | ||
| 233 | + } | ||
| 234 | + ], | ||
| 235 | + "outputs": [ | ||
| 236 | + { | ||
| 237 | + "name": "values", | ||
| 238 | + "index": 0, | ||
| 239 | + "dtype": "float16", | ||
| 240 | + "format": "ND", | ||
| 241 | + "paramType": "required", | ||
| 242 | + "shape": [ | ||
| 243 | + -2 | ||
| 244 | + ] | ||
| 245 | + }, | ||
| 246 | + { | ||
| 247 | + "name": "indices", | ||
| 248 | + "index": 1, | ||
| 249 | + "dtype": "int64", | ||
| 250 | + "format": "ND", | ||
| 251 | + "paramType": "required", | ||
| 252 | + "shape": [ | ||
| 253 | + -2 | ||
| 254 | + ] | ||
| 255 | + } | ||
| 256 | + ], | ||
| 257 | + "attrs": [ | ||
| 258 | + { | ||
| 259 | + "name": "sorted", | ||
| 260 | + "dtype": "bool", | ||
| 261 | + "value": null | ||
| 262 | + }, | ||
| 263 | + { | ||
| 264 | + "name": "dim", | ||
| 265 | + "dtype": "int", | ||
| 266 | + "value": null | ||
| 267 | + }, | ||
| 268 | + { | ||
| 269 | + "name": "largest", | ||
| 270 | + "dtype": "bool", | ||
| 271 | + "value": null | ||
| 272 | + }, | ||
| 273 | + { | ||
| 274 | + "name": "indices_dtype", | ||
| 275 | + "dtype": "int", | ||
| 276 | + "value": 9 | ||
| 277 | + } | ||
| 278 | + ] | ||
| 279 | + } | ||
| 142 | ] | 280 | ] |
| 143 | -} | 281 | + } |
| @@ -1,143 +1,281 @@ | |||
| 1 | { | 1 | { |
| 2 | - "op_type": "RadixTopK", | 2 | + "op_type": "RadixTopK", |
| 3 | - "op_list": [ | 3 | + "op_list": [ |
| 4 | + { | ||
| 5 | + "bin_filename": "RadixTopK_x_bfloat16_indices_int32_k_int32", | ||
| 6 | + "inputs": [ | ||
| 4 | { | 7 | { |
| 5 | - "bin_filename": "RadixTopK_x_bfloat16_indices_int32_k_int32", | 8 | + "name": "x", |
| 6 | - "inputs": [ | 9 | + "index": 0, |
| 7 | - { | 10 | + "dtype": "bfloat16", |
| 8 | - "name": "x", | 11 | + "format": "ND", |
| 9 | - "index": 0, | 12 | + "paramType": "required", |
| 10 | - "dtype": "bfloat16", | 13 | + "shape": [ |
| 11 | - "format": "ND", | 14 | + -2 |
| 12 | - "paramType": "required", | 15 | + ] |
| 13 | - "shape": [ | ||
| 14 | - -2 | ||
| 15 | - ] | ||
| 16 | - }, | ||
| 17 | - { | ||
| 18 | - "name": "k", | ||
| 19 | - "index": 1, | ||
| 20 | - "dtype": "int32", | ||
| 21 | - "format": "ND", | ||
| 22 | - "paramType": "required", | ||
| 23 | - "shape": [ | ||
| 24 | - -2 | ||
| 25 | - ] | ||
| 26 | - } | ||
| 27 | - ], | ||
| 28 | - "outputs": [ | ||
| 29 | - { | ||
| 30 | - "name": "values", | ||
| 31 | - "index": 0, | ||
| 32 | - "dtype": "bfloat16", | ||
| 33 | - "format": "ND", | ||
| 34 | - "paramType": "required", | ||
| 35 | - "shape": [ | ||
| 36 | - -2 | ||
| 37 | - ] | ||
| 38 | - }, | ||
| 39 | - { | ||
| 40 | - "name": "indices", | ||
| 41 | - "index": 1, | ||
| 42 | - "dtype": "int32", | ||
| 43 | - "format": "ND", | ||
| 44 | - "paramType": "required", | ||
| 45 | - "shape": [ | ||
| 46 | - -2 | ||
| 47 | - ] | ||
| 48 | - } | ||
| 49 | - ], | ||
| 50 | - "attrs": [ | ||
| 51 | - { | ||
| 52 | - "name": "sorted", | ||
| 53 | - "dtype": "bool", | ||
| 54 | - "value": null | ||
| 55 | - }, | ||
| 56 | - { | ||
| 57 | - "name": "dim", | ||
| 58 | - "dtype": "int", | ||
| 59 | - "value": null | ||
| 60 | - }, | ||
| 61 | - { | ||
| 62 | - "name": "largest", | ||
| 63 | - "dtype": "bool", | ||
| 64 | - "value": null | ||
| 65 | - }, | ||
| 66 | - { | ||
| 67 | - "name": "indices_dtype", | ||
| 68 | - "dtype": "int", | ||
| 69 | - "value": 3 | ||
| 70 | - } | ||
| 71 | - ] | ||
| 72 | }, | 16 | }, |
| 73 | { | 17 | { |
| 74 | - "bin_filename": "RadixTopK_x_float16_indices_int32_k_int32", | 18 | + "name": "k", |
| 75 | - "inputs": [ | 19 | + "index": 1, |
| 76 | - { | 20 | + "dtype": "int32", |
| 77 | - "name": "x", | 21 | + "format": "ND", |
| 78 | - "index": 0, | 22 | + "paramType": "required", |
| 79 | - "dtype": "float16", | 23 | + "shape": [ |
| 80 | - "format": "ND", | 24 | + -2 |
| 81 | - "paramType": "required", | 25 | + ] |
| 82 | - "shape": [ | ||
| 83 | - -2 | ||
| 84 | - ] | ||
| 85 | - }, | ||
| 86 | - { | ||
| 87 | - "name": "k", | ||
| 88 | - "index": 1, | ||
| 89 | - "dtype": "int32", | ||
| 90 | - "format": "ND", | ||
| 91 | - "paramType": "required", | ||
| 92 | - "shape": [ | ||
| 93 | - -2 | ||
| 94 | - ] | ||
| 95 | - } | ||
| 96 | - ], | ||
| 97 | - "outputs": [ | ||
| 98 | - { | ||
| 99 | - "name": "values", | ||
| 100 | - "index": 0, | ||
| 101 | - "dtype": "float16", | ||
| 102 | - "format": "ND", | ||
| 103 | - "paramType": "required", | ||
| 104 | - "shape": [ | ||
| 105 | - -2 | ||
| 106 | - ] | ||
| 107 | - }, | ||
| 108 | - { | ||
| 109 | - "name": "indices", | ||
| 110 | - "index": 1, | ||
| 111 | - "dtype": "int32", | ||
| 112 | - "format": "ND", | ||
| 113 | - "paramType": "required", | ||
| 114 | - "shape": [ | ||
| 115 | - -2 | ||
| 116 | - ] | ||
| 117 | - } | ||
| 118 | - ], | ||
| 119 | - "attrs": [ | ||
| 120 | - { | ||
| 121 | - "name": "sorted", | ||
| 122 | - "dtype": "bool", | ||
| 123 | - "value": null | ||
| 124 | - }, | ||
| 125 | - { | ||
| 126 | - "name": "dim", | ||
| 127 | - "dtype": "int", | ||
| 128 | - "value": null | ||
| 129 | - }, | ||
| 130 | - { | ||
| 131 | - "name": "largest", | ||
| 132 | - "dtype": "bool", | ||
| 133 | - "value": null | ||
| 134 | - }, | ||
| 135 | - { | ||
| 136 | - "name": "indices_dtype", | ||
| 137 | - "dtype": "int", | ||
| 138 | - "value": 3 | ||
| 139 | - } | ||
| 140 | - ] | ||
| 141 | } | 26 | } |
| 142 | - ] | 27 | + ], |
| 143 | -} | 28 | + "outputs": [ |
| 29 | + { | ||
| 30 | + "name": "values", | ||
| 31 | + "index": 0, | ||
| 32 | + "dtype": "bfloat16", | ||
| 33 | + "format": "ND", | ||
| 34 | + "paramType": "required", | ||
| 35 | + "shape": [ | ||
| 36 | + -2 | ||
| 37 | + ] | ||
| 38 | + }, | ||
| 39 | + { | ||
| 40 | + "name": "indices", | ||
| 41 | + "index": 1, | ||
| 42 | + "dtype": "int32", | ||
| 43 | + "format": "ND", | ||
| 44 | + "paramType": "required", | ||
| 45 | + "shape": [ | ||
| 46 | + -2 | ||
| 47 | + ] | ||
| 48 | + } | ||
| 49 | + ], | ||
| 50 | + "attrs": [ | ||
| 51 | + { | ||
| 52 | + "name": "sorted", | ||
| 53 | + "dtype": "bool", | ||
| 54 | + "value": null | ||
| 55 | + }, | ||
| 56 | + { | ||
| 57 | + "name": "dim", | ||
| 58 | + "dtype": "int", | ||
| 59 | + "value": null | ||
| 60 | + }, | ||
| 61 | + { | ||
| 62 | + "name": "largest", | ||
| 63 | + "dtype": "bool", | ||
| 64 | + "value": null | ||
| 65 | + }, | ||
| 66 | + { | ||
| 67 | + "name": "indices_dtype", | ||
| 68 | + "dtype": "int", | ||
| 69 | + "value": 3 | ||
| 70 | + } | ||
| 71 | + ] | ||
| 72 | + }, | ||
| 73 | + { | ||
| 74 | + "bin_filename": "RadixTopK_x_float16_indices_int32_k_int32", | ||
| 75 | + "inputs": [ | ||
| 76 | + { | ||
| 77 | + "name": "x", | ||
| 78 | + "index": 0, | ||
| 79 | + "dtype": "float16", | ||
| 80 | + "format": "ND", | ||
| 81 | + "paramType": "required", | ||
| 82 | + "shape": [ | ||
| 83 | + -2 | ||
| 84 | + ] | ||
| 85 | + }, | ||
| 86 | + { | ||
| 87 | + "name": "k", | ||
| 88 | + "index": 1, | ||
| 89 | + "dtype": "int32", | ||
| 90 | + "format": "ND", | ||
| 91 | + "paramType": "required", | ||
| 92 | + "shape": [ | ||
| 93 | + -2 | ||
| 94 | + ] | ||
| 95 | + } | ||
| 96 | + ], | ||
| 97 | + "outputs": [ | ||
| 98 | + { | ||
| 99 | + "name": "values", | ||
| 100 | + "index": 0, | ||
| 101 | + "dtype": "float16", | ||
| 102 | + "format": "ND", | ||
| 103 | + "paramType": "required", | ||
| 104 | + "shape": [ | ||
| 105 | + -2 | ||
| 106 | + ] | ||
| 107 | + }, | ||
| 108 | + { | ||
| 109 | + "name": "indices", | ||
| 110 | + "index": 1, | ||
| 111 | + "dtype": "int32", | ||
| 112 | + "format": "ND", | ||
| 113 | + "paramType": "required", | ||
| 114 | + "shape": [ | ||
| 115 | + -2 | ||
| 116 | + ] | ||
| 117 | + } | ||
| 118 | + ], | ||
| 119 | + "attrs": [ | ||
| 120 | + { | ||
| 121 | + "name": "sorted", | ||
| 122 | + "dtype": "bool", | ||
| 123 | + "value": null | ||
| 124 | + }, | ||
| 125 | + { | ||
| 126 | + "name": "dim", | ||
| 127 | + "dtype": "int", | ||
| 128 | + "value": null | ||
| 129 | + }, | ||
| 130 | + { | ||
| 131 | + "name": "largest", | ||
| 132 | + "dtype": "bool", | ||
| 133 | + "value": null | ||
| 134 | + }, | ||
| 135 | + { | ||
| 136 | + "name": "indices_dtype", | ||
| 137 | + "dtype": "int", | ||
| 138 | + "value": 3 | ||
| 139 | + } | ||
| 140 | + ] | ||
| 141 | + }, | ||
| 142 | + { | ||
| 143 | + "bin_filename": "RadixTopK_x_bfloat16_indices_int64_k_int32", | ||
| 144 | + "inputs": [ | ||
| 145 | + { | ||
| 146 | + "name": "x", | ||
| 147 | + "index": 0, | ||
| 148 | + "dtype": "bfloat16", | ||
| 149 | + "format": "ND", | ||
| 150 | + "paramType": "required", | ||
| 151 | + "shape": [ | ||
| 152 | + -2 | ||
| 153 | + ] | ||
| 154 | + }, | ||
| 155 | + { | ||
| 156 | + "name": "k", | ||
| 157 | + "index": 1, | ||
| 158 | + "dtype": "int32", | ||
| 159 | + "format": "ND", | ||
| 160 | + "paramType": "required", | ||
| 161 | + "shape": [ | ||
| 162 | + -2 | ||
| 163 | + ] | ||
| 164 | + } | ||
| 165 | + ], | ||
| 166 | + "outputs": [ | ||
| 167 | + { | ||
| 168 | + "name": "values", | ||
| 169 | + "index": 0, | ||
| 170 | + "dtype": "bfloat16", | ||
| 171 | + "format": "ND", | ||
| 172 | + "paramType": "required", | ||
| 173 | + "shape": [ | ||
| 174 | + -2 | ||
| 175 | + ] | ||
| 176 | + }, | ||
| 177 | + { | ||
| 178 | + "name": "indices", | ||
| 179 | + "index": 1, | ||
| 180 | + "dtype": "int64", | ||
| 181 | + "format": "ND", | ||
| 182 | + "paramType": "required", | ||
| 183 | + "shape": [ | ||
| 184 | + -2 | ||
| 185 | + ] | ||
| 186 | + } | ||
| 187 | + ], | ||
| 188 | + "attrs": [ | ||
| 189 | + { | ||
| 190 | + "name": "sorted", | ||
| 191 | + "dtype": "bool", | ||
| 192 | + "value": null | ||
| 193 | + }, | ||
| 194 | + { | ||
| 195 | + "name": "dim", | ||
| 196 | + "dtype": "int", | ||
| 197 | + "value": null | ||
| 198 | + }, | ||
| 199 | + { | ||
| 200 | + "name": "largest", | ||
| 201 | + "dtype": "bool", | ||
| 202 | + "value": null | ||
| 203 | + }, | ||
| 204 | + { | ||
| 205 | + "name": "indices_dtype", | ||
| 206 | + "dtype": "int", | ||
| 207 | + "value": 9 | ||
| 208 | + } | ||
| 209 | + ] | ||
| 210 | + }, | ||
| 211 | + { | ||
| 212 | + "bin_filename": "RadixTopK_x_float16_indices_int64_k_int32", | ||
| 213 | + "inputs": [ | ||
| 214 | + { | ||
| 215 | + "name": "x", | ||
| 216 | + "index": 0, | ||
| 217 | + "dtype": "float16", | ||
| 218 | + "format": "ND", | ||
| 219 | + "paramType": "required", | ||
| 220 | + "shape": [ | ||
| 221 | + -2 | ||
| 222 | + ] | ||
| 223 | + }, | ||
| 224 | + { | ||
| 225 | + "name": "k", | ||
| 226 | + "index": 1, | ||
| 227 | + "dtype": "int32", | ||
| 228 | + "format": "ND", | ||
| 229 | + "paramType": "required", | ||
| 230 | + "shape": [ | ||
| 231 | + -2 | ||
| 232 | + ] | ||
| 233 | + } | ||
| 234 | + ], | ||
| 235 | + "outputs": [ | ||
| 236 | + { | ||
| 237 | + "name": "values", | ||
| 238 | + "index": 0, | ||
| 239 | + "dtype": "float16", | ||
| 240 | + "format": "ND", | ||
| 241 | + "paramType": "required", | ||
| 242 | + "shape": [ | ||
| 243 | + -2 | ||
| 244 | + ] | ||
| 245 | + }, | ||
| 246 | + { | ||
| 247 | + "name": "indices", | ||
| 248 | + "index": 1, | ||
| 249 | + "dtype": "int64", | ||
| 250 | + "format": "ND", | ||
| 251 | + "paramType": "required", | ||
| 252 | + "shape": [ | ||
| 253 | + -2 | ||
| 254 | + ] | ||
| 255 | + } | ||
| 256 | + ], | ||
| 257 | + "attrs": [ | ||
| 258 | + { | ||
| 259 | + "name": "sorted", | ||
| 260 | + "dtype": "bool", | ||
| 261 | + "value": null | ||
| 262 | + }, | ||
| 263 | + { | ||
| 264 | + "name": "dim", | ||
| 265 | + "dtype": "int", | ||
| 266 | + "value": null | ||
| 267 | + }, | ||
| 268 | + { | ||
| 269 | + "name": "largest", | ||
| 270 | + "dtype": "bool", | ||
| 271 | + "value": null | ||
| 272 | + }, | ||
| 273 | + { | ||
| 274 | + "name": "indices_dtype", | ||
| 275 | + "dtype": "int", | ||
| 276 | + "value": 9 | ||
| 277 | + } | ||
| 278 | + ] | ||
| 279 | + } | ||
| 280 | + ] | ||
| 281 | +} | ||
| @@ -13,6 +13,8 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | +#include <cstdint> | ||
| 17 | +#include <limits> | ||
| 16 | #include "tiling/tiling_api.h" | 18 | #include "tiling/tiling_api.h" |
| 17 | #include "register/tilingdata_base.h" | 19 | #include "register/tilingdata_base.h" |
| 18 | #include "register/op_def_registry.h" | 20 | #include "register/op_def_registry.h" |
| @@ -31,6 +33,13 @@ static constexpr uint64_t MAX_TILE_LEN = 12032; | |||
| 31 | static constexpr uint64_t LARGE_SORT_LEN = 1440000000; | 33 | static constexpr uint64_t LARGE_SORT_LEN = 1440000000; |
| 32 | static constexpr uint64_t LARGE_TILE_LEN = 10496; | 34 | static constexpr uint64_t LARGE_TILE_LEN = 10496; |
| 33 | static constexpr uint64_t MAX_TILE_NUM_IN_UB = 6144; | 35 | static constexpr uint64_t MAX_TILE_NUM_IN_UB = 6144; |
| 36 | +// sortLen 超过 int32 上限时切换 int64 模板(索引值域与跨核计数可达 sortLen) | ||
| 37 | +static constexpr uint64_t INT32_SORT_LEN_MAX = static_cast<uint64_t>(std::numeric_limits<int32_t>::max()); | ||
| 38 | +// checkUb 中 UB 占用按「每 tile 元素数 × 系数」估算: | ||
| 39 | +// int32 索引:xBuf ping/pong(fp32) + outValue(fp32) + outIndex(int32) = 16 元素/tile; | ||
| 40 | +// int64 索引:outIndexBuf 翻倍(int64),系数 16 -> 20。 | ||
| 41 | +static constexpr uint64_t UB_COEF_PER_TILE = 16; | ||
| 42 | +static constexpr uint64_t UB_COEF_PER_TILE_I64 = 20; | ||
| 34 | static constexpr uint64_t INPUT_X = 0; | 43 | static constexpr uint64_t INPUT_X = 0; |
| 35 | static constexpr uint64_t NUM_VALUE_2BIT = 4; | 44 | static constexpr uint64_t NUM_VALUE_2BIT = 4; |
| 36 | static constexpr uint64_t BIT_PER_BYTE = 8; | 45 | static constexpr uint64_t BIT_PER_BYTE = 8; |
| @@ -79,6 +88,16 @@ ge::graphStatus RadixTopKTiling::GetShapeAttrsInfo() | |||
| 79 | tilingData_.sorted = *attrs->GetAttrPointer<bool>(ATTR_INDEX_SORTED); | 88 | tilingData_.sorted = *attrs->GetAttrPointer<bool>(ATTR_INDEX_SORTED); |
| 80 | tilingData_.largest = *attrs->GetAttrPointer<bool>(ATTR_INDEX_LARGEST); | 89 | tilingData_.largest = *attrs->GetAttrPointer<bool>(ATTR_INDEX_LARGEST); |
| 81 | isLargeShape_ = tilingData_.sortLen > LARGE_SORT_LEN; | 90 | isLargeShape_ = tilingData_.sortLen > LARGE_SORT_LEN; |
| 91 | + isInt64Shape_ = tilingData_.sortLen > INT32_SORT_LEN_MAX; | ||
| 92 | + // int64 模板输出 int64 索引,防御性校验输出 dtype 匹配(int32 组合不允许超 int32 shape) | ||
| 93 | + if (isInt64Shape_) { | ||
| 94 | + auto indicesDesc = context_->GetOutputDesc(1); | ||
| 95 | + OP_CHECK_NULL_WITH_CONTEXT(context_, indicesDesc); | ||
| 96 | + OP_CHECK_IF((indicesDesc->GetDataType() != ge::DT_INT64), | ||
| 97 | + OP_LOGE(opName_, "RadixTopK sortLen [%lu] exceeds int32, indices output must be int64.", | ||
| 98 | + tilingData_.sortLen), | ||
| 99 | + return ge::GRAPH_FAILED); | ||
| 100 | + } | ||
| 82 | 101 | ||
| 83 | xDType_ = context_->GetInputDesc(INPUT_X)->GetDataType(); | 102 | xDType_ = context_->GetInputDesc(INPUT_X)->GetDataType(); |
| 84 | xDtypeSize_ = ge::GetSizeByDataType(xDType_); | 103 | xDtypeSize_ = ge::GetSizeByDataType(xDType_); |
| @@ -91,21 +110,30 @@ ge::graphStatus RadixTopKTiling::DoOpTiling() | |||
| 91 | { | 110 | { |
| 92 | OP_LOGD(opName_, "RadixTopKTiling DoOpTiling."); | 111 | OP_LOGD(opName_, "RadixTopKTiling DoOpTiling."); |
| 93 | bool isCalcSuccess = false; | 112 | bool isCalcSuccess = false; |
| 94 | - if (!isLargeShape_) { | 113 | + if (isInt64Shape_) { |
| 114 | + isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen, UB_COEF_PER_TILE_I64); | ||
| 115 | + } else if (!isLargeShape_) { | ||
| 95 | isCalcSuccess = CalcTilingParams(tilingData_.sortLen); | 116 | isCalcSuccess = CalcTilingParams(tilingData_.sortLen); |
| 96 | if (!isCalcSuccess) { | 117 | if (!isCalcSuccess) { |
| 97 | isLargeShape_ = true; | 118 | isLargeShape_ = true; |
| 98 | - isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen); | 119 | + isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen, UB_COEF_PER_TILE); |
| 99 | } | 120 | } |
| 100 | } else { | 121 | } else { |
| 101 | - isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen); | 122 | + isCalcSuccess = CalcLargeTilingParams(tilingData_.sortLen, UB_COEF_PER_TILE); |
| 102 | } | 123 | } |
| 103 | OP_CHECK_IF((!isCalcSuccess), OP_LOGE(opName_, "RadixTopK Tiling failed."), return ge::GRAPH_FAILED); | 124 | OP_CHECK_IF((!isCalcSuccess), OP_LOGE(opName_, "RadixTopK Tiling failed."), return ge::GRAPH_FAILED); |
| 125 | + // int64 模板的 coreTopK/边界计数等 int32 通道要求 kValue 不超 int32 上限 | ||
| 126 | + OP_CHECK_IF((isInt64Shape_ && tilingData_.kValue > INT32_SORT_LEN_MAX), | ||
| 127 | + OP_LOGE(opName_, "RadixTopK kValue [%lu] exceeds int32 limit for int64 shape.", tilingData_.kValue), | ||
| 128 | + return ge::GRAPH_FAILED); | ||
| 129 | + // per-core 直方图计数保持 int32:per-core 元素数须 < 2^31(S < 2^31*coreNum,物理内存内恒成立,防御性校验) | ||
| 130 | + OP_CHECK_IF((isInt64Shape_ && tilingData_.formerTileNum * tilingData_.formerTileLen > INT32_SORT_LEN_MAX), | ||
| 131 | + OP_LOGE(opName_, "RadixTopK per-core element num exceeds int32 limit."), return ge::GRAPH_FAILED); | ||
| 104 | 132 | ||
| 105 | blockDim_ = tilingData_.formerCoreNum + tilingData_.tailCoreNum; | 133 | blockDim_ = tilingData_.formerCoreNum + tilingData_.tailCoreNum; |
| 106 | tilingData_.coreNum = blockDim_; | 134 | tilingData_.coreNum = blockDim_; |
| 107 | - // 仅 UB 变体可复用 indices 内存(WS 变体的 tileTopK/tileHist 与 CopyOutResult 跨 core 冲突) | 135 | + // 仅 UB 变体可复用 indices 内存(WS/INT64 变体的 tileTopK/tileHist 与 CopyOutResult 跨 core 冲突) |
| 108 | - if (isLargeShape_) { | 136 | + if (isLargeShape_ || isInt64Shape_) { |
| 109 | tilingData_.needWorkspace = true; | 137 | tilingData_.needWorkspace = true; |
| 110 | } else { | 138 | } else { |
| 111 | uint64_t wsInt32Cnt = tilingData_.coreNum * (NUM_VALUE_2BIT + 2); | 139 | uint64_t wsInt32Cnt = tilingData_.coreNum * (NUM_VALUE_2BIT + 2); |
| @@ -124,10 +152,11 @@ uint64_t RadixTopKTiling::GetTilingKey() const | |||
| 124 | { | 152 | { |
| 125 | OP_LOGD(opName_, "RadixTopKTiling GetTilingKey."); | 153 | OP_LOGD(opName_, "RadixTopKTiling GetTilingKey."); |
| 126 | 154 | ||
| 127 | - const uint64_t tilingKey = GET_TPL_TILING_KEY(tilingData_.sorted, tilingData_.largest, isLargeShape_); | 155 | + const uint64_t tilingKey = GET_TPL_TILING_KEY(tilingData_.sorted, tilingData_.largest, isLargeShape_, |
| 156 | + isInt64Shape_); | ||
| 128 | OP_LOGD(opName_, "tilingKey is: [%lu]", tilingKey); | 157 | OP_LOGD(opName_, "tilingKey is: [%lu]", tilingKey); |
| 129 | - OP_LOGD(opName_, "sorted, largest, isLargeShape is: [%d, %d, %d]", tilingData_.sorted, tilingData_.largest, | 158 | + OP_LOGD(opName_, "sorted, largest, isLargeShape, isInt64Shape is: [%d, %d, %d, %d]", tilingData_.sorted, |
| 130 | - isLargeShape_); | 159 | + tilingData_.largest, isLargeShape_, isInt64Shape_); |
| 131 | 160 | ||
| 132 | return tilingKey; | 161 | return tilingKey; |
| 133 | } | 162 | } |
| @@ -139,7 +168,7 @@ ge::graphStatus RadixTopKTiling::GetWorkspaceSize() | |||
| 139 | workspaceSize_ = 0; | 168 | workspaceSize_ = 0; |
| 140 | return ge::GRAPH_SUCCESS; | 169 | return ge::GRAPH_SUCCESS; |
| 141 | } | 170 | } |
| 142 | - if (isLargeShape_) { | 171 | + if (isLargeShape_ || isInt64Shape_) { |
| 143 | workspaceSize_ = (NUM_VALUE_2BIT * tilingData_.coreNum + tilingData_.coreNum + tilingData_.coreNum + | 172 | workspaceSize_ = (NUM_VALUE_2BIT * tilingData_.coreNum + tilingData_.coreNum + tilingData_.coreNum + |
| 144 | WS_INT32_PER_TILE * tilingData_.totalTileNum) * | 173 | WS_INT32_PER_TILE * tilingData_.totalTileNum) * |
| 145 | sizeof(int32_t); | 174 | sizeof(int32_t); |
| @@ -199,6 +228,7 @@ void RadixTopKTiling::PrintTilingData() | |||
| 199 | OP_LOGD(opName_, "kValue: %lu.", tilingData_.kValue); | 228 | OP_LOGD(opName_, "kValue: %lu.", tilingData_.kValue); |
| 200 | OP_LOGD(opName_, "largest: %d.", tilingData_.largest); | 229 | OP_LOGD(opName_, "largest: %d.", tilingData_.largest); |
| 201 | OP_LOGD(opName_, "sorted: %d.", tilingData_.sorted); | 230 | OP_LOGD(opName_, "sorted: %d.", tilingData_.sorted); |
| 231 | + OP_LOGD(opName_, "isInt64Shape: %d.", isInt64Shape_); | ||
| 202 | OP_LOGD(opName_, "needWorkspace: %d.", tilingData_.needWorkspace); | 232 | OP_LOGD(opName_, "needWorkspace: %d.", tilingData_.needWorkspace); |
| 203 | } | 233 | } |
| 204 | 234 | ||
| @@ -235,14 +265,19 @@ bool RadixTopKTiling::TryCalcTileDistribution(const uint64_t& dataNum, uint64_t | |||
| 235 | return false; | 265 | return false; |
| 236 | } | 266 | } |
| 237 | 267 | ||
| 238 | -bool RadixTopKTiling::CalcLargeTilingParams(const uint64_t& dataNum) | 268 | +/** |
| 269 | + * @brief 大 shape / int64 shape 的 tile 切分(两者仅 UB 系数不同,合并为一个带参函数) | ||
| 270 | + * @param dataNum 单 core 处理的数据量 | ||
| 271 | + * @param ubCoefPerTile UB 占用中「每 tile 元素数」的系数(int32 索引 16、int64 索引 20) | ||
| 272 | + */ | ||
| 273 | +bool RadixTopKTiling::CalcLargeTilingParams(const uint64_t& dataNum, const uint64_t& ubCoefPerTile) | ||
| 239 | { | 274 | { |
| 240 | uint64_t dataAlign = CACHE_LINE / xDtypeSize_; | 275 | uint64_t dataAlign = CACHE_LINE / xDtypeSize_; |
| 241 | uint64_t minTileLen = dataAlign; | 276 | uint64_t minTileLen = dataAlign; |
| 242 | 277 | ||
| 243 | - auto checkUb = [this](uint64_t tileLen) -> bool { | 278 | + auto checkUb = [this, ubCoefPerTile](uint64_t tileLen) -> bool { |
| 244 | uint64_t cmpMaskSize = tileLen / 8; | 279 | uint64_t cmpMaskSize = tileLen / 8; |
| 245 | - uint64_t ubUsed = 16 * tileLen + cmpMaskSize + MAX_TILE_NUM_IN_UB * sizeof(int32_t) + BLOCK_SIZE; | 280 | + uint64_t ubUsed = ubCoefPerTile * tileLen + cmpMaskSize + MAX_TILE_NUM_IN_UB * sizeof(int32_t) + BLOCK_SIZE; |
| 246 | return ubUsed <= ubSize_; | 281 | return ubUsed <= ubSize_; |
| 247 | }; | 282 | }; |
| 248 | 283 | ||
| @@ -47,7 +47,7 @@ protected: | |||
| 47 | // 7、保存Tiling数据 | 47 | // 7、保存Tiling数据 |
| 48 | ge::graphStatus PostTiling() override; | 48 | ge::graphStatus PostTiling() override; |
| 49 | 49 | ||
| 50 | - bool CalcLargeTilingParams(const uint64_t& dataNum); | 50 | + bool CalcLargeTilingParams(const uint64_t& dataNum, const uint64_t& ubCoefPerTile); |
| 51 | bool CalcTilingParams(const uint64_t& dataNum); | 51 | bool CalcTilingParams(const uint64_t& dataNum); |
| 52 | void PrintTilingData(); | 52 | void PrintTilingData(); |
| 53 | 53 | ||
| @@ -63,6 +63,7 @@ private: | |||
| 63 | uint32_t xDtypeSize_{0}; | 63 | uint32_t xDtypeSize_{0}; |
| 64 | uint64_t xAlign_{0}; | 64 | uint64_t xAlign_{0}; |
| 65 | bool isLargeShape_{false}; | 65 | bool isLargeShape_{false}; |
| 66 | + bool isInt64Shape_{false}; | ||
| 66 | RadixTopKTilingData tilingData_{}; | 67 | RadixTopKTilingData tilingData_{}; |
| 67 | 68 | ||
| 68 | void SetTilingData(); | 69 | void SetTilingData(); |
| @@ -10,23 +10,29 @@ | |||
| 10 | 10 | ||
| 11 | /*! | 11 | /*! |
| 12 | * \file radix_top_k.cpp | 12 | * \file radix_top_k.cpp |
| 13 | - * \brief radix top k op kernel | 13 | + * \brief radix top k op kernel |
| 14 | */ | 14 | */ |
| 15 | #include "kernel_operator.h" | 15 | #include "kernel_operator.h" |
| 16 | #include "radix_top_k_struct.h" | 16 | #include "radix_top_k_struct.h" |
| 17 | #include "radix_top_k_tiling_key.h" | 17 | #include "radix_top_k_tiling_key.h" |
| 18 | #include "radix_top_k_ub.h" | 18 | #include "radix_top_k_ub.h" |
| 19 | #include "radix_top_k_ws.h" | 19 | #include "radix_top_k_ws.h" |
| 20 | +#include "radix_top_k_ws_i64.h" | ||
| 20 | using namespace AscendC; | 21 | using namespace AscendC; |
| 21 | 22 | ||
| 22 | -template <bool sorted, bool largest, bool isLargeShape> | 23 | +template <bool sorted, bool largest, bool isLargeShape, bool isInt64Shape> |
| 23 | -__global__ __aicore__ void radix_top_k(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) | 24 | +__global__ __aicore__ void radix_top_k(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace, |
| 25 | + GM_ADDR tiling) | ||
| 24 | { | 26 | { |
| 25 | TPipe tpipe; | 27 | TPipe tpipe; |
| 26 | REGISTER_TILING_DEFAULT(RadixTopKTilingData); | 28 | REGISTER_TILING_DEFAULT(RadixTopKTilingData); |
| 27 | GET_TILING_DATA(tilingData, tiling); | 29 | GET_TILING_DATA(tilingData, tiling); |
| 28 | #if (defined(DTYPE_X)) | 30 | #if (defined(DTYPE_X)) |
| 29 | - if constexpr (isLargeShape) { | 31 | + if constexpr (isInt64Shape) { |
| 32 | + RadixTopK::RadixTopKWsI64<DTYPE_X, largest> op(tpipe, tilingData); | ||
| 33 | + op.Init(x, k, values, indices, workspace); | ||
| 34 | + op.Process(); | ||
| 35 | + } else if constexpr (isLargeShape) { | ||
| 30 | RadixTopK::RadixTopKWs<DTYPE_X, largest> op(tpipe, tilingData); | 36 | RadixTopK::RadixTopKWs<DTYPE_X, largest> op(tpipe, tilingData); |
| 31 | op.Init(x, k, values, indices, workspace); | 37 | op.Init(x, k, values, indices, workspace); |
| 32 | op.Process(); | 38 | op.Process(); |
| @@ -40,10 +46,9 @@ __global__ __aicore__ void radix_top_k(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ | |||
| 40 | 46 | ||
| 41 | // op_kernel UT 链接桩:仅提供 extern "C" 符号供链接,不参与实际执行逻辑。 | 47 | // op_kernel UT 链接桩:仅提供 extern "C" 符号供链接,不参与实际执行逻辑。 |
| 42 | #ifdef __CCE_UT_TEST__ | 48 | #ifdef __CCE_UT_TEST__ |
| 43 | -extern "C" __global__ __aicore__ void radix_top_k( | 49 | +extern "C" __global__ __aicore__ void radix_top_k(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, |
| 44 | - GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, | 50 | + GM_ADDR workspace, GM_ADDR tiling) |
| 45 | - GM_ADDR workspace, GM_ADDR tiling) | ||
| 46 | { | 51 | { |
| 47 | - radix_top_k<true, true, false>(x, k, values, indices, workspace, tiling); | 52 | + radix_top_k<true, true, false, false>(x, k, values, indices, workspace, tiling); |
| 48 | } | 53 | } |
| 49 | -#endif | 54 | +#endif |
| @@ -59,7 +59,7 @@ using namespace AscendC; | |||
| 59 | #define PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileKVal, vecIndexExpr) \ | 59 | #define PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileKVal, vecIndexExpr) \ |
| 60 | if (curTileKVal > 0) { \ | 60 | if (curTileKVal > 0) { \ |
| 61 | auto xLocal = curBuf.template Get<T>(); \ | 61 | auto xLocal = curBuf.template Get<T>(); \ |
| 62 | - auto cmpMaskTensor = tempBuf_.template Get<uint8_t>(); \ | 62 | + auto cmpMaskTensor = this->tempBuf_.template Get<uint8_t>(); \ |
| 63 | auto vecIndex = (vecIndexExpr); \ | 63 | auto vecIndex = (vecIndexExpr); \ |
| 64 | if constexpr (IsSameType<T, bfloat16_t>::value) { \ | 64 | if constexpr (IsSameType<T, bfloat16_t>::value) { \ |
| 65 | auto xLocalFp32 = xLocal.template ReinterpretCast<float>(); \ | 65 | auto xLocalFp32 = xLocal.template ReinterpretCast<float>(); \ |
| @@ -68,6 +68,9 @@ using namespace AscendC; | |||
| 68 | cmpMaskTensor, vecIndex, tileOffset); \ | 68 | cmpMaskTensor, vecIndex, tileOffset); \ |
| 69 | } else if constexpr (IsSameType<T, half>::value) { \ | 69 | } else if constexpr (IsSameType<T, half>::value) { \ |
| 70 | auto xLocalHalf = xLocal[this->tileLen_]; \ | 70 | auto xLocalHalf = xLocal[this->tileLen_]; \ |
| 71 | + if constexpr (largest) { \ | ||
| 72 | + this->CanonicalizeDataForLargest(curBuf, curTileLen); \ | ||
| 73 | + } \ | ||
| 71 | this->template ProcessOneTileTopK<half>(xLocalHalf, curTileLen, curTileKVal, coreTopKOffset, maxIndexLen, \ | 74 | this->template ProcessOneTileTopK<half>(xLocalHalf, curTileLen, curTileKVal, coreTopKOffset, maxIndexLen, \ |
| 72 | cmpMaskTensor, vecIndex, tileOffset); \ | 75 | cmpMaskTensor, vecIndex, tileOffset); \ |
| 73 | } \ | 76 | } \ |
| @@ -101,7 +104,7 @@ protected: | |||
| 101 | const int32_t& tileHistBinStride, const int32_t& tileHistBinOffset, | 104 | const int32_t& tileHistBinStride, const int32_t& tileHistBinOffset, |
| 102 | const uint64_t& curTileLen); | 105 | const uint64_t& curTileLen); |
| 103 | 106 | ||
| 104 | - __aicore__ inline void NegateDataForLargest(TBuf<TPosition::VECIN>& curBuf, const uint64_t& curTileLen); | 107 | + __aicore__ inline void CanonicalizeDataForLargest(TBuf<TPosition::VECIN>& curBuf, const uint64_t& curTileLen); |
| 105 | 108 | ||
| 106 | template <bool isInit> | 109 | template <bool isInit> |
| 107 | __aicore__ inline void CopyOut2Ws(const uint64_t& maskLen, const uint64_t& gmOffset); | 110 | __aicore__ inline void CopyOut2Ws(const uint64_t& maskLen, const uint64_t& gmOffset); |
| @@ -143,6 +146,13 @@ protected: | |||
| 143 | template <typename CT> | 146 | template <typename CT> |
| 144 | __aicore__ inline CT ExtractBoundaryNumber(); | 147 | __aicore__ inline CT ExtractBoundaryNumber(); |
| 145 | 148 | ||
| 149 | + /** | ||
| 150 | + * @brief 边界值是否为 NaN(位模式判定,不做浮点自比较以免触发浮点相等告警) | ||
| 151 | + * NaN ⟺ 指数位全 1 且尾数非 0(对 ±NaN、任意 payload 都成立) | ||
| 152 | + */ | ||
| 153 | + template <typename CT> | ||
| 154 | + __aicore__ inline bool IsNaNValue(const CT& value); | ||
| 155 | + | ||
| 146 | __aicore__ inline uint64_t CalcBlockOffset(uint64_t batchId); | 156 | __aicore__ inline uint64_t CalcBlockOffset(uint64_t batchId); |
| 147 | 157 | ||
| 148 | protected: | 158 | protected: |
| @@ -271,6 +281,16 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::TwiddleInB16(TBuf<TPosit | |||
| 271 | Not(temp1, temp1, calcLen); | 281 | Not(temp1, temp1, calcLen); |
| 272 | PipeBarrier<PIPE_V>(); | 282 | PipeBarrier<PIPE_V>(); |
| 273 | And(xLocalInt16, xLocalInt16, temp1, calcLen); | 283 | And(xLocalInt16, xLocalInt16, temp1, calcLen); |
| 284 | + if constexpr (!largest && (sizeof(T) == sizeof(int16_t))) { | ||
| 285 | + // largest=false 的降序由键域取补承担:先做升序键(上面的 twiddle),再整体取补。 | ||
| 286 | + // 对全部 65536 个 16 位模式恒有 ~twiddle(x) == twiddle(bits(-x)),故除 NaN 外本步 | ||
| 287 | + // 与「先取反再 twiddle」逐位一致,序键语义不变;仅 NaN 受影响: | ||
| 288 | + // CanonicalizeDataForLargest 已把 ±NaN 规范化为同一个正 NaN(实测 fp16 与 bf16 都是 | ||
| 289 | + // 0x7FFF,与输入 NaN 的符号无关),其升序键为 0xFFFF,取补后为 0x0000,低于 -Inf 的 | ||
| 290 | + // 0x007F(bf16)/0x03FF(fp16),落到序键最底端 ⇒ 在计数相不被计入 k 槽、提取相也不会被选中。 | ||
| 291 | + PipeBarrier<PIPE_V>(); | ||
| 292 | + Not(xLocalInt16, xLocalInt16, calcLen); | ||
| 293 | + } | ||
| 274 | } | 294 | } |
| 275 | 295 | ||
| 276 | /** | 296 | /** |
| @@ -339,23 +359,49 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::CalcCumsumHistogram16( | |||
| 339 | } | 359 | } |
| 340 | 360 | ||
| 341 | /** | 361 | /** |
| 342 | - * @brief 当 largest=false 时对数据取反,将求最小值转为求最大值 | 362 | + * @brief ±NaN 规范化(两个方向都要做):把 ±NaN 统一变成同一个正 NaN(16 位类型的原域), |
| 363 | + * 使 TwiddleInB16 键域对 NaN 的记账与提取相的掩码口径一致;有限值与 ±Inf 逐位不变 | ||
| 364 | + * (fp16/bf16 → fp32 → ×1 → 回 16 位是精确往返)。 | ||
| 365 | + * - largest=false:规范化后 NaN 键为 0xFFFF,经 TwiddleInB16 末尾的键域取补落 0x0000 | ||
| 366 | + * (低于 -Inf 的 0x007F/0x03FF)⇒ 计数相不计入 k 槽、提取相 <LT> 也不匹配 | ||
| 367 | + * ⇒ NaN 不入选(与 torch 语义一致)。降序本身由该键域取补承担,故此处不取反。 | ||
| 368 | + * - largest=true:规范化后 NaN 键为 0xFFFF(最大端)⇒ 计数相把 NaN 计入该 tile 的 | ||
| 369 | + * k 配额,与提取相 NOT(LE) = (x > b) ∪ isNaN 口径一致 ⇒ NaN 必入选(与 torch 一致)。 | ||
| 370 | + * **缺了这一步会丢 −NaN**:−NaN 原样 twiddle 后键为 ~0xFE00=0x01FF(fp16) / | ||
| 371 | + * ~0xFFC0=0x003F(bf16),落在 −Inf 之下 ⇒ 直方图不计入配额,但提取掩码仍匹配 | ||
| 372 | + * ⇒ rsvdCntV 超出该 tile 配额,被 SubTopKAndCopyOut 的 min(rsvdCntV, curTileK) | ||
| 373 | + * 截断(GatherMask 按 lane 序保留靠前的匹配)⇒ −NaN 槽丢空、背景值顶替。 | ||
| 374 | + * 实测(S=1.2e7+4 / k=1001 / 尾块 8 个 ±NaN / largest=true):fp16 只中 +NaN 半边、 | ||
| 375 | + * −NaN 0/4、values 多重集错位 410/1001;bf16 同类(−NaN 0/4,错位 8/1001,量化粗 | ||
| 376 | + * 使错位值多为等值)。全 +NaN 输入(8/8、错位 0)与 largest=false(NaN 0 入选、 | ||
| 377 | + * 错位 0)是对照。 | ||
| 343 | * @param curBuf 当前数据缓冲区 | 378 | * @param curBuf 当前数据缓冲区 |
| 344 | * @param curTileLen 当前 tile 数据长度 | 379 | * @param curTileLen 当前 tile 数据长度 |
| 345 | */ | 380 | */ |
| 346 | template <typename T, bool largest> | 381 | template <typename T, bool largest> |
| 347 | -__aicore__ inline void RadixTopKBaseKernel<T, largest>::NegateDataForLargest(TBuf<TPosition::VECIN>& curBuf, | 382 | +__aicore__ inline void RadixTopKBaseKernel<T, largest>::CanonicalizeDataForLargest(TBuf<TPosition::VECIN>& curBuf, |
| 348 | - const uint64_t& curTileLen) | 383 | + const uint64_t& curTileLen) |
| 349 | { | 384 | { |
| 350 | - if constexpr (!largest) { | 385 | + LocalTensor<T> xCanon = curBuf.Get<T>(); |
| 351 | - LocalTensor<T> xNgt = curBuf.Get<T>(); | 386 | + PipeBarrier<PIPE_V>(); |
| 387 | + if constexpr (IsSameType<T, bfloat16_t>::value) { | ||
| 388 | + // bf16:Muls 不支持 bf16(编译报错),只能借 fp32 域做规范化 —— 实测该往返把 ±NaN 归成 | ||
| 389 | + // 0x7FFF,±Inf/±0/次正规码无损。 | ||
| 390 | + // 注意这里是乘 +1 而非 −1:降序由 TwiddleInB16 末尾的键域取补承担,本步只需把 ±NaN | ||
| 391 | + // 统一成同一个正 NaN(否则 −NaN 取补后会落到序键顶端而被误选,即 q7 缺陷)。 | ||
| 392 | + // (坑位记录:当年 bf16 的取反是「把位模式 reinterpret 成 half 再乘 −1」, | ||
| 393 | + // |x|≥2^121 与 ±Inf/NaN 会落进 half 的非规格区退化为 NaN ⇒ D64/S2。) | ||
| 394 | + LocalTensor<float> xFp32 = outValueBuf_.Get<float>(); | ||
| 395 | + Cast(xFp32, xCanon[tileLen_], RoundMode::CAST_NONE, curTileLen); | ||
| 352 | PipeBarrier<PIPE_V>(); | 396 | PipeBarrier<PIPE_V>(); |
| 353 | - if constexpr (IsSameType<T, bfloat16_t>::value) { | 397 | + Muls(xFp32, xFp32, (float)1, curTileLen); |
| 354 | - Muls(xNgt[tileLen_].template ReinterpretCast<half>(), xNgt[tileLen_].template ReinterpretCast<half>(), | 398 | + PipeBarrier<PIPE_V>(); |
| 355 | - (half)-1, curTileLen); | 399 | + Cast(xCanon[tileLen_], xFp32, RoundMode::CAST_RINT, curTileLen); |
| 356 | - } else { | 400 | + } else if constexpr (IsSameType<T, half>::value) { |
| 357 | - Muls(xNgt[tileLen_], xNgt[tileLen_], (T)-1, curTileLen); | 401 | + // fp16:原生同类型 Muls(x, 1)。实测:与「Cast→fp32→Muls(+1)→Cast 回」对 |
| 358 | - } | 402 | + // ±NaN/±Inf/±0/次正规码等全部边界输入逐位一致(±NaN 都规范化成 0x7FFF)。 |
| 403 | + // 省掉 2 条向量指令与 2 次 PipeBarrier,也不再需要 fp32 暂存。 | ||
| 404 | + Muls(xCanon[tileLen_], xCanon[tileLen_], (half)1, curTileLen); | ||
| 359 | } | 405 | } |
| 360 | } | 406 | } |
| 361 | 407 | ||
| @@ -407,6 +453,23 @@ __aicore__ inline uint32_t RadixTopKBaseKernel<T, largest>::CreateVecIndex4TopK( | |||
| 407 | return maxIndexLen; | 453 | return maxIndexLen; |
| 408 | } | 454 | } |
| 409 | 455 | ||
| 456 | +/** | ||
| 457 | + * @brief 单个 tile 内按 cmpMode 提取匹配 boundaryNumber 的元素并输出 | ||
| 458 | + */ | ||
| 459 | +template <typename T, bool largest> | ||
| 460 | +template <typename CT> | ||
| 461 | +__aicore__ inline bool RadixTopKBaseKernel<T, largest>::IsNaNValue(const CT& value) | ||
| 462 | +{ | ||
| 463 | + float valueFp32; | ||
| 464 | + if constexpr (IsSameType<CT, float>::value) { | ||
| 465 | + valueFp32 = value; | ||
| 466 | + } else { | ||
| 467 | + valueFp32 = static_cast<float>(value); // half -> fp32,NaN 语义保持 | ||
| 468 | + } | ||
| 469 | + uint32_t bits = *reinterpret_cast<uint32_t*>(&valueFp32); | ||
| 470 | + return (bits & 0x7FFFFFFFU) > 0x7F800000U; // 指数位全 1 且尾数非 0 ⇒ NaN | ||
| 471 | +} | ||
| 472 | + | ||
| 410 | /** | 473 | /** |
| 411 | * @brief 单个 tile 内按 cmpMode 提取匹配 boundaryNumber 的元素并输出 | 474 | * @brief 单个 tile 内按 cmpMode 提取匹配 boundaryNumber 的元素并输出 |
| 412 | * 先 Compares → GatherMask values/indices → CopyOutResult | 475 | * 先 Compares → GatherMask values/indices → CopyOutResult |
| @@ -438,7 +501,37 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::SubTopKAndCopyOut( | |||
| 438 | uint64_t curTileLenAlign = Ops::Base::CeilAlign(curTileLen, REPEAT_SIZE / sizeof(CT)); | 501 | uint64_t curTileLenAlign = Ops::Base::CeilAlign(curTileLen, REPEAT_SIZE / sizeof(CT)); |
| 439 | 502 | ||
| 440 | PipeBarrier<PIPE_V>(); | 503 | PipeBarrier<PIPE_V>(); |
| 441 | - Compares<CT, uint8_t>(cmpMaskTensor, xLocal, boundaryNumber, cmpMode, curTileLenAlign); | 504 | + if constexpr (largest && cmpMode != AscendC::CMPMODE::EQ && |
| 505 | + (IsSameType<CT, float>::value || IsSameType<CT, half>::value)) { | ||
| 506 | + if (IsNaNValue(boundaryNumber)) { | ||
| 507 | + // 边界值本身就是 NaN:候选入选集合只有 NaN(key 域 0xFFFF 组,计数相已把 ±NaN 归入同一 | ||
| 508 | + // bin 并按 NaN 个数分配配额)。此时上式的 NOT(LE) 会退化成「全真」——x <= NaN 对**有限值 | ||
| 509 | + // 同样为 false**,取反后掩码全 1,GatherMask 于是把 tile 前 curTileK 个(有限的)元素当成 | ||
| 510 | + // 入选者拷出;实测混合数据 + largest=true 时表现为 top-k 里 NaN 只剩零星几个(1015 个 NaN | ||
| 511 | + // 只选中 3 个,其余槽被背景元素顶替)。 | ||
| 512 | + // 改取 isNaN 掩码:NaN == NaN 恒为 false,Compare(x, x, EQ) 逐位取反即 isNaN, | ||
| 513 | + // 对 ±NaN 与任意 payload 都成立,与计数相的配额口径完全一致。 | ||
| 514 | + Compare<CT, uint8_t>(cmpMaskTensor, xLocal, xLocal, AscendC::CMPMODE::EQ, curTileLenAlign); | ||
| 515 | + PipeBarrier<PIPE_V>(); | ||
| 516 | + Not(cmpMaskTensor.template ReinterpretCast<int16_t>(), cmpMaskTensor.template ReinterpretCast<int16_t>(), | ||
| 517 | + static_cast<uint32_t>((curTileLenAlign >> 3) / sizeof(int16_t))); | ||
| 518 | + } else { | ||
| 519 | + // largest=true 的「必然入选」通过:NaN 的序键在 twiddle 域位于最大端(key ∈ [0xFF81, 0xFFFF]), | ||
| 520 | + // 直方图按 key 统计时已把 NaN 计入该 tile 的配额,但 x > b 对 NaN 恒为 false, | ||
| 521 | + // 于是 NaN 不被拷出、配额反被背景元素顶替(S1:被顶替槽数精确等于注入的 NaN 数)。 | ||
| 522 | + // 这里不直接 Compares(GT),而取 NOT(LE):LE 对 NaN 恒为 false,逐位取反即为 true, | ||
| 523 | + // 掩码 = (x > b) ∪ isNaN,与 key 域的配额统计口径一致(逐位运算,全值域有效)。 | ||
| 524 | + // largest=false 不加:key 域中 +NaN 不计入入选,且 torch 语义要求 NaN 不入选; | ||
| 525 | + // 整型 CT 无 NaN,走下面的直通分支。 | ||
| 526 | + Compares<CT, uint8_t>(cmpMaskTensor, xLocal, boundaryNumber, AscendC::CMPMODE::LE, curTileLenAlign); | ||
| 527 | + PipeBarrier<PIPE_V>(); | ||
| 528 | + Not(cmpMaskTensor.ReinterpretCast<int16_t>(), cmpMaskTensor.ReinterpretCast<int16_t>(), | ||
| 529 | + static_cast<uint32_t>((curTileLenAlign >> 3) / sizeof(int16_t))); | ||
| 530 | + } | ||
| 531 | + } else { | ||
| 532 | + PipeBarrier<PIPE_V>(); | ||
| 533 | + Compares<CT, uint8_t>(cmpMaskTensor, xLocal, boundaryNumber, cmpMode, curTileLenAlign); | ||
| 534 | + } | ||
| 442 | uint64_t rsvdCntV = 0, rsvdCntI = 0; | 535 | uint64_t rsvdCntV = 0, rsvdCntI = 0; |
| 443 | WaitFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForValue_); | 536 | WaitFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForValue_); |
| 444 | PipeBarrier<PIPE_V>(); | 537 | PipeBarrier<PIPE_V>(); |
| @@ -467,7 +560,11 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::SubTopKAndCopyOut( | |||
| 467 | {1, 1, 0, 0}, rsvdCntI); | 560 | {1, 1, 0, 0}, rsvdCntI); |
| 468 | } | 561 | } |
| 469 | VToSSync(); | 562 | VToSSync(); |
| 470 | - uint32_t count = (cmpMode != AscendC::CMPMODE::EQ) ? rsvdCntV : curTileK; | 563 | + // 非 EQ 通过按 tile 配额封顶:NaN 掩码并入后 rsvdCntV 可能略多于直方图统计出的配额 |
| 564 | + // (例如混入 key 域未计入的 -NaN),封顶可避免拷出到 k 槽之外的越界写 | ||
| 565 | + uint64_t passRsvdCnt = (cmpMode != AscendC::CMPMODE::EQ) ? rsvdCntV : static_cast<uint64_t>(curTileK); | ||
| 566 | + uint32_t count = static_cast<uint32_t>( | ||
| 567 | + passRsvdCnt < static_cast<uint64_t>(curTileK) ? passRsvdCnt : static_cast<uint64_t>(curTileK)); | ||
| 471 | SToVSync(); | 568 | SToVSync(); |
| 472 | if constexpr (IsSameType<T, bfloat16_t>::value) { | 569 | if constexpr (IsSameType<T, bfloat16_t>::value) { |
| 473 | PipeBarrier<PIPE_V>(); | 570 | PipeBarrier<PIPE_V>(); |
| @@ -493,12 +590,16 @@ __aicore__ inline void RadixTopKBaseKernel<T, largest>::CopyOutResult(const int3 | |||
| 493 | { | 590 | { |
| 494 | LocalTensor<T> outValueLocal = outValueBuf_.Get<T>(); | 591 | LocalTensor<T> outValueLocal = outValueBuf_.Get<T>(); |
| 495 | LocalTensor<int32_t> outIndexLocal = outIndexBuf_.Get<int32_t>(); | 592 | LocalTensor<int32_t> outIndexLocal = outIndexBuf_.Get<int32_t>(); |
| 496 | - | 593 | + if (count > 0) { |
| 497 | - DataCopyPad(valueGm_[coreTopKOffset], outValueLocal, | 594 | + DataCopyPad(valueGm_[coreTopKOffset], outValueLocal, |
| 498 | - DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(T) * count), 0, 0, 0}); | 595 | + DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(T) * count), 0, 0, 0}); |
| 596 | + } | ||
| 499 | SetFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForValue_); | 597 | SetFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForValue_); |
| 500 | - DataCopyPad(indexGm_[coreTopKOffset], outIndexLocal, | 598 | + if (count > 0) { |
| 501 | - DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * count), 0, 0, 0}); | 599 | + DataCopyPad( |
| 600 | + indexGm_[coreTopKOffset], outIndexLocal, | ||
| 601 | + DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * count), 0, 0, 0}); | ||
| 602 | + } | ||
| 502 | SetFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForIndex_); | 603 | SetFlag<HardEvent::MTE3_V>(eventIDMTE3ToVForIndex_); |
| 503 | } | 604 | } |
| 504 | 605 | ||
| @@ -22,23 +22,23 @@ | |||
| 22 | * 包含数据切分、核分配、tile 参数等编译期确定的常量信息 | 22 | * 包含数据切分、核分配、tile 参数等编译期确定的常量信息 |
| 23 | */ | 23 | */ |
| 24 | struct RadixTopKTilingData { | 24 | struct RadixTopKTilingData { |
| 25 | - uint64_t coreNum; /**< 总 core 数 */ | 25 | + uint64_t coreNum; /**< 总 core 数 */ |
| 26 | - uint64_t formerCoreNum; /**< 主核数量(处理 formerTileNum 个 tile) */ | 26 | + uint64_t formerCoreNum; /**< 主核数量(处理 formerTileNum 个 tile) */ |
| 27 | - uint64_t tailCoreNum; /**< 尾核数量(处理 tailTileNum 个 tile) */ | 27 | + uint64_t tailCoreNum; /**< 尾核数量(处理 tailTileNum 个 tile) */ |
| 28 | - uint64_t totalTileNum; /**< 总 tile 数 */ | 28 | + uint64_t totalTileNum; /**< 总 tile 数 */ |
| 29 | - uint64_t formerTileNum; /**< 主核 tile 数 */ | 29 | + uint64_t formerTileNum; /**< 主核 tile 数 */ |
| 30 | - uint64_t tailTileNum; /**< 尾核 tile 数 */ | 30 | + uint64_t tailTileNum; /**< 尾核 tile 数 */ |
| 31 | - uint64_t formerTileLen; /**< 每个 tile 的数据长度(主核/尾核常规tile) */ | 31 | + uint64_t formerTileLen; /**< 每个 tile 的数据长度(主核/尾核常规tile) */ |
| 32 | - uint64_t tailTileLen; /**< 最后一个 tail tile 的数据长度 */ | 32 | + uint64_t tailTileLen; /**< 最后一个 tail tile 的数据长度 */ |
| 33 | - uint64_t batch; /**< batch 数(除最后一维外各维度乘积) */ | 33 | + uint64_t batch; /**< batch 数(除最后一维外各维度乘积) */ |
| 34 | uint64_t sortLen; /**< 排序轴长度(输入最后一维长度) */ | 34 | uint64_t sortLen; /**< 排序轴长度(输入最后一维长度) */ |
| 35 | - uint64_t kValue; /**< topK 值 */ | 35 | + uint64_t kValue; /**< topK 值 */ |
| 36 | 36 | ||
| 37 | - bool largest; /**< 是否求最大 k 值(true: largest, false: smallest) */ | 37 | + bool largest; /**< 是否求最大 k 值(true: largest, false: smallest) */ |
| 38 | - bool sorted; /**< 输出结果是否排序(当前仅支持 false) */ | 38 | + bool sorted; /**< 输出结果是否排序(当前仅支持 false) */ |
| 39 | - bool needWorkspace; /**< 是否使用 workspace(false: 复用 indices 内存做暂存) */ | 39 | + bool needWorkspace; /**< 是否使用 workspace(false: 复用 indices 内存做暂存) */ |
| 40 | }; | 40 | }; |
| 41 | 41 | ||
| 42 | #pragma pack(pop) | 42 | #pragma pack(pop) |
| 43 | 43 | ||
| 44 | -#endif | 44 | +#endif |
| @@ -24,23 +24,18 @@ | |||
| 24 | * sorted: 输出是否排序(0/1) | 24 | * sorted: 输出是否排序(0/1) |
| 25 | * largest: 是否求最大 k 值(0/1) | 25 | * largest: 是否求最大 k 值(0/1) |
| 26 | * isLargeShape: 是否大 shape 场景(0/1),用于选择 UB/WS 变体 | 26 | * isLargeShape: 是否大 shape 场景(0/1),用于选择 UB/WS 变体 |
| 27 | + * isInt64Shape: sortLen 超过 int32 上限(0/1),选择 int64 变体(追加在最高位, | ||
| 28 | + * 现有 3 参数 key 编码值不变) | ||
| 27 | */ | 29 | */ |
| 28 | -ASCENDC_TPL_ARGS_DECL(RadixTopK, | 30 | +ASCENDC_TPL_ARGS_DECL(RadixTopK, ASCENDC_TPL_BOOL_DECL(sorted, 0, 1), ASCENDC_TPL_BOOL_DECL(largest, 0, 1), |
| 29 | - ASCENDC_TPL_BOOL_DECL(sorted, 0, 1), | 31 | + ASCENDC_TPL_BOOL_DECL(isLargeShape, 0, 1), ASCENDC_TPL_BOOL_DECL(isInt64Shape, 0, 1)); |
| 30 | - ASCENDC_TPL_BOOL_DECL(largest, 0, 1), | ||
| 31 | - ASCENDC_TPL_BOOL_DECL(isLargeShape, 0, 1) | ||
| 32 | -); | ||
| 33 | 32 | ||
| 34 | /** | 33 | /** |
| 35 | * @brief 模板参数组合选择 | 34 | * @brief 模板参数组合选择 |
| 36 | * 用于 GET_TPL_TILING_KEY 获取 TilingKey 时校验 TilingKey 合法性 | 35 | * 用于 GET_TPL_TILING_KEY 获取 TilingKey 时校验 TilingKey 合法性 |
| 37 | */ | 36 | */ |
| 38 | -ASCENDC_TPL_SEL( | 37 | +ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_BOOL_SEL(sorted, 0, 1), ASCENDC_TPL_BOOL_SEL(largest, 0, 1), |
| 39 | - ASCENDC_TPL_ARGS_SEL( | 38 | + ASCENDC_TPL_BOOL_SEL(isLargeShape, 0, 1), |
| 40 | - ASCENDC_TPL_BOOL_SEL(sorted, 0, 1), | 39 | + ASCENDC_TPL_BOOL_SEL(isInt64Shape, 0, 1)), ); |
| 41 | - ASCENDC_TPL_BOOL_SEL(largest, 0, 1), | ||
| 42 | - ASCENDC_TPL_BOOL_SEL(isLargeShape, 0, 1) | ||
| 43 | - ), | ||
| 44 | -); | ||
| 45 | 40 | ||
| 46 | #endif // RADIX_TOP_K_TILING_KEY_H | 41 | #endif // RADIX_TOP_K_TILING_KEY_H |
| @@ -73,7 +73,7 @@ private: | |||
| 73 | template <typename T, bool largest> | 73 | template <typename T, bool largest> |
| 74 | __aicore__ inline void RadixTopKUb<T, largest>::Process() | 74 | __aicore__ inline void RadixTopKUb<T, largest>::Process() |
| 75 | { | 75 | { |
| 76 | - for (int32_t batchId = 0; batchId < this->batch_; batchId++) { | 76 | + for (uint64_t batchId = 0; batchId < this->batch_; batchId++) { |
| 77 | SubProcess(batchId); | 77 | SubProcess(batchId); |
| 78 | SyncAll(); | 78 | SyncAll(); |
| 79 | } | 79 | } |
| @@ -183,7 +183,7 @@ __aicore__ inline void RadixTopKUb<T, largest>::SubProcess(uint64_t batchId) | |||
| 183 | for (int32_t tileId = 0; tileId < this->tileNum_; tileId++) { | 183 | for (int32_t tileId = 0; tileId < this->tileNum_; tileId++) { |
| 184 | PINGPONG_TILE_BEGIN(tileId, blockOffset); | 184 | PINGPONG_TILE_BEGIN(tileId, blockOffset); |
| 185 | 185 | ||
| 186 | - this->NegateDataForLargest(curBuf, curTileLen); | 186 | + this->CanonicalizeDataForLargest(curBuf, curTileLen); |
| 187 | this->TwiddleInB16(curBuf, curTileLen); | 187 | this->TwiddleInB16(curBuf, curTileLen); |
| 188 | this->DoAndMask(curBuf, curTileLen); | 188 | this->DoAndMask(curBuf, curTileLen); |
| 189 | LocalTensor<int32_t> tileHist = tempBuf_.Get<int32_t>(); | 189 | LocalTensor<int32_t> tileHist = tempBuf_.Get<int32_t>(); |
| @@ -294,6 +294,8 @@ __aicore__ inline void RadixTopKUb<T, largest>::HandleLastRoundBoundary(LocalTen | |||
| 294 | LocalTensor<float>& tileHistFp32) | 294 | LocalTensor<float>& tileHistFp32) |
| 295 | { | 295 | { |
| 296 | PipeBarrier<PIPE_V>(); | 296 | PipeBarrier<PIPE_V>(); |
| 297 | + // 覆盖两个分支:ReduceGlobalHist 末尾的 Scalar SetValue 写过 resTensor, | ||
| 298 | + // 后续 V pipe 的 Sub(写 resTensor) 与 else 分支 Cast(写 tileHistFp32=resTensor) 均需 S->V 可见性 | ||
| 297 | SToVSync(); | 299 | SToVSync(); |
| 298 | if (this->boundaryBinPrev < this->numValue_) { | 300 | if (this->boundaryBinPrev < this->numValue_) { |
| 299 | Sub(resTensor, tileHist[this->boundaryBin * this->tileNumAlign_], | 301 | Sub(resTensor, tileHist[this->boundaryBin * this->tileNumAlign_], |
| @@ -387,6 +389,13 @@ __aicore__ inline void RadixTopKUb<T, largest>::TileTopK(const uint64_t& batchId | |||
| 387 | uint64_t firstLen = (firstId == this->tileNum_ - 1) ? this->tailTileLen_ : this->tileLen_; | 389 | uint64_t firstLen = (firstId == this->tileNum_ - 1) ? this->tailTileLen_ : this->tileLen_; |
| 388 | this->CopyIn(this->xBufPing_, firstLen, blockOffset + firstId * this->tileLen_); | 390 | this->CopyIn(this->xBufPing_, firstLen, blockOffset + firstId * this->tileLen_); |
| 389 | uint32_t maxIndexLen = CreateVecIndex4TopK(); | 391 | uint32_t maxIndexLen = CreateVecIndex4TopK(); |
| 392 | + // maxIndexLen == 0 表示 tempBuf_ 剩余空间放不下 512 个预生成索引,此时 | ||
| 393 | + // SubTopKAndCopyOut 会走「现算索引」兜底分支,vecIndex 不会被使用。 | ||
| 394 | + // 所以必须判空后再取句柄,0 时用 buffer 首地址占位。 | ||
| 395 | + LocalTensor<int32_t> vecIndexTopK = maxIndexLen > 0 ? | ||
| 396 | + tempBuf_.GetWithOffset<int32_t>( | ||
| 397 | + maxIndexLen, this->tileLen_ * sizeof(uint8_t) / BYTE_SIZE) : | ||
| 398 | + tempBuf_.Get<int32_t>(); // 占位句柄,0 时不会被使用 | ||
| 390 | 399 | ||
| 391 | this->InitTopKEvent(); | 400 | this->InitTopKEvent(); |
| 392 | 401 | ||
| @@ -398,8 +407,7 @@ __aicore__ inline void RadixTopKUb<T, largest>::TileTopK(const uint64_t& batchId | |||
| 398 | 407 | ||
| 399 | int32_t curTileK = tileTopK.GetValue(tileId); | 408 | int32_t curTileK = tileTopK.GetValue(tileId); |
| 400 | PipeBarrier<PIPE_V>(); | 409 | PipeBarrier<PIPE_V>(); |
| 401 | - PROCESS_ONE_TILE_TOPK_OR_SKIP( | 410 | + PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileK, vecIndexTopK); |
| 402 | - curTileK, tempBuf_.GetWithOffset<int32_t>(maxIndexLen, this->tileLen_ * sizeof(uint8_t) / BYTE_SIZE)); | ||
| 403 | 411 | ||
| 404 | PINGPONG_TILE_END(tileId); | 412 | PINGPONG_TILE_END(tileId); |
| 405 | } | 413 | } |
| @@ -30,36 +30,20 @@ public: | |||
| 30 | 30 | ||
| 31 | __aicore__ inline void Init(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace) | 31 | __aicore__ inline void Init(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace) |
| 32 | { | 32 | { |
| 33 | - InitParams(); | 33 | + InitWsGm(x, k, values, indices, workspace); |
| 34 | - this->xGm_.SetGlobalBuffer((__gm__ T*)x); | 34 | + InitBuffers(sizeof(int32_t)); // 索引输出 int32;int64 大 shape 子类改传 sizeof(int64_t) |
| 35 | - this->valueGm_.SetGlobalBuffer((__gm__ T*)values); | ||
| 36 | - this->kGm_.SetGlobalBuffer((__gm__ int32_t*)k); | ||
| 37 | - this->indexGm_.SetGlobalBuffer((__gm__ int32_t*)indices); | ||
| 38 | - this->globalHistGm_.SetGlobalBuffer((__gm__ int32_t*)workspace); | ||
| 39 | - this->boundaryBinCumSumGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_); | ||
| 40 | - this->coreTopKGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + | ||
| 41 | - this->coreNum_); | ||
| 42 | - uint32_t tileTopKOffset = 0; | ||
| 43 | - if (this->blockIdx_ < this->formerCoreNum_) { | ||
| 44 | - tileTopKOffset = this->blockIdx_ * this->formerTileNum_; | ||
| 45 | - } else { | ||
| 46 | - tileTopKOffset = this->formerCoreNum_ * this->formerTileNum_ + | ||
| 47 | - (this->blockIdx_ - this->formerCoreNum_) * this->tailTileNum_; | ||
| 48 | - } | ||
| 49 | - uint32_t tileHistOffset = this->numValue_ * tileTopKOffset; | ||
| 50 | - tileTopKGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_ * 2 + | ||
| 51 | - tileTopKOffset); | ||
| 52 | - tileHistGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_ * 2 + | ||
| 53 | - this->totalTileNum_ + tileHistOffset); | ||
| 54 | - InitBuffers(); | ||
| 55 | } | 35 | } |
| 56 | 36 | ||
| 57 | __aicore__ inline void SubProcess(uint64_t batchId); | 37 | __aicore__ inline void SubProcess(uint64_t batchId); |
| 58 | __aicore__ inline void Process(); | 38 | __aicore__ inline void Process(); |
| 59 | 39 | ||
| 60 | -private: | 40 | + // int64 大 shape 子类 RadixTopKWsI64 复用以下成员与 WS 搬运工具(仅访问级别,无逻辑变化) |
| 41 | +protected: | ||
| 42 | + /** @brief 初始化 GM/WS 布局(含 indexGm_ 的 int32 视图);int64 子类复用后再补设 indexGm64_ */ | ||
| 43 | + __aicore__ inline void InitWsGm(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace); | ||
| 61 | __aicore__ inline void InitParams(); | 44 | __aicore__ inline void InitParams(); |
| 62 | - __aicore__ inline void InitBuffers(); | 45 | + /** @brief 形参为索引输出元素字节数:int32 变体 sizeof(int32_t)、int64 变体 sizeof(int64_t) */ |
| 46 | + __aicore__ inline void InitBuffers(const uint32_t& indexElemSize); | ||
| 63 | __aicore__ inline void ClearTileTopKInWs(); | 47 | __aicore__ inline void ClearTileTopKInWs(); |
| 64 | __aicore__ inline void ClearHistInWs(const int32_t& roundId); | 48 | __aicore__ inline void ClearHistInWs(const int32_t& roundId); |
| 65 | __aicore__ inline void ClearTileHistInWs(const uint32_t& dstOffset, const uint32_t& dataNum); | 49 | __aicore__ inline void ClearTileHistInWs(const uint32_t& dstOffset, const uint32_t& dataNum); |
| @@ -86,6 +70,8 @@ private: | |||
| 86 | __aicore__ inline void AddTileHistToTileTopK(LocalTensor<int32_t>& globalHist); | 70 | __aicore__ inline void AddTileHistToTileTopK(LocalTensor<int32_t>& globalHist); |
| 87 | __aicore__ inline void HandleLastRoundBoundary(LocalTensor<int32_t>& resTensor, LocalTensor<float>& tileHistFp32); | 71 | __aicore__ inline void HandleLastRoundBoundary(LocalTensor<int32_t>& resTensor, LocalTensor<float>& tileHistFp32); |
| 88 | __aicore__ inline void WriteCoreTopKFromWs(LocalTensor<int32_t>& resTensor, LocalTensor<float>& tileHistFp32); | 72 | __aicore__ inline void WriteCoreTopKFromWs(LocalTensor<int32_t>& resTensor, LocalTensor<float>& tileHistFp32); |
| 73 | + /** @brief 前驱 core 边界 bin 累计数(int64 累加,总量可达 sortLen > 2^31;int32 形状下同样安全) */ | ||
| 74 | + __aicore__ inline void ComputeCumSumPrevI64(LocalTensor<int32_t>& resTensor, int64_t& cumSumValuePrev); | ||
| 89 | 75 | ||
| 90 | GlobalTensor<int32_t> tileTopKGm_; /**< WS 中 tileTopK 段的 Global Tensor */ | 76 | GlobalTensor<int32_t> tileTopKGm_; /**< WS 中 tileTopK 段的 Global Tensor */ |
| 91 | GlobalTensor<int32_t> tileHistGm_; /**< WS 中 tileHist 段的 Global Tensor */ | 77 | GlobalTensor<int32_t> tileHistGm_; /**< WS 中 tileHist 段的 Global Tensor */ |
| @@ -107,13 +93,43 @@ private: | |||
| 107 | template <typename T, bool largest> | 93 | template <typename T, bool largest> |
| 108 | __aicore__ inline void RadixTopKWs<T, largest>::Process() | 94 | __aicore__ inline void RadixTopKWs<T, largest>::Process() |
| 109 | { | 95 | { |
| 110 | - for (int32_t batchId = 0; batchId < this->batch_; batchId++) { | 96 | + for (uint64_t batchId = 0; batchId < this->batch_; batchId++) { |
| 111 | SubProcess(batchId); | 97 | SubProcess(batchId); |
| 112 | SyncAll(); | 98 | SyncAll(); |
| 113 | } | 99 | } |
| 114 | GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(this->eventIDMTE2ToVForX_); | 100 | GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(this->eventIDMTE2ToVForX_); |
| 115 | } | 101 | } |
| 116 | 102 | ||
| 103 | +/** | ||
| 104 | + * @brief 初始化 GM/WS 布局(含 indexGm_ 的 int32 视图) | ||
| 105 | + * int64 大 shape 子类 RadixTopKWsI64 复用本函数后再补设 indexGm64_,不重复整段布局代码 | ||
| 106 | + */ | ||
| 107 | +template <typename T, bool largest> | ||
| 108 | +__aicore__ inline void RadixTopKWs<T, largest>::InitWsGm(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, | ||
| 109 | + GM_ADDR workspace) | ||
| 110 | +{ | ||
| 111 | + InitParams(); | ||
| 112 | + this->xGm_.SetGlobalBuffer((__gm__ T*)x); | ||
| 113 | + this->valueGm_.SetGlobalBuffer((__gm__ T*)values); | ||
| 114 | + this->kGm_.SetGlobalBuffer((__gm__ int32_t*)k); | ||
| 115 | + this->indexGm_.SetGlobalBuffer((__gm__ int32_t*)indices); | ||
| 116 | + this->globalHistGm_.SetGlobalBuffer((__gm__ int32_t*)workspace); | ||
| 117 | + this->boundaryBinCumSumGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_); | ||
| 118 | + this->coreTopKGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_); | ||
| 119 | + uint32_t tileTopKOffset = 0; | ||
| 120 | + if (this->blockIdx_ < this->formerCoreNum_) { | ||
| 121 | + tileTopKOffset = this->blockIdx_ * this->formerTileNum_; | ||
| 122 | + } else { | ||
| 123 | + tileTopKOffset = this->formerCoreNum_ * this->formerTileNum_ + | ||
| 124 | + (this->blockIdx_ - this->formerCoreNum_) * this->tailTileNum_; | ||
| 125 | + } | ||
| 126 | + uint32_t tileHistOffset = this->numValue_ * tileTopKOffset; | ||
| 127 | + tileTopKGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_ * 2 + | ||
| 128 | + tileTopKOffset); | ||
| 129 | + tileHistGm_.SetGlobalBuffer((__gm__ int32_t*)workspace + this->numValue_ * this->coreNum_ + this->coreNum_ * 2 + | ||
| 130 | + this->totalTileNum_ + tileHistOffset); | ||
| 131 | +} | ||
| 132 | + | ||
| 117 | /** | 133 | /** |
| 118 | * @brief 初始化基本参数(调用基类)+ WS 变体特有参数(tileNum 分段次数) | 134 | * @brief 初始化基本参数(调用基类)+ WS 变体特有参数(tileNum 分段次数) |
| 119 | */ | 135 | */ |
| @@ -132,9 +148,10 @@ __aicore__ inline void RadixTopKWs<T, largest>::InitParams() | |||
| 132 | /** | 148 | /** |
| 133 | * @brief 初始化 UB 缓冲区:xBuf Ping-Pong、tileHistAndTopKBuf、tempBuf、outBuf、globalHistBuf | 149 | * @brief 初始化 UB 缓冲区:xBuf Ping-Pong、tileHistAndTopKBuf、tempBuf、outBuf、globalHistBuf |
| 134 | * WS 变体中 tileHist/tileTopK 存 GM,UB 只用作分段读写缓冲 | 150 | * WS 变体中 tileHist/tileTopK 存 GM,UB 只用作分段读写缓冲 |
| 151 | + * @param indexElemSize 索引输出元素字节数(int32 变体 4、int64 变体 8) | ||
| 135 | */ | 152 | */ |
| 136 | template <typename T, bool largest> | 153 | template <typename T, bool largest> |
| 137 | -__aicore__ inline void RadixTopKWs<T, largest>::InitBuffers() | 154 | +__aicore__ inline void RadixTopKWs<T, largest>::InitBuffers(const uint32_t& indexElemSize) |
| 138 | { | 155 | { |
| 139 | uint32_t cmpMaskSize = this->tileLen_ * sizeof(uint8_t) / BYTE_SIZE; | 156 | uint32_t cmpMaskSize = this->tileLen_ * sizeof(uint8_t) / BYTE_SIZE; |
| 140 | this->tpipe_.InitBuffer(this->xBufPing_, this->tileLen_ * sizeof(float)); | 157 | this->tpipe_.InitBuffer(this->xBufPing_, this->tileLen_ * sizeof(float)); |
| @@ -142,7 +159,7 @@ __aicore__ inline void RadixTopKWs<T, largest>::InitBuffers() | |||
| 142 | this->tpipe_.InitBuffer(tileHistAndTopKBuf_, MAX_TILE_NUM_IN_UB * sizeof(int32_t)); | 159 | this->tpipe_.InitBuffer(tileHistAndTopKBuf_, MAX_TILE_NUM_IN_UB * sizeof(int32_t)); |
| 143 | this->tpipe_.InitBuffer(tempBuf_, cmpMaskSize); | 160 | this->tpipe_.InitBuffer(tempBuf_, cmpMaskSize); |
| 144 | this->tpipe_.InitBuffer(this->outValueBuf_, this->tileLen_ * sizeof(float)); | 161 | this->tpipe_.InitBuffer(this->outValueBuf_, this->tileLen_ * sizeof(float)); |
| 145 | - this->tpipe_.InitBuffer(this->outIndexBuf_, this->tileLen_ * sizeof(int32_t)); | 162 | + this->tpipe_.InitBuffer(this->outIndexBuf_, this->tileLen_ * indexElemSize); |
| 146 | this->tpipe_.InitBuffer(this->globalHistBuf_, this->numValue_ * sizeof(int32_t)); | 163 | this->tpipe_.InitBuffer(this->globalHistBuf_, this->numValue_ * sizeof(int32_t)); |
| 147 | } | 164 | } |
| 148 | 165 | ||
| @@ -181,7 +198,7 @@ __aicore__ inline void RadixTopKWs<T, largest>::SubProcess(uint64_t batchId) | |||
| 181 | for (int32_t tileId = startTileId; tileId < endTileId; tileId++) { | 198 | for (int32_t tileId = startTileId; tileId < endTileId; tileId++) { |
| 182 | PINGPONG_TILE_BEGIN(tileId, blockOffset); | 199 | PINGPONG_TILE_BEGIN(tileId, blockOffset); |
| 183 | 200 | ||
| 184 | - this->NegateDataForLargest(curBuf, curTileLen); | 201 | + this->CanonicalizeDataForLargest(curBuf, curTileLen); |
| 185 | this->TwiddleInB16(curBuf, curTileLen); | 202 | this->TwiddleInB16(curBuf, curTileLen); |
| 186 | this->DoAndMask(curBuf, curTileLen); | 203 | this->DoAndMask(curBuf, curTileLen); |
| 187 | LocalTensor<int32_t> tileHist = tileHistAndTopKBuf_.Get<int32_t>(); | 204 | LocalTensor<int32_t> tileHist = tileHistAndTopKBuf_.Get<int32_t>(); |
| @@ -206,6 +223,8 @@ __aicore__ inline void RadixTopKWs<T, largest>::SubProcess(uint64_t batchId) | |||
| 206 | this->andMask16_ >>= BITS_PER_ROUND; | 223 | this->andMask16_ >>= BITS_PER_ROUND; |
| 207 | if (Update(roundId)) | 224 | if (Update(roundId)) |
| 208 | break; | 225 | break; |
| 226 | + // 与 ub 模板对齐:确保所有核完成 Update 内 CopyInGlobalHist 对 globalHistGm_ 的读取, | ||
| 227 | + // 再允许快核进入下一轮 CopyOut2Ws<true> 覆盖 | ||
| 209 | SyncAll(); | 228 | SyncAll(); |
| 210 | } | 229 | } |
| 211 | SyncAll(); | 230 | SyncAll(); |
| @@ -261,8 +280,13 @@ __aicore__ inline void RadixTopKWs<T, largest>::AddTileHistToTileTopK(LocalTenso | |||
| 261 | } | 280 | } |
| 262 | 281 | ||
| 263 | /** | 282 | /** |
| 264 | - * @brief WS 变体:处理最后一轮边界 bin 的元素分配 | 283 | + * @brief 处理最后一轮边界 bin 的元素分配:从 WS 读取各 tile 的边界 bin / 前驱 bin 计数, |
| 265 | - * 从 WS 读取各 tile 边界 bin 数据,计算每个 core 的元素分配 | 284 | + * 按 (core, tile) 顺序把剩余配额 remainK_ 分配到各 tile 的 tileTopK |
| 285 | + * | ||
| 286 | + * 计数与配额统一用 int64 标量(跨核总量可达 sortLen > 2^31;int32 形状下同样安全), | ||
| 287 | + * 分配按 MAX_TILE_NUM_IN_UB_BY3 分段:段内先求差分总和 segSum,若剩余配额能吞下整段 | ||
| 288 | + * 则按元素向量 Add 给段内各 tile 加各自差分、跳过逐 tile 标量扫描,只在配额耗尽的那个 | ||
| 289 | + * 段内退化为逐 tile 标量扫描。两条变体共用本实现(int64 变体不再单独 override)。 | ||
| 266 | * @param resTensor 临时 Tensor | 290 | * @param resTensor 临时 Tensor |
| 267 | * @param tileHistFp32 tileHist 的 fp32 视图 | 291 | * @param tileHistFp32 tileHist 的 fp32 视图 |
| 268 | */ | 292 | */ |
| @@ -270,7 +294,9 @@ template <typename T, bool largest> | |||
| 270 | __aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTensor<int32_t>& resTensor, | 294 | __aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTensor<int32_t>& resTensor, |
| 271 | LocalTensor<float>& tileHistFp32) | 295 | LocalTensor<float>& tileHistFp32) |
| 272 | { | 296 | { |
| 273 | - uint32_t reduceSumValue = 0; | 297 | + int64_t reduceSumValue = 0; |
| 298 | + // 首个迭代前,ReduceGlobalHist 末尾用 Scalar SetValue 写过 resTensor[0..3], | ||
| 299 | + // 需保证 S 写对后续 MTE2 覆写 resTensor 可见(与 ub 模板 HandleLastRoundBoundary 的 SToVSync 对齐) | ||
| 274 | SToMTE2Sync(); | 300 | SToMTE2Sync(); |
| 275 | for (int32_t i = 0; i <= tileNumBy2RepeatTimes_; i++) { | 301 | for (int32_t i = 0; i <= tileNumBy2RepeatTimes_; i++) { |
| 276 | uint32_t dataNum = i == tileNumBy2RepeatTimes_ ? tileNumBy2Remain_ : MAX_TILE_NUM_IN_UB_BY2; | 302 | uint32_t dataNum = i == tileNumBy2RepeatTimes_ ? tileNumBy2Remain_ : MAX_TILE_NUM_IN_UB_BY2; |
| @@ -295,17 +321,19 @@ __aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTen | |||
| 295 | SToVSync(); | 321 | SToVSync(); |
| 296 | ReduceSum(tileHistFp32, tileHistFp32[segOff], tileHistFp32[segOff], curSegLen); | 322 | ReduceSum(tileHistFp32, tileHistFp32[segOff], tileHistFp32[segOff], curSegLen); |
| 297 | VToSSync(); | 323 | VToSSync(); |
| 298 | - reduceSumValue += static_cast<int32_t>(tileHistFp32.GetValue(0)); | 324 | + // 分段和 <= 2^24 精确;累加器 int64(跨段总量 = per-core 边界 bin 元素数) |
| 325 | + reduceSumValue += static_cast<int64_t>(tileHistFp32.GetValue(0)); | ||
| 299 | } | 326 | } |
| 300 | } | 327 | } |
| 301 | - resTensor.SetValue(0, reduceSumValue); | 328 | + // per-core 边界 bin 计数 < 2^31(per-core 元素数上限),int32 写回安全 |
| 329 | + resTensor.SetValue(0, static_cast<int32_t>(reduceSumValue)); | ||
| 302 | SToMTE3Sync(); | 330 | SToMTE3Sync(); |
| 303 | this->CopyOutBoundaryBinCumSum(resTensor); | 331 | this->CopyOutBoundaryBinCumSum(resTensor); |
| 304 | SyncAll(); | 332 | SyncAll(); |
| 305 | 333 | ||
| 306 | - int32_t cumSumValuePrev = 0; | 334 | + int64_t cumSumValuePrev = 0; |
| 307 | - this->ComputeCumSumPrev(resTensor, cumSumValuePrev); | 335 | + ComputeCumSumPrevI64(resTensor, cumSumValuePrev); |
| 308 | - int32_t remainCoreBoundaryNum = this->remainK_ - cumSumValuePrev; | 336 | + int64_t remainCoreBoundaryNum = static_cast<int64_t>(this->remainK_) - cumSumValuePrev; |
| 309 | if (remainCoreBoundaryNum > 0) { | 337 | if (remainCoreBoundaryNum > 0) { |
| 310 | LocalTensor<int32_t> tileTopK = tileHistAndTopKBuf_.Get<int32_t>(); | 338 | LocalTensor<int32_t> tileTopK = tileHistAndTopKBuf_.Get<int32_t>(); |
| 311 | LocalTensor<int32_t> tileHistBoundary = tileTopK[MAX_TILE_NUM_IN_UB_BY3]; | 339 | LocalTensor<int32_t> tileHistBoundary = tileTopK[MAX_TILE_NUM_IN_UB_BY3]; |
| @@ -320,19 +348,61 @@ __aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTen | |||
| 320 | CopyTileHistWs2Ub(tileHistBoundary, this->boundaryBin * this->tileNum_ + startTileId, dataNum); | 348 | CopyTileHistWs2Ub(tileHistBoundary, this->boundaryBin * this->tileNum_ + startTileId, dataNum); |
| 321 | CopyTileHistWs2Ub(tileHistBoundaryPrev, this->boundaryBinPrev * this->tileNum_ + startTileId, dataNum); | 349 | CopyTileHistWs2Ub(tileHistBoundaryPrev, this->boundaryBinPrev * this->tileNum_ + startTileId, dataNum); |
| 322 | CopyTileTopKWs2Ub(tileTopK, startTileId, dataNum); | 350 | CopyTileTopKWs2Ub(tileTopK, startTileId, dataNum); |
| 323 | - MTE2ToSSync(); | 351 | + MTE2ToVSync(); |
| 324 | - for (int32_t tileId = startTileId; tileId < endTileId; tileId++) { | 352 | + // 段级定位:先算本段边界 bin 差分总和 segSum(向量),若剩余配额能吞下整段, |
| 325 | - int32_t curTileBoundaryNum = tileHistBoundary.GetValue(tileId - startTileId); | 353 | + // 则按元素向量 Add 给段内各 tile 加各自差分(tileTopK[i] += diff[i]), |
| 326 | - if (this->boundaryBinPrev < this->numValue_) { | 354 | + // 跳过逐 tile 标量扫描——标量扫描只在配额耗尽的那个段内进行(≤2048 次后必然 break)。 |
| 327 | - curTileBoundaryNum -= tileHistBoundaryPrev.GetValue(tileId - startTileId); | 355 | + // 注意:整段吞下不是加常量 segSum(那会给每个 tile 超发全段总和的配额, |
| 356 | + // 导致 curTileK 超过 tile 实际 EQ 元素数、拷出陈旧 lane——NaN/-1 输出的根源) | ||
| 357 | + if (this->boundaryBinPrev < this->numValue_) { | ||
| 358 | + Sub(tileHistBoundary, tileHistBoundary, tileHistBoundaryPrev, dataNum); | ||
| 359 | + PipeBarrier<PIPE_V>(); | ||
| 360 | + } | ||
| 361 | + int32_t int32Align = BLOCK_SIZE / sizeof(int32_t); | ||
| 362 | + int32_t safeSumTileNum = FLOAT32_SAFE_INT / this->tileLen_ / int32Align * int32Align; | ||
| 363 | + // fp32 视图借用已被 Sub 消费的 prev 区域,保持 tileHistBoundary 的 int32 差分 | ||
| 364 | + // 不被 Cast 覆写(整段 Add 与段内标量扫描都还要复用它) | ||
| 365 | + LocalTensor<float> segFp32 = tileHistBoundaryPrev.template ReinterpretCast<float>(); | ||
| 366 | + PipeBarrier<PIPE_V>(); | ||
| 367 | + Cast(segFp32, tileHistBoundary, RoundMode::CAST_NONE, dataNum); | ||
| 368 | + PipeBarrier<PIPE_V>(); | ||
| 369 | + int64_t segSum = 0; | ||
| 370 | + int32_t segSize = dataNum < static_cast<uint32_t>(safeSumTileNum) ? static_cast<int32_t>(dataNum) : | ||
| 371 | + safeSumTileNum; | ||
| 372 | + for (int32_t segOff = 0; segOff < static_cast<int32_t>(dataNum); segOff += segSize) { | ||
| 373 | + int32_t curSegLen = AscendC::Std::min(segSize, static_cast<int32_t>(dataNum - segOff)); | ||
| 374 | + SToVSync(); | ||
| 375 | + ReduceSum(segFp32, segFp32[segOff], segFp32[segOff], curSegLen); | ||
| 376 | + VToSSync(); | ||
| 377 | + segSum += static_cast<int64_t>(segFp32.GetValue(0)); | ||
| 378 | + } | ||
| 379 | + if (remainCoreBoundaryNum >= segSum) { | ||
| 380 | + // 整段吞下:tileTopK += diff 向量按元素加(int32 Add) | ||
| 381 | + SToVSync(); | ||
| 382 | + PipeBarrier<PIPE_V>(); | ||
| 383 | + Add(tileTopK, tileTopK, tileHistBoundary, dataNum); | ||
| 384 | + PipeBarrier<PIPE_V>(); | ||
| 385 | + remainCoreBoundaryNum -= segSum; | ||
| 386 | + SToMTE3Sync(); | ||
| 387 | + CopyTileTopKUb2Ws(tileTopK, startTileId, dataNum); | ||
| 388 | + MTE3ToMTE2Sync(); | ||
| 389 | + if (remainCoreBoundaryNum == 0) { | ||
| 390 | + break; | ||
| 328 | } | 391 | } |
| 392 | + continue; | ||
| 393 | + } | ||
| 394 | + // 段内部分配:逐 tile 标量扫描(本段必是配额耗尽段;差分已在 tileHistBoundary 内, | ||
| 395 | + // 标量侧无需再减 prev——prev 区域已被 segFp32 借用覆写) | ||
| 396 | + MTE2ToSSync(); | ||
| 397 | + for (int32_t tileId = startTileId; tileId < static_cast<int32_t>(endTileId); tileId++) { | ||
| 398 | + int64_t curTileBoundaryNum = tileHistBoundary.GetValue(tileId - startTileId); | ||
| 329 | if (curTileBoundaryNum < remainCoreBoundaryNum) { | 399 | if (curTileBoundaryNum < remainCoreBoundaryNum) { |
| 330 | remainCoreBoundaryNum -= curTileBoundaryNum; | 400 | remainCoreBoundaryNum -= curTileBoundaryNum; |
| 331 | - tileTopK.SetValue(tileId - startTileId, | 401 | + tileTopK.SetValue(tileId - startTileId, tileTopK.GetValue(tileId - startTileId) + |
| 332 | - tileTopK.GetValue(tileId - startTileId) + curTileBoundaryNum); | 402 | + static_cast<int32_t>(curTileBoundaryNum)); |
| 333 | } else { | 403 | } else { |
| 334 | - tileTopK.SetValue(tileId - startTileId, | 404 | + tileTopK.SetValue(tileId - startTileId, tileTopK.GetValue(tileId - startTileId) + |
| 335 | - tileTopK.GetValue(tileId - startTileId) + remainCoreBoundaryNum); | 405 | + static_cast<int32_t>(remainCoreBoundaryNum)); |
| 336 | remainCoreBoundaryNum = 0; | 406 | remainCoreBoundaryNum = 0; |
| 337 | break; | 407 | break; |
| 338 | } | 408 | } |
| @@ -345,7 +415,8 @@ __aicore__ inline void RadixTopKWs<T, largest>::HandleLastRoundBoundary(LocalTen | |||
| 345 | } | 415 | } |
| 346 | 416 | ||
| 347 | /** | 417 | /** |
| 348 | - * @brief WS 变体:从 WS 分段读取 tileTopK 累加得到本 core 总 TopK 计数并写回 WS | 418 | + * @brief 从 WS 分段读取 tileTopK 累加得到本 core 总 TopK 计数并写回 WS |
| 419 | + * 累加器 int64(int32 形状下同样安全),两条变体共用 | ||
| 349 | * @param resTensor 临时 Tensor | 420 | * @param resTensor 临时 Tensor |
| 350 | * @param tileHistFp32 tileHist 的 fp32 视图 | 421 | * @param tileHistFp32 tileHist 的 fp32 视图 |
| 351 | */ | 422 | */ |
| @@ -354,7 +425,7 @@ __aicore__ inline void RadixTopKWs<T, largest>::WriteCoreTopKFromWs(LocalTensor< | |||
| 354 | LocalTensor<float>& tileHistFp32) | 425 | LocalTensor<float>& tileHistFp32) |
| 355 | { | 426 | { |
| 356 | LocalTensor<int32_t> tileTopK = tileHistAndTopKBuf_.Get<int32_t>(); | 427 | LocalTensor<int32_t> tileTopK = tileHistAndTopKBuf_.Get<int32_t>(); |
| 357 | - int32_t totalTileTopKInCore = 0; | 428 | + int64_t totalTileTopKInCore = 0; |
| 358 | int32_t int32Align = BLOCK_SIZE / sizeof(int32_t); | 429 | int32_t int32Align = BLOCK_SIZE / sizeof(int32_t); |
| 359 | int32_t safeSumTileNum = FLOAT32_SAFE_INT / this->tileLen_ / int32Align * int32Align; | 430 | int32_t safeSumTileNum = FLOAT32_SAFE_INT / this->tileLen_ / int32Align * int32Align; |
| 360 | for (int32_t i = 0; i <= tileNumRepeatTimes_; i++) { | 431 | for (int32_t i = 0; i <= tileNumRepeatTimes_; i++) { |
| @@ -371,10 +442,29 @@ __aicore__ inline void RadixTopKWs<T, largest>::WriteCoreTopKFromWs(LocalTensor< | |||
| 371 | PipeBarrier<PIPE_V>(); | 442 | PipeBarrier<PIPE_V>(); |
| 372 | ReduceSum(tileHistFp32, tileHistFp32, tileHistFp32, curSegLen); | 443 | ReduceSum(tileHistFp32, tileHistFp32, tileHistFp32, curSegLen); |
| 373 | VToSSync(); | 444 | VToSSync(); |
| 374 | - totalTileTopKInCore += static_cast<int32_t>(tileHistFp32.GetValue(0)); | 445 | + totalTileTopKInCore += static_cast<int64_t>(tileHistFp32.GetValue(0)); |
| 375 | } | 446 | } |
| 376 | } | 447 | } |
| 377 | - this->CopyOutCoreTopK(resTensor, totalTileTopKInCore); | 448 | + // per-core 计数 < 2^31(per-core 元素数上限),coreTopK 通道保持 int32 |
| 449 | + this->CopyOutCoreTopK(resTensor, static_cast<int32_t>(totalTileTopKInCore)); | ||
| 450 | +} | ||
| 451 | + | ||
| 452 | +/** | ||
| 453 | + * @brief 计算排在当前 core 之前所有 core 的边界 bin 累计元素数(int64 累加,总量可达 sortLen) | ||
| 454 | + */ | ||
| 455 | +template <typename T, bool largest> | ||
| 456 | +__aicore__ inline void RadixTopKWs<T, largest>::ComputeCumSumPrevI64(LocalTensor<int32_t>& resTensor, | ||
| 457 | + int64_t& cumSumValuePrev) | ||
| 458 | +{ | ||
| 459 | + DataCopyPad( | ||
| 460 | + resTensor, this->boundaryBinCumSumGm_, | ||
| 461 | + DataCopyExtParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * this->coreNum_), 0, 0, 0}, | ||
| 462 | + DataCopyPadExtParams<int32_t>{true, 0, 0, 0}); | ||
| 463 | + MTE2ToSSync(); | ||
| 464 | + cumSumValuePrev = 0; | ||
| 465 | + for (int32_t coreId = 0; coreId < this->blockIdx_; coreId++) { | ||
| 466 | + cumSumValuePrev += resTensor.GetValue(coreId); | ||
| 467 | + } | ||
| 378 | } | 468 | } |
| 379 | 469 | ||
| 380 | /** | 470 | /** |
| @@ -446,6 +536,12 @@ __aicore__ inline void RadixTopKWs<T, largest>::TileTopK(const uint64_t& batchId | |||
| 446 | uint64_t firstLen = (firstId == this->tileNum_ - 1) ? this->tailTileLen_ : this->tileLen_; | 536 | uint64_t firstLen = (firstId == this->tileNum_ - 1) ? this->tailTileLen_ : this->tileLen_; |
| 447 | this->CopyIn(this->xBufPing_, firstLen, blockOffset + firstId * this->tileLen_); | 537 | this->CopyIn(this->xBufPing_, firstLen, blockOffset + firstId * this->tileLen_); |
| 448 | uint32_t maxIndexLen = CreateVecIndex4TopK(); | 538 | uint32_t maxIndexLen = CreateVecIndex4TopK(); |
| 539 | + // 同 UB 变体:maxIndexLen == 0 时 SubTopKAndCopyOut 走现算索引兜底分支、 | ||
| 540 | + // 不使用 vecIndex;因此 0 时改用 buffer 首地址占位,不做零长度取句柄。 | ||
| 541 | + LocalTensor<int32_t> vecIndexTopK = maxIndexLen > 0 ? | ||
| 542 | + tileHistAndTopKBuf_.GetWithOffset<int32_t>( | ||
| 543 | + maxIndexLen, this->tileNumAlign_ * sizeof(int32_t)) : | ||
| 544 | + tileHistAndTopKBuf_.Get<int32_t>(); // 占位句柄,0 时不会被使用 | ||
| 449 | 545 | ||
| 450 | this->InitTopKEvent(); | 546 | this->InitTopKEvent(); |
| 451 | uint64_t startRepeatId = firstId / MAX_TILE_NUM_IN_UB; | 547 | uint64_t startRepeatId = firstId / MAX_TILE_NUM_IN_UB; |
| @@ -467,8 +563,7 @@ __aicore__ inline void RadixTopKWs<T, largest>::TileTopK(const uint64_t& batchId | |||
| 467 | 563 | ||
| 468 | int32_t curTileK = tileTopK.GetValue(tileId - repeatId * MAX_TILE_NUM_IN_UB); | 564 | int32_t curTileK = tileTopK.GetValue(tileId - repeatId * MAX_TILE_NUM_IN_UB); |
| 469 | PipeBarrier<PIPE_V>(); | 565 | PipeBarrier<PIPE_V>(); |
| 470 | - PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileK, tileHistAndTopKBuf_.GetWithOffset<int32_t>( | 566 | + PROCESS_ONE_TILE_TOPK_OR_SKIP(curTileK, vecIndexTopK); |
| 471 | - maxIndexLen, this->tileNumAlign_ * sizeof(int32_t))); | ||
| 472 | 567 | ||
| 473 | PINGPONG_TILE_END(tileId); | 568 | PINGPONG_TILE_END(tileId); |
| 474 | } | 569 | } |
| @@ -0,0 +1,646 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file radix_top_k_ws_i64.h | ||
| 13 | + * \brief Radix TopK kernel — int64 大 shape 变体(sortLen > INT32_MAX) | ||
| 14 | + * | ||
| 15 | + * 计算逻辑与 RadixTopKWs 完全一致(继承复用其全部 WS 搬运工具与同步结构), | ||
| 16 | + * 仅以下位置 int64 化。910B 向量指令(Add/Adds/Duplicate/GatherMask)不支持 | ||
| 17 | + * int64,唯一可用的 int64 向量能力是 Cast 的 s32<->s64 转换通道,且不能承担 | ||
| 18 | + * 加法/合并,因此 int64 算术一律走标量: | ||
| 19 | + * 1) 跨核 globalHist 归约与边界查找(总量可达 sortLen > 2^31):标量 int64 累加, | ||
| 20 | + * 每轮 coreNum*numValue_(<=200)次,代价可忽略; | ||
| 21 | + * 2) 边界计数跨核前缀和(ComputeCumSumPrevI64)与 remainCoreBoundaryNum:int64 标量; | ||
| 22 | + * 3) 索引输出:GatherMask 仍在 int32 域压缩 tile 内索引,逐元素标量加 int64 | ||
| 23 | + * tileOffset 后就地后向扩展写 int64,成本 O(kValue)。 | ||
| 24 | + * | ||
| 25 | + * 与基类 RadixTopKWs 的复用边界(避免两处同步修改): | ||
| 26 | + * - 复用(本类不重写):InitWsGm(GM/WS 布局)、InitBuffers(差一个索引元素字节数)、 | ||
| 27 | + * HandleLastRoundBoundary(int64 计数 + 段级快速路径)、WriteCoreTopKFromWs、 | ||
| 28 | + * ComputeCumSumPrevI64、全部 WS 搬运/清除/同步工具; | ||
| 29 | + * - 必须重写:Process/SubProcess(非虚成员 + 静态派发,基类体内会静态绑定到基类 Update)、 | ||
| 30 | + * Update 及其 int64 归约/边界查找/neighbor 累加、TileTopK、索引输出相关函数。 | ||
| 31 | + * 若要进一步合并 Process/SubProcess,需要引入 CRTP 或把基类模板化为 <T, largest, IndexT>, | ||
| 32 | + * 属结构性重构,未在本 PR 处理。 | ||
| 33 | + * | ||
| 34 | + * int32 安全域(保持不变):per-core globalHist/tileHist/tileTopK 计数与 coreTopK | ||
| 35 | + * (per-core 元素数 = sortLen/coreNum < 2^31,由 tiling 保证,物理内存内恒成立; | ||
| 36 | + * kValue <= INT32_MAX 由 tiling 校验)、vecIndex(< tileLen)、x/value 数据通路。 | ||
| 37 | + */ | ||
| 38 | + | ||
| 39 | +#ifndef RADIX_TOP_K_WS_I64_H | ||
| 40 | +#define RADIX_TOP_K_WS_I64_H | ||
| 41 | + | ||
| 42 | +#include "radix_top_k_ws.h" | ||
| 43 | + | ||
| 44 | +namespace RadixTopK { | ||
| 45 | +using namespace AscendC; | ||
| 46 | + | ||
| 47 | +/** | ||
| 48 | + * @brief i64 变体专用的 tile 循环前导:与 PROCESS_ONE_TILE_TOPK_OR_SKIP 的唯一区别是 | ||
| 49 | + * 不传 vecIndex/maxIndexLen——i64 路径的索引一律在 SubTopKAndCopyOutI64 内现算 | ||
| 50 | + * (VectorMergeI64),预生成索引链路整条不适用。 | ||
| 51 | + * 前置条件:curBuf, coreTopKOffset, tileOffset, curTileLen 在作用域内 | ||
| 52 | + */ | ||
| 53 | +#define PROCESS_ONE_TILE_TOPK_OR_SKIP_I64(curTileKVal) \ | ||
| 54 | + if (curTileKVal > 0) { \ | ||
| 55 | + auto xLocal = curBuf.template Get<T>(); \ | ||
| 56 | + auto cmpMaskTensor = this->tempBuf_.template Get<uint8_t>(); \ | ||
| 57 | + if constexpr (IsSameType<T, bfloat16_t>::value) { \ | ||
| 58 | + auto xLocalFp32 = xLocal.template ReinterpretCast<float>(); \ | ||
| 59 | + Cast(xLocalFp32, xLocal[this->tileLen_], RoundMode::CAST_NONE, curTileLen); \ | ||
| 60 | + this->template ProcessOneTileTopK<float>(xLocalFp32, curTileLen, curTileKVal, coreTopKOffset, \ | ||
| 61 | + cmpMaskTensor, tileOffset); \ | ||
| 62 | + } else if constexpr (IsSameType<T, half>::value) { \ | ||
| 63 | + auto xLocalHalf = xLocal[this->tileLen_]; \ | ||
| 64 | + if constexpr (largest) { \ | ||
| 65 | + this->CanonicalizeDataForLargest(curBuf, curTileLen); \ | ||
| 66 | + } \ | ||
| 67 | + this->template ProcessOneTileTopK<half>(xLocalHalf, curTileLen, curTileKVal, coreTopKOffset, \ | ||
| 68 | + cmpMaskTensor, tileOffset); \ | ||
| 69 | + } \ | ||
| 70 | + } else { \ | ||
| 71 | + PipeBarrier<PIPE_MTE2>(); \ | ||
| 72 | + } | ||
| 73 | + | ||
| 74 | +template <typename T, bool largest> | ||
| 75 | +class RadixTopKWsI64 : public RadixTopKWs<T, largest> { | ||
| 76 | + using Base = RadixTopKWs<T, largest>; | ||
| 77 | + | ||
| 78 | +public: | ||
| 79 | + __aicore__ inline RadixTopKWsI64(TPipe& tpipe, const RadixTopKTilingData& tilingData) : Base(tpipe, tilingData) {} | ||
| 80 | + | ||
| 81 | + __aicore__ inline void Init(GM_ADDR x, GM_ADDR k, GM_ADDR values, GM_ADDR indices, GM_ADDR workspace); | ||
| 82 | + __aicore__ inline void SubProcess(uint64_t batchId); | ||
| 83 | + __aicore__ inline void Process(); | ||
| 84 | + | ||
| 85 | +private: | ||
| 86 | + __aicore__ inline void ReduceGlobalHistI64(LocalTensor<int32_t>& globalHist, LocalTensor<int64_t>& reduced); | ||
| 87 | + __aicore__ inline void FindBoundaryBinI64(const LocalTensor<int64_t>& reduced, int32_t roundId); | ||
| 88 | + __aicore__ inline void AddTileHistToTileTopKI64(const LocalTensor<int64_t>& reduced); | ||
| 89 | + __aicore__ inline bool Update(const int32_t& roundId); | ||
| 90 | + __aicore__ inline void CopyOutResultI64(const int32_t& count, const uint64_t& coreTopKOffset); | ||
| 91 | + | ||
| 92 | + template <AscendC::CMPMODE cmpMode, typename CT> | ||
| 93 | + __aicore__ inline void SubTopKAndCopyOutI64(LocalTensor<CT>& xLocal, const CT& boundaryNumber, | ||
| 94 | + const uint64_t& curTileLen, int32_t& curTileK, uint64_t& coreTopKOffset, | ||
| 95 | + LocalTensor<uint8_t>& cmpMaskTensor, const int64_t& tileOffset); | ||
| 96 | + | ||
| 97 | + template <typename CT> | ||
| 98 | + __aicore__ inline void ProcessOneTileTopK(LocalTensor<CT>& xLocalTyped, const uint64_t& curTileLen, | ||
| 99 | + int32_t& curTileK, uint64_t& coreTopKOffset, | ||
| 100 | + LocalTensor<uint8_t>& cmpMaskTensor, const int64_t& tileOffset); | ||
| @@ -131,6 +131,9 @@ __aicore__ inline void RadixTopKWs<T, largest>::CopyTileHistInWs(const uint32_t& | |||
| 131 | DataCopyExtParams tileHistOutParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * dataNum), | 131 | DataCopyExtParams tileHistOutParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t) * dataNum), |
| 132 | 0, 0, 0}; | 132 | 0, 0, 0}; |
| 133 | DataCopyPad(tileHistGm_[dstOffset + i * MAX_TILE_NUM_IN_UB], tileHist, tileHistOutParams); | 133 | DataCopyPad(tileHistGm_[dstOffset + i * MAX_TILE_NUM_IN_UB], tileHist, tileHistOutParams); |
| 134 | + // 迭代间必须等 MTE3 搬完 UB 再发起下一块 MTE2(复用同一 UB 段), | ||
| 135 | + // 否则下一迭代的 GM→UB 覆盖尚未搬走的头部数据,GM 目标段头部被后块源污染 | ||
| 136 | + // (i64 场景 tileNum>6144 进入多块循环才触发,实测 K=240000 出现 tile 级 ±1 配额错位) | ||
| 134 | MTE3ToMTE2Sync(); | 137 | MTE3ToMTE2Sync(); |
| 135 | } | 138 | } |
| 136 | } | 139 | } |
| @@ -1,10 +1,10 @@ | |||
| 1 | # --------------------------------------------------------------------------------------------------------- | 1 | # --------------------------------------------------------------------------------------------------------- |
| 2 | # Copyright (c) 2026 Huawei Technologies Co., Ltd. | 2 | # Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 3 | -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 4 | # CANN Open Software License Agreement Version 2.0 (the "License"). | 4 | # CANN Open Software License Agreement Version 2.0 (the "License"). |
| 5 | # Please refer to the License for details. You may not use this file except in compliance with the License. | 5 | # Please refer to the License for details. You may not use this file except in compliance with the License. |
| 6 | -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 7 | -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # --------------------------------------------------------------------------------------------------------- | 9 | # --------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| @@ -15,4 +15,14 @@ if(UT_TEST_ALL OR OP_KERNEL_UT) | |||
| 15 | AddOpTestCase(radix_top_k "ascend910b" | 15 | AddOpTestCase(radix_top_k "ascend910b" |
| 16 | "-DDTYPE_X=half -D__CCE_UT_TEST__ -DTestUtDefaultTilingStruct=RadixTopKTilingData" | 16 | "-DDTYPE_X=half -D__CCE_UT_TEST__ -DTestUtDefaultTilingStruct=RadixTopKTilingData" |
| 17 | "${radix_top_k_tiling_files}") | 17 | "${radix_top_k_tiling_files}") |
| 18 | + | ||
| 19 | + if(TARGET ${OP_KERNEL_MODULE_NAME}_common_obj) | ||
| 20 | + target_compile_definitions(${OP_KERNEL_MODULE_NAME}_common_obj | ||
| 21 | + PRIVATE BUILD_SOC_VERSION=ascend910b) | ||
| 22 | + message(STATUS "[radix_top_k UT] BUILD_SOC_VERSION=ascend910b -> ${OP_KERNEL_MODULE_NAME}_common_obj") | ||
| 23 | + else() | ||
| 24 | + cmake_language(DEFER DIRECTORY ${CMAKE_SOURCE_DIR} CALL target_compile_definitions | ||
| 25 | + ${OP_KERNEL_MODULE_NAME}_common_obj PRIVATE BUILD_SOC_VERSION=ascend910b) | ||
| 26 | + message(STATUS "[radix_top_k UT] BUILD_SOC_VERSION=ascend910b -> ${OP_KERNEL_MODULE_NAME}_common_obj (deferred)") | ||
| 27 | + endif() | ||
| 18 | endif() | 28 | endif() |
| @@ -119,7 +119,7 @@ static bool IsAiCoreSupport(const aclTensor* self, int64_t k) | |||
| 119 | return self->GetDataType() == op::DataType::DT_FLOAT16; | 119 | return self->GetDataType() == op::DataType::DT_FLOAT16; |
| 120 | } | 120 | } |
| 121 | 121 | ||
| 122 | -static bool IsRadixTopKSupport(const aclTensor* self, int64_t k) | 122 | +static bool IsRadixTopKSupport(const aclTensor* self, int64_t k, op::DataType indicesDType) |
| 123 | { | 123 | { |
| 124 | SocVersion version = GetCurrentPlatformInfo().GetSocVersion(); | 124 | SocVersion version = GetCurrentPlatformInfo().GetSocVersion(); |
| 125 | auto inputShape = self->GetViewShape(); | 125 | auto inputShape = self->GetViewShape(); |
| @@ -127,10 +127,13 @@ static bool IsRadixTopKSupport(const aclTensor* self, int64_t k) | |||
| 127 | int64_t sortLen = inputShape.GetDim(dimNum - 1); | 127 | int64_t sortLen = inputShape.GetDim(dimNum - 1); |
| 128 | bool socCheck = version == SocVersion::ASCEND910B || version == SocVersion::ASCEND910_93; | 128 | bool socCheck = version == SocVersion::ASCEND910B || version == SocVersion::ASCEND910_93; |
| 129 | bool dtypeCheck = CheckType(self->GetDataType(), {op::DataType::DT_FLOAT16, op::DataType::DT_BF16}); | 129 | bool dtypeCheck = CheckType(self->GetDataType(), {op::DataType::DT_FLOAT16, op::DataType::DT_BF16}); |
| 130 | - bool shapeCheck = false; | 130 | + // sortLen 超 int32 上限 → int64 大 shape 模板:只要求 indices 为 int64 且 k > 0, |
| 131 | + // 不套用下面的 k 比例门限,故此处直接判定 | ||
| 131 | if (sortLen > MAX_INT32_INPUTSIZE) { | 132 | if (sortLen > MAX_INT32_INPUTSIZE) { |
| 132 | - shapeCheck = false; | 133 | + return socCheck && dtypeCheck && (indicesDType == op::DataType::DT_INT64) && k > 0; |
| 133 | - } else if (sortLen >= RADIX_TOP_K_S_THRESHOLD_2) { | 134 | + } |
| 135 | + bool shapeCheck = false; | ||
| 136 | + if (sortLen >= RADIX_TOP_K_S_THRESHOLD_2) { | ||
| 134 | shapeCheck = sortLen > RADIX_TOP_K_S_K_RATIO_2 * k; | 137 | shapeCheck = sortLen > RADIX_TOP_K_S_K_RATIO_2 * k; |
| 135 | } else if (sortLen >= RADIX_TOP_K_S_THRESHOLD_1) { | 138 | } else if (sortLen >= RADIX_TOP_K_S_THRESHOLD_1) { |
| 136 | shapeCheck = sortLen > RADIX_TOP_K_S_K_RATIO_1 * k; | 139 | shapeCheck = sortLen > RADIX_TOP_K_S_K_RATIO_1 * k; |
| @@ -230,9 +233,13 @@ std::tuple<aclTensor*, aclTensor*> TopkCommon(const aclTensor* self, int64_t k, | |||
| 230 | 233 | ||
| 231 | // AICore执行路径 | 234 | // AICore执行路径 |
| 232 | if (IsAiCoreSupport(self, k)) { | 235 | if (IsAiCoreSupport(self, k)) { |
| 236 | + int64_t topkDimNum = static_cast<int64_t>(self->GetViewShape().GetDimNum()); | ||
| 237 | + int64_t sortLen = self->GetViewShape().GetDim(topkDimNum - 1); | ||
| 233 | aclTensor* indicesOut = nullptr; | 238 | aclTensor* indicesOut = nullptr; |
| 234 | if (IsRegBase() && !IsSortWithIndex(self, k, sorted)) { | 239 | if (IsRegBase() && !IsSortWithIndex(self, k, sorted)) { |
| 235 | indicesOut = executor->AllocTensor(outShape, indicesDType, self->GetStorageFormat()); | 240 | indicesOut = executor->AllocTensor(outShape, indicesDType, self->GetStorageFormat()); |
| 241 | + } else if (sortLen > MAX_INT32_INPUTSIZE && indicesDType == op::DataType::DT_INT64) { | ||
| 242 | + indicesOut = executor->AllocTensor(outShape, op::DataType::DT_INT64, self->GetStorageFormat()); | ||
| 236 | } else { | 243 | } else { |
| 237 | indicesOut = executor->AllocTensor(outShape, op::DataType::DT_INT32, self->GetStorageFormat()); | 244 | indicesOut = executor->AllocTensor(outShape, op::DataType::DT_INT32, self->GetStorageFormat()); |
| 238 | } | 245 | } |
| @@ -243,7 +250,7 @@ std::tuple<aclTensor*, aclTensor*> TopkCommon(const aclTensor* self, int64_t k, | |||
| 243 | if (IsRegBase()) { | 250 | if (IsRegBase()) { |
| 244 | return TopkV2AiCoreForDavid(self, kTensor, dim, largest, sorted, valuesOut, indicesOut, indicesDType, | 251 | return TopkV2AiCoreForDavid(self, kTensor, dim, largest, sorted, valuesOut, indicesOut, indicesDType, |
| 245 | sortPolicy, executor); | 252 | sortPolicy, executor); |
| 246 | - } else if (IsRadixTopKSupport(self, k)) { | 253 | + } else if (IsRadixTopKSupport(self, k, indicesDType)) { |
| 247 | return RadixTopK(self, kTensor, dim, largest, sorted, valuesOut, indicesOut, indicesDType, executor); | 254 | return RadixTopK(self, kTensor, dim, largest, sorted, valuesOut, indicesOut, indicesDType, executor); |
| 248 | } else { | 255 | } else { |
| 249 | return TopkV2AiCore(self, kTensor, dim, largest, sorted, ge::DT_INT32, sortPolicy, valuesOut, | 256 | return TopkV2AiCore(self, kTensor, dim, largest, sorted, ge::DT_INT32, sortPolicy, valuesOut, |