Merged
perf(segsum): optimize short-sequence batching #7354
zsq_888Add on 4 days ago
perf(segsum): optimize short-sequence batching #7354
Merged
Total: 2 files changed+201-2
| @@ -77,6 +77,36 @@ void SegsumTilingArch35::GetTilingKey(uint64_t ubSizePlatform) | |||
| 77 | rowLen = CeilA2B(tailDimLength, blockElement) * blockElement; | 77 | rowLen = CeilA2B(tailDimLength, blockElement) * blockElement; |
| 78 | 78 | ||
| 79 | const int64_t ubAvailable = static_cast<int64_t>(ubSizePlatform - RESERVED_UB); | 79 | const int64_t ubAvailable = static_cast<int64_t>(ubSizePlatform - RESERVED_UB); |
| 80 | + // All buffers coexist: input/output in T and, for narrow T, their FP32 views. | ||
| 81 | + // Multiples of 64 sequences keep every UB buffer and core output boundary aligned. | ||
| 82 | + constexpr int64_t batchAlign = 64; | ||
| 83 | + constexpr int64_t minBatch = 256; | ||
| 84 | + constexpr int64_t maxBatch = 1024; | ||
| 85 | + constexpr int64_t maxFillBatch = 32768; | ||
| 86 | + // Choose packed sequences when the peak live buffers can hold at least one batch tile. | ||
| 87 | + // Divide before comparing to avoid forming length squared for large fallback inputs. | ||
| 88 | + const int64_t packedScalarBytes = dataTypeSize + (isFloat ? 0 : BYTE_LEN_4); | ||
| 89 | + const int64_t packedElementBudget = ubAvailable / (minBatch * packedScalarBytes); | ||
| 90 | + const bool packedFits = tailDimLength == 1 || | ||
| 91 | + (tailDimLength > 1 && packedElementBudget / tailDimLength > tailDimLength); | ||
| 92 | + if (packedFits && averageBatches >= minBatch) { | ||
| 93 | + const int64_t bytesPerBatch = tailDimLength == 1 ? dataTypeSize : | ||
| 94 | + (tailDimLength + tailDimLength * tailDimLength) * | ||
| 95 | + (dataTypeSize + (isFloat ? 0 : BYTE_LEN_4)); | ||
| 96 | + const int64_t workLimit = CeilA2B(averageBatches, batchAlign) * batchAlign; | ||
| 97 | + int64_t batchTile = std::min(ubAvailable / bytesPerBatch, workLimit); | ||
| 98 | + batchTile = std::min(batchTile, tailDimLength == 1 ? maxFillBatch : maxBatch); | ||
| 99 | + batchTile = batchTile / batchAlign * batchAlign; | ||
| 100 | + if (batchTile >= minBatch) { | ||
| 101 | + averageBatches = CeilA2B(averageBatches, blockElement) * blockElement; | ||
| 102 | + needCoreNum = static_cast<uint32_t>(CeilA2B(batches, averageBatches)); | ||
| 103 | + // Normal row blocks have rowNum <= length; larger values encode a packed tile. | ||
| 104 | + rowNum = batchTile * tailDimLength; | ||
| 105 | + stripeLen = 0; | ||
| 106 | + tilingKey = ROW_BLOCK_TILING_KEY; | ||
| 107 | + return; | ||
| 108 | + } | ||
| 109 | + } | ||
| 80 | // TilingKey 1: x row (plus its fp32 view for non-float dtypes) and the fp32 carry are resident, | 110 | // TilingKey 1: x row (plus its fp32 view for non-float dtypes) and the fp32 carry are resident, |
| 81 | // each extra row costs one fp32 accumulator plus its output buffer (for float the exp buffer | 111 | // each extra row costs one fp32 accumulator plus its output buffer (for float the exp buffer |
| 82 | // doubles as the output buffer). | 112 | // doubles as the output buffer). |
| @@ -30,6 +30,109 @@ using namespace AscendC; | |||
| 30 | constexpr float ZERO_FLOAT = 0; | 30 | constexpr float ZERO_FLOAT = 0; |
| 31 | constexpr int32_t NO_BUFFER_NUM = 1; | 31 | constexpr int32_t NO_BUFFER_NUM = 1; |
| 32 | 32 | ||
| 33 | +constexpr Reg::CastTrait EXP_FP32_TO_S32_RINT = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT, | ||
| 34 | + Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT}; | ||
| 35 | +constexpr Reg::CastTrait EXP_S32_TO_FP32 = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT, Reg::MaskMergeMode::ZEROING, | ||
| 36 | + RoundMode::CAST_NONE}; | ||
| 37 | + | ||
| 38 | +// Preserve the carry input. Normal range reduction keeps the polynomial near | ||
| 39 | +// one; exceptional, overflow and subnormal ranges retain the original Exp. | ||
| 40 | +__simd_callee__ inline void ShortExpAccurate(Reg::RegTensor<float>& result, Reg::RegTensor<float>& input, | ||
| 41 | + Reg::MaskReg& mask) | ||
| 42 | +{ | ||
| 43 | + Reg::RegTensor<float> absolute, zero, safe, scaled, powerFloat, negativePower, reduced, coefficient, polynomial; | ||
| 44 | + Reg::RegTensor<int32_t> power; | ||
| 45 | + Reg::MaskReg normal; | ||
| 46 | + Reg::Exp(result, input, mask); | ||
| 47 | + Reg::Abs(absolute, input, mask); | ||
| 48 | + Reg::CompareScalar<float, CMPMODE::LE>(normal, absolute, 80.0f, mask); | ||
| 49 | + Reg::Duplicate(zero, 0.0f, mask); | ||
| 50 | + Reg::Select(safe, input, zero, normal); | ||
| 51 | + Reg::Muls(scaled, safe, 1.4426950216293335f, mask); | ||
| 52 | + Reg::Cast<int32_t, float, EXP_FP32_TO_S32_RINT>(power, scaled, mask); | ||
| 53 | + Reg::Cast<float, int32_t, EXP_S32_TO_FP32>(powerFloat, power, mask); | ||
| 54 | + Reg::Muls(negativePower, powerFloat, -1.0f, mask); | ||
| 55 | + Reg::Muls(reduced, safe, 1.0f, mask); | ||
| 56 | + Reg::Duplicate(coefficient, 0.693145751953125f, mask); | ||
| 57 | + Reg::MulAddDst(reduced, negativePower, coefficient, mask); | ||
| 58 | + Reg::Duplicate(coefficient, 1.428606765330187e-6f, mask); | ||
| 59 | + Reg::MulAddDst(reduced, negativePower, coefficient, mask); | ||
| 60 | + Reg::Duplicate(polynomial, 2.4801587642286904e-5f, mask); | ||
| 61 | + Reg::Duplicate(coefficient, 1.9841270113829523e-4f, mask); | ||
| 62 | + Reg::MulDstAdd(polynomial, reduced, coefficient, mask); | ||
| 63 | + Reg::Duplicate(coefficient, 1.3888889225199819e-3f, mask); | ||
| 64 | + Reg::MulDstAdd(polynomial, reduced, coefficient, mask); | ||
| 65 | + Reg::Duplicate(coefficient, 8.333333767950535e-3f, mask); | ||
| 66 | + Reg::MulDstAdd(polynomial, reduced, coefficient, mask); | ||
| 67 | + Reg::Duplicate(coefficient, 4.16666679084301e-2f, mask); | ||
| 68 | + Reg::MulDstAdd(polynomial, reduced, coefficient, mask); | ||
| 69 | + Reg::Duplicate(coefficient, 1.666666716337204e-1f, mask); | ||
| 70 | + Reg::MulDstAdd(polynomial, reduced, coefficient, mask); | ||
| 71 | + Reg::Duplicate(coefficient, 0.5f, mask); | ||
| 72 | + Reg::MulDstAdd(polynomial, reduced, coefficient, mask); | ||
| 73 | + Reg::Duplicate(coefficient, 1.0f, mask); | ||
| 74 | + Reg::MulDstAdd(polynomial, reduced, coefficient, mask); | ||
| 75 | + Reg::Duplicate(coefficient, 1.0f, mask); | ||
| 76 | + Reg::MulDstAdd(polynomial, reduced, coefficient, mask); | ||
| 77 | + Reg::Adds(power, power, 127, mask); | ||
| 78 | + Reg::ShiftLefts(power, power, static_cast<int16_t>(23), mask); | ||
| 79 | + Reg::Mul(polynomial, polynomial, (Reg::RegTensor<float>&)power, mask); | ||
| 80 | + Reg::Select(result, polynomial, result, normal); | ||
| 81 | +} | ||
| 82 | + | ||
| 83 | +__simd_vf__ inline void ComputeAccurateExpVf(__ubuf__ float* output, __ubuf__ float* input, uint32_t count) | ||
| 84 | +{ | ||
| 85 | + constexpr uint32_t lanes = VECTOR_REG_WIDTH / sizeof(float); | ||
| 86 | + const uint16_t loops = static_cast<uint16_t>((count + lanes - 1) / lanes); | ||
| 87 | + Reg::RegTensor<int32_t> signedIndex; | ||
| 88 | + Reg::RegTensor<float> value, result; | ||
| 89 | + Reg::Arange(signedIndex, 0); | ||
| 90 | + for (uint16_t block = 0; block < loops; ++block) { | ||
| 91 | + auto mask = Reg::UpdateMask<float>(count); | ||
| 92 | + Reg::Gather(value, input + block * lanes, (Reg::RegTensor<uint32_t>&)signedIndex, mask); | ||
| 93 | + ShortExpAccurate(result, value, mask); | ||
| 94 | + Reg::Scatter(output + block * lanes, result, (Reg::RegTensor<uint32_t>&)signedIndex, mask); | ||
| 95 | + } | ||
| 96 | +} | ||
| 97 | + | ||
| 98 | +// Each lane owns one sequence. Preserve the original per-column addition order. | ||
| 99 | +template <bool PRECISE_EXP> | ||
| 100 | +__simd_vf__ inline void ComputeShortBatchVf(__ubuf__ float* x, __ubuf__ float* y, uint32_t count, uint32_t length) | ||
| 101 | +{ | ||
| 102 | + constexpr uint32_t lanes = VECTOR_REG_WIDTH / sizeof(float); | ||
| 103 | + const uint16_t loops = static_cast<uint16_t>((count + lanes - 1) / lanes); | ||
| 104 | + Reg::RegTensor<int32_t> lane; | ||
| 105 | + Reg::RegTensor<uint32_t> inputBase, outputBase, inputIndex, outputIndex; | ||
| 106 | + Reg::RegTensor<float> one, sum, value, result; | ||
| 107 | + auto full = Reg::CreateMask<float, Reg::MaskPattern::ALL>(); | ||
| 108 | + Reg::Arange(lane, 0); | ||
| 109 | + Reg::Muls(inputBase, (Reg::RegTensor<uint32_t>&)lane, length, full); | ||
| 110 | + Reg::Muls(outputBase, (Reg::RegTensor<uint32_t>&)lane, length * length, full); | ||
| 111 | + Reg::Duplicate(one, 1.0f, full); | ||
| 112 | + for (uint16_t i = 0; i < loops; ++i) { | ||
| 113 | + auto mask = Reg::UpdateMask<float>(count); | ||
| 114 | + auto xTile = x + i * lanes * length; | ||
| 115 | + auto yTile = y + i * lanes * length * length; | ||
| 116 | + for (uint16_t col = 0; col < length; ++col) { | ||
| 117 | + Reg::Adds(outputIndex, outputBase, col * (length + 1), mask); | ||
| 118 | + Reg::Scatter(yTile, one, outputIndex, mask); | ||
| 119 | + Reg::Duplicate(sum, 0.0f, mask); | ||
| 120 | + for (uint16_t row = col + 1; row < length; ++row) { | ||
| 121 | + Reg::Adds(inputIndex, inputBase, static_cast<uint32_t>(row), mask); | ||
| 122 | + Reg::Gather(value, xTile, inputIndex, mask); | ||
| 123 | + Reg::Add(sum, sum, value, mask); | ||
| 124 | + if constexpr (PRECISE_EXP) { | ||
| 125 | + ShortExpAccurate(result, sum, mask); | ||
| 126 | + } else { | ||
| 127 | + Reg::Exp(result, sum, mask); | ||
| 128 | + } | ||
| 129 | + Reg::Adds(outputIndex, outputBase, row * length + col, mask); | ||
| 130 | + Reg::Scatter(yTile, result, outputIndex, mask); | ||
| 131 | + } | ||
| 132 | + } | ||
| 133 | + } | ||
| 134 | +} | ||
| 135 | + | ||
| 33 | template <typename T, int32_t MODE> | 136 | template <typename T, int32_t MODE> |
| 34 | class Segsum { | 137 | class Segsum { |
| 35 | public: | 138 | public: |
| @@ -71,6 +174,7 @@ private: | |||
| 71 | __aicore__ inline void ParseTilingData(const SegsumTilingDataArch35* tilingData); | 174 | __aicore__ inline void ParseTilingData(const SegsumTilingDataArch35* tilingData); |
| 72 | __aicore__ inline void LoadX(int64_t offset, int64_t num); | 175 | __aicore__ inline void LoadX(int64_t offset, int64_t num); |
| 73 | __aicore__ inline void ComputeRowBlocks(); | 176 | __aicore__ inline void ComputeRowBlocks(); |
| 177 | + __aicore__ inline void ComputeShortBatch(); | ||
| 74 | __aicore__ inline void ComputeColumnStripes(); | 178 | __aicore__ inline void ComputeColumnStripes(); |
| 75 | __aicore__ inline void CopyOutRows(int64_t offset, int64_t rows, uint32_t bytesPerRow); | 179 | __aicore__ inline void CopyOutRows(int64_t offset, int64_t rows, uint32_t bytesPerRow); |
| 76 | 180 | ||
| @@ -122,6 +226,17 @@ __aicore__ inline void Segsum<T, MODE>::Init(GM_ADDR x, GM_ADDR y, const SegsumT | |||
| 122 | yGm.SetGlobalBuffer((__gm__ T*)y); | 226 | yGm.SetGlobalBuffer((__gm__ T*)y); |
| 123 | 227 | ||
| 124 | if constexpr (MODE == 1) { | 228 | if constexpr (MODE == 1) { |
| 229 | + if (rowNum > tailDimLength) { | ||
| 230 | + if (tailDimLength > 1) { | ||
| 231 | + pipe.InitBuffer(xBuf, rowNum * sizeof(T)); | ||
| 232 | + if constexpr (!IS_FLOAT) { | ||
| 233 | + pipe.InitBuffer(xFloatBuf, rowNum * sizeof(float)); | ||
| 234 | + pipe.InitBuffer(expBuf, rowNum * tailDimLength * sizeof(float)); | ||
| 235 | + } | ||
| 236 | + } | ||
| 237 | + pipe.InitBuffer(yQue, NO_BUFFER_NUM, rowNum * tailDimLength * sizeof(T)); | ||
| 238 | + return; | ||
| 239 | + } | ||
| 125 | pipe.InitBuffer(xBuf, rowLen * sizeof(T)); | 240 | pipe.InitBuffer(xBuf, rowLen * sizeof(T)); |
| 126 | if constexpr (!IS_FLOAT) { | 241 | if constexpr (!IS_FLOAT) { |
| 127 | pipe.InitBuffer(xFloatBuf, rowLen * sizeof(float)); | 242 | pipe.InitBuffer(xFloatBuf, rowLen * sizeof(float)); |
| @@ -149,6 +264,10 @@ __aicore__ inline void Segsum<T, MODE>::Process() | |||
| 149 | return; | 264 | return; |
| 150 | } | 265 | } |
| 151 | if constexpr (MODE == 1) { | 266 | if constexpr (MODE == 1) { |
| 267 | + if (rowNum > tailDimLength) { | ||
| 268 | + ComputeShortBatch(); | ||
| 269 | + return; | ||
| 270 | + } | ||
| 152 | ComputeRowBlocks(); | 271 | ComputeRowBlocks(); |
| 153 | } else { | 272 | } else { |
| 154 | ComputeColumnStripes(); | 273 | ComputeColumnStripes(); |
| @@ -196,6 +315,46 @@ __aicore__ inline void Segsum<T, MODE>::CopyOutRows(int64_t offset, int64_t rows | |||
| 196 | yQue.FreeTensor(yTensor); | 315 | yQue.FreeTensor(yTensor); |
| 197 | } | 316 | } |
| 198 | 317 | ||
| 318 | +template <typename T, int32_t MODE> | ||
| 319 | +__aicore__ inline void Segsum<T, MODE>::ComputeShortBatch() | ||
| 320 | +{ | ||
| 321 | + const int64_t tile = rowNum / tailDimLength; | ||
| 322 | + if (tailDimLength == 1) { | ||
| 323 | + // The 1x1 output is exp(0) regardless of the input, including NaN/Inf. | ||
| 324 | + LocalTensor<T> ones = yQue.AllocTensor<T>(); | ||
| 325 | + Duplicate(ones, static_cast<T>(1.0f), static_cast<int32_t>(tile)); | ||
| 326 | + yQue.EnQue(ones); | ||
| 327 | + ones = yQue.DeQue<T>(); | ||
| 328 | + for (int64_t start = batchStart; start < batchEnd; start += tile) { | ||
| 329 | + const uint32_t count = static_cast<uint32_t>(Min(tile, batchEnd - start)); | ||
| 330 | + DataCopyExtParams copyParams{1, static_cast<uint32_t>(count * sizeof(T)), 0, 0, 0}; | ||
| 331 | + DataCopyPad(yGm[start], ones, copyParams); | ||
| 332 | + } | ||
| 333 | + yQue.FreeTensor(ones); | ||
| 334 | + return; | ||
| 335 | + } | ||
| 336 | + LocalTensor<float> xFloat = GetXFloat(); | ||
| 337 | + for (int64_t start = batchStart; start < batchEnd; start += tile) { | ||
| 338 | + const uint32_t count = static_cast<uint32_t>(Min(tile, batchEnd - start)); | ||
| 339 | + LoadX(tailDimLength * start, tailDimLength * count); | ||
| 340 | + LocalTensor<T> yTensor = yQue.AllocTensor<T>(); | ||
| 341 | + LocalTensor<float> yFloat = GetExpTensor(yTensor); | ||
| 342 | + const uint32_t outputCount = static_cast<uint32_t>(tailDimLength * tailDimLength * count); | ||
| 343 | + Duplicate(yFloat, 0.0f, outputCount); | ||
| 344 | + PipeBarrier<PIPE_V>(); | ||
| 345 | + asc_vf_call<ComputeShortBatchVf<IS_FLOAT>>((__ubuf__ float*)xFloat.GetPhyAddr(), | ||
| 346 | + (__ubuf__ float*)yFloat.GetPhyAddr(), count, | ||
| 347 | + static_cast<uint32_t>(tailDimLength)); | ||
| 348 | + PipeBarrier<PIPE_V>(); | ||
| 349 | + if constexpr (!IS_FLOAT) { | ||
| 350 | + Cast(yTensor, yFloat, RoundMode::CAST_RINT, outputCount); | ||
| 351 | + PipeBarrier<PIPE_V>(); | ||
| 352 | + } | ||
| 353 | + yQue.EnQue(yTensor); | ||
| 354 | + CopyOutRows(tailDimLength * tailDimLength * start, 1, outputCount * sizeof(T)); | ||
| 355 | + } | ||
| 356 | +} | ||
| 357 | + | ||
| 199 | /* | 358 | /* |
| 200 | * TilingKey 1: a whole row fits in UB. rowNum consecutive rows are chained inside the fp32 | 359 | * TilingKey 1: a whole row fits in UB. rowNum consecutive rows are chained inside the fp32 |
| 201 | * accumulator buffer, exponentiated in place and written back with a single contiguous copy. | 360 | * accumulator buffer, exponentiated in place and written back with a single contiguous copy. |
| @@ -233,7 +392,12 @@ __aicore__ inline void Segsum<T, MODE>::ComputeRowBlocks() | |||
| 233 | Adds(curRow, prevRow, xFloat.GetValue(rowIdx), rowIdx); | 392 | Adds(curRow, prevRow, xFloat.GetValue(rowIdx), rowIdx); |
| 234 | PipeBarrier<PIPE_V>(); | 393 | PipeBarrier<PIPE_V>(); |
| 235 | } | 394 | } |
| 236 | - Exp(expRows[r * rowLen], curRow, rowIdx + 1); | 395 | + if constexpr (IS_FLOAT) { |
| 396 | + asc_vf_call<ComputeAccurateExpVf>((__ubuf__ float*)expRows[r * rowLen].GetPhyAddr(), | ||
| 397 | + (__ubuf__ float*)curRow.GetPhyAddr(), rowIdx + 1); | ||
| 398 | + } else { | ||
| 399 | + Exp(expRows[r * rowLen], curRow, rowIdx + 1); | ||
| 400 | + } | ||
| 237 | PipeBarrier<PIPE_V>(); | 401 | PipeBarrier<PIPE_V>(); |
| 238 | } | 402 | } |
| 239 | 403 | ||
| @@ -293,7 +457,12 @@ __aicore__ inline void Segsum<T, MODE>::ComputeColumnStripes() | |||
| 293 | PipeBarrier<PIPE_V>(); | 457 | PipeBarrier<PIPE_V>(); |
| 294 | } | 458 | } |
| 295 | if (validCols > 0) { | 459 | if (validCols > 0) { |
| 296 | - Exp(expRow, curRow, validCols); | 460 | + if constexpr (IS_FLOAT) { |
| 461 | + asc_vf_call<ComputeAccurateExpVf>((__ubuf__ float*)expRow.GetPhyAddr(), | ||
| 462 | + (__ubuf__ float*)curRow.GetPhyAddr(), validCols); | ||
| 463 | + } else { | ||
| 464 | + Exp(expRow, curRow, validCols); | ||
| 465 | + } | ||
| 297 | PipeBarrier<PIPE_V>(); | 466 | PipeBarrier<PIPE_V>(); |
| 298 | } | 467 | } |
| 299 | 468 | ||