Merged
perf(segsum): optimize short-sequence batching #7354
perf(segsum): optimize short-sequence batching #7354
Merged
zsq_888Add on 4 days ago
Total: 2 files changed+201-2
@@ -77,6 +77,36 @@ void SegsumTilingArch35::GetTilingKey(uint64_t ubSizePlatform)
77 rowLen = CeilA2B(tailDimLength, blockElement) * blockElement;77 rowLen = CeilA2B(tailDimLength, blockElement) * blockElement;
78 78 
79 const int64_t ubAvailable = static_cast<int64_t>(ubSizePlatform - RESERVED_UB);79 const int64_t ubAvailable = static_cast<int64_t>(ubSizePlatform - RESERVED_UB);
80+ // All buffers coexist: input/output in T and, for narrow T, their FP32 views.
81+ // Multiples of 64 sequences keep every UB buffer and core output boundary aligned.
82+ constexpr int64_t batchAlign = 64;
83+ constexpr int64_t minBatch = 256;
84+ constexpr int64_t maxBatch = 1024;
85+ constexpr int64_t maxFillBatch = 32768;
86+ // Choose packed sequences when the peak live buffers can hold at least one batch tile.
87+ // Divide before comparing to avoid forming length squared for large fallback inputs.
88+ const int64_t packedScalarBytes = dataTypeSize + (isFloat ? 0 : BYTE_LEN_4);
89+ const int64_t packedElementBudget = ubAvailable / (minBatch * packedScalarBytes);
90+ const bool packedFits = tailDimLength == 1 ||
91+ (tailDimLength > 1 && packedElementBudget / tailDimLength > tailDimLength);
92+ if (packedFits && averageBatches >= minBatch) {
93+ const int64_t bytesPerBatch = tailDimLength == 1 ? dataTypeSize :
94+ (tailDimLength + tailDimLength * tailDimLength) *
95+ (dataTypeSize + (isFloat ? 0 : BYTE_LEN_4));
96+ const int64_t workLimit = CeilA2B(averageBatches, batchAlign) * batchAlign;
97+ int64_t batchTile = std::min(ubAvailable / bytesPerBatch, workLimit);
98+ batchTile = std::min(batchTile, tailDimLength == 1 ? maxFillBatch : maxBatch);
99+ batchTile = batchTile / batchAlign * batchAlign;
100+ if (batchTile >= minBatch) {
101+ averageBatches = CeilA2B(averageBatches, blockElement) * blockElement;
102+ needCoreNum = static_cast<uint32_t>(CeilA2B(batches, averageBatches));
103+ // Normal row blocks have rowNum <= length; larger values encode a packed tile.
104+ rowNum = batchTile * tailDimLength;
105+ stripeLen = 0;
106+ tilingKey = ROW_BLOCK_TILING_KEY;
107+ return;
108+ }
109+ }
80 // TilingKey 1: x row (plus its fp32 view for non-float dtypes) and the fp32 carry are resident,110 // TilingKey 1: x row (plus its fp32 view for non-float dtypes) and the fp32 carry are resident,
81 // each extra row costs one fp32 accumulator plus its output buffer (for float the exp buffer111 // each extra row costs one fp32 accumulator plus its output buffer (for float the exp buffer
82 // doubles as the output buffer).112 // doubles as the output buffer).
@@ -30,6 +30,109 @@ using namespace AscendC;
30constexpr float ZERO_FLOAT = 0;30constexpr float ZERO_FLOAT = 0;
31constexpr int32_t NO_BUFFER_NUM = 1;31constexpr int32_t NO_BUFFER_NUM = 1;
32 32 
33+constexpr Reg::CastTrait EXP_FP32_TO_S32_RINT = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT,
34+ Reg::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
35+constexpr Reg::CastTrait EXP_S32_TO_FP32 = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT, Reg::MaskMergeMode::ZEROING,
36+ RoundMode::CAST_NONE};
37+ 
38+// Preserve the carry input. Normal range reduction keeps the polynomial near
39+// one; exceptional, overflow and subnormal ranges retain the original Exp.
40+__simd_callee__ inline void ShortExpAccurate(Reg::RegTensor<float>& result, Reg::RegTensor<float>& input,
41+ Reg::MaskReg& mask)
42+{
43+ Reg::RegTensor<float> absolute, zero, safe, scaled, powerFloat, negativePower, reduced, coefficient, polynomial;
44+ Reg::RegTensor<int32_t> power;
45+ Reg::MaskReg normal;
46+ Reg::Exp(result, input, mask);
47+ Reg::Abs(absolute, input, mask);
48+ Reg::CompareScalar<float, CMPMODE::LE>(normal, absolute, 80.0f, mask);
49+ Reg::Duplicate(zero, 0.0f, mask);
50+ Reg::Select(safe, input, zero, normal);
51+ Reg::Muls(scaled, safe, 1.4426950216293335f, mask);
52+ Reg::Cast<int32_t, float, EXP_FP32_TO_S32_RINT>(power, scaled, mask);
53+ Reg::Cast<float, int32_t, EXP_S32_TO_FP32>(powerFloat, power, mask);
54+ Reg::Muls(negativePower, powerFloat, -1.0f, mask);
55+ Reg::Muls(reduced, safe, 1.0f, mask);
56+ Reg::Duplicate(coefficient, 0.693145751953125f, mask);
57+ Reg::MulAddDst(reduced, negativePower, coefficient, mask);
58+ Reg::Duplicate(coefficient, 1.428606765330187e-6f, mask);
59+ Reg::MulAddDst(reduced, negativePower, coefficient, mask);
60+ Reg::Duplicate(polynomial, 2.4801587642286904e-5f, mask);
61+ Reg::Duplicate(coefficient, 1.9841270113829523e-4f, mask);
62+ Reg::MulDstAdd(polynomial, reduced, coefficient, mask);
63+ Reg::Duplicate(coefficient, 1.3888889225199819e-3f, mask);
64+ Reg::MulDstAdd(polynomial, reduced, coefficient, mask);
65+ Reg::Duplicate(coefficient, 8.333333767950535e-3f, mask);
66+ Reg::MulDstAdd(polynomial, reduced, coefficient, mask);
67+ Reg::Duplicate(coefficient, 4.16666679084301e-2f, mask);
68+ Reg::MulDstAdd(polynomial, reduced, coefficient, mask);
69+ Reg::Duplicate(coefficient, 1.666666716337204e-1f, mask);
70+ Reg::MulDstAdd(polynomial, reduced, coefficient, mask);
71+ Reg::Duplicate(coefficient, 0.5f, mask);
72+ Reg::MulDstAdd(polynomial, reduced, coefficient, mask);
73+ Reg::Duplicate(coefficient, 1.0f, mask);
74+ Reg::MulDstAdd(polynomial, reduced, coefficient, mask);
75+ Reg::Duplicate(coefficient, 1.0f, mask);
76+ Reg::MulDstAdd(polynomial, reduced, coefficient, mask);
77+ Reg::Adds(power, power, 127, mask);
78+ Reg::ShiftLefts(power, power, static_cast<int16_t>(23), mask);
79+ Reg::Mul(polynomial, polynomial, (Reg::RegTensor<float>&)power, mask);
80+ Reg::Select(result, polynomial, result, normal);
81+}
82+ 
83+__simd_vf__ inline void ComputeAccurateExpVf(__ubuf__ float* output, __ubuf__ float* input, uint32_t count)
84+{
85+ constexpr uint32_t lanes = VECTOR_REG_WIDTH / sizeof(float);
86+ const uint16_t loops = static_cast<uint16_t>((count + lanes - 1) / lanes);
87+ Reg::RegTensor<int32_t> signedIndex;
88+ Reg::RegTensor<float> value, result;
89+ Reg::Arange(signedIndex, 0);
90+ for (uint16_t block = 0; block < loops; ++block) {
91+ auto mask = Reg::UpdateMask<float>(count);
92+ Reg::Gather(value, input + block * lanes, (Reg::RegTensor<uint32_t>&)signedIndex, mask);
93+ ShortExpAccurate(result, value, mask);
94+ Reg::Scatter(output + block * lanes, result, (Reg::RegTensor<uint32_t>&)signedIndex, mask);
95+ }
96+}
97+ 
98+// Each lane owns one sequence. Preserve the original per-column addition order.
99+template <bool PRECISE_EXP>
100+__simd_vf__ inline void ComputeShortBatchVf(__ubuf__ float* x, __ubuf__ float* y, uint32_t count, uint32_t length)
101+{
102+ constexpr uint32_t lanes = VECTOR_REG_WIDTH / sizeof(float);
103+ const uint16_t loops = static_cast<uint16_t>((count + lanes - 1) / lanes);
104+ Reg::RegTensor<int32_t> lane;
105+ Reg::RegTensor<uint32_t> inputBase, outputBase, inputIndex, outputIndex;
106+ Reg::RegTensor<float> one, sum, value, result;
107+ auto full = Reg::CreateMask<float, Reg::MaskPattern::ALL>();
108+ Reg::Arange(lane, 0);
109+ Reg::Muls(inputBase, (Reg::RegTensor<uint32_t>&)lane, length, full);
110+ Reg::Muls(outputBase, (Reg::RegTensor<uint32_t>&)lane, length * length, full);
111+ Reg::Duplicate(one, 1.0f, full);
112+ for (uint16_t i = 0; i < loops; ++i) {
113+ auto mask = Reg::UpdateMask<float>(count);
114+ auto xTile = x + i * lanes * length;
115+ auto yTile = y + i * lanes * length * length;
116+ for (uint16_t col = 0; col < length; ++col) {
117+ Reg::Adds(outputIndex, outputBase, col * (length + 1), mask);
118+ Reg::Scatter(yTile, one, outputIndex, mask);
119+ Reg::Duplicate(sum, 0.0f, mask);
120+ for (uint16_t row = col + 1; row < length; ++row) {
121+ Reg::Adds(inputIndex, inputBase, static_cast<uint32_t>(row), mask);
122+ Reg::Gather(value, xTile, inputIndex, mask);
123+ Reg::Add(sum, sum, value, mask);
124+ if constexpr (PRECISE_EXP) {
125+ ShortExpAccurate(result, sum, mask);
126+ } else {
127+ Reg::Exp(result, sum, mask);
128+ }
129+ Reg::Adds(outputIndex, outputBase, row * length + col, mask);
130+ Reg::Scatter(yTile, result, outputIndex, mask);
131+ }
132+ }
133+ }
134+}
135+ 
33template <typename T, int32_t MODE>136template <typename T, int32_t MODE>
34class Segsum {137class Segsum {
35public:138public:
@@ -71,6 +174,7 @@ private:
71 __aicore__ inline void ParseTilingData(const SegsumTilingDataArch35* tilingData);174 __aicore__ inline void ParseTilingData(const SegsumTilingDataArch35* tilingData);
72 __aicore__ inline void LoadX(int64_t offset, int64_t num);175 __aicore__ inline void LoadX(int64_t offset, int64_t num);
73 __aicore__ inline void ComputeRowBlocks();176 __aicore__ inline void ComputeRowBlocks();
177+ __aicore__ inline void ComputeShortBatch();
74 __aicore__ inline void ComputeColumnStripes();178 __aicore__ inline void ComputeColumnStripes();
75 __aicore__ inline void CopyOutRows(int64_t offset, int64_t rows, uint32_t bytesPerRow);179 __aicore__ inline void CopyOutRows(int64_t offset, int64_t rows, uint32_t bytesPerRow);
76 180 
@@ -122,6 +226,17 @@ __aicore__ inline void Segsum<T, MODE>::Init(GM_ADDR x, GM_ADDR y, const SegsumT
122 yGm.SetGlobalBuffer((__gm__ T*)y);226 yGm.SetGlobalBuffer((__gm__ T*)y);
123 227 
124 if constexpr (MODE == 1) {228 if constexpr (MODE == 1) {
229+ if (rowNum > tailDimLength) {
230+ if (tailDimLength > 1) {
231+ pipe.InitBuffer(xBuf, rowNum * sizeof(T));
232+ if constexpr (!IS_FLOAT) {
233+ pipe.InitBuffer(xFloatBuf, rowNum * sizeof(float));
234+ pipe.InitBuffer(expBuf, rowNum * tailDimLength * sizeof(float));
235+ }
236+ }
237+ pipe.InitBuffer(yQue, NO_BUFFER_NUM, rowNum * tailDimLength * sizeof(T));
238+ return;
239+ }
125 pipe.InitBuffer(xBuf, rowLen * sizeof(T));240 pipe.InitBuffer(xBuf, rowLen * sizeof(T));
126 if constexpr (!IS_FLOAT) {241 if constexpr (!IS_FLOAT) {
127 pipe.InitBuffer(xFloatBuf, rowLen * sizeof(float));242 pipe.InitBuffer(xFloatBuf, rowLen * sizeof(float));
@@ -149,6 +264,10 @@ __aicore__ inline void Segsum<T, MODE>::Process()
149 return;264 return;
150 }265 }
151 if constexpr (MODE == 1) {266 if constexpr (MODE == 1) {
267+ if (rowNum > tailDimLength) {
268+ ComputeShortBatch();
269+ return;
270+ }
152 ComputeRowBlocks();271 ComputeRowBlocks();
153 } else {272 } else {
154 ComputeColumnStripes();273 ComputeColumnStripes();
@@ -196,6 +315,46 @@ __aicore__ inline void Segsum<T, MODE>::CopyOutRows(int64_t offset, int64_t rows
196 yQue.FreeTensor(yTensor);315 yQue.FreeTensor(yTensor);
197}316}
198 317 
318+template <typename T, int32_t MODE>
319+__aicore__ inline void Segsum<T, MODE>::ComputeShortBatch()
320+{
321+ const int64_t tile = rowNum / tailDimLength;
322+ if (tailDimLength == 1) {
323+ // The 1x1 output is exp(0) regardless of the input, including NaN/Inf.
324+ LocalTensor<T> ones = yQue.AllocTensor<T>();
325+ Duplicate(ones, static_cast<T>(1.0f), static_cast<int32_t>(tile));
326+ yQue.EnQue(ones);
327+ ones = yQue.DeQue<T>();
328+ for (int64_t start = batchStart; start < batchEnd; start += tile) {
329+ const uint32_t count = static_cast<uint32_t>(Min(tile, batchEnd - start));
330+ DataCopyExtParams copyParams{1, static_cast<uint32_t>(count * sizeof(T)), 0, 0, 0};
331+ DataCopyPad(yGm[start], ones, copyParams);
332+ }
333+ yQue.FreeTensor(ones);
334+ return;
335+ }
336+ LocalTensor<float> xFloat = GetXFloat();
337+ for (int64_t start = batchStart; start < batchEnd; start += tile) {
338+ const uint32_t count = static_cast<uint32_t>(Min(tile, batchEnd - start));
339+ LoadX(tailDimLength * start, tailDimLength * count);
340+ LocalTensor<T> yTensor = yQue.AllocTensor<T>();
341+ LocalTensor<float> yFloat = GetExpTensor(yTensor);
342+ const uint32_t outputCount = static_cast<uint32_t>(tailDimLength * tailDimLength * count);
343+ Duplicate(yFloat, 0.0f, outputCount);
344+ PipeBarrier<PIPE_V>();
345+ asc_vf_call<ComputeShortBatchVf<IS_FLOAT>>((__ubuf__ float*)xFloat.GetPhyAddr(),
346+ (__ubuf__ float*)yFloat.GetPhyAddr(), count,
347+ static_cast<uint32_t>(tailDimLength));
348+ PipeBarrier<PIPE_V>();
349+ if constexpr (!IS_FLOAT) {
350+ Cast(yTensor, yFloat, RoundMode::CAST_RINT, outputCount);
351+ PipeBarrier<PIPE_V>();
352+ }
353+ yQue.EnQue(yTensor);
354+ CopyOutRows(tailDimLength * tailDimLength * start, 1, outputCount * sizeof(T));
355+ }
356+}
357+ 
199/*358/*
200 * TilingKey 1: a whole row fits in UB. rowNum consecutive rows are chained inside the fp32359 * TilingKey 1: a whole row fits in UB. rowNum consecutive rows are chained inside the fp32
201 * accumulator buffer, exponentiated in place and written back with a single contiguous copy.360 * accumulator buffer, exponentiated in place and written back with a single contiguous copy.
@@ -233,7 +392,12 @@ __aicore__ inline void Segsum<T, MODE>::ComputeRowBlocks()
233 Adds(curRow, prevRow, xFloat.GetValue(rowIdx), rowIdx);392 Adds(curRow, prevRow, xFloat.GetValue(rowIdx), rowIdx);
234 PipeBarrier<PIPE_V>();393 PipeBarrier<PIPE_V>();
235 }394 }
236- Exp(expRows[r * rowLen], curRow, rowIdx + 1);395+ if constexpr (IS_FLOAT) {
396+ asc_vf_call<ComputeAccurateExpVf>((__ubuf__ float*)expRows[r * rowLen].GetPhyAddr(),
397+ (__ubuf__ float*)curRow.GetPhyAddr(), rowIdx + 1);
398+ } else {
399+ Exp(expRows[r * rowLen], curRow, rowIdx + 1);
400+ }
237 PipeBarrier<PIPE_V>();401 PipeBarrier<PIPE_V>();
238 }402 }
239 403 
@@ -293,7 +457,12 @@ __aicore__ inline void Segsum<T, MODE>::ComputeColumnStripes()
293 PipeBarrier<PIPE_V>();457 PipeBarrier<PIPE_V>();
294 }458 }
295 if (validCols > 0) {459 if (validCols > 0) {
296- Exp(expRow, curRow, validCols);460+ if constexpr (IS_FLOAT) {
461+ asc_vf_call<ComputeAccurateExpVf>((__ubuf__ float*)expRow.GetPhyAddr(),
462+ (__ubuf__ float*)curRow.GetPhyAddr(), validCols);
463+ } else {
464+ Exp(expRow, curRow, validCols);
465+ }
297 PipeBarrier<PIPE_V>();466 PipeBarrier<PIPE_V>();
298 }467 }
299 468