Star908
1826
代码
代码
Issues226
Pull Requests308
Actions
项目讨论
Wiki
分析
Star908
1826
已合并
dynamic_mx_quant fix aicerr #6938
季骏创建于 7月2日
dynamic_mx_quant fix aicerr #6938
已合并
季骏创建于 7月2日
从ji-jun1/ops-nnmaster合入到cann/ops-nnmaster
共 4 个文件变更+21-22
Mquant/dynamic_mx_quant/op_host/arch35/dynamic_mx_quant_tail_axis_tiling_arch35.cpp+9-10
@@ -34,6 +34,7 @@ constexpr int64_t BYTES_OF_SCALE_TYPE = sizeof(uint8_t);
34constexpr int64_t BYTES_OF_INVERSE_SCALE_TYPE = sizeof(uint16_t);34constexpr int64_t BYTES_OF_INVERSE_SCALE_TYPE = sizeof(uint16_t);
35constexpr int64_t DB_BUFFER = 2;35constexpr int64_t DB_BUFFER = 2;
36constexpr float LOAD_BALANCE_THRESHOLD = 0.75; // 负载均衡的阈值36constexpr float LOAD_BALANCE_THRESHOLD = 0.75; // 负载均衡的阈值
37+constexpr int64_t RESERVED_UB_SIZE = 2 * 1024; // 预留空间
37 38 
38ge::graphStatus DynamicMxQuantTailAxisTiling::SetTilingDataForTailAxis()39ge::graphStatus DynamicMxQuantTailAxisTiling::SetTilingDataForTailAxis()
39{40{
@@ -203,23 +204,21 @@ ge::graphStatus DynamicMxQuantTailAxisTiling::DoTiling()
203 204 
204 // 计算UB可以放下的block（1×256）数量205 // 计算UB可以放下的block（1×256）数量
205 // maxUbBlockNum * SLICE_SIZE_256 * (BYTES_OF_INPUT_TYPE * DB_BUFFER + BYTES_OF_OUTPUT_TYPE * DB_BUFFER) +206 // maxUbBlockNum * SLICE_SIZE_256 * (BYTES_OF_INPUT_TYPE * DB_BUFFER + BYTES_OF_OUTPUT_TYPE * DB_BUFFER) +
206- // maxUbBlockNum * DIGIT_EIGHT * (BYTES_OF_MAX_VALUE_TYPE + BYTES_OF_SCALE_TYPE * DB_BUFFER +207+ // maxUbBlockNum * DIGIT_EIGHT * DB_BUFFER * (BYTES_OF_MAX_VALUE_TYPE + BYTES_OF_SCALE_TYPE +
207 // BYTES_OF_INVERSE_SCALE_TYPE)208 // BYTES_OF_INVERSE_SCALE_TYPE)
208- // <= ubSize209+ // <= ubSize - RESERVED_UB_SIZE
209 // fp16/bf16 输入 DB_BUFFER 总字节数 = 2*2 = 4；fp32 输入需要 fp32 数据 4*2 = 8210 // fp16/bf16 输入 DB_BUFFER 总字节数 = 2*2 = 4；fp32 输入需要 fp32 数据 4*2 = 8
210 int64_t bytesInputDbBuffer = DB_BUFFER * tilingParam_.inputDtypeSize;211 int64_t bytesInputDbBuffer = DB_BUFFER * tilingParam_.inputDtypeSize;
211 // fp32 输入时 maxExpBuffer_ 使用 sizeof(float)=4，其余使用 sizeof(uint16_t)=2212 // fp32 输入时 maxExpBuffer_ 使用 sizeof(float)=4，其余使用 sizeof(uint16_t)=2
212 int64_t bytesOfMaxValueType = tilingParam_.inputDtypeSize;213 int64_t bytesOfMaxValueType = tilingParam_.inputDtypeSize;
213 if (tilingParam_.dstType == ge::DT_FLOAT4_E2M1 || tilingParam_.dstType == ge::DT_FLOAT4_E1M2) { // Y FP4214 if (tilingParam_.dstType == ge::DT_FLOAT4_E2M1 || tilingParam_.dstType == ge::DT_FLOAT4_E1M2) { // Y FP4
214- tilingParam_.maxUbBlockNum = tilingParam_.ubSize /215+ tilingParam_.maxUbBlockNum = (tilingParam_.ubSize - RESERVED_UB_SIZE) /
215- (SLICE_SIZE_256 * (bytesInputDbBuffer + BYTES_OF_OUTPUT_FP4_DB_BUFFER_TYPE) +216+ (SLICE_SIZE_256 * (bytesInputDbBuffer + BYTES_OF_OUTPUT_FP4_DB_BUFFER_TYPE) +
216- DIGIT_EIGHT * (bytesOfMaxValueType + BYTES_OF_SCALE_TYPE * DB_BUFFER +217+ DIGIT_EIGHT * DB_BUFFER * (bytesOfMaxValueType + BYTES_OF_SCALE_TYPE + BYTES_OF_INVERSE_SCALE_TYPE));
217- BYTES_OF_INVERSE_SCALE_TYPE));
218 } else if (tilingParam_.dstType == ge::DT_FLOAT8_E5M2 || tilingParam_.dstType == ge::DT_FLOAT8_E4M3FN) { // Y FP8218 } else if (tilingParam_.dstType == ge::DT_FLOAT8_E5M2 || tilingParam_.dstType == ge::DT_FLOAT8_E4M3FN) { // Y FP8
219- tilingParam_.maxUbBlockNum = tilingParam_.ubSize /219+ tilingParam_.maxUbBlockNum = (tilingParam_.ubSize - RESERVED_UB_SIZE) /
220- (SLICE_SIZE_256 * (bytesInputDbBuffer + BYTES_OF_OUTPUT_FP8_DB_BUFFER_TYPE) +220+ (SLICE_SIZE_256 * (bytesInputDbBuffer + BYTES_OF_OUTPUT_FP8_DB_BUFFER_TYPE) +
221- DIGIT_EIGHT * (bytesOfMaxValueType + BYTES_OF_SCALE_TYPE * DB_BUFFER +221+ DIGIT_EIGHT * DB_BUFFER * (bytesOfMaxValueType + BYTES_OF_SCALE_TYPE + BYTES_OF_INVERSE_SCALE_TYPE));
222- BYTES_OF_INVERSE_SCALE_TYPE));
223 }222 }
224 tilingParam_.maxUbBlockNum *= DIGIT_EIGHT; // 转换成UB可以放下的block（1×32）数量223 tilingParam_.maxUbBlockNum *= DIGIT_EIGHT; // 转换成UB可以放下的block（1×32）数量
225 224 
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_tail_axis.h+2-2
@@ -185,8 +185,8 @@ __aicore__ inline void DynamicMxQuantTailAxis<T, U, SCALE_ALG>::Init(GM_ADDR x,
185 pipe_.InitBuffer(inQueue_, DB_BUFFER, maxUbBlockNum_ * blockSize_ * sizeof(T));185 pipe_.InitBuffer(inQueue_, DB_BUFFER, maxUbBlockNum_ * blockSize_ * sizeof(T));
186 pipe_.InitBuffer(outQueue_, DB_BUFFER, maxUbBlockNum_ * blockSize_ * sizeof(uint8_t) / DIGIT_TWO);186 pipe_.InitBuffer(outQueue_, DB_BUFFER, maxUbBlockNum_ * blockSize_ * sizeof(uint8_t) / DIGIT_TWO);
187 pipe_.InitBuffer(mxScaleQueue_, DB_BUFFER, maxUbBlockNum_ * sizeof(uint8_t));187 pipe_.InitBuffer(mxScaleQueue_, DB_BUFFER, maxUbBlockNum_ * sizeof(uint8_t));
188- pipe_.InitBuffer(maxExpBuffer_, maxUbBlockNum_ * sizeof(T));188+ pipe_.InitBuffer(maxExpBuffer_, maxUbBlockNum_ * sizeof(T) * 2);
189- pipe_.InitBuffer(recipScaleBuffer_, maxUbBlockNum_ * sizeof(uint16_t));189+ pipe_.InitBuffer(recipScaleBuffer_, maxUbBlockNum_ * sizeof(uint16_t) * 2);
190 190 
191 xGm_.SetGlobalBuffer((__gm__ T*)x + xGmOffset_);191 xGm_.SetGlobalBuffer((__gm__ T*)x + xGmOffset_);
192 yGm_.SetGlobalBuffer((__gm__ uint8_t*)y + xGmOffset_ / DIGIT_TWO);192 yGm_.SetGlobalBuffer((__gm__ uint8_t*)y + xGmOffset_ / DIGIT_TWO);
Mquant/dynamic_mx_quant/op_kernel/arch35/dynamic_mx_quant_tail_axis_fp8.h+2-2
@@ -131,8 +131,8 @@ __aicore__ inline void DynamicMxQuantTailAxisFP8<T, U, SCALE_ALG>::Init(
131 pipe_.InitBuffer(inQueue_, DB_BUFFER, maxUbBlockNum_ * blockSize_ * sizeof(T));131 pipe_.InitBuffer(inQueue_, DB_BUFFER, maxUbBlockNum_ * blockSize_ * sizeof(T));
132 pipe_.InitBuffer(outQueue_, DB_BUFFER, maxUbBlockNum_ * blockSize_ * sizeof(uint8_t));132 pipe_.InitBuffer(outQueue_, DB_BUFFER, maxUbBlockNum_ * blockSize_ * sizeof(uint8_t));
133 pipe_.InitBuffer(mxScaleQueue_, DB_BUFFER, maxUbBlockNum_ * sizeof(uint8_t));133 pipe_.InitBuffer(mxScaleQueue_, DB_BUFFER, maxUbBlockNum_ * sizeof(uint8_t));
134- pipe_.InitBuffer(maxExpBuffer_, maxUbBlockNum_ * sizeof(T));134+ pipe_.InitBuffer(maxExpBuffer_, maxUbBlockNum_ * sizeof(T) * 2);
135- pipe_.InitBuffer(recipScaleBuffer_, maxUbBlockNum_ * sizeof(uint16_t));135+ pipe_.InitBuffer(recipScaleBuffer_, maxUbBlockNum_ * sizeof(uint16_t) * 2);
136 136 
137 xGm_.SetGlobalBuffer((__gm__ T*)x + xGmOffset_);137 xGm_.SetGlobalBuffer((__gm__ T*)x + xGmOffset_);
138 yGm_.SetGlobalBuffer((__gm__ uint8_t*)y + xGmOffset_);138 yGm_.SetGlobalBuffer((__gm__ uint8_t*)y + xGmOffset_);
Mquant/dynamic_mx_quant/tests/ut/op_host/arch35/test_dynamic_mx_quant_tiling.cpp+8-8
@@ -154,7 +154,7 @@ TEST_F(DynamicMxQuantTiling, DynamicMxQuant_tiling_ascendc_bfloat16_fp4e2m1_tail
154 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};154 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};
155 int64_t axis = -1;155 int64_t axis = -1;
156 int64_t blockSize = 32;156 int64_t blockSize = 32;
157- string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 1528 0 0 0";157+ string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 1480 0 0 0";
158 158 
159 ExecuteTestCase(ge::DT_BF16, ge::DT_FLOAT4_E2M1, shape, scaleShape, axis, blockSize, expectTilingData);159 ExecuteTestCase(ge::DT_BF16, ge::DT_FLOAT4_E2M1, shape, scaleShape, axis, blockSize, expectTilingData);
160}160}
@@ -187,7 +187,7 @@ TEST_F(DynamicMxQuantTiling, DynamicMxQuant_tiling_ascendc_bfloat16_fp4e1m2_tail
187 gert::StorageShape scaleShape = {{1024, 512, 64, 1, 2}, {1024, 512, 64, 1, 2}};187 gert::StorageShape scaleShape = {{1024, 512, 64, 1, 2}, {1024, 512, 64, 1, 2}};
188 int64_t axis = 3;188 int64_t axis = 3;
189 int64_t blockSize = 32;189 int64_t blockSize = 32;
190- string expectTilingData = "33 253952 4 32 64 64 64 1 33554432 32 2 32 524288 524288 1528 0 0 0";190+ string expectTilingData = "33 253952 4 32 64 64 64 1 33554432 32 2 32 524288 524288 1480 0 0 0";
191 191 
192 ExecuteTestCase(ge::DT_BF16, ge::DT_FLOAT4_E1M2, shape, scaleShape, axis, blockSize, expectTilingData);192 ExecuteTestCase(ge::DT_BF16, ge::DT_FLOAT4_E1M2, shape, scaleShape, axis, blockSize, expectTilingData);
193}193}
@@ -276,7 +276,7 @@ TEST_F(DynamicMxQuantTiling, DynamicMxQuant_tiling_ascendc_bfloat16_fp8e4m3fn_ta
276 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};276 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};
277 int64_t axis = -1;277 int64_t axis = -1;
278 int64_t blockSize = 32;278 int64_t blockSize = 32;
279- string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 1280 0 0 0";279+ string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 1240 0 0 0";
280 280 
281 ExecuteTestCase(ge::DT_BF16, ge::DT_FLOAT8_E4M3FN, shape, scaleShape, axis, blockSize, expectTilingData);281 ExecuteTestCase(ge::DT_BF16, ge::DT_FLOAT8_E4M3FN, shape, scaleShape, axis, blockSize, expectTilingData);
282}282}
@@ -320,7 +320,7 @@ TEST_F(DynamicMxQuantTiling, DynamicMxQuant_tiling_ascendc_bfloat16_fp8e5m2_tail
320 gert::StorageShape scaleShape = {{15, 60, 16, 2, 2}, {15, 60, 16, 2, 2}};320 gert::StorageShape scaleShape = {{15, 60, 16, 2, 2}, {15, 60, 16, 2, 2}};
321 int64_t axis = -1;321 int64_t axis = -1;
322 int64_t blockSize = 32;322 int64_t blockSize = 32;
323- string expectTilingData = "33 253952 4 32 64 64 64 1 14400 128 2 128 225 225 1280 0 0 0";323+ string expectTilingData = "33 253952 4 32 64 64 64 1 14400 128 2 128 225 225 1240 0 0 0";
324 324 
325 ExecuteTestCase(ge::DT_BF16, ge::DT_FLOAT8_E5M2, shape, scaleShape, axis, blockSize, expectTilingData);325 ExecuteTestCase(ge::DT_BF16, ge::DT_FLOAT8_E5M2, shape, scaleShape, axis, blockSize, expectTilingData);
326}326}
@@ -368,7 +368,7 @@ TEST_F(DynamicMxQuantTiling, DynamicMxQuant_tiling_ascendc_float32_fp4e2m1_tail_
368 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};368 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};
369 int64_t axis = -1;369 int64_t axis = -1;
370 int64_t blockSize = 32;370 int64_t blockSize = 32;
371- string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 856 0 0 0";371+ string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 832 0 0 0";
372 372 
373 ExecuteTestCase(ge::DT_FLOAT, ge::DT_FLOAT4_E2M1, shape, scaleShape, axis, blockSize, expectTilingData);373 ExecuteTestCase(ge::DT_FLOAT, ge::DT_FLOAT4_E2M1, shape, scaleShape, axis, blockSize, expectTilingData);
374}374}
@@ -379,7 +379,7 @@ TEST_F(DynamicMxQuantTiling, DynamicMxQuant_tiling_ascendc_float32_fp4e1m2_tail_
379 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};379 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};
380 int64_t axis = -1;380 int64_t axis = -1;
381 int64_t blockSize = 32;381 int64_t blockSize = 32;
382- string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 856 0 0 0";382+ string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 832 0 0 0";
383 383 
384 ExecuteTestCase(ge::DT_FLOAT, ge::DT_FLOAT4_E1M2, shape, scaleShape, axis, blockSize, expectTilingData);384 ExecuteTestCase(ge::DT_FLOAT, ge::DT_FLOAT4_E1M2, shape, scaleShape, axis, blockSize, expectTilingData);
385}385}
@@ -390,7 +390,7 @@ TEST_F(DynamicMxQuantTiling, DynamicMxQuant_tiling_ascendc_float32_fp8e4m3fn_tai
390 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};390 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};
391 int64_t axis = -1;391 int64_t axis = -1;
392 int64_t blockSize = 32;392 int64_t blockSize = 32;
393- string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 768 0 0 0";393+ string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 752 0 0 0";
394 394 
395 ExecuteTestCase(ge::DT_FLOAT, ge::DT_FLOAT8_E4M3FN, shape, scaleShape, axis, blockSize, expectTilingData);395 ExecuteTestCase(ge::DT_FLOAT, ge::DT_FLOAT8_E4M3FN, shape, scaleShape, axis, blockSize, expectTilingData);
396}396}
@@ -401,7 +401,7 @@ TEST_F(DynamicMxQuantTiling, DynamicMxQuant_tiling_ascendc_float32_fp8e5m2_tail_
401 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};401 gert::StorageShape scaleShape = {{60, 14, 16, 2, 2}, {60, 14, 16, 2, 2}};
402 int64_t axis = -1;402 int64_t axis = -1;
403 int64_t blockSize = 32;403 int64_t blockSize = 32;
404- string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 768 0 0 0";404+ string expectTilingData = "33 253952 4 32 64 64 64 1 13440 128 2 128 210 210 752 0 0 0";
405 405 
406 ExecuteTestCase(ge::DT_FLOAT, ge::DT_FLOAT8_E5M2, shape, scaleShape, axis, blockSize, expectTilingData);406 ExecuteTestCase(ge::DT_FLOAT, ge::DT_FLOAT8_E5M2, shape, scaleShape, axis, blockSize, expectTilingData);
407}407}