已合并
cleancode重复代码修复 #2655
Chen_HaoWen创建于 3月13日
cleancode重复代码修复 #2655
已合并
共 63 个文件变更+1380-1363
| @@ -224,16 +224,16 @@ bool IsGammaBetaTensorDescAsExpect(const GNode& layer_norm_v4_node, const Shape& | |||
| 224 | 224 | ||
| 225 | bool IsAllInputShapeDtypeRight(const std::unique_ptr<MatchResult>& match_result) | 225 | bool IsAllInputShapeDtypeRight(const std::unique_ptr<MatchResult>& match_result) |
| 226 | { | 226 | { |
| 227 | - NodeIo add1_output_0; | 227 | + NodeIo add1OutNode; |
 | |||
| 228 | - NodeIo add2_output_0; | 228 | + NodeIo add2OutNode; |
| 229 | OP_LOGE_IF( | 229 | OP_LOGE_IF( |
| 230 | - match_result->GetCapturedTensor(kAdd1CaptureIdx, add1_output_0)!= SUCCESS, false, kPassName, | 230 | + match_result->GetCapturedTensor(kAdd1CaptureIdx, add1OutNode)!= SUCCESS, false, kPassName, |
代码结构与可维护性: 函数调用OP_LOGE_IF的第三个参数类型不匹配。OP_LOGE_IF宏的第三个参数预期是一个字符串（const char*），但此处直接传递了kPassName（类型为std::string）。虽然std::string可以隐式转换为const char*（通过c_str()），但直接传递std::string对象可能在某些编译环境或宏定义下导致问题，或者降低代码的清晰度。最佳实践是显式调用c_str()方法以确保类型安全。在同一文件的第99、135、140行等位置，都正确使用了kPassName.c_str()。 问题类型: 代码结构与可维护性 文件路径: norm/add_layer_norm/op_graph/fusion_pass/add_layer_norm_v4_fusion_pass.cpp行号: 230 问题代码: match_result->GetCapturedTensor(kAdd1CaptureIdx, add1OutNode)!= SUCCESS, false, kPassName, 修改建议: 将kPassName修改为kPassName.c_str()，以明确传递C风格字符串，确保与OP_LOGE_IF宏的参数类型要求一致，并保持代码风格统一。修改后：
OP_LOGE_IF(
match_result->GetCapturedTensor(kAdd1CaptureIdx, add1OutNode)!= SUCCESS, false, kPassName.c_str(),
"Failed to GetCaptrue tensor");
同样，第233行的kPassName也应修改为kPassName.c_str()。--- 此评论由代码审查工具自动生成   | |||
| 231 | "Failed to GetCaptrue tensor"); | 231 | "Failed to GetCaptrue tensor"); |
| 232 | OP_LOGE_IF( | 232 | OP_LOGE_IF( |
| 233 | - match_result->GetCapturedTensor(kAdd2CaptureIdx, add2_output_0) != SUCCESS, false, kPassName, | 233 | + match_result->GetCapturedTensor(kAdd2CaptureIdx, add2OutNode) != SUCCESS, false, kPassName, |
| 234 | "Failed to GetCaptrue tensor"); | 234 | "Failed to GetCaptrue tensor"); |
| 235 | - auto add1_node = add1_output_0.node; | 235 | + auto add1_node = add1OutNode.node; |
| 236 | - auto add2_node = add2_output_0.node; | 236 | + auto add2_node = add2OutNode.node; |
| 237 | AscendString add1_node_name; | 237 | AscendString add1_node_name; |
| 238 | add1_node.GetName(add1_node_name); | 238 | add1_node.GetName(add1_node_name); |
| 239 | AscendString add2_node_name; | 239 | AscendString add2_node_name; |
| @@ -299,15 +299,15 @@ inline TILING_TYPE AddLayerNormTilingImpl( | |||
| 299 | } | 299 | } |
| 300 | } | 300 | } |
| 301 | 301 | ||
| 302 | - // when x1/x2: fp16/bf16 beta/gamma:fp32. | 302 | + // when x1/x2: fp16/bf16 beta/gamma:fp32 - case 2 |
| 303 | bool isSingleRowLessTensor2 = (!is310P) && (isNoBias || isBroadcastBias) && isXB16GammaB32 && enableXOut; | 303 | bool isSingleRowLessTensor2 = (!is310P) && (isNoBias || isBroadcastBias) && isXB16GammaB32 && enableXOut; |
| 304 | if (isSingleRowLessTensor2) { | 304 | if (isSingleRowLessTensor2) { |
| 305 | - auto oneRowX1X2YBufSize = dtSize * 1 * numCol + 4 * 1 * numCol; | 305 | + auto oneRowX1X2YBufSize2 = dtSize * 1 * numCol + 4 * 1 * numCol; |
| 306 | - auto oneRowMeanRstdBufSize = 4 * 1 * 2; | 306 | + auto oneRowMeanRstdBufSize2 = 4 * 1 * 2; |
| 307 | - auto oneRowTmpBufSize = 4 * 2 * numCol; | 307 | + auto oneRowTmpBufSize2 = 4 * 2 * numCol; |
| 308 | - auto oneRowReduceBufSize = 4 * 64; | 308 | + auto oneRowReduceBufSize2 = 4 * 64; |
| 309 | tmpUbSize = static_cast<uint32_t>( | 309 | tmpUbSize = static_cast<uint32_t>( |
| 310 | - oneRowX1X2YBufSize + oneRowMeanRstdBufSize + oneRowTmpBufSize + oneRowReduceBufSize + UB_RESERVED_BYTE); | 310 | + oneRowX1X2YBufSize2 + oneRowMeanRstdBufSize2 + oneRowTmpBufSize2 + oneRowReduceBufSize2 + UB_RESERVED_BYTE); |
| 311 | if (tmpUbSize < maxUbSize) { | 311 | if (tmpUbSize < maxUbSize) { |
| 312 | rowPerTime = 1U; | 312 | rowPerTime = 1U; |
| 313 | colPerTime = numCol; | 313 | colPerTime = numCol; |
| @@ -39,17 +39,17 @@ constexpr uint32_t LAYER_NUM_TWO = 2; | |||
| 39 | 39 | ||
| 40 | 40 | ||
| 41 | 41 | ||
| 42 | -template <typename Tp, Tp v> | 42 | +template <typename Tp2, Tp2 v> |
| 43 | struct integral_constant { | 43 | struct integral_constant { |
| 44 | - static constexpr Tp value = v; | 44 | + static constexpr Tp2 value = v; |
| 45 | }; | 45 | }; |
| 46 | using true_type = integral_constant<bool, true>; | 46 | using true_type = integral_constant<bool, true>; |
| 47 | using false_type = integral_constant<bool, false>; | 47 | using false_type = integral_constant<bool, false>; |
| 48 | template <typename, typename> | 48 | template <typename, typename> |
| 49 | struct is_same : public false_type { | 49 | struct is_same : public false_type { |
| 50 | }; | 50 | }; |
| 51 | -template <typename Tp> | 51 | +template <typename Tp3> |
| 52 | -struct is_same<Tp, Tp> : public true_type { | 52 | +struct is_same<Tp3, Tp3> : public true_type { |
| 53 | }; | 53 | }; |
| 54 | 54 | ||
| 55 | template <typename T, template <typename U> typename R, template <typename U> typename S> | 55 | template <typename T, template <typename U> typename R, template <typename U> typename S> |
| @@ -325,53 +325,53 @@ __aicore__ inline void Level0AddFp32Short( | |||
| 325 | const LocalTensor<float>& dstLocal, const LocalTensor<float>& src0Local, const LocalTensor<float>& src1Local, | 325 | const LocalTensor<float>& dstLocal, const LocalTensor<float>& src0Local, const LocalTensor<float>& src1Local, |
| 326 | uint32_t alignElem, uint32_t repeat, uint32_t processElem) | 326 | uint32_t alignElem, uint32_t repeat, uint32_t processElem) |
| 327 | { | 327 | { |
| 328 | - uint32_t maxElemFp32 = ELEM_PER_REP_FP32; | 328 | + uint32_t addMaxElemFp32 = ELEM_PER_REP_FP32; |
| 329 | - uint8_t repStride = alignElem / FLOAT_BLOCK_ELEM; | 329 | + uint8_t addRepStride = alignElem / FLOAT_BLOCK_ELEM; |
| 330 | - uint32_t tailCount = processElem % maxElemFp32; | 330 | + uint32_t addTailCount = processElem % addMaxElemFp32; |
| 331 | 331 | ||
| 332 | - uint32_t repeatTimes = repeat / MAX_REP_NUM; | 332 | + uint32_t addRepeatTimes = repeat / MAX_REP_NUM; |
| 333 | 333 | ||
| 334 | - uint32_t index = 0; | 334 | + uint32_t addIndex = 0; |
| 335 | - uint32_t elemIndex = 0; | 335 | + uint32_t addElemIndex = 0; |
| 336 | - if (likely(repeatTimes == 0)) { | 336 | + if (likely(addRepeatTimes == 0)) { |
| 337 | - elemIndex = 0; | 337 | + addElemIndex = 0; |
| 338 | - for (; elemIndex + maxElemFp32 <= processElem; elemIndex += maxElemFp32) { | 338 | + for (; addElemIndex + addMaxElemFp32 <= processElem; addElemIndex += addMaxElemFp32) { |
| 339 | - Add(dstLocal[elemIndex], src0Local[elemIndex], src1Local[elemIndex], maxElemFp32, repeat, | 339 | + Add(dstLocal[addElemIndex], src0Local[addElemIndex], src1Local[addElemIndex], addMaxElemFp32, repeat, |
| 340 | - {1, 1, 1, repStride, 0, repStride}); | 340 | + {1, 1, 1, addRepStride, 0, addRepStride}); |
| 341 | } | 341 | } |
| 342 | - if (tailCount != 0) { | 342 | + if (addTailCount != 0) { |
| 343 | - Add(dstLocal[elemIndex], src0Local[elemIndex], src1Local[elemIndex], tailCount, repeat, | 343 | + Add(dstLocal[addElemIndex], src0Local[addElemIndex], src1Local[addElemIndex], addTailCount, repeat, |
| 344 | - {1, 1, 1, repStride, 0, repStride}); | 344 | + {1, 1, 1, addRepStride, 0, addRepStride}); |
| 345 | } | 345 | } |
| 346 | } else { | 346 | } else { |
| 347 | - uint32_t repTailNum = repeat % MAX_REP_NUM; | 347 | + uint32_t addRepTailNum = repeat % MAX_REP_NUM; |
| 348 | - uint32_t repIndex = 0; | 348 | + uint32_t addRepIndex = 0; |
| 349 | - uint32_t repElem; | 349 | + uint32_t addRepElem; |
| 350 | - for (; repIndex + MAX_REP_NUM <= repeat; repIndex += MAX_REP_NUM) { | 350 | + for (; addRepIndex + MAX_REP_NUM <= repeat; addRepIndex += MAX_REP_NUM) { |
| 351 | - elemIndex = 0; | 351 | + addElemIndex = 0; |
| 352 | - repElem = repIndex * alignElem; | 352 | + addRepElem = addRepIndex * alignElem; |
| 353 | - for (; elemIndex + maxElemFp32 <= processElem; elemIndex += maxElemFp32) { | 353 | + for (; addElemIndex + addMaxElemFp32 <= processElem; addElemIndex += addMaxElemFp32) { |
| 354 | - index = repElem + elemIndex; | 354 | + addIndex = addRepElem + addElemIndex; |
| 355 | - Add(dstLocal[elemIndex], src0Local[index], src1Local[elemIndex], maxElemFp32, MAX_REP_NUM, | 355 | + Add(dstLocal[addElemIndex], src0Local[addIndex], src1Local[addElemIndex], addMaxElemFp32, MAX_REP_NUM, |
| 356 | - {1, 1, 1, repStride, 0, repStride}); | 356 | + {1, 1, 1, addRepStride, 0, addRepStride}); |
| 357 | } | 357 | } |
| 358 | - if (tailCount != 0) { | 358 | + if (addTailCount != 0) { |
| 359 | - index = repElem + elemIndex; | 359 | + addIndex = addRepElem + addElemIndex; |
| 360 | - Add(dstLocal[elemIndex], src0Local[index], src1Local[elemIndex], tailCount, MAX_REP_NUM, | 360 | + Add(dstLocal[addElemIndex], src0Local[addIndex], src1Local[addElemIndex], addTailCount, MAX_REP_NUM, |
| 361 | - {1, 1, 1, repStride, 0, repStride}); | 361 | + {1, 1, 1, addRepStride, 0, addRepStride}); |
| 362 | } | 362 | } |
| 363 | } | 363 | } |
| 364 | - if (repTailNum != 0) { | 364 | + if (addRepTailNum != 0) { |
| 365 | - elemIndex = 0; | 365 | + addElemIndex = 0; |
| 366 | - for (; elemIndex + maxElemFp32 <= processElem; elemIndex += maxElemFp32) { | 366 | + for (; addElemIndex + addMaxElemFp32 <= processElem; addElemIndex += addMaxElemFp32) { |
| 367 | - index = repElem + elemIndex; | 367 | + addIndex = addRepElem + addElemIndex; |
| 368 | - Add(dstLocal[elemIndex], src0Local[index], src1Local[elemIndex], maxElemFp32, repTailNum, | 368 | + Add(dstLocal[addElemIndex], src0Local[addIndex], src1Local[addElemIndex], addMaxElemFp32, addRepTailNum, |
| 369 | - {1, 1, 1, repStride, 0, repStride}); | 369 | + {1, 1, 1, addRepStride, 0, addRepStride}); |
| 370 | } | 370 | } |
| 371 | - if (tailCount != 0) { | 371 | + if (addTailCount != 0) { |
| 372 | - index = repElem + elemIndex; | 372 | + addIndex = addRepElem + addElemIndex; |
| 373 | - Add(dstLocal[elemIndex], src0Local[index], src1Local[elemIndex], tailCount, repTailNum, | 373 | + Add(dstLocal[addElemIndex], src0Local[addIndex], src1Local[addElemIndex], addTailCount, addRepTailNum, |
| 374 | - {1, 1, 1, repStride, 0, repStride}); | 374 | + {1, 1, 1, addRepStride, 0, addRepStride}); |
| 375 | } | 375 | } |
| 376 | } | 376 | } |
| 377 | } | 377 | } |
| @@ -53,12 +53,12 @@ public: | |||
| 53 | return (x + ONE_BLK_SIZE - 1) / ONE_BLK_SIZE * ONE_BLK_SIZE; | 53 | return (x + ONE_BLK_SIZE - 1) / ONE_BLK_SIZE * ONE_BLK_SIZE; |
| 54 | } | 54 | } |
| 55 | 55 | ||
| 56 | - __aicore__ inline uint32_t MIN(uint32_t x, uint32_t y) | 56 | + __aicore__ inline uint32_t MIN_KERNEL(uint32_t x, uint32_t y) |
| 57 | { | 57 | { |
| 58 | return x < y ? x : y; | 58 | return x < y ? x : y; |
| 59 | } | 59 | } |
| 60 | 60 | ||
| 61 | - __aicore__ inline uint32_t MAX(uint32_t x, uint32_t y) | 61 | + __aicore__ inline uint32_t MAX_KERNEL(uint32_t x, uint32_t y) |
| 62 | { | 62 | { |
| 63 | return x > y ? x : y; | 63 | return x > y ? x : y; |
| 64 | } | 64 | } |
| @@ -86,7 +86,7 @@ public: | |||
| 86 | row_step = first_dim_per_time; | 86 | row_step = first_dim_per_time; |
| 87 | } else { | 87 | } else { |
| 88 | row_work = l_first_dim_per_core; | 88 | row_work = l_first_dim_per_core; |
| 89 | - row_step = MIN(first_dim_per_time, row_work); | 89 | + row_step = MIN_KERNEL(first_dim_per_time, row_work); |
| 90 | } | 90 | } |
| 91 | row_tail_ = (row_work % row_step == 0) ? row_step : (row_work % row_step); | 91 | row_tail_ = (row_work % row_step == 0) ? row_step : (row_work % row_step); |
| 92 | gm_offset_ = nl_first_dim_per_core * num_last_dim; | 92 | gm_offset_ = nl_first_dim_per_core * num_last_dim; |
| @@ -362,25 +362,25 @@ public: | |||
| 362 | ave_tmp += y_local_fp32.GetValue(0); | 362 | ave_tmp += y_local_fp32.GetValue(0); |
| 363 | } | 363 | } |
| 364 | // 2. Reduce Var | 364 | // 2. Reduce Var |
| 365 | - float var_tmp = 0; | 365 | + float var_tmp_2 = 0; |
| 366 | - for (int32_t col_idx = 0; col_idx < col_move_cnt; ++col_idx) { | 366 | + for (int32_t col_idx_2 = 0; col_idx_2 < col_move_cnt; ++col_idx_2) { |
| 367 | - auto col_offset = col_idx * last_dim_per_time; | 367 | + auto col_offset_2 = col_idx_2 * last_dim_per_time; |
| 368 | - int process_count = col_idx < col_move_cnt - 1 ? last_dim_per_time : col_tail; | 368 | + int process_count_2 = col_idx_2 < col_move_cnt - 1 ? last_dim_per_time : col_tail; |
| 369 | - Adds(x_local_fp32[col_offset], x_local_fp32[col_offset], -1 * ave_tmp, process_count); | 369 | + Adds(x_local_fp32[col_offset_2], x_local_fp32[col_offset_2], -1 * ave_tmp, process_count_2); |
| 370 | PipeBarrier<PIPE_V>(); | 370 | PipeBarrier<PIPE_V>(); |
| 371 | - Mul(y_local_fp32, x_local_fp32[col_offset], x_local_fp32[col_offset], process_count); | 371 | + Mul(y_local_fp32, x_local_fp32[col_offset_2], x_local_fp32[col_offset_2], process_count_2); |
| 372 | PipeBarrier<PIPE_V>(); | 372 | PipeBarrier<PIPE_V>(); |
| 373 | - Muls(y_local_fp32, y_local_fp32, aveNum, process_count); | 373 | + Muls(y_local_fp32, y_local_fp32, aveNum, process_count_2); |
| 374 | PipeBarrier<PIPE_V>(); | 374 | PipeBarrier<PIPE_V>(); |
| 375 | - ReduceSum(y_local_fp32, y_local_fp32, y_local_fp32, process_count); | 375 | + ReduceSum(y_local_fp32, y_local_fp32, y_local_fp32, process_count_2); |
| 376 | SetFlag<HardEvent::V_S>(EVENT_ID0); | 376 | SetFlag<HardEvent::V_S>(EVENT_ID0); |
| 377 | WaitFlag<HardEvent::V_S>(EVENT_ID0); | 377 | WaitFlag<HardEvent::V_S>(EVENT_ID0); |
| 378 | - var_tmp += y_local_fp32.GetValue(0); | 378 | + var_tmp_2 += y_local_fp32.GetValue(0); |
| 379 | } | 379 | } |
| 380 | - float rstd_tmp = 1 / sqrt(var_tmp + eps); | 380 | + float rstd_tmp_2 = 1 / sqrt(var_tmp_2 + eps); |
| 381 | 381 | ||
| 382 | mean_local.SetValue(0, ave_tmp); | 382 | mean_local.SetValue(0, ave_tmp); |
| 383 | - rstd_local.SetValue(0, rstd_tmp); | 383 | + rstd_local.SetValue(0, rstd_tmp_2); |
| 384 | 384 | ||
| 385 | // 3. Compute result | 385 | // 3. Compute result |
| 386 | for (int32_t col_idx = 0; col_idx < col_move_cnt; ++col_idx) { | 386 | for (int32_t col_idx = 0; col_idx < col_move_cnt; ++col_idx) { |
| @@ -388,7 +388,7 @@ public: | |||
| 388 | int process_count = col_idx < col_move_cnt - 1 ? last_dim_per_time : col_tail; | 388 | int process_count = col_idx < col_move_cnt - 1 ? last_dim_per_time : col_tail; |
| 389 | LocalTensor<T> y_local = y_que.template AllocTensor<T>(); | 389 | LocalTensor<T> y_local = y_que.template AllocTensor<T>(); |
| 390 | // x_local_fp32[col_offset] = (x - ave) | 390 | // x_local_fp32[col_offset] = (x - ave) |
| 391 | - Muls(y_local_fp32, x_local_fp32[col_offset], rstd_tmp, process_count); | 391 | + Muls(y_local_fp32, x_local_fp32[col_offset], rstd_tmp_2, process_count); |
| 392 | PipeBarrier<PIPE_V>(); | 392 | PipeBarrier<PIPE_V>(); |
| 393 | if constexpr (IS_BETAGAMMA_NEEDCAST) { | 393 | if constexpr (IS_BETAGAMMA_NEEDCAST) { |
| 394 | if constexpr (is_same<T, float>::value) { | 394 | if constexpr (is_same<T, float>::value) { |
| @@ -447,49 +447,49 @@ public: | |||
| 447 | __aicore__ inline void CopyInAddSlice(int32_t row_idx, int32_t process_count, int32_t col_offset) | 447 | __aicore__ inline void CopyInAddSlice(int32_t row_idx, int32_t process_count, int32_t col_offset) |
| 448 | { | 448 | { |
| 449 | LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); | 449 | LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); |
| 450 | - LocalTensor<float> y_local_fp32 = y_buf_fp32.Get<float>(); | 450 | + LocalTensor<float> y_local_fp32_2 = y_buf_fp32.Get<float>(); |
| 451 | - LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); | 451 | + LocalTensor<float> z_local_fp32_2 = z_buf_fp32.Get<float>(); |
| 452 | CopyInSlicePhase1(row_idx, process_count, col_offset); | 452 | CopyInSlicePhase1(row_idx, process_count, col_offset); |
| 453 | - LocalTensor<T> x1_local = x1_que.template DeQue<T>(); | 453 | + LocalTensor<T> x1_local_2 = x1_que.template DeQue<T>(); |
| 454 | - LocalTensor<T> x2_local = x2_que.template DeQue<T>(); | 454 | + LocalTensor<T> x2_local_2 = x2_que.template DeQue<T>(); |
| 455 | if constexpr (is_same<T, float>::value) { | 455 | if constexpr (is_same<T, float>::value) { |
| 456 | if constexpr (IS_X1_NEEDCAST) { | 456 | if constexpr (IS_X1_NEEDCAST) { |
| 457 | - auto y_local_buffer = y_local_fp32.template ReinterpretCast<T_X1>(); | 457 | + auto y_local_buffer_2 = y_local_fp32_2.template ReinterpretCast<T_X1>(); |
| 458 | - Cast(x1_local, y_local_buffer, RoundMode::CAST_NONE, process_count); | 458 | + Cast(x1_local_2, y_local_buffer_2, RoundMode::CAST_NONE, process_count); |
| 459 | PipeBarrier<PIPE_V>(); | 459 | PipeBarrier<PIPE_V>(); |
| 460 | } | 460 | } |
| 461 | if constexpr (IS_X2_NEEDCAST) { | 461 | if constexpr (IS_X2_NEEDCAST) { |
| 462 | - auto y_local_buffer = y_local_fp32.template ReinterpretCast<T_X2>(); | 462 | + auto y_local_buffer_2 = y_local_fp32_2.template ReinterpretCast<T_X2>(); |
| 463 | - Cast(x2_local, y_local_buffer, RoundMode::CAST_NONE, process_count); | 463 | + Cast(x2_local_2, y_local_buffer_2, RoundMode::CAST_NONE, process_count); |
| 464 | PipeBarrier<PIPE_V>(); | 464 | PipeBarrier<PIPE_V>(); |
| 465 | } | 465 | } |
| 466 | - Add(x_local_fp32, x1_local, x2_local, process_count); | 466 | + Add(x_local_fp32, x1_local_2, x2_local_2, process_count); |
| 467 | PipeBarrier<PIPE_V>(); | 467 | PipeBarrier<PIPE_V>(); |
| 468 | } else { | 468 | } else { |
| 469 | - Cast(x_local_fp32, x1_local, RoundMode::CAST_NONE, process_count); | 469 | + Cast(x_local_fp32, x1_local_2, RoundMode::CAST_NONE, process_count); |
| 470 | - Cast(y_local_fp32, x2_local, RoundMode::CAST_NONE, process_count); | 470 | + Cast(y_local_fp32_2, x2_local_2, RoundMode::CAST_NONE, process_count); |
| 471 | PipeBarrier<PIPE_V>(); | 471 | PipeBarrier<PIPE_V>(); |
| 472 | - Add(x_local_fp32, x_local_fp32, y_local_fp32, process_count); | 472 | + Add(x_local_fp32, x_local_fp32, y_local_fp32_2, process_count); |
| 473 | PipeBarrier<PIPE_V>(); | 473 | PipeBarrier<PIPE_V>(); |
| 474 | } | 474 | } |
| 475 | - x1_que.FreeTensor(x1_local); | 475 | + x1_que.FreeTensor(x1_local_2); |
| 476 | - x2_que.FreeTensor(x2_local); | 476 | + x2_que.FreeTensor(x2_local_2); |
| 477 | if constexpr (IS_BIAS_PRESENT) { | 477 | if constexpr (IS_BIAS_PRESENT) { |
| 478 | - LocalTensor<T> x3_in = x1_que.template AllocTensor<T>(); | 478 | + LocalTensor<T> x3_in_2 = x1_que.template AllocTensor<T>(); |
| 479 | - uint32_t gm_offset = row_idx * row_step * num_last_dim + col_offset; | 479 | + uint32_t gm_offset_2 = row_idx * row_step * num_last_dim + col_offset; |
| 480 | - DataCopyEx(x3_in, bias_gm[gm_offset], process_count); | 480 | + DataCopyEx(x3_in_2, bias_gm[gm_offset_2], process_count); |
| 481 | - x1_que.EnQue(x3_in); | 481 | + x1_que.EnQue(x3_in_2); |
| 482 | - auto x3_local = x1_que.template DeQue<T>(); | 482 | + auto x3_local_2 = x1_que.template DeQue<T>(); |
| 483 | if constexpr (is_same<T, float>::value) { | 483 | if constexpr (is_same<T, float>::value) { |
| 484 | - Add(x_local_fp32, x3_local, x_local_fp32, process_count); | 484 | + Add(x_local_fp32, x3_local_2, x_local_fp32, process_count); |
| 485 | PipeBarrier<PIPE_V>(); | 485 | PipeBarrier<PIPE_V>(); |
| 486 | } else { | 486 | } else { |
| 487 | - Cast(y_local_fp32, x3_local, RoundMode::CAST_NONE, process_count); | 487 | + Cast(y_local_fp32_2, x3_local_2, RoundMode::CAST_NONE, process_count); |
| 488 | PipeBarrier<PIPE_V>(); | 488 | PipeBarrier<PIPE_V>(); |
| 489 | - Add(x_local_fp32, y_local_fp32, x_local_fp32, process_count); | 489 | + Add(x_local_fp32, y_local_fp32_2, x_local_fp32, process_count); |
| 490 | PipeBarrier<PIPE_V>(); | 490 | PipeBarrier<PIPE_V>(); |
| 491 | } | 491 | } |
| 492 | - x1_que.FreeTensor(x3_local); | 492 | + x1_que.FreeTensor(x3_local_2); |
| 493 | } else if constexpr (IS_BIAS_BROADCAST) { | 493 | } else if constexpr (IS_BIAS_BROADCAST) { |
| 494 | LocalTensor<T> bias_local_in = bias_que.template AllocTensor<T>(); | 494 | LocalTensor<T> bias_local_in = bias_que.template AllocTensor<T>(); |
| 495 | DataCopyEx(bias_local_in, bias_gm[col_offset], process_count); | 495 | DataCopyEx(bias_local_in, bias_gm[col_offset], process_count); |
| @@ -499,9 +499,9 @@ public: | |||
| 499 | Add(x_local_fp32, bias_local, x_local_fp32, process_count); | 499 | Add(x_local_fp32, bias_local, x_local_fp32, process_count); |
| 500 | PipeBarrier<PIPE_V>(); | 500 | PipeBarrier<PIPE_V>(); |
| 501 | } else { | 501 | } else { |
| 502 | - Cast(y_local_fp32, bias_local, RoundMode::CAST_NONE, process_count); | 502 | + Cast(y_local_fp32_2, bias_local, RoundMode::CAST_NONE, process_count); |
| 503 | PipeBarrier<PIPE_V>(); | 503 | PipeBarrier<PIPE_V>(); |
| 504 | - Add(x_local_fp32, y_local_fp32, x_local_fp32, process_count); | 504 | + Add(x_local_fp32, y_local_fp32_2, x_local_fp32, process_count); |
| 505 | PipeBarrier<PIPE_V>(); | 505 | PipeBarrier<PIPE_V>(); |
| 506 | } | 506 | } |
| 507 | bias_que.FreeTensor(bias_local); | 507 | bias_que.FreeTensor(bias_local); |
| @@ -936,22 +936,22 @@ private: | |||
| 936 | DataCopyEx(x2_local_in, x2_gm[gm_offset], size); | 936 | DataCopyEx(x2_local_in, x2_gm[gm_offset], size); |
| 937 | } | 937 | } |
| 938 | x2_que.EnQue(x2_local_in); | 938 | x2_que.EnQue(x2_local_in); |
| 939 | - auto x2_local = x2_que.template DeQue<T>(); | 939 | + auto x2_local_2 = x2_que.template DeQue<T>(); |
| 940 | if constexpr (is_same<float, T>::value) { | 940 | if constexpr (is_same<float, T>::value) { |
| 941 | if constexpr (IS_X2_NEEDCAST) { | 941 | if constexpr (IS_X2_NEEDCAST) { |
| 942 | - auto y_local_buffer = y_buf_local.template ReinterpretCast<T_X2>(); | 942 | + auto y_local_buffer_2 = y_buf_local.template ReinterpretCast<T_X2>(); |
| 943 | - Cast(x2_local, y_local_buffer, RoundMode::CAST_NONE, size); | 943 | + Cast(x2_local_2, y_local_buffer_2, RoundMode::CAST_NONE, size); |
| 944 | PipeBarrier<PIPE_V>(); | 944 | PipeBarrier<PIPE_V>(); |
| 945 | } | 945 | } |
| 946 | - Add(add_buf_local, x2_local, add_buf_local, size); | 946 | + Add(add_buf_local, x2_local_2, add_buf_local, size); |
| 947 | PipeBarrier<PIPE_V>(); | 947 | PipeBarrier<PIPE_V>(); |
| 948 | } else { | 948 | } else { |
| 949 | - Cast(y_buf_local, x2_local, RoundMode::CAST_NONE, size); | 949 | + Cast(y_buf_local, x2_local_2, RoundMode::CAST_NONE, size); |
| 950 | PipeBarrier<PIPE_V>(); | 950 | PipeBarrier<PIPE_V>(); |
| 951 | Add(add_buf_local, y_buf_local, add_buf_local, size); | 951 | Add(add_buf_local, y_buf_local, add_buf_local, size); |
| 952 | PipeBarrier<PIPE_V>(); | 952 | PipeBarrier<PIPE_V>(); |
| 953 | } | 953 | } |
| 954 | - x2_que.FreeTensor(x2_local); | 954 | + x2_que.FreeTensor(x2_local_2); |
| 955 | } | 955 | } |
| 956 | 956 | ||
| 957 | if constexpr (IS_BIAS_PRESENT || IS_BIAS_BROADCAST) { | 957 | if constexpr (IS_BIAS_PRESENT || IS_BIAS_BROADCAST) { |
| @@ -30,6 +30,11 @@ public: | |||
| 30 | Ppipe = pipe; | 30 | Ppipe = pipe; |
| 31 | } | 31 | } |
| 32 | 32 | ||
| 33 | + __aicore__ inline uint32_t ROUND_UP32(uint32_t x) | ||
| 34 | + { | ||
| 35 | + return (x + ONE_BLK_SIZE - 1) / ONE_BLK_SIZE * ONE_BLK_SIZE; | ||
| 36 | + } | ||
| 37 | + | ||
| 33 | __aicore__ inline uint32_t CEIL_DIV(uint32_t x, uint32_t y) | 38 | __aicore__ inline uint32_t CEIL_DIV(uint32_t x, uint32_t y) |
| 34 | { | 39 | { |
| 35 | if (y > 0) { | 40 | if (y > 0) { |
| @@ -38,11 +43,6 @@ public: | |||
| 38 | return 0; | 43 | return 0; |
| 39 | } | 44 | } |
| 40 | 45 | ||
| 41 | - __aicore__ inline uint32_t ROUND_UP32(uint32_t x) | ||
| 42 | - { | ||
| 43 | - return (x + ONE_BLK_SIZE - 1) / ONE_BLK_SIZE * ONE_BLK_SIZE; | ||
| 44 | - } | ||
| 45 | - | ||
| 46 | __aicore__ inline uint32_t BlockAlign(uint32_t x, uint32_t blockElem) | 46 | __aicore__ inline uint32_t BlockAlign(uint32_t x, uint32_t blockElem) |
| 47 | { | 47 | { |
| 48 | if (blockElem > 0) { | 48 | if (blockElem > 0) { |
| @@ -51,14 +51,14 @@ public: | |||
| 51 | return 0; | 51 | return 0; |
| 52 | } | 52 | } |
| 53 | 53 | ||
| 54 | - __aicore__ inline uint32_t MIN(uint32_t x, uint32_t y) | 54 | + __aicore__ inline uint32_t MIN(uint32_t a, uint32_t b) |
| 55 | { | 55 | { |
| 56 | - return x < y ? x : y; | 56 | + return a < b ? a : b; |
| 57 | } | 57 | } |
| 58 | 58 | ||
| 59 | - __aicore__ inline uint32_t MAX(uint32_t x, uint32_t y) | 59 | + __aicore__ inline uint32_t MAX(uint32_t a, uint32_t b) |
| 60 | { | 60 | { |
| 61 | - return x > y ? x : y; | 61 | + return a > b ? a : b; |
| 62 | } | 62 | } |
| 63 | 63 | ||
| 64 | __aicore__ inline void Init( | 64 | __aicore__ inline void Init( |
| @@ -150,15 +150,15 @@ public: | |||
| 150 | padParams.rightPadding = numLastDimAligned - numLastDim; | 150 | padParams.rightPadding = numLastDimAligned - numLastDim; |
| 151 | } | 151 | } |
| 152 | 152 | ||
| 153 | - LocalTensor<float> betaLocal = betaBuf.template Get<float>(); | 153 | + LocalTensor<float> betaLocalReduce = betaBuf.template Get<float>(); |
| 154 | - LocalTensor<float> gammaLocal = gammaBuf.template Get<float>(); | 154 | + LocalTensor<float> gammaLocalReduce = gammaBuf.template Get<float>(); |
| 155 | 155 | ||
| 156 | if constexpr (is_same<float, T>::value) { | 156 | if constexpr (is_same<float, T>::value) { |
| 157 | - DataCopyEx(betaLocal, betaGm, numLastDim); | 157 | + DataCopyEx(betaLocalReduce, betaGm, numLastDim); |
| 158 | - DataCopyEx(gammaLocal, gammaGm, numLastDim); | 158 | + DataCopyEx(gammaLocalReduce, gammaGm, numLastDim); |
| 159 | } else { | 159 | } else { |
| 160 | - auto betaLocalHalf = betaLocal.ReinterpretCast<T>(); | 160 | + auto betaLocalHalf = betaLocalReduce.ReinterpretCast<T>(); |
| 161 | - auto gammaLocalHalf = gammaLocal.ReinterpretCast<T>(); | 161 | + auto gammaLocalHalf = gammaLocalReduce.ReinterpretCast<T>(); |
| 162 | DataCopyEx(betaLocalHalf[numLastDimAligned], betaGm, numLastDim); | 162 | DataCopyEx(betaLocalHalf[numLastDimAligned], betaGm, numLastDim); |
| 163 | DataCopyEx(gammaLocalHalf[numLastDimAligned], gammaGm, numLastDim); | 163 | DataCopyEx(gammaLocalHalf[numLastDimAligned], gammaGm, numLastDim); |
| 164 | } | 164 | } |
| @@ -180,8 +180,8 @@ public: | |||
| 180 | auto x1x2Local = x1x2Que.template DeQue<T>(); | 180 | auto x1x2Local = x1x2Que.template DeQue<T>(); |
| 181 | 181 | ||
| 182 | if constexpr (!is_same<T, float>::value) { | 182 | if constexpr (!is_same<T, float>::value) { |
| 183 | - Cast(gammaLocal, gammaLocal.ReinterpretCast<T>()[numLastDimAligned], RoundMode::CAST_NONE, numLastDim); | 183 | + Cast(gammaLocalReduce, gammaLocalReduce.ReinterpretCast<T>()[numLastDimAligned], RoundMode::CAST_NONE, numLastDim); |
| 184 | - Cast(betaLocal, betaLocal.ReinterpretCast<T>()[numLastDimAligned], RoundMode::CAST_NONE, numLastDim); | 184 | + Cast(betaLocalReduce, betaLocalReduce.ReinterpretCast<T>()[numLastDimAligned], RoundMode::CAST_NONE, numLastDim); |
| 185 | } | 185 | } |
| 186 | 186 | ||
| 187 | if constexpr (IS_BIAS_BROADCAST) { | 187 | if constexpr (IS_BIAS_BROADCAST) { |
| @@ -191,9 +191,9 @@ public: | |||
| 191 | } | 191 | } |
| 192 | CopyOutAdditionalOutput(0, rowStep); | 192 | CopyOutAdditionalOutput(0, rowStep); |
| 193 | if constexpr (IS_NORMAL_SPECIAL_REDUCE_BIG_N_CASE) { | 193 | if constexpr (IS_NORMAL_SPECIAL_REDUCE_BIG_N_CASE) { |
| 194 | - PrecisionComputeBigN(rowStep, gammaLocal, betaLocal); | 194 | + PrecisionComputeBigN(rowStep, gammaLocalReduce, betaLocalReduce); |
| 195 | } else { | 195 | } else { |
| 196 | - PrecisionCompute(rowStep, gammaLocal, betaLocal, elementCount); | 196 | + PrecisionCompute(rowStep, gammaLocalReduce, betaLocalReduce, elementCount); |
| 197 | } | 197 | } |
| 198 | CopyOut(0, rowStep); | 198 | CopyOut(0, rowStep); |
| 199 | gmOffset += rowStep * numLastDim; | 199 | gmOffset += rowStep * numLastDim; |
| @@ -212,9 +212,9 @@ public: | |||
| 212 | 212 | ||
| 213 | CopyOutAdditionalOutput(rowIdx, rowStep); | 213 | CopyOutAdditionalOutput(rowIdx, rowStep); |
| 214 | if constexpr (IS_NORMAL_SPECIAL_REDUCE_BIG_N_CASE) { | 214 | if constexpr (IS_NORMAL_SPECIAL_REDUCE_BIG_N_CASE) { |
| 215 | - PrecisionComputeBigN(rowStep, gammaLocal, betaLocal); | 215 | + PrecisionComputeBigN(rowStep, gammaLocalReduce, betaLocalReduce); |
| 216 | } else { | 216 | } else { |
| 217 | - PrecisionCompute(rowStep, gammaLocal, betaLocal, elementCount); | 217 | + PrecisionCompute(rowStep, gammaLocalReduce, betaLocalReduce, elementCount); |
| 218 | } | 218 | } |
| 219 | CopyOut(rowIdx, rowStep); | 219 | CopyOut(rowIdx, rowStep); |
| 220 | gmOffset += rowStep * numLastDim; | 220 | gmOffset += rowStep * numLastDim; |
| @@ -237,9 +237,9 @@ public: | |||
| 237 | 237 | ||
| 238 | CopyOutAdditionalOutput(rowIdx, rowTail_); | 238 | CopyOutAdditionalOutput(rowIdx, rowTail_); |
| 239 | if constexpr (IS_NORMAL_SPECIAL_REDUCE_BIG_N_CASE) { | 239 | if constexpr (IS_NORMAL_SPECIAL_REDUCE_BIG_N_CASE) { |
| 240 | - PrecisionComputeBigN(rowTail_, gammaLocal, betaLocal); | 240 | + PrecisionComputeBigN(rowTail_, gammaLocalReduce, betaLocalReduce); |
| 241 | } else { | 241 | } else { |
| 242 | - PrecisionCompute(rowTail_, gammaLocal, betaLocal, elementCount); | 242 | + PrecisionCompute(rowTail_, gammaLocalReduce, betaLocalReduce, elementCount); |
| 243 | } | 243 | } |
| 244 | CopyOut(rowIdx, rowTail_); | 244 | CopyOut(rowIdx, rowTail_); |
| 245 | } | 245 | } |
| @@ -622,7 +622,7 @@ private: | |||
| 622 | PipeBarrier<PIPE_V>(); | 622 | PipeBarrier<PIPE_V>(); |
| 623 | 623 | ||
| 624 | // 8. y = (x - mean) / rstd * beta + gamma | 624 | // 8. y = (x - mean) / rstd * beta + gamma |
| 625 | - LocalTensor<T> yLocal = yQue.template AllocTensor<T>(); | 625 | + LocalTensor<T> yLocalBigN = yQue.template AllocTensor<T>(); |
| 626 | if constexpr (!is_same<T, float>::value) { | 626 | if constexpr (!is_same<T, float>::value) { |
| 627 | Level0MulFp32Short(zLocalFp32, gammaLocal, zLocalFp32, numLastDimAligned, nums, numLastDim); | 627 | Level0MulFp32Short(zLocalFp32, gammaLocal, zLocalFp32, numLastDimAligned, nums, numLastDim); |
| 628 | PipeBarrier<PIPE_V>(); | 628 | PipeBarrier<PIPE_V>(); |
| @@ -630,39 +630,39 @@ private: | |||
| 630 | PipeBarrier<PIPE_V>(); | 630 | PipeBarrier<PIPE_V>(); |
| 631 | 631 | ||
| 632 | if constexpr (is_same<T, half>::value) { | 632 | if constexpr (is_same<T, half>::value) { |
| 633 | - Cast(yLocal, zLocalFp32, RoundMode::CAST_NONE, elementNum); | 633 | + Cast(yLocalBigN, zLocalFp32, RoundMode::CAST_NONE, elementNum); |
| 634 | } else { | 634 | } else { |
| 635 | - Cast(yLocal, zLocalFp32, RoundMode::CAST_RINT, elementNum); | 635 | + Cast(yLocalBigN, zLocalFp32, RoundMode::CAST_RINT, elementNum); |
| 636 | } | 636 | } |
| 637 | PipeBarrier<PIPE_V>(); | 637 | PipeBarrier<PIPE_V>(); |
| 638 | } else { | 638 | } else { |
| 639 | - Level0MulFp32Short(yLocal, gammaLocal, zLocalFp32, numLastDimAligned, nums, numLastDim); | 639 | + Level0MulFp32Short(yLocalBigN, gammaLocal, zLocalFp32, numLastDimAligned, nums, numLastDim); |
| 640 | PipeBarrier<PIPE_V>(); | 640 | PipeBarrier<PIPE_V>(); |
| 641 | - Level0AddFp32Short(yLocal, betaLocal, yLocal, numLastDimAligned, nums, numLastDim); | 641 | + Level0AddFp32Short(yLocalBigN, betaLocal, yLocalBigN, numLastDimAligned, nums, numLastDim); |
| 642 | PipeBarrier<PIPE_V>(); | 642 | PipeBarrier<PIPE_V>(); |
| 643 | } | 643 | } |
| 644 | 644 | ||
| 645 | meanQue.EnQue(meanLocal); | 645 | meanQue.EnQue(meanLocal); |
| 646 | rstdQue.EnQue(rstdLocal); | 646 | rstdQue.EnQue(rstdLocal); |
| 647 | - yQue.EnQue(yLocal); | 647 | + yQue.EnQue(yLocalBigN); |
| 648 | } | 648 | } |
| 649 | 649 | ||
| 650 | 650 | ||
| 651 | __aicore__ inline void CopyOut(int32_t rowIdx, int32_t rowCount) | 651 | __aicore__ inline void CopyOut(int32_t rowIdx, int32_t rowCount) |
| 652 | { | 652 | { |
| 653 | - LocalTensor<T> res = yQue.template DeQue<T>(); | 653 | + LocalTensor<T> resReduce = yQue.template DeQue<T>(); |
| 654 | - uint32_t gmOffset = rowIdx * rowStep * numLastDim; | 654 | + uint32_t gmOffsetReduce = rowIdx * rowStep * numLastDim; |
| 655 | - DataCopyEx(yGm[gmOffset], res, numLastDim, rowCount); | 655 | + DataCopyEx(yGm[gmOffsetReduce], resReduce, numLastDim, rowCount); |
| 656 | - yQue.FreeTensor(res); | 656 | + yQue.FreeTensor(resReduce); |
| 657 | 657 | ||
| 658 | 658 | ||
| 659 | - uint32_t gmOffsetMean = rowIdx * rowStep; | 659 | + uint32_t gmOffsetMeanReduce = rowIdx * rowStep; |
| 660 | - LocalTensor<float> mean = meanQue.template DeQue<float>(); | 660 | + LocalTensor<float> meanReduce = meanQue.template DeQue<float>(); |
| 661 | - LocalTensor<float> rstd = rstdQue.template DeQue<float>(); | 661 | + LocalTensor<float> rstdReduce = rstdQue.template DeQue<float>(); |
| 662 | - DataCopyEx(meanGm[gmOffsetMean], mean, rowCount); | 662 | + DataCopyEx(meanGm[gmOffsetMeanReduce], meanReduce, rowCount); |
| 663 | - DataCopyEx(rstdGm[gmOffsetMean], rstd, rowCount); | 663 | + DataCopyEx(rstdGm[gmOffsetMeanReduce], rstdReduce, rowCount); |
| 664 | - meanQue.FreeTensor(mean); | 664 | + meanQue.FreeTensor(meanReduce); |
| 665 | - rstdQue.FreeTensor(rstd); | 665 | + rstdQue.FreeTensor(rstdReduce); |
| 666 | 666 | ||
| 667 | } | 667 | } |
| 668 | 668 | ||
| @@ -697,8 +697,8 @@ private: | |||
| 697 | GlobalTensor<float> rstdGm; | 697 | GlobalTensor<float> rstdGm; |
| 698 | GlobalTensor<float> workspaceGm; | 698 | GlobalTensor<float> workspaceGm; |
| 699 | uint32_t numCore; | 699 | uint32_t numCore; |
| 700 | - uint32_t numFirstDim; | ||
| 701 | uint32_t numLastDim; | 700 | uint32_t numLastDim; |
| 701 | + uint32_t numFirstDim; | ||
| 702 | uint32_t rowStep; | 702 | uint32_t rowStep; |
| 703 | uint32_t rowWork; | 703 | uint32_t rowWork; |
| 704 | uint32_t gmOffset_; | 704 | uint32_t gmOffset_; |
| @@ -43,16 +43,16 @@ public: | |||
| 43 | return (x + ONE_BLK_SIZE - 1) / ONE_BLK_SIZE * ONE_BLK_SIZE; | 43 | return (x + ONE_BLK_SIZE - 1) / ONE_BLK_SIZE * ONE_BLK_SIZE; |
| 44 | } | 44 | } |
| 45 | 45 | ||
| 46 | - __aicore__ inline uint32_t MIN(uint32_t x, uint32_t y) | ||
| 47 | - { | ||
| 48 | - return x < y ? x : y; | ||
| 49 | - } | ||
| 50 | - | ||
| 51 | __aicore__ inline uint32_t MAX(uint32_t x, uint32_t y) | 46 | __aicore__ inline uint32_t MAX(uint32_t x, uint32_t y) |
| 52 | { | 47 | { |
| 53 | return x > y ? x : y; | 48 | return x > y ? x : y; |
| 54 | } | 49 | } |
| 55 | 50 | ||
| 51 | + __aicore__ inline uint32_t MIN(uint32_t x, uint32_t y) | ||
| 52 | + { | ||
| 53 | + return x < y ? x : y; | ||
| 54 | + } | ||
| 55 | + | ||
| 56 | __aicore__ inline void InitVar( | 56 | __aicore__ inline void InitVar( |
| 57 | uint32_t num_core_, uint32_t num_Last_dim_, uint32_t num_first_dim_, uint32_t nl_first_dim_per_core_, | 57 | uint32_t num_core_, uint32_t num_Last_dim_, uint32_t num_first_dim_, uint32_t nl_first_dim_per_core_, |
| 58 | uint32_t l_first_dim_per_core_, uint32_t first_dim_per_time_, uint32_t last_dim_per_time_, float eps_, | 58 | uint32_t l_first_dim_per_core_, uint32_t first_dim_per_time_, uint32_t last_dim_per_time_, float eps_, |
| @@ -125,16 +125,16 @@ public: | |||
| 125 | } | 125 | } |
| 126 | 126 | ||
| 127 | 127 | ||
| 128 | - Ppipe->InitBuffer(meanQue, BUFFER_NUM, ROUND_UP32(rowStep * sizeof(float))); | ||
| 129 | Ppipe->InitBuffer(rstdQue, BUFFER_NUM, ROUND_UP32(rowStep * sizeof(float))); | 128 | Ppipe->InitBuffer(rstdQue, BUFFER_NUM, ROUND_UP32(rowStep * sizeof(float))); |
| 129 | + Ppipe->InitBuffer(meanQue, BUFFER_NUM, ROUND_UP32(rowStep * sizeof(float))); | ||
| 130 | 130 | ||
| 131 | } | 131 | } |
| 132 | 132 | ||
| 133 | __aicore__ inline void Process() | 133 | __aicore__ inline void Process() |
| 134 | { | 134 | { |
| 135 | + DataCopyPadParams padParams; | ||
| 135 | int32_t rowMoveCnt = CEIL_DIV(rowWork, rowStep); | 136 | int32_t rowMoveCnt = CEIL_DIV(rowWork, rowStep); |
| 136 | 137 | ||
| 137 | - DataCopyPadParams padParams; | ||
| 138 | if (lastDimPad) { | 138 | if (lastDimPad) { |
| 139 | padParams.isPad = true; | 139 | padParams.isPad = true; |
| 140 | padParams.paddingValue = 0; | 140 | padParams.paddingValue = 0; |
| @@ -224,29 +224,29 @@ private: | |||
| 224 | int32_t procId, int32_t rowCount, LocalTensor<T>& biasLocal, LocalTensor<T>& x1x2Local, uint32_t elementCount) | 224 | int32_t procId, int32_t rowCount, LocalTensor<T>& biasLocal, LocalTensor<T>& x1x2Local, uint32_t elementCount) |
| 225 | { | 225 | { |
| 226 | LocalTensor<float> addBufLocal = xBufFp32.Get<float>(); | 226 | LocalTensor<float> addBufLocal = xBufFp32.Get<float>(); |
| 227 | - LocalTensor<float> yBufLocal = yBufFp32.Get<float>(); | 227 | + LocalTensor<float> yBufLocalSpecial = yBufFp32.Get<float>(); |
| 228 | 228 | ||
| 229 | - auto x1Local = x1x2Local[0]; | 229 | + auto x1LocalSpecial = x1x2Local[0]; |
| 230 | - auto x2Local = x1x2Local[elementCount]; | 230 | + auto x2LocalSpecial = x1x2Local[elementCount]; |
| 231 | 231 | ||
| 232 | // Use add as | 232 | // Use add as |
| 233 | if constexpr (is_same<float, T>::value) { | 233 | if constexpr (is_same<float, T>::value) { |
| 234 | - Add(addBufLocal, x2Local, x1Local, elementCount); | 234 | + Add(addBufLocal, x2LocalSpecial, x1LocalSpecial, elementCount); |
| 235 | PipeBarrier<PIPE_V>(); | 235 | PipeBarrier<PIPE_V>(); |
| 236 | for (int i = 0; i < rowCount; i++) { | 236 | for (int i = 0; i < rowCount; i++) { |
| 237 | Add(addBufLocal[i * numLastDimAligned], biasLocal, addBufLocal[i * numLastDimAligned], numLastDim); | 237 | Add(addBufLocal[i * numLastDimAligned], biasLocal, addBufLocal[i * numLastDimAligned], numLastDim); |
| 238 | } | 238 | } |
| 239 | PipeBarrier<PIPE_V>(); | 239 | PipeBarrier<PIPE_V>(); |
| 240 | } else { | 240 | } else { |
| 241 | - Cast(addBufLocal, x1Local, RoundMode::CAST_NONE, elementCount); | 241 | + Cast(addBufLocal, x1LocalSpecial, RoundMode::CAST_NONE, elementCount); |
| 242 | - Cast(yBufLocal, x2Local, RoundMode::CAST_NONE, elementCount); | 242 | + Cast(yBufLocalSpecial, x2LocalSpecial, RoundMode::CAST_NONE, elementCount); |
| 243 | PipeBarrier<PIPE_V>(); | 243 | PipeBarrier<PIPE_V>(); |
| 244 | - Add(yBufLocal, addBufLocal, yBufLocal, elementCount); | 244 | + Add(yBufLocalSpecial, addBufLocal, yBufLocalSpecial, elementCount); |
| 245 | Cast(x1x2Local.template ReinterpretCast<float>(), biasLocal, RoundMode::CAST_NONE, numLastDim); | 245 | Cast(x1x2Local.template ReinterpretCast<float>(), biasLocal, RoundMode::CAST_NONE, numLastDim); |
| 246 | PipeBarrier<PIPE_V>(); | 246 | PipeBarrier<PIPE_V>(); |
| 247 | for (int i = 0; i < rowCount; i++) { | 247 | for (int i = 0; i < rowCount; i++) { |
| 248 | Add(addBufLocal[i * numLastDimAligned], x1x2Local.template ReinterpretCast<float>(), | 248 | Add(addBufLocal[i * numLastDimAligned], x1x2Local.template ReinterpretCast<float>(), |
| 249 | - yBufLocal[i * numLastDimAligned], numLastDim); | 249 | + yBufLocalSpecial[i * numLastDimAligned], numLastDim); |
| 250 | } | 250 | } |
| 251 | PipeBarrier<PIPE_V>(); | 251 | PipeBarrier<PIPE_V>(); |
| 252 | } | 252 | } |
| @@ -346,23 +346,23 @@ private: | |||
| 346 | __aicore__ inline void CopyOutAdditionalOutput(int32_t procId, int32_t rowCount) | 346 | __aicore__ inline void CopyOutAdditionalOutput(int32_t procId, int32_t rowCount) |
| 347 | { | 347 | { |
| 348 | if constexpr (IS_ADDITIONAL_OUTPUT_ENABLE) { | 348 | if constexpr (IS_ADDITIONAL_OUTPUT_ENABLE) { |
| 349 | - LocalTensor<float> addBufLocal = xBufFp32.Get<float>(); | 349 | + LocalTensor<float> addBufLocalSpecial = xBufFp32.Get<float>(); |
| 350 | - uint32_t gmOffset = procId * rowStep * numLastDim; | 350 | + uint32_t gmOffsetSpecial = procId * rowStep * numLastDim; |
| 351 | - auto elementCount = numLastDimAligned * rowCount; | 351 | + auto elementCountSpecial = numLastDimAligned * rowCount; |
| 352 | - auto xLocal = yQue.template AllocTensor<T>(); | 352 | + auto xLocalSpecial = yQue.template AllocTensor<T>(); |
| 353 | if constexpr (is_same<T, float>::value) { | 353 | if constexpr (is_same<T, float>::value) { |
| 354 | - Adds(xLocal, addBufLocal, ZERO, elementCount); | 354 | + Adds(xLocalSpecial, addBufLocalSpecial, ZERO, elementCountSpecial); |
| 355 | } else if constexpr (is_same<T, half>::value) { | 355 | } else if constexpr (is_same<T, half>::value) { |
| 356 | - Cast(xLocal, addBufLocal, RoundMode::CAST_NONE, elementCount); | 356 | + Cast(xLocalSpecial, addBufLocalSpecial, RoundMode::CAST_NONE, elementCountSpecial); |
| 357 | } else { | 357 | } else { |
| 358 | - Cast(xLocal, addBufLocal, RoundMode::CAST_RINT, elementCount); | 358 | + Cast(xLocalSpecial, addBufLocalSpecial, RoundMode::CAST_RINT, elementCountSpecial); |
| 359 | } | 359 | } |
| 360 | PipeBarrier<PIPE_V>(); | 360 | PipeBarrier<PIPE_V>(); |
| 361 | - yQue.template EnQue<T>(xLocal); | 361 | + yQue.template EnQue<T>(xLocalSpecial); |
| 362 | - auto x = yQue.template DeQue<T>(); | 362 | + auto xSpecial = yQue.template DeQue<T>(); |
| 363 | 363 | ||
| 364 | - DataCopyEx(xGm[gmOffset], x, numLastDim, rowCount); | 364 | + DataCopyEx(xGm[gmOffsetSpecial], xSpecial, numLastDim, rowCount); |
| 365 | - yQue.FreeTensor(x); | 365 | + yQue.FreeTensor(xSpecial); |
| 366 | } | 366 | } |
| 367 | } | 367 | } |
| 368 | 368 | ||
| @@ -30,14 +30,14 @@ public: | |||
| 30 | __aicore__ inline AddLayerNormGradDeterminsticCompute(){}; | 30 | __aicore__ inline AddLayerNormGradDeterminsticCompute(){}; |
| 31 | __aicore__ inline void initBuffer(TPipe& pipe,GlobalTensor<float>& pdGammaOutTensorGM,GlobalTensor<float>& pdBetaOutTensorGM,GlobalTensor<float>& workspaceGM,int64_t workspaceNum) { | 31 | __aicore__ inline void initBuffer(TPipe& pipe,GlobalTensor<float>& pdGammaOutTensorGM,GlobalTensor<float>& pdBetaOutTensorGM,GlobalTensor<float>& workspaceGM,int64_t workspaceNum) { |
| 32 | pipe_ = pipe; | 32 | pipe_ = pipe; |
| 33 | - pipe_.InitBuffer(queueGammaIn_, DOUBLE_BUFFER, ROW_TEMPLATE * COL_TEMPLATE * sizeof(float)); | ||
| 34 | pipe_.InitBuffer(queueGammaOut_, DOUBLE_BUFFER, COL_TEMPLATE * sizeof(float)); | 33 | pipe_.InitBuffer(queueGammaOut_, DOUBLE_BUFFER, COL_TEMPLATE * sizeof(float)); |
| 35 | - pipe_.InitBuffer(queueBetaIn_, DOUBLE_BUFFER, ROW_TEMPLATE * COL_TEMPLATE * sizeof(float)); | 34 | + pipe_.InitBuffer(queueGammaIn_, DOUBLE_BUFFER, ROW_TEMPLATE * COL_TEMPLATE * sizeof(float)); |
| 36 | pipe_.InitBuffer(queueBetaOut_, DOUBLE_BUFFER, COL_TEMPLATE * sizeof(float)); | 35 | pipe_.InitBuffer(queueBetaOut_, DOUBLE_BUFFER, COL_TEMPLATE * sizeof(float)); |
| 37 | - pdGammaOutTensorGM_ = pdGammaOutTensorGM; | 36 | + pipe_.InitBuffer(queueBetaIn_, DOUBLE_BUFFER, ROW_TEMPLATE * COL_TEMPLATE * sizeof(float)); |
| 38 | pdBetaOutTensorGM_ = pdBetaOutTensorGM; | 37 | pdBetaOutTensorGM_ = pdBetaOutTensorGM; |
| 39 | - workspaceGM_ = workspaceGM; | 38 | + pdGammaOutTensorGM_ = pdGammaOutTensorGM; |
| 40 | workspaceNum_ = workspaceNum; | 39 | workspaceNum_ = workspaceNum; |
| 40 | + workspaceGM_ = workspaceGM; | ||
| 41 | } | 41 | } |
| 42 | 42 | ||
| 43 | __aicore__ inline void FinalProcessDeterministic(int64_t tcolAlignV, int64_t tblockNum, int64_t tcol) { | 43 | __aicore__ inline void FinalProcessDeterministic(int64_t tcolAlignV, int64_t tblockNum, int64_t tcol) { |
| @@ -48,20 +48,20 @@ public: | |||
| 48 | buffer2_ = queueGammaOut_.AllocTensor<float>(); | 48 | buffer2_ = queueGammaOut_.AllocTensor<float>(); |
| 49 | buffer3_ = queueBetaIn_.AllocTensor<float>(); | 49 | buffer3_ = queueBetaIn_.AllocTensor<float>(); |
| 50 | buffer4_ = queueBetaOut_.AllocTensor<float>(); | 50 | buffer4_ = queueBetaOut_.AllocTensor<float>(); |
| 51 | - int64_t colcycleCount = (colAlignV_ + COL_TEMPLATE - 1) / COL_TEMPLATE; | 51 | + int64_t colCycleCount = (colAlignV_ + COL_TEMPLATE - 1) / COL_TEMPLATE; |
| 52 | - int64_t colcyclePerBlockCount = (colcycleCount + GetBlockNum() - 1) / GetBlockNum(); | 52 | + int64_t colCyclePerBlockCount = (colCycleCount + GetBlockNum() - 1) / GetBlockNum(); |
| 53 | - int64_t rowcycleCount = (row_ + ROW_TEMPLATE - 1) / ROW_TEMPLATE; | 53 | + int64_t rowCycleCount = (row_ + ROW_TEMPLATE - 1) / ROW_TEMPLATE; |
| 54 | int64_t colSize = COL_TEMPLATE; | 54 | int64_t colSize = COL_TEMPLATE; |
| 55 | int64_t rowSize = ROW_TEMPLATE; | 55 | int64_t rowSize = ROW_TEMPLATE; |
| 56 | int64_t taskId = 0; | 56 | int64_t taskId = 0; |
| 57 | - for (int64_t blocktaskId = 0; blocktaskId < colcyclePerBlockCount; blocktaskId++) { | 57 | + for (int64_t blocktaskId = 0; blocktaskId < colCyclePerBlockCount; blocktaskId++) { |
| 58 | taskId = blocktaskId * GetBlockNum() + GetBlockIdx(); | 58 | taskId = blocktaskId * GetBlockNum() + GetBlockIdx(); |
| 59 | - if (taskId < colcycleCount) { | 59 | + if (taskId < colCycleCount) { |
| 60 | - if (taskId == colcycleCount - 1) { | 60 | + if (taskId == colCycleCount - 1) { |
| 61 | colSize = col_ - COL_TEMPLATE * taskId; | 61 | colSize = col_ - COL_TEMPLATE * taskId; |
| 62 | } | 62 | } |
| 63 | - for(int64_t i = 0; i < rowcycleCount; i++) { | 63 | + for(int64_t i = 0; i < rowCycleCount; i++) { |
| 64 | - if (i == rowcycleCount - 1) { | 64 | + if (i == rowCycleCount - 1) { |
| 65 | rowSize = row_ - ROW_TEMPLATE * i; | 65 | rowSize = row_ - ROW_TEMPLATE * i; |
| 66 | } | 66 | } |
| 67 | copyIn(taskId,i,colSize,rowSize); | 67 | copyIn(taskId,i,colSize,rowSize); |
| @@ -73,10 +73,10 @@ public: | |||
| 73 | break; | 73 | break; |
| 74 | } | 74 | } |
| 75 | } | 75 | } |
| 76 | - queueGammaIn_.FreeTensor(buffer1_); | ||
| 77 | queueGammaOut_.FreeTensor(buffer2_); | 76 | queueGammaOut_.FreeTensor(buffer2_); |
| 78 | - queueBetaIn_.FreeTensor(buffer3_); | 77 | + queueGammaIn_.FreeTensor(buffer1_); |
| 79 | queueBetaOut_.FreeTensor(buffer4_); | 78 | queueBetaOut_.FreeTensor(buffer4_); |
| 79 | + queueBetaIn_.FreeTensor(buffer3_); | ||
| 80 | } | 80 | } |
| 81 | 81 | ||
| 82 | __aicore__ inline void copyIn(int64_t colIndex, int64_t rowIndex, int64_t colSize, int64_t rowSize) { | 82 | __aicore__ inline void copyIn(int64_t colIndex, int64_t rowIndex, int64_t colSize, int64_t rowSize) { |
| @@ -171,14 +171,14 @@ public: | |||
| 171 | if (colSizeMod != 0) { | 171 | if (colSizeMod != 0) { |
| 172 | colSizeAlign += FLOAT_ALIGN - colSizeMod; | 172 | colSizeAlign += FLOAT_ALIGN - colSizeMod; |
| 173 | } | 173 | } |
| 174 | - TEventID eventID = GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>(); | 174 | + TEventID eventId = GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>(); |
| 175 | - SetFlag<HardEvent::MTE2_V>(eventID); | 175 | + SetFlag<HardEvent::MTE2_V>(eventId); |
| 176 | - WaitFlag<HardEvent::MTE2_V>(eventID); | 176 | + WaitFlag<HardEvent::MTE2_V>(eventId); |
| 177 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventID); | 177 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventId); |
| 178 | - TEventID eventID1 = GetTPipePtr()->AllocEventID<HardEvent::MTE3_V>(); | 178 | + TEventID eventId1 = GetTPipePtr()->AllocEventID<HardEvent::MTE3_V>(); |
| 179 | - SetFlag<HardEvent::MTE3_V>(eventID1); | 179 | + SetFlag<HardEvent::MTE3_V>(eventId1); |
| 180 | - WaitFlag<HardEvent::MTE3_V>(eventID1); | 180 | + WaitFlag<HardEvent::MTE3_V>(eventId1); |
| 181 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE3_V>(eventID1); | 181 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE3_V>(eventId1); |
| 182 | Duplicate(buffer2_, static_cast<float>(0.0), COL_TEMPLATE); | 182 | Duplicate(buffer2_, static_cast<float>(0.0), COL_TEMPLATE); |
| 183 | Duplicate(buffer4_, static_cast<float>(0.0), COL_TEMPLATE); | 183 | Duplicate(buffer4_, static_cast<float>(0.0), COL_TEMPLATE); |
| 184 | PipeBarrier<PIPE_V>(); | 184 | PipeBarrier<PIPE_V>(); |
| @@ -236,8 +236,8 @@ private: | |||
| 236 | TPipe pipe_; | 236 | TPipe pipe_; |
| 237 | TQue<QuePosition::VECOUT, DOUBLE_BUFFER> queueGammaOut_; | 237 | TQue<QuePosition::VECOUT, DOUBLE_BUFFER> queueGammaOut_; |
| 238 | TQue<QuePosition::VECOUT, DOUBLE_BUFFER> queueBetaOut_; | 238 | TQue<QuePosition::VECOUT, DOUBLE_BUFFER> queueBetaOut_; |
| 239 | - TQue<QuePosition::VECIN, DOUBLE_BUFFER> queueGammaIn_; | ||
| 240 | TQue<QuePosition::VECIN, DOUBLE_BUFFER> queueBetaIn_; | 239 | TQue<QuePosition::VECIN, DOUBLE_BUFFER> queueBetaIn_; |
| 240 | + TQue<QuePosition::VECIN, DOUBLE_BUFFER> queueGammaIn_; | ||
| 241 | LocalTensor<float> buffer1_; | 241 | LocalTensor<float> buffer1_; |
| 242 | LocalTensor<float> buffer2_; | 242 | LocalTensor<float> buffer2_; |
| 243 | LocalTensor<float> buffer3_; | 243 | LocalTensor<float> buffer3_; |
| @@ -557,8 +557,8 @@ private: | |||
| 557 | GlobalTensor<T> x1Gm; | 557 | GlobalTensor<T> x1Gm; |
| 558 | GlobalTensor<T> x2Gm; | 558 | GlobalTensor<T> x2Gm; |
| 559 | GlobalTensor<T> gammaGm; | 559 | GlobalTensor<T> gammaGm; |
| 560 | - GlobalTensor<T> dXGm; | ||
| 561 | GlobalTensor<T> dSumGm; | 560 | GlobalTensor<T> dSumGm; |
| 561 | + GlobalTensor<T> dXGm; | ||
| 562 | GlobalTensor<int32_t> syncGlobal_; | 562 | GlobalTensor<int32_t> syncGlobal_; |
| 563 | GlobalTensor<int32_t> syncGlobal1_; | 563 | GlobalTensor<int32_t> syncGlobal1_; |
| 564 | 564 | ||
| @@ -50,11 +50,11 @@ ge::graphStatus AddLayerNormQuantEmptyTiling::GetAttrs() | |||
| 50 | 50 | ||
| 51 | ge::graphStatus AddLayerNormQuantEmptyTiling::CheckShapeAllPositive(gert::Shape& shape) | 51 | ge::graphStatus AddLayerNormQuantEmptyTiling::CheckShapeAllPositive(gert::Shape& shape) |
| 52 | { | 52 | { |
| 53 | - for (size_t i = 0; i < shape.GetDimNum(); i++) { | 53 | + for (size_t idx = 0; idx < shape.GetDimNum(); idx++) { |
| 54 | OP_CHECK_IF( | 54 | OP_CHECK_IF( |
| 55 | - shape.GetDim(i) < 0, | 55 | + shape.GetDim(idx) < 0, |
| 56 | OP_LOGE( | 56 | OP_LOGE( |
| 57 | - context_->GetNodeName(), "Dim %lu of input should be positive, but actual %ld.", i, shape.GetDim(i)), | 57 | + context_->GetNodeName(), "Dim %lu of input should be positive, but actual %ld.", idx, shape.GetDim(idx)), |
| 58 | return ge::GRAPH_FAILED); | 58 | return ge::GRAPH_FAILED); |
| 59 | } | 59 | } |
| 60 | return ge::GRAPH_SUCCESS; | 60 | return ge::GRAPH_SUCCESS; |
| @@ -86,9 +86,9 @@ static ge::graphStatus CanUseRegbase(gert::TilingContext* context, bool& useRegb | |||
| 86 | auto platformInfo = context->GetPlatformInfo(); | 86 | auto platformInfo = context->GetPlatformInfo(); |
| 87 | if (platformInfo != nullptr) { | 87 | if (platformInfo != nullptr) { |
| 88 | auto ascendcPlatform = platform_ascendc::PlatformAscendC(platformInfo); | 88 | auto ascendcPlatform = platform_ascendc::PlatformAscendC(platformInfo); |
| 89 | - auto npuArch = ascendcPlatform.GetCurNpuArch(); | 89 | + auto npuArchType = ascendcPlatform.GetCurNpuArch(); |
| 90 | useRegbase = (IsRegbaseSocVersion(context) || | 90 | useRegbase = (IsRegbaseSocVersion(context) || |
| 91 | - npuArch == NpuArch::DAV_5102); | 91 | + npuArchType == NpuArch::DAV_5102); |
| 92 | } else { | 92 | } else { |
| 93 | auto compileInfo = reinterpret_cast<const AddLayerNormQuantCompileInfo*>(context->GetCompileInfo()); | 93 | auto compileInfo = reinterpret_cast<const AddLayerNormQuantCompileInfo*>(context->GetCompileInfo()); |
| 94 | OP_CHECK_NULL_WITH_CONTEXT(context, compileInfo); | 94 | OP_CHECK_NULL_WITH_CONTEXT(context, compileInfo); |
| @@ -58,11 +58,11 @@ static ge::graphStatus InferShape4AddRmsNorm(gert::InferShapeContext* context) | |||
| 58 | return GRAPH_FAILED); | 58 | return GRAPH_FAILED); |
| 59 | 59 | ||
| 60 | rstdShape->SetDimNum(xDimNum); | 60 | rstdShape->SetDimNum(xDimNum); |
| 61 | - for (size_t i = 0; i < xDimNum; i++) { | 61 | + for (size_t rmsIdx = 0; rmsIdx < xDimNum; rmsIdx++) { |
| 62 | - if (i < xDimNum - gammaDimNum) { | 62 | + if (rmsIdx < xDimNum - gammaDimNum) { |
| 63 | - rstdShape->SetDim(i, x1Shape->GetDim(i)); | 63 | + rstdShape->SetDim(rmsIdx, x1Shape->GetDim(rmsIdx)); |
| 64 | } else { | 64 | } else { |
| 65 | - rstdShape->SetDim(i, 1); | 65 | + rstdShape->SetDim(rmsIdx, 1); |
| 66 | } | 66 | } |
| 67 | } | 67 | } |
| 68 | 68 | ||
| @@ -36,12 +36,12 @@ constexpr uint32_t MODE_SPLIT_D = 1; | |||
| 36 | constexpr uint32_t MODE_MERGE_N = 2; | 36 | constexpr uint32_t MODE_MERGE_N = 2; |
| 37 | constexpr uint32_t MODE_SINGLE_N = 3; | 37 | constexpr uint32_t MODE_SINGLE_N = 3; |
| 38 | constexpr uint32_t MODE_MULTI_N = 4; | 38 | constexpr uint32_t MODE_MULTI_N = 4; |
| 39 | -constexpr int32_t INPUT_X1_INDEX = 0; | 39 | +constexpr int32_t RMS_INPUT_X1_INDEX = 0; |
| 40 | -constexpr int32_t INPUT_X2_INDEX = 1; | 40 | +constexpr int32_t RMS_INPUT_X2_INDEX = 1; |
| 41 | -constexpr int32_t INPUT_GAMMA_INDEX = 2; | 41 | +constexpr int32_t RMS_INPUT_GAMMA_INDEX = 2; |
| 42 | -constexpr int32_t OUTPUT_Y_INDEX = 0; | 42 | +constexpr int32_t RMS_OUTPUT_Y_INDEX = 0; |
| 43 | -constexpr int32_t OUTPUT_RSTD_INDEX = 1; | 43 | +constexpr int32_t RMS_OUTPUT_RSTD_INDEX = 1; |
| 44 | -constexpr int32_t OUTPUT_X_INDEX = 2; | 44 | +constexpr int32_t RMS_OUTPUT_X_INDEX = 2; |
| 45 | constexpr size_t MAX_DIM_NUM = 8; | 45 | constexpr size_t MAX_DIM_NUM = 8; |
| 46 | constexpr size_t MIN_DIM_X = 1; | 46 | constexpr size_t MIN_DIM_X = 1; |
| 47 | constexpr size_t MIN_DIM_GAMMA = 1; | 47 | constexpr size_t MIN_DIM_GAMMA = 1; |
| @@ -102,12 +102,12 @@ static void SetByDtype(ge::DataType dataType, uint32_t& dtypeKey, uint32_t& data | |||
| 102 | } | 102 | } |
| 103 | static bool CheckNullptr(const gert::TilingContext* context) | 103 | static bool CheckNullptr(const gert::TilingContext* context) |
| 104 | { | 104 | { |
| 105 | - const gert::StorageShape* x1_shape = context->GetInputShape(INPUT_X1_INDEX); | 105 | + const gert::StorageShape* x1_shape = context->GetInputShape(RMS_INPUT_X1_INDEX); |
| 106 | - const gert::StorageShape* x2_shape = context->GetInputShape(INPUT_X2_INDEX); | 106 | + const gert::StorageShape* x2_shape = context->GetInputShape(RMS_INPUT_X2_INDEX); |
| 107 | - const gert::StorageShape* gamma_shape = context->GetInputShape(INPUT_GAMMA_INDEX); | 107 | + const gert::StorageShape* gamma_shape = context->GetInputShape(RMS_INPUT_GAMMA_INDEX); |
| 108 | - const gert::StorageShape* y_shape = context->GetOutputShape(OUTPUT_Y_INDEX); | 108 | + const gert::StorageShape* y_shape = context->GetOutputShape(RMS_OUTPUT_Y_INDEX); |
| 109 | - const gert::StorageShape* rstd_shape = context->GetOutputShape(OUTPUT_RSTD_INDEX); | 109 | + const gert::StorageShape* rstd_shape = context->GetOutputShape(RMS_OUTPUT_RSTD_INDEX); |
| 110 | - const gert::StorageShape* x_shape = context->GetOutputShape(OUTPUT_X_INDEX); | 110 | + const gert::StorageShape* x_shape = context->GetOutputShape(RMS_OUTPUT_X_INDEX); |
| 111 | 111 | ||
| 112 | norm_key = RMS_NORM_KEY; | 112 | norm_key = RMS_NORM_KEY; |
| 113 | if(rstd_shape->GetOriginShape().GetShapeSize() <= 0 && x_shape->GetOriginShape().GetShapeSize() <= 0){ | 113 | if(rstd_shape->GetOriginShape().GetShapeSize() <= 0 && x_shape->GetOriginShape().GetShapeSize() <= 0){ |
| @@ -131,12 +131,12 @@ static bool CheckNullptr(const gert::TilingContext* context) | |||
| 131 | } | 131 | } |
| 132 | static bool CheckInputOutputDim(const gert::TilingContext* context) | 132 | static bool CheckInputOutputDim(const gert::TilingContext* context) |
| 133 | { | 133 | { |
| 134 | - const gert::StorageShape* x1_shape = context->GetInputShape(INPUT_X1_INDEX); | 134 | + const gert::StorageShape* x1_shape = context->GetInputShape(RMS_INPUT_X1_INDEX); |
| 135 | - const gert::StorageShape* x2_shape = context->GetInputShape(INPUT_X2_INDEX); | 135 | + const gert::StorageShape* x2_shape = context->GetInputShape(RMS_INPUT_X2_INDEX); |
| 136 | - const gert::StorageShape* gamma_shape = context->GetInputShape(INPUT_GAMMA_INDEX); | 136 | + const gert::StorageShape* gamma_shape = context->GetInputShape(RMS_INPUT_GAMMA_INDEX); |
| 137 | - const gert::StorageShape* y_shape = context->GetOutputShape(OUTPUT_Y_INDEX); | 137 | + const gert::StorageShape* y_shape = context->GetOutputShape(RMS_OUTPUT_Y_INDEX); |
| 138 | - const gert::StorageShape* rstd_shape = context->GetOutputShape(OUTPUT_RSTD_INDEX); | 138 | + const gert::StorageShape* rstd_shape = context->GetOutputShape(RMS_OUTPUT_RSTD_INDEX); |
| 139 | - const gert::StorageShape* x_shape = context->GetOutputShape(OUTPUT_X_INDEX); | 139 | + const gert::StorageShape* x_shape = context->GetOutputShape(RMS_OUTPUT_X_INDEX); |
| 140 | 140 | ||
| 141 | size_t x1DimNum = x1_shape->GetStorageShape().GetDimNum(); | 141 | size_t x1DimNum = x1_shape->GetStorageShape().GetDimNum(); |
| 142 | size_t x2DimNum = x2_shape->GetStorageShape().GetDimNum(); | 142 | size_t x2DimNum = x2_shape->GetStorageShape().GetDimNum(); |
| @@ -174,12 +174,12 @@ static bool CheckInputOutputDim(const gert::TilingContext* context) | |||
| 174 | static bool CheckInputOutputShape(const gert::TilingContext* context) | 174 | static bool CheckInputOutputShape(const gert::TilingContext* context) |
| 175 | { | 175 | { |
| 176 | OP_CHECK_IF(!CheckInputOutputDim(context), OP_LOGE(context, "Input Dim invalid."), return false); | 176 | OP_CHECK_IF(!CheckInputOutputDim(context), OP_LOGE(context, "Input Dim invalid."), return false); |
| 177 | - const gert::StorageShape* x1_shape = context->GetInputShape(INPUT_X1_INDEX); | 177 | + const gert::StorageShape* x1_shape = context->GetInputShape(RMS_INPUT_X1_INDEX); |
| 178 | - const gert::StorageShape* x2_shape = context->GetInputShape(INPUT_X2_INDEX); | 178 | + const gert::StorageShape* x2_shape = context->GetInputShape(RMS_INPUT_X2_INDEX); |
| 179 | - const gert::StorageShape* gamma_shape = context->GetInputShape(INPUT_GAMMA_INDEX); | 179 | + const gert::StorageShape* gamma_shape = context->GetInputShape(RMS_INPUT_GAMMA_INDEX); |
| 180 | - const gert::StorageShape* y_shape = context->GetOutputShape(OUTPUT_Y_INDEX); | 180 | + const gert::StorageShape* y_shape = context->GetOutputShape(RMS_OUTPUT_Y_INDEX); |
| 181 | - const gert::StorageShape* rstd_shape = context->GetOutputShape(OUTPUT_RSTD_INDEX); | 181 | + const gert::StorageShape* rstd_shape = context->GetOutputShape(RMS_OUTPUT_RSTD_INDEX); |
| 182 | - const gert::StorageShape* x_shape = context->GetOutputShape(OUTPUT_X_INDEX); | 182 | + const gert::StorageShape* x_shape = context->GetOutputShape(RMS_OUTPUT_X_INDEX); |
| 183 | 183 | ||
| 184 | size_t x1DimNum = x1_shape->GetStorageShape().GetDimNum(); | 184 | size_t x1DimNum = x1_shape->GetStorageShape().GetDimNum(); |
| 185 | size_t gammaDimNum = gamma_shape->GetStorageShape().GetDimNum(); | 185 | size_t gammaDimNum = gamma_shape->GetStorageShape().GetDimNum(); |
| @@ -36,26 +36,26 @@ public: | |||
| 36 | this->rowFactor = tiling->row_factor; | 36 | this->rowFactor = tiling->row_factor; |
| 37 | this->ubFactor = tiling->ub_factor; | 37 | this->ubFactor = tiling->ub_factor; |
| 38 | this->epsilon = tiling->epsilon; | 38 | this->epsilon = tiling->epsilon; |
| 39 | - this->avgFactor = (numCol != 0) ? (float)1.0 / numCol : 0; | 39 | + this->avgFactor = (this->numCol != 0) ? (float)1.0 / this->numCol : 0; |
| 40 | 40 | ||
| 41 | blockIdx_ = GetBlockIdx(); | 41 | blockIdx_ = GetBlockIdx(); |
| 42 | if (blockIdx_ < GetBlockNum() - 1) { | 42 | if (blockIdx_ < GetBlockNum() - 1) { |
| 43 | - this->rowWork = blockFactor; | 43 | + this->rowWork = this->blockFactor; |
| 44 | } else if (blockIdx_ == GetBlockNum() - 1) { | 44 | } else if (blockIdx_ == GetBlockNum() - 1) { |
| 45 | - this->rowWork = numRow - (GetBlockNum() - 1) * blockFactor; | 45 | + this->rowWork = this->numRow - (GetBlockNum() - 1) * this->blockFactor; |
| 46 | } | 46 | } |
| 47 | // get start index for current core, core parallel | 47 | // get start index for current core, core parallel |
| 48 | - x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 48 | + x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 49 | - x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 49 | + x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 50 | - gammaGm.SetGlobalBuffer((__gm__ T*)gamma, numCol); | 50 | + gammaGm.SetGlobalBuffer((__gm__ T*)gamma, this->numCol); |
| 51 | - yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 51 | + yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 52 | 52 | ||
| 53 | if constexpr (MODE == ADD_RMS_NORM_MODE) { | 53 | if constexpr (MODE == ADD_RMS_NORM_MODE) { |
| 54 | - rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_ * blockFactor, blockFactor); | 54 | + rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_ * this->blockFactor, this->blockFactor); |
| 55 | - xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 55 | + xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 56 | } | 56 | } |
| 57 | if constexpr (MODE == PRE_RMS_NORM_MODE) { | 57 | if constexpr (MODE == PRE_RMS_NORM_MODE) { |
| 58 | - xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 58 | + xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 59 | } | 59 | } |
| 60 | 60 | ||
| 61 | // pipe alloc memory to queue, the unit is Bytes | 61 | // pipe alloc memory to queue, the unit is Bytes |
| @@ -76,11 +76,11 @@ public: | |||
| 76 | CopyInGamma(); | 76 | CopyInGamma(); |
| 77 | LocalTensor<T> gammaLocal = inQueueGamma.DeQue<T>(); | 77 | LocalTensor<T> gammaLocal = inQueueGamma.DeQue<T>(); |
| 78 | 78 | ||
| 79 | - uint32_t i_o_max = RmsNorm::CeilDiv(rowWork, rowFactor); | 79 | + uint32_t i_o_max = RmsNorm::CeilDiv(this->rowWork, this->rowFactor); |
| 80 | - uint32_t row_tail = rowWork - (i_o_max - 1) * rowFactor; | 80 | + uint32_t row_tail = this->rowWork - (i_o_max - 1) * this->rowFactor; |
| 81 | 81 | ||
| 82 | for (uint32_t i_o = 0; i_o < i_o_max - 1; i_o++) { | 82 | for (uint32_t i_o = 0; i_o < i_o_max - 1; i_o++) { |
| 83 | - SubProcess(i_o, rowFactor, gammaLocal); | 83 | + SubProcess(i_o, this->rowFactor, gammaLocal); |
| 84 | } | 84 | } |
| 85 | SubProcess(i_o_max - 1, row_tail, gammaLocal); | 85 | SubProcess(i_o_max - 1, row_tail, gammaLocal); |
| 86 | inQueueGamma.FreeTensor(gammaLocal); | 86 | inQueueGamma.FreeTensor(gammaLocal); |
| @@ -90,7 +90,7 @@ public: | |||
| 90 | { | 90 | { |
| 91 | LocalTensor<float> rstdLocal = outQueueRstd.AllocTensor<float>(); | 91 | LocalTensor<float> rstdLocal = outQueueRstd.AllocTensor<float>(); |
| 92 | for (uint32_t i_i = 0; i_i < calc_row_num; i_i++) { | 92 | for (uint32_t i_i = 0; i_i < calc_row_num; i_i++) { |
| 93 | - uint32_t gm_bias = (i_o * rowFactor + i_i) * numCol; | 93 | + uint32_t gm_bias = (i_o * this->rowFactor + i_i) * this->numCol; |
| 94 | CopyIn(gm_bias); | 94 | CopyIn(gm_bias); |
| 95 | Compute(i_i, gammaLocal, rstdLocal); | 95 | Compute(i_i, gammaLocal, rstdLocal); |
| 96 | CopyOutY(gm_bias); | 96 | CopyOutY(gm_bias); |
| @@ -207,12 +207,12 @@ private: | |||
| 207 | Mul(sqx, x_fp32, x_fp32, numCol); | 207 | Mul(sqx, x_fp32, x_fp32, numCol); |
| 208 | PipeBarrier<PIPE_V>(); | 208 | PipeBarrier<PIPE_V>(); |
| 209 | 209 | ||
| 210 | - Muls(sqx, sqx, avgFactor, numCol); | 210 | + Muls(sqx, sqx, this->avgFactor, this->numCol); |
| 211 | PipeBarrier<PIPE_V>(); | 211 | PipeBarrier<PIPE_V>(); |
| 212 | - ReduceSumCustom(sqx, sqx, reduce_buf_local, numCol); | 212 | + ReduceSumCustom(sqx, sqx, reduce_buf_local, this->numCol); |
| 213 | PipeBarrier<PIPE_V>(); | 213 | PipeBarrier<PIPE_V>(); |
| 214 | 214 | ||
| 215 | - Adds(sqx, sqx, epsilon, 1); | 215 | + Adds(sqx, sqx, this->epsilon, 1); |
| 216 | PipeBarrier<PIPE_V>(); | 216 | PipeBarrier<PIPE_V>(); |
| 217 | 217 | ||
| 218 | Sqrt(sqx, sqx, 1); | 218 | Sqrt(sqx, sqx, 1); |
| @@ -220,21 +220,21 @@ private: | |||
| 220 | PipeBarrier<PIPE_V>(); | 220 | PipeBarrier<PIPE_V>(); |
| 221 | Div(sqx, reduce_buf_local, sqx, 1); | 221 | Div(sqx, reduce_buf_local, sqx, 1); |
| 222 | PipeBarrier<PIPE_V>(); | 222 | PipeBarrier<PIPE_V>(); |
| 223 | - event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 223 | + event_t event_v_s2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 224 | - SetFlag<HardEvent::V_S>(event_v_s); | 224 | + SetFlag<HardEvent::V_S>(event_v_s2); |
| 225 | - WaitFlag<HardEvent::V_S>(event_v_s); | 225 | + WaitFlag<HardEvent::V_S>(event_v_s2); |
| 226 | - float rstdValue = sqx.GetValue(0); | 226 | + float rstdValue2 = sqx.GetValue(0); |
| 227 | - event_t event_s_v = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | 227 | + event_t event_s_v2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); |
| 228 | - SetFlag<HardEvent::S_V>(event_s_v); | 228 | + SetFlag<HardEvent::S_V>(event_s_v2); |
| 229 | - WaitFlag<HardEvent::S_V>(event_s_v); | 229 | + WaitFlag<HardEvent::S_V>(event_s_v2); |
| 230 | - rstdLocal.SetValue(inner_progress, rstdValue); | 230 | + rstdLocal.SetValue(inner_progress, rstdValue2); |
| 231 | PipeBarrier<PIPE_V>(); | 231 | PipeBarrier<PIPE_V>(); |
| 232 | - Muls(x_fp32, x_fp32, rstdValue, numCol); | 232 | + Muls(x_fp32, x_fp32, rstdValue2, this->numCol); |
| 233 | PipeBarrier<PIPE_V>(); | 233 | PipeBarrier<PIPE_V>(); |
| 234 | LocalTensor<bfloat16_t> yLocal = outQueueY.AllocTensor<bfloat16_t>(); | 234 | LocalTensor<bfloat16_t> yLocal = outQueueY.AllocTensor<bfloat16_t>(); |
| 235 | - Cast(yLocal, x_fp32, RoundMode::CAST_RINT, numCol); | 235 | + Cast(yLocal, x_fp32, RoundMode::CAST_RINT, this->numCol); |
| 236 | PipeBarrier<PIPE_V>(); | 236 | PipeBarrier<PIPE_V>(); |
| 237 | - Cast(x_fp32, yLocal, RoundMode::CAST_NONE, numCol); | 237 | + Cast(x_fp32, yLocal, RoundMode::CAST_NONE, this->numCol); |
| 238 | PipeBarrier<PIPE_V>(); | 238 | PipeBarrier<PIPE_V>(); |
| 239 | Cast(sqx, gammaLocal, RoundMode::CAST_NONE, numCol); // gamma_fp32 reuse sqx | 239 | Cast(sqx, gammaLocal, RoundMode::CAST_NONE, numCol); // gamma_fp32 reuse sqx |
| 240 | PipeBarrier<PIPE_V>(); | 240 | PipeBarrier<PIPE_V>(); |
| @@ -256,16 +256,16 @@ private: | |||
| 256 | LocalTensor<float> sqx = sqxBuf.Get<float>(); | 256 | LocalTensor<float> sqx = sqxBuf.Get<float>(); |
| 257 | LocalTensor<float> reduce_buf_local = reduceFp32Buf.Get<float>(); | 257 | LocalTensor<float> reduce_buf_local = reduceFp32Buf.Get<float>(); |
| 258 | 258 | ||
| 259 | - Mul(sqx, x_fp32, x_fp32, numCol); | 259 | + Mul(sqx, x_fp32, x_fp32, this->numCol); |
| 260 | PipeBarrier<PIPE_V>(); | 260 | PipeBarrier<PIPE_V>(); |
| 261 | 261 | ||
| 262 | - Muls(sqx, sqx, avgFactor, numCol); | 262 | + Muls(sqx, sqx, this->avgFactor, this->numCol); |
| 263 | PipeBarrier<PIPE_V>(); | 263 | PipeBarrier<PIPE_V>(); |
| 264 | 264 | ||
| 265 | - ReduceSumCustom(sqx, sqx, reduce_buf_local, numCol); | 265 | + ReduceSumCustom(sqx, sqx, reduce_buf_local, this->numCol); |
| 266 | PipeBarrier<PIPE_V>(); | 266 | PipeBarrier<PIPE_V>(); |
| 267 | 267 | ||
| 268 | - Adds(sqx, sqx, epsilon, 1); | 268 | + Adds(sqx, sqx, this->epsilon, 1); |
| 269 | PipeBarrier<PIPE_V>(); | 269 | PipeBarrier<PIPE_V>(); |
| 270 | 270 | ||
| 271 | Sqrt(sqx, sqx, 1); | 271 | Sqrt(sqx, sqx, 1); |
| @@ -273,19 +273,19 @@ private: | |||
| 273 | PipeBarrier<PIPE_V>(); | 273 | PipeBarrier<PIPE_V>(); |
| 274 | Div(sqx, reduce_buf_local, sqx, 1); | 274 | Div(sqx, reduce_buf_local, sqx, 1); |
| 275 | PipeBarrier<PIPE_V>(); | 275 | PipeBarrier<PIPE_V>(); |
| 276 | - event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 276 | + event_t event_v_s3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 277 | - SetFlag<HardEvent::V_S>(event_v_s); | 277 | + SetFlag<HardEvent::V_S>(event_v_s3); |
| 278 | - WaitFlag<HardEvent::V_S>(event_v_s); | 278 | + WaitFlag<HardEvent::V_S>(event_v_s3); |
| 279 | - float rstdValue = sqx.GetValue(0); | 279 | + float rstdValue3 = sqx.GetValue(0); |
| 280 | - event_t event_s_v = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | 280 | + event_t event_s_v3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); |
| 281 | - SetFlag<HardEvent::S_V>(event_s_v); | 281 | + SetFlag<HardEvent::S_V>(event_s_v3); |
| 282 | - WaitFlag<HardEvent::S_V>(event_s_v); | 282 | + WaitFlag<HardEvent::S_V>(event_s_v3); |
| 283 | - rstdLocal.SetValue(inner_progress, rstdValue); | 283 | + rstdLocal.SetValue(inner_progress, rstdValue3); |
| 284 | PipeBarrier<PIPE_V>(); | 284 | PipeBarrier<PIPE_V>(); |
| 285 | - Muls(x_fp32, x_fp32, rstdValue, numCol); | 285 | + Muls(x_fp32, x_fp32, rstdValue3, this->numCol); |
| 286 | PipeBarrier<PIPE_V>(); | 286 | PipeBarrier<PIPE_V>(); |
| 287 | LocalTensor<half> yLocal = outQueueY.AllocTensor<half>(); | 287 | LocalTensor<half> yLocal = outQueueY.AllocTensor<half>(); |
| 288 | - Cast(yLocal, x_fp32, RoundMode::CAST_NONE, numCol); | 288 | + Cast(yLocal, x_fp32, RoundMode::CAST_NONE, this->numCol); |
| 289 | 289 | ||
| 290 | event_t event_v_mte = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE2)); | 290 | event_t event_v_mte = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE2)); |
| 291 | SetFlag<HardEvent::V_MTE2>(event_v_mte); | 291 | SetFlag<HardEvent::V_MTE2>(event_v_mte); |
| @@ -308,7 +308,7 @@ private: | |||
| 308 | { | 308 | { |
| 309 | LocalTensor<float> rstdLocal = outQueueRstd.DeQue<float>(); | 309 | LocalTensor<float> rstdLocal = outQueueRstd.DeQue<float>(); |
| 310 | 310 | ||
| 311 | - DataCopyCustom<float>(rstdGm[outer_progress * rowFactor], rstdLocal, num); | 311 | + DataCopyCustom<float>(rstdGm[outer_progress * this->rowFactor], rstdLocal, num); |
| 312 | 312 | ||
| 313 | outQueueRstd.FreeTensor(rstdLocal); | 313 | outQueueRstd.FreeTensor(rstdLocal); |
| 314 | } | 314 | } |
| @@ -334,7 +334,7 @@ private: | |||
| 334 | 334 | ||
| 335 | uint32_t numRow; | 335 | uint32_t numRow; |
| 336 | uint32_t numCol; | 336 | uint32_t numCol; |
| 337 | - uint32_t blockFactor; // number of calculations rows on each core | 337 | + uint32_t blockFactor; |
| 338 | uint32_t rowFactor; | 338 | uint32_t rowFactor; |
| 339 | uint32_t ubFactor; | 339 | uint32_t ubFactor; |
| 340 | float epsilon; | 340 | float epsilon; |
| @@ -23,16 +23,16 @@ extern "C" __global__ __aicore__ void add_rms_norm( | |||
| 23 | GM_ADDR x1, GM_ADDR x2, GM_ADDR gamma, GM_ADDR y, GM_ADDR rstd, GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) | 23 | GM_ADDR x1, GM_ADDR x2, GM_ADDR gamma, GM_ADDR y, GM_ADDR rstd, GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) |
| 24 | { | 24 | { |
| 25 | KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_AIV_ONLY); | 25 | KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_AIV_ONLY); |
| 26 | - TPipe pipe; | 26 | + TPipe aptPipe; |
| 27 | if (TILING_KEY_IS(1000)) { | 27 | if (TILING_KEY_IS(1000)) { |
| 28 | - GET_TILING_DATA_WITH_STRUCT(AddRMSNormRegbaseRFullLoadTilingData, tilingDataIn, tiling); | 28 | + GET_TILING_DATA_WITH_STRUCT(AddRMSNormRegbaseRFullLoadTilingData, aptTilingDataIn, tiling); |
| 29 | - KernelAddRmsNormRegBase<DTYPE_X1> op(&pipe); | 29 | + KernelAddRmsNormRegBase<DTYPE_X1> op(&aptPipe); |
| 30 | - op.Init(x1, x2, gamma, y, rstd, x, &tilingDataIn); | 30 | + op.Init(x1, x2, gamma, y, rstd, x, &aptTilingDataIn); |
| 31 | op.Process(); | 31 | op.Process(); |
| 32 | } else if (TILING_KEY_IS(2000)) { | 32 | } else if (TILING_KEY_IS(2000)) { |
| 33 | - GET_TILING_DATA_WITH_STRUCT(AddRMSNormRegbaseTilingData, tilingDataIn, tiling); | 33 | + GET_TILING_DATA_WITH_STRUCT(AddRMSNormRegbaseTilingData, aptTilingDataIn, tiling); |
| 34 | - KernelAddRmsNormRegBaseSplitD<DTYPE_X1> op(&pipe); | 34 | + KernelAddRmsNormRegBaseSplitD<DTYPE_X1> op(&aptPipe); |
| 35 | - op.Init(x1, x2, gamma, y, rstd, x, &tilingDataIn); | 35 | + op.Init(x1, x2, gamma, y, rstd, x, &aptTilingDataIn); |
| 36 | op.Process(); | 36 | op.Process(); |
| 37 | } | 37 | } |
| 38 | } | 38 | } |
| @@ -52,28 +52,28 @@ __aicore__ inline void ReduceSumFP32ToBlock( | |||
| 52 | int32_t count) | 52 | int32_t count) |
| 53 | { | 53 | { |
| 54 | // count need smaller than 255 repeat | 54 | // count need smaller than 255 repeat |
| 55 | - uint64_t mask = NUM_PER_REP_FP32; | 55 | + uint64_t reduceMask = NUM_PER_REP_FP32; |
| 56 | - int32_t repeatTimes = count / NUM_PER_REP_FP32; | 56 | + int32_t reduceRepeatTimes = count / NUM_PER_REP_FP32; |
| 57 | - int32_t tailCount = count % NUM_PER_REP_FP32; | 57 | + int32_t reduceTailCount = count % NUM_PER_REP_FP32; |
| 58 | - int32_t bodyCount = repeatTimes * NUM_PER_REP_FP32; | 58 | + int32_t reduceBodyCount = reduceRepeatTimes * NUM_PER_REP_FP32; |
| 59 | - BinaryRepeatParams repeatParams; | 59 | + BinaryRepeatParams reduceRepeatParams; |
| 60 | - repeatParams.src0RepStride = ONE_REPEAT_BYTE_SIZE / ONE_BLK_SIZE; | 60 | + reduceRepeatParams.src0RepStride = ONE_REPEAT_BYTE_SIZE / ONE_BLK_SIZE; |
| 61 | - repeatParams.src0BlkStride = 1; | 61 | + reduceRepeatParams.src0BlkStride = 1; |
| 62 | - repeatParams.src1RepStride = 0; | 62 | + reduceRepeatParams.src1RepStride = 0; |
| 63 | - repeatParams.src1BlkStride = 1; | 63 | + reduceRepeatParams.src1BlkStride = 1; |
| 64 | - repeatParams.dstRepStride = 0; | 64 | + reduceRepeatParams.dstRepStride = 0; |
| 65 | - repeatParams.dstBlkStride = 1; | 65 | + reduceRepeatParams.dstBlkStride = 1; |
| 66 | Duplicate(work_local, ZERO, NUM_PER_REP_FP32); | 66 | Duplicate(work_local, ZERO, NUM_PER_REP_FP32); |
| 67 | PipeBarrier<PIPE_V>(); | 67 | PipeBarrier<PIPE_V>(); |
| 68 | - if (likely(repeatTimes > 0)) { | 68 | + if (likely(reduceRepeatTimes > 0)) { |
| 69 | - Add(work_local, src_local, work_local, mask, repeatTimes, repeatParams); | 69 | + Add(work_local, src_local, work_local, reduceMask, reduceRepeatTimes, reduceRepeatParams); |
| 70 | PipeBarrier<PIPE_V>(); | 70 | PipeBarrier<PIPE_V>(); |
| 71 | } | 71 | } |
| 72 | - if (unlikely(tailCount != 0)) { | 72 | + if (unlikely(reduceTailCount != 0)) { |
| 73 | - Add(work_local, src_local[bodyCount], work_local, tailCount, 1, repeatParams); | 73 | + Add(work_local, src_local[reduceBodyCount], work_local, reduceTailCount, 1, reduceRepeatParams); |
| 74 | PipeBarrier<PIPE_V>(); | 74 | PipeBarrier<PIPE_V>(); |
| 75 | } | 75 | } |
| 76 | - BlockReduceSum(dst_local, work_local, 1, mask, 1, 1, DEFAULT_REPEAT_STRIDE); | 76 | + BlockReduceSum(dst_local, work_local, 1, reduceMask, 1, 1, DEFAULT_REPEAT_STRIDE); |
| 77 | PipeBarrier<PIPE_V>(); | 77 | PipeBarrier<PIPE_V>(); |
| 78 | } | 78 | } |
| 79 | 79 | ||
| @@ -81,16 +81,16 @@ __aicore__ inline void BlockReduceSumFP32( | |||
| 81 | const LocalTensor<float>& dst_local, const LocalTensor<float>& src_local, int32_t count) | 81 | const LocalTensor<float>& dst_local, const LocalTensor<float>& src_local, int32_t count) |
| 82 | { | 82 | { |
| 83 | // count need multiple of 8 | 83 | // count need multiple of 8 |
| 84 | - int32_t repeatTimes = count / NUM_PER_REP_FP32; | 84 | + int32_t blockRepeatTimes = count / NUM_PER_REP_FP32; |
| 85 | - int32_t tailCount = count % NUM_PER_REP_FP32; | 85 | + int32_t blockTailCount = count % NUM_PER_REP_FP32; |
| 86 | - int32_t dstAddr = repeatTimes * 8; | 86 | + int32_t dstAddr = blockRepeatTimes * 8; |
| 87 | - int32_t srcAddr = repeatTimes * NUM_PER_REP_FP32; | 87 | + int32_t srcAddr = blockRepeatTimes * NUM_PER_REP_FP32; |
| 88 | - if (likely(repeatTimes > 0)) { | 88 | + if (likely(blockRepeatTimes > 0)) { |
| 89 | - BlockReduceSum(dst_local, src_local, repeatTimes, NUM_PER_REP_FP32, 1, 1, DEFAULT_REPEAT_STRIDE); | 89 | + BlockReduceSum(dst_local, src_local, blockRepeatTimes, NUM_PER_REP_FP32, 1, 1, DEFAULT_REPEAT_STRIDE); |
| 90 | PipeBarrier<PIPE_V>(); | 90 | PipeBarrier<PIPE_V>(); |
| 91 | } | 91 | } |
| 92 | - if (tailCount != 0) { | 92 | + if (blockTailCount != 0) { |
| 93 | - BlockReduceSum(dst_local[dstAddr], src_local[srcAddr], 1, tailCount, 1, 1, DEFAULT_REPEAT_STRIDE); | 93 | + BlockReduceSum(dst_local[dstAddr], src_local[srcAddr], 1, blockTailCount, 1, 1, DEFAULT_REPEAT_STRIDE); |
| 94 | PipeBarrier<PIPE_V>(); | 94 | PipeBarrier<PIPE_V>(); |
| 95 | } | 95 | } |
| 96 | } | 96 | } |
| @@ -99,31 +99,31 @@ template <typename T, typename U, typename R> | |||
| 99 | __aicore__ inline void DataCopyCustom(const U& dstTensor, const R& srcTensor, const uint32_t count) | 99 | __aicore__ inline void DataCopyCustom(const U& dstTensor, const R& srcTensor, const uint32_t count) |
| 100 | { | 100 | { |
| 101 | 101 | ||
| 102 | - DataCopyParams copyParams; | 102 | + DataCopyParams customCopyParams; |
| 103 | - copyParams.blockLen = count * sizeof(T); | 103 | + customCopyParams.blockLen = count * sizeof(T); |
| 104 | - copyParams.blockCount = 1; | 104 | + customCopyParams.blockCount = 1; |
| 105 | if constexpr (is_same<U, AscendC::LocalTensor<T>>::value) { | 105 | if constexpr (is_same<U, AscendC::LocalTensor<T>>::value) { |
| 106 | - DataCopyPadParams padParams; | 106 | + DataCopyPadParams customPadParams; |
| 107 | - DataCopyPad(dstTensor, srcTensor, copyParams, padParams); | 107 | + DataCopyPad(dstTensor, srcTensor, customCopyParams, customPadParams); |
| 108 | } else { | 108 | } else { |
| 109 | - DataCopyPad(dstTensor, srcTensor, copyParams); | 109 | + DataCopyPad(dstTensor, srcTensor, customCopyParams); |
| 110 | } | 110 | } |
| 111 | 111 | ||
| 112 | // only support count greater than 32byte | 112 | // only support count greater than 32byte |
| 113 | - int32_t numPerBlock = ONE_BLK_SIZE / sizeof(T); | 113 | + int32_t customNumPerBlock = ONE_BLK_SIZE / sizeof(T); |
| 114 | - if (count % numPerBlock == 0) { | 114 | + if (count % customNumPerBlock == 0) { |
| 115 | DataCopy(dstTensor, srcTensor, count); | 115 | DataCopy(dstTensor, srcTensor, count); |
| 116 | } else { | 116 | } else { |
| 117 | if constexpr (is_same<U, AscendC::LocalTensor<T>>::value) { | 117 | if constexpr (is_same<U, AscendC::LocalTensor<T>>::value) { |
| 118 | - int32_t num = AlignUp(count, numPerBlock); | 118 | + int32_t customNum = AlignUp(count, customNumPerBlock); |
| 119 | - DataCopy(dstTensor, srcTensor, num); | 119 | + DataCopy(dstTensor, srcTensor, customNum); |
| 120 | } else { | 120 | } else { |
| 121 | - int32_t num = count / numPerBlock * numPerBlock; | 121 | + int32_t customNum = count / customNumPerBlock * customNumPerBlock; |
| 122 | - DataCopy(dstTensor, srcTensor, num); | 122 | + DataCopy(dstTensor, srcTensor, customNum); |
| 123 | SetFlag<HardEvent::MTE3_S>(EVENT_ID0); | 123 | SetFlag<HardEvent::MTE3_S>(EVENT_ID0); |
| 124 | WaitFlag<HardEvent::MTE3_S>(EVENT_ID0); | 124 | WaitFlag<HardEvent::MTE3_S>(EVENT_ID0); |
| 125 | - for (int32_t i = 0; i < numPerBlock; i++) { | 125 | + for (int32_t i = 0; i < customNumPerBlock; i++) { |
| 126 | - T tensorValue = srcTensor.GetValue(count - numPerBlock + i); | 126 | + T tensorValue = srcTensor.GetValue(count - customNumPerBlock + i); |
| 127 | srcTensor.SetValue(i, tensorValue); | 127 | srcTensor.SetValue(i, tensorValue); |
| 128 | } | 128 | } |
| 129 | SetFlag<HardEvent::S_MTE3>(EVENT_ID0); | 129 | SetFlag<HardEvent::S_MTE3>(EVENT_ID0); |
| @@ -63,21 +63,21 @@ public: | |||
| 63 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); | 63 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); |
| 64 | this->InitParams(tiling); | 64 | this->InitParams(tiling); |
| 65 | // get start index for current core, core parallel | 65 | // get start index for current core, core parallel |
| 66 | - x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 66 | + x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 67 | - x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 67 | + x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 68 | - gammaGm.SetGlobalBuffer((__gm__ T*)gamma, numCol); | 68 | + gammaGm.SetGlobalBuffer((__gm__ T*)gamma, this->numCol); |
| 69 | - yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 69 | + yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 70 | if constexpr (MODE == ADD_RMS_NORM_MODE) { | 70 | if constexpr (MODE == ADD_RMS_NORM_MODE) { |
| 71 | - rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_ * blockFactor, blockFactor); | 71 | + rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_ * this->blockFactor, this->blockFactor); |
| 72 | - xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 72 | + xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 73 | } | 73 | } |
| 74 | if constexpr (MODE == PRE_RMS_NORM_MODE) { | 74 | if constexpr (MODE == PRE_RMS_NORM_MODE) { |
| 75 | - xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 75 | + xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 76 | } | 76 | } |
| 77 | 77 | ||
| 78 | // pipe alloc memory to queue, the unit is Bytes | 78 | // pipe alloc memory to queue, the unit is Bytes |
| 79 | - Ppipe->InitBuffer(inQueueX, DOUBLE_BUFFER_NUM, ubFactor * sizeof(T)); | 79 | + Ppipe->InitBuffer(inQueueX, DOUBLE_BUFFER_NUM, this->ubFactor * sizeof(T)); |
| 80 | - Ppipe->InitBuffer(inQueueGamma, BUFFER_NUM, ubFactor * sizeof(T)); | 80 | + Ppipe->InitBuffer(inQueueGamma, BUFFER_NUM, this->ubFactor * sizeof(T)); |
| 81 | Ppipe->InitBuffer(outQueueY, DOUBLE_BUFFER_NUM, ubFactor * sizeof(T)); | 81 | Ppipe->InitBuffer(outQueueY, DOUBLE_BUFFER_NUM, ubFactor * sizeof(T)); |
| 82 | 82 | ||
| 83 | Ppipe->InitBuffer(outQueueRstd, BUFFER_NUM, rowFactor * sizeof(float)); | 83 | Ppipe->InitBuffer(outQueueRstd, BUFFER_NUM, rowFactor * sizeof(float)); |
| @@ -383,7 +383,7 @@ private: | |||
| 383 | uint32_t numRow; | 383 | uint32_t numRow; |
| 384 | uint32_t numCol; | 384 | uint32_t numCol; |
| 385 | uint32_t numColAlign; | 385 | uint32_t numColAlign; |
| 386 | - uint32_t blockFactor; // number of calculations rows on each core | 386 | + uint32_t blockFactor; |
| 387 | uint32_t rowFactor; | 387 | uint32_t rowFactor; |
| 388 | uint32_t ubFactor; | 388 | uint32_t ubFactor; |
| 389 | float epsilon; | 389 | float epsilon; |
| @@ -41,7 +41,7 @@ public: | |||
| 41 | 41 | ||
| 42 | blockIdx_ = GetBlockIdx(); | 42 | blockIdx_ = GetBlockIdx(); |
| 43 | if (blockIdx_ < GetBlockNum() - 1) { | 43 | if (blockIdx_ < GetBlockNum() - 1) { |
| 44 | - this->rowWork = blockFactor; | 44 | + this->rowWork = this->blockFactor; |
| 45 | this->rowLoop = tiling->row_loop; | 45 | this->rowLoop = tiling->row_loop; |
| 46 | this->rowTail = tiling->row_tail; | 46 | this->rowTail = tiling->row_tail; |
| 47 | } else if (blockIdx_ == GetBlockNum() - 1) { | 47 | } else if (blockIdx_ == GetBlockNum() - 1) { |
| @@ -50,10 +50,10 @@ public: | |||
| 50 | this->rowTail = tiling->last_block_row_tail; | 50 | this->rowTail = tiling->last_block_row_tail; |
| 51 | } | 51 | } |
| 52 | // get start index for current core, core parallel | 52 | // get start index for current core, core parallel |
| 53 | - x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 53 | + x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 54 | - x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 54 | + x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 55 | - gammaGm.SetGlobalBuffer((__gm__ T*)gamma, numCol); | 55 | + gammaGm.SetGlobalBuffer((__gm__ T*)gamma, this->numCol); |
| 56 | - yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 56 | + yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 57 | if constexpr (MODE == ADD_RMS_NORM_MODE) { | 57 | if constexpr (MODE == ADD_RMS_NORM_MODE) { |
| 58 | rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_ * blockFactor, blockFactor); | 58 | rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_ * blockFactor, blockFactor); |
| 59 | xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 59 | xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * blockFactor * numCol, rowWork * numCol); |
| @@ -63,33 +63,33 @@ public: | |||
| 63 | } | 63 | } |
| 64 | 64 | ||
| 65 | // pipe alloc memory to queue, the unit is Bytes | 65 | // pipe alloc memory to queue, the unit is Bytes |
| 66 | - Ppipe->InitBuffer(inQueueX, DOUBLE_BUFFER_NUM, ubFactor * sizeof(T)); | 66 | + Ppipe->InitBuffer(inQueueX, DOUBLE_BUFFER_NUM, this->ubFactor * sizeof(T)); |
| 67 | - Ppipe->InitBuffer(inQueueGamma, BUFFER_NUM, numColAlign * sizeof(T)); | 67 | + Ppipe->InitBuffer(inQueueGamma, BUFFER_NUM, this->numColAlign * sizeof(T)); |
| 68 | - Ppipe->InitBuffer(outQueueY, DOUBLE_BUFFER_NUM, ubFactor * sizeof(T)); | 68 | + Ppipe->InitBuffer(outQueueY, DOUBLE_BUFFER_NUM, this->ubFactor * sizeof(T)); |
| 69 | 69 | ||
| 70 | - Ppipe->InitBuffer(outQueueRstd, BUFFER_NUM, rowFactor * NUM_PER_BLK_FP32 * sizeof(float)); | 70 | + Ppipe->InitBuffer(outQueueRstd, BUFFER_NUM, this->rowFactor * NUM_PER_BLK_FP32 * sizeof(float)); |
| 71 | 71 | ||
| 72 | - Ppipe->InitBuffer(rstdBuf, rowFactor * NUM_PER_BLK_FP32 * sizeof(float)); | 72 | + Ppipe->InitBuffer(rstdBuf, this->rowFactor * NUM_PER_BLK_FP32 * sizeof(float)); |
| 73 | 73 | ||
| 74 | if constexpr (is_same<T, half>::value || is_same<T, bfloat16_t>::value) { | 74 | if constexpr (is_same<T, half>::value || is_same<T, bfloat16_t>::value) { |
| 75 | - Ppipe->InitBuffer(xFp32Buf, ubFactor * sizeof(float)); | 75 | + Ppipe->InitBuffer(xFp32Buf, this->ubFactor * sizeof(float)); |
| 76 | } | 76 | } |
| 77 | - Ppipe->InitBuffer(sqxBuf, ubFactor * sizeof(float)); | 77 | + Ppipe->InitBuffer(sqxBuf, this->ubFactor * sizeof(float)); |
| 78 | Ppipe->InitBuffer(reduceFp32Buf, NUM_PER_REP_FP32 * sizeof(float)); | 78 | Ppipe->InitBuffer(reduceFp32Buf, NUM_PER_REP_FP32 * sizeof(float)); |
| 79 | - Ppipe->InitBuffer(offsetBuf, rowFactor * NUM_PER_BLK_FP32 * sizeof(uint32_t)); | 79 | + Ppipe->InitBuffer(offsetBuf, this->rowFactor * NUM_PER_BLK_FP32 * sizeof(uint32_t)); |
| 80 | } | 80 | } |
| 81 | __aicore__ inline void Process() | 81 | __aicore__ inline void Process() |
| 82 | { | 82 | { |
| 83 | CopyInGamma(); | 83 | CopyInGamma(); |
| 84 | LocalTensor<T> gammaLocal = inQueueGamma.DeQue<T>(); | 84 | LocalTensor<T> gammaLocal = inQueueGamma.DeQue<T>(); |
| 85 | LocalTensor<uint32_t> offsetLocal = offsetBuf.Get<uint32_t>(); | 85 | LocalTensor<uint32_t> offsetLocal = offsetBuf.Get<uint32_t>(); |
| 86 | - for (uint32_t i = 0; i < rowFactor; i++) { | 86 | + for (uint32_t i = 0; i < this->rowFactor; i++) { |
| 87 | Duplicate(offsetLocal[i * NUM_PER_BLK_FP32], i * ONE_BLK_SIZE, NUM_PER_BLK_FP32); | 87 | Duplicate(offsetLocal[i * NUM_PER_BLK_FP32], i * ONE_BLK_SIZE, NUM_PER_BLK_FP32); |
| 88 | } | 88 | } |
| 89 | - for (uint32_t i_o = 0; i_o < rowLoop - 1; i_o++) { | 89 | + for (uint32_t i_o = 0; i_o < this->rowLoop - 1; i_o++) { |
| 90 | - SubProcessHalf(i_o, rowFactor, gammaLocal); | 90 | + SubProcessHalf(i_o, this->rowFactor, gammaLocal); |
| 91 | } | 91 | } |
| 92 | - SubProcessHalf(rowLoop - 1, rowTail, gammaLocal); | 92 | + SubProcessHalf(this->rowLoop - 1, this->rowTail, gammaLocal); |
| 93 | inQueueGamma.FreeTensor(gammaLocal); | 93 | inQueueGamma.FreeTensor(gammaLocal); |
| 94 | } | 94 | } |
| 95 | 95 | ||
| @@ -124,16 +124,16 @@ private: | |||
| 124 | __aicore__ inline void CopyInX(uint32_t gm_bias, uint32_t calc_row_num) | 124 | __aicore__ inline void CopyInX(uint32_t gm_bias, uint32_t calc_row_num) |
| 125 | { | 125 | { |
| 126 | LocalTensor<T> x1Local = inQueueX.AllocTensor<T>(); | 126 | LocalTensor<T> x1Local = inQueueX.AllocTensor<T>(); |
| 127 | - DataCopyCustom<T>(x1Local, x1Gm[gm_bias], calc_row_num * numCol); | 127 | + DataCopyCustom<T>(x1Local, x1Gm[gm_bias], calc_row_num * this->numCol); |
| 128 | inQueueX.EnQue(x1Local); | 128 | inQueueX.EnQue(x1Local); |
| 129 | LocalTensor<T> x2Local = inQueueX.AllocTensor<T>(); | 129 | LocalTensor<T> x2Local = inQueueX.AllocTensor<T>(); |
| 130 | - DataCopyCustom<T>(x2Local, x2Gm[gm_bias], calc_row_num * numCol); | 130 | + DataCopyCustom<T>(x2Local, x2Gm[gm_bias], calc_row_num * this->numCol); |
| 131 | inQueueX.EnQue(x2Local); | 131 | inQueueX.EnQue(x2Local); |
| 132 | } | 132 | } |
| 133 | 133 | ||
| 134 | __aicore__ inline LocalTensor<T> ComputeX(uint32_t calc_row_num) | 134 | __aicore__ inline LocalTensor<T> ComputeX(uint32_t calc_row_num) |
| 135 | { | 135 | { |
| 136 | - uint32_t calc_num = calc_row_num * numColAlign; | 136 | + uint32_t calc_num = calc_row_num * this->numColAlign; |
| 137 | LocalTensor<T> x1Local = inQueueX.DeQue<T>(); | 137 | LocalTensor<T> x1Local = inQueueX.DeQue<T>(); |
| 138 | LocalTensor<T> x2Local = inQueueX.DeQue<T>(); | 138 | LocalTensor<T> x2Local = inQueueX.DeQue<T>(); |
| 139 | LocalTensor<T> xLocal = outQueueY.AllocTensor<T>(); | 139 | LocalTensor<T> xLocal = outQueueY.AllocTensor<T>(); |
| @@ -33,22 +33,22 @@ public: | |||
| 33 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); | 33 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); |
| 34 | 34 | ||
| 35 | this->numCol = tiling->num_col; | 35 | this->numCol = tiling->num_col; |
| 36 | - this->blockFactor = 1; // in this case, blockFactor = 1 | 36 | + this->blockFactor = 1; |
| 37 | this->ubFactor = tiling->ub_factor; | 37 | this->ubFactor = tiling->ub_factor; |
| 38 | this->epsilon = tiling->epsilon; | 38 | this->epsilon = tiling->epsilon; |
| 39 | - this->avgFactor = (numCol != 0) ? (float)1.0 / numCol : 0; | 39 | + this->avgFactor = (this->numCol != 0) ? (float)1.0 / this->numCol : 0; |
| 40 | 40 | ||
| 41 | this->rowWork = 1; | 41 | this->rowWork = 1; |
| 42 | blockIdx_ = GetBlockIdx(); | 42 | blockIdx_ = GetBlockIdx(); |
| 43 | // get start index for current core, core parallel | 43 | // get start index for current core, core parallel |
| 44 | - x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * numCol, numCol); | 44 | + x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * this->numCol, this->numCol); |
| 45 | - x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * numCol, numCol); | 45 | + x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * this->numCol, this->numCol); |
| 46 | - gammaGm.SetGlobalBuffer((__gm__ T*)gamma, numCol); | 46 | + gammaGm.SetGlobalBuffer((__gm__ T*)gamma, this->numCol); |
| 47 | - yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * numCol, numCol); | 47 | + yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * this->numCol, this->numCol); |
| 48 | 48 | ||
| 49 | if constexpr (MODE == ADD_RMS_NORM_MODE) { | 49 | if constexpr (MODE == ADD_RMS_NORM_MODE) { |
| 50 | rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_, 1); | 50 | rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_, 1); |
| 51 | - xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * numCol, numCol); | 51 | + xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * this->numCol, this->numCol); |
| 52 | } | 52 | } |
| 53 | if constexpr (MODE == PRE_RMS_NORM_MODE) { | 53 | if constexpr (MODE == PRE_RMS_NORM_MODE) { |
| 54 | xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * numCol, numCol); | 54 | xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * numCol, numCol); |
| @@ -108,15 +108,15 @@ private: | |||
| 108 | event_t eventMTE3V = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); | 108 | event_t eventMTE3V = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); |
| 109 | SetFlag<HardEvent::MTE3_V>(eventMTE3V); | 109 | SetFlag<HardEvent::MTE3_V>(eventMTE3V); |
| 110 | 110 | ||
| 111 | - Cast(xFp32Local, x1Local, RoundMode::CAST_NONE, numCol); | 111 | + Cast(xFp32Local, x1Local, RoundMode::CAST_NONE, this->numCol); |
| 112 | PipeBarrier<PIPE_V>(); | 112 | PipeBarrier<PIPE_V>(); |
| 113 | - Mul(sqxLocal, xFp32Local, xFp32Local, numCol); | 113 | + Mul(sqxLocal, xFp32Local, xFp32Local, this->numCol); |
| 114 | PipeBarrier<PIPE_V>(); | 114 | PipeBarrier<PIPE_V>(); |
| 115 | - Muls(sqxLocal, sqxLocal, avgFactor, numCol); | 115 | + Muls(sqxLocal, sqxLocal, this->avgFactor, this->numCol); |
| 116 | PipeBarrier<PIPE_V>(); | 116 | PipeBarrier<PIPE_V>(); |
| 117 | - ReduceSumCustom(sqxLocal, sqxLocal, tmpLocal, numCol); | 117 | + ReduceSumCustom(sqxLocal, sqxLocal, tmpLocal, this->numCol); |
| 118 | PipeBarrier<PIPE_V>(); | 118 | PipeBarrier<PIPE_V>(); |
| 119 | - Adds(sqxLocal, sqxLocal, epsilon, 1); | 119 | + Adds(sqxLocal, sqxLocal, this->epsilon, 1); |
| 120 | PipeBarrier<PIPE_V>(); | 120 | PipeBarrier<PIPE_V>(); |
| 121 | Sqrt(sqxLocal, sqxLocal, 1); | 121 | Sqrt(sqxLocal, sqxLocal, 1); |
| 122 | Duplicate(tmpLocal, ONE, 1); | 122 | Duplicate(tmpLocal, ONE, 1); |
| @@ -132,24 +132,24 @@ private: | |||
| 132 | DataCopyCustom<float>(rstdGm, sqxLocal, 1); | 132 | DataCopyCustom<float>(rstdGm, sqxLocal, 1); |
| 133 | } | 133 | } |
| 134 | 134 | ||
| 135 | - event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 135 | + event_t eventVS_FP32 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 136 | - SetFlag<HardEvent::V_S>(eventVS); | 136 | + SetFlag<HardEvent::V_S>(eventVS_FP32); |
| 137 | - WaitFlag<HardEvent::V_S>(eventVS); | 137 | + WaitFlag<HardEvent::V_S>(eventVS_FP32); |
| 138 | - float rstdValue = sqxLocal.GetValue(0); | 138 | + float rstdValueFp32 = sqxLocal.GetValue(0); |
| 139 | - event_t eventSV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | 139 | + event_t eventSV_FP32 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); |
| 140 | - SetFlag<HardEvent::S_V>(eventSV); | 140 | + SetFlag<HardEvent::S_V>(eventSV_FP32); |
| 141 | - WaitFlag<HardEvent::S_V>(eventSV); | 141 | + WaitFlag<HardEvent::S_V>(eventSV_FP32); |
| 142 | 142 | ||
| 143 | - Muls(xFp32Local, xFp32Local, rstdValue, numCol); | 143 | + Muls(xFp32Local, xFp32Local, rstdValueFp32, this->numCol); |
| 144 | PipeBarrier<PIPE_V>(); | 144 | PipeBarrier<PIPE_V>(); |
| 145 | WaitFlag<HardEvent::MTE3_V>(eventMTE3V); | 145 | WaitFlag<HardEvent::MTE3_V>(eventMTE3V); |
| 146 | - Cast(x1Local, xFp32Local, RoundMode::CAST_NONE, numCol); | 146 | + Cast(x1Local, xFp32Local, RoundMode::CAST_NONE, this->numCol); |
| 147 | PipeBarrier<PIPE_V>(); | 147 | PipeBarrier<PIPE_V>(); |
| 148 | WaitFlag<HardEvent::MTE2_V>(eventMTE2V2); | 148 | WaitFlag<HardEvent::MTE2_V>(eventMTE2V2); |
| 149 | - Mul(x1Local, x1Local, x2Local, numCol); | 149 | + Mul(x1Local, x1Local, x2Local, this->numCol); |
| 150 | SetFlag<HardEvent::V_MTE3>(eventVMTE3); | 150 | SetFlag<HardEvent::V_MTE3>(eventVMTE3); |
| 151 | WaitFlag<HardEvent::V_MTE3>(eventVMTE3); | 151 | WaitFlag<HardEvent::V_MTE3>(eventVMTE3); |
| 152 | - DataCopyCustom<T>(yGm, x1Local, numCol); | 152 | + DataCopyCustom<T>(yGm, x1Local, this->numCol); |
| 153 | } | 153 | } |
| 154 | 154 | ||
| 155 | __aicore__ inline void ProcessFp32() | 155 | __aicore__ inline void ProcessFp32() |
| @@ -189,11 +189,11 @@ private: | |||
| 189 | 189 | ||
| 190 | Mul(sqxLocal, x1Local, x1Local, numCol); | 190 | Mul(sqxLocal, x1Local, x1Local, numCol); |
| 191 | PipeBarrier<PIPE_V>(); | 191 | PipeBarrier<PIPE_V>(); |
| 192 | - Muls(sqxLocal, sqxLocal, avgFactor, numCol); | 192 | + Muls(sqxLocal, sqxLocal, this->avgFactor, this->numCol); |
| 193 | PipeBarrier<PIPE_V>(); | 193 | PipeBarrier<PIPE_V>(); |
| 194 | - ReduceSumCustom(sqxLocal, sqxLocal, tmpLocal, numCol); | 194 | + ReduceSumCustom(sqxLocal, sqxLocal, tmpLocal, this->numCol); |
| 195 | PipeBarrier<PIPE_V>(); | 195 | PipeBarrier<PIPE_V>(); |
| 196 | - Adds(sqxLocal, sqxLocal, epsilon, 1); | 196 | + Adds(sqxLocal, sqxLocal, this->epsilon, 1); |
| 197 | PipeBarrier<PIPE_V>(); | 197 | PipeBarrier<PIPE_V>(); |
| 198 | Sqrt(sqxLocal, sqxLocal, 1); | 198 | Sqrt(sqxLocal, sqxLocal, 1); |
| 199 | Duplicate(tmpLocal, ONE, 1); | 199 | Duplicate(tmpLocal, ONE, 1); |
| @@ -207,13 +207,13 @@ private: | |||
| 207 | WaitFlag<HardEvent::V_MTE3>(eventVMTE3); | 207 | WaitFlag<HardEvent::V_MTE3>(eventVMTE3); |
| 208 | DataCopyCustom<float>(rstdGm, sqxLocal, 1); | 208 | DataCopyCustom<float>(rstdGm, sqxLocal, 1); |
| 209 | 209 | ||
| 210 | - event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 210 | + event_t eventVS_FP16 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 211 | - SetFlag<HardEvent::V_S>(eventVS); | 211 | + SetFlag<HardEvent::V_S>(eventVS_FP16); |
| 212 | - WaitFlag<HardEvent::V_S>(eventVS); | 212 | + WaitFlag<HardEvent::V_S>(eventVS_FP16); |
| 213 | float rstdValue = sqxLocal.GetValue(0); | 213 | float rstdValue = sqxLocal.GetValue(0); |
| 214 | - event_t eventSV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | 214 | + event_t eventSV_FP16 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); |
| 215 | - SetFlag<HardEvent::S_V>(eventSV); | 215 | + SetFlag<HardEvent::S_V>(eventSV_FP16); |
| 216 | - WaitFlag<HardEvent::S_V>(eventSV); | 216 | + WaitFlag<HardEvent::S_V>(eventSV_FP16); |
| 217 | WaitFlag<HardEvent::MTE3_V>(eventMTE3V); | 217 | WaitFlag<HardEvent::MTE3_V>(eventMTE3V); |
| 218 | Muls(x1Local, x1Local, rstdValue, numCol); | 218 | Muls(x1Local, x1Local, rstdValue, numCol); |
| 219 | PipeBarrier<PIPE_V>(); | 219 | PipeBarrier<PIPE_V>(); |
| @@ -36,23 +36,23 @@ public: | |||
| 36 | this->rowFactor = tiling->row_factor; | 36 | this->rowFactor = tiling->row_factor; |
| 37 | this->ubFactor = tiling->ub_factor; | 37 | this->ubFactor = tiling->ub_factor; |
| 38 | this->epsilon = tiling->epsilon; | 38 | this->epsilon = tiling->epsilon; |
| 39 | - this->avgFactor = (numCol != 0) ? (float)1.0 / numCol : 0; | 39 | + this->avgFactor = (this->numCol != 0) ? (float)1.0 / this->numCol : 0; |
| 40 | 40 | ||
| 41 | blockIdx_ = GetBlockIdx(); | 41 | blockIdx_ = GetBlockIdx(); |
| 42 | if (blockIdx_ < GetBlockNum() - 1) { | 42 | if (blockIdx_ < GetBlockNum() - 1) { |
| 43 | - this->rowWork = blockFactor; | 43 | + this->rowWork = this->blockFactor; |
| 44 | } else if (blockIdx_ == GetBlockNum() - 1) { | 44 | } else if (blockIdx_ == GetBlockNum() - 1) { |
| 45 | - this->rowWork = numRow - (GetBlockNum() - 1) * blockFactor; | 45 | + this->rowWork = this->numRow - (GetBlockNum() - 1) * this->blockFactor; |
| 46 | } else { | 46 | } else { |
| 47 | } | 47 | } |
| 48 | // get start index for current core, core parallel | 48 | // get start index for current core, core parallel |
| 49 | - x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 49 | + x1Gm.SetGlobalBuffer((__gm__ T*)x1 + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 50 | - x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 50 | + x2Gm.SetGlobalBuffer((__gm__ T*)x2 + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 51 | - gammaGm.SetGlobalBuffer((__gm__ T*)gamma, numCol); | 51 | + gammaGm.SetGlobalBuffer((__gm__ T*)gamma, this->numCol); |
| 52 | - yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 52 | + yGm.SetGlobalBuffer((__gm__ T*)y + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 53 | if constexpr (MODE == ADD_RMS_NORM_MODE) { | 53 | if constexpr (MODE == ADD_RMS_NORM_MODE) { |
| 54 | - rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_ * blockFactor, blockFactor); | 54 | + rstdGm.SetGlobalBuffer((__gm__ float*)rstd + blockIdx_ * this->blockFactor, this->blockFactor); |
| 55 | - xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 55 | + xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * this->blockFactor * this->numCol, this->rowWork * this->numCol); |
| 56 | } | 56 | } |
| 57 | if constexpr (MODE == PRE_RMS_NORM_MODE) { | 57 | if constexpr (MODE == PRE_RMS_NORM_MODE) { |
| 58 | xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * blockFactor * numCol, rowWork * numCol); | 58 | xGm.SetGlobalBuffer((__gm__ T*)x + blockIdx_ * blockFactor * numCol, rowWork * numCol); |
| @@ -118,51 +118,51 @@ public: | |||
| 118 | private: | 118 | private: |
| 119 | __aicore__ inline void CopyInAndAdd(uint32_t i_idx, uint32_t j_idx, uint32_t num) | 119 | __aicore__ inline void CopyInAndAdd(uint32_t i_idx, uint32_t j_idx, uint32_t num) |
| 120 | { | 120 | { |
| 121 | - LocalTensor<T> x1x2_in = inQueueX.AllocTensor<T>(); | 121 | + LocalTensor<T> splitX1X2In = inQueueX.AllocTensor<T>(); |
| 122 | - LocalTensor<T> x1_in = x1x2_in[0]; | 122 | + LocalTensor<T> splitX1In = splitX1X2In[0]; |
| 123 | - LocalTensor<T> x2_in = x1x2_in[ubFactor]; | 123 | + LocalTensor<T> splitX2In = splitX1X2In[this->ubFactor]; |
| 124 | - DataCopyCustom<T>(x1_in, x1Gm[i_idx * numCol + j_idx * ubFactor], num); | 124 | + DataCopyCustom<T>(splitX1In, x1Gm[i_idx * this->numCol + j_idx * this->ubFactor], num); |
| 125 | - DataCopyCustom<T>(x2_in, x2Gm[i_idx * numCol + j_idx * ubFactor], num); | 125 | + DataCopyCustom<T>(splitX2In, x2Gm[i_idx * this->numCol + j_idx * this->ubFactor], num); |
| 126 | - inQueueX.EnQue(x1x2_in); | 126 | + inQueueX.EnQue(splitX1X2In); |
| 127 | - LocalTensor<T> x1x2Local = inQueueX.DeQue<T>(); | 127 | + LocalTensor<T> splitX1X2Local = inQueueX.DeQue<T>(); |
| 128 | 128 | ||
| 129 | - auto x1Local = x1x2Local[0]; | 129 | + auto splitX1Local = splitX1X2Local[0]; |
| 130 | - auto x2Local = x1x2Local[ubFactor]; | 130 | + auto splitX2Local = splitX1X2Local[this->ubFactor]; |
| 131 | 131 | ||
| 132 | - LocalTensor<T> xLocal = outQueueY.AllocTensor<T>(); | 132 | + LocalTensor<T> splitXLocal = outQueueY.AllocTensor<T>(); |
| 133 | 133 | ||
| 134 | if constexpr (is_same<T, half>::value) { | 134 | if constexpr (is_same<T, half>::value) { |
| 135 | - LocalTensor<float> x1_fp32 = xFp32Buf.Get<float>(); | 135 | + LocalTensor<float> splitX1Fp32 = xFp32Buf.Get<float>(); |
| 136 | 136 | ||
| 137 | - Add(xLocal, x1Local, x2Local, num); | 137 | + Add(splitXLocal, splitX1Local, splitX2Local, num); |
| 138 | PipeBarrier<PIPE_V>(); | 138 | PipeBarrier<PIPE_V>(); |
| 139 | - Cast(x1_fp32, xLocal, RoundMode::CAST_NONE, num); | 139 | + Cast(splitX1Fp32, splitXLocal, RoundMode::CAST_NONE, num); |
| 140 | PipeBarrier<PIPE_V>(); | 140 | PipeBarrier<PIPE_V>(); |
| 141 | // x1+x2 saved in x1_fp32 | 141 | // x1+x2 saved in x1_fp32 |
| 142 | } else if constexpr (is_same<T, bfloat16_t>::value) { | 142 | } else if constexpr (is_same<T, bfloat16_t>::value) { |
| 143 | LocalTensor<float> x1_fp32 = xFp32Buf.Get<float>(); | 143 | LocalTensor<float> x1_fp32 = xFp32Buf.Get<float>(); |
| 144 | - LocalTensor<float> x2_fp32 = x1x2Local.template ReinterpretCast<float>(); | 144 | + LocalTensor<float> x2_fp32 = splitX1X2Local.template ReinterpretCast<float>(); |
| 145 | 145 | ||
| 146 | - Cast(x1_fp32, x1Local, RoundMode::CAST_NONE, num); | 146 | + Cast(x1_fp32, splitX1Local, RoundMode::CAST_NONE, num); |
| 147 | PipeBarrier<PIPE_V>(); | 147 | PipeBarrier<PIPE_V>(); |
| 148 | - Cast(x2_fp32, x2Local, RoundMode::CAST_NONE, num); | 148 | + Cast(x2_fp32, splitX2Local, RoundMode::CAST_NONE, num); |
| 149 | PipeBarrier<PIPE_V>(); | 149 | PipeBarrier<PIPE_V>(); |
| 150 | 150 | ||
| 151 | Add(x1_fp32, x1_fp32, x2_fp32, num); | 151 | Add(x1_fp32, x1_fp32, x2_fp32, num); |
| 152 | PipeBarrier<PIPE_V>(); | 152 | PipeBarrier<PIPE_V>(); |
| 153 | - Cast(xLocal, x1_fp32, RoundMode::CAST_RINT, num); | 153 | + Cast(splitXLocal, x1_fp32, RoundMode::CAST_RINT, num); |
| 154 | PipeBarrier<PIPE_V>(); | 154 | PipeBarrier<PIPE_V>(); |
| 155 | // x1+x2 saved in x1_fp32 | 155 | // x1+x2 saved in x1_fp32 |
| 156 | } else { | 156 | } else { |
| 157 | - Add(x1Local, x1Local, x2Local, num); | 157 | + Add(splitX1Local, splitX1Local, splitX2Local, num); |
| 158 | PipeBarrier<PIPE_V>(); | 158 | PipeBarrier<PIPE_V>(); |
| 159 | - Adds(xLocal, x1Local, (float)0.0, num); | 159 | + Adds(splitXLocal, splitX1Local, (float)0.0, num); |
| 160 | // x1+x2 saved in inQueueX | 160 | // x1+x2 saved in inQueueX |
| 161 | } | 161 | } |
| 162 | - inQueueX.FreeTensor(x1x2Local); | 162 | + inQueueX.FreeTensor(splitX1X2Local); |
| 163 | 163 | ||
| 164 | // copy out to workspace && x_out | 164 | // copy out to workspace && x_out |
| 165 | - outQueueY.EnQue(xLocal); | 165 | + outQueueY.EnQue(splitXLocal); |
| 166 | auto x_out = outQueueY.DeQue<T>(); | 166 | auto x_out = outQueueY.DeQue<T>(); |
| 167 | if constexpr (MODE == ADD_RMS_NORM_MODE || MODE == PRE_RMS_NORM_MODE) { | 167 | if constexpr (MODE == ADD_RMS_NORM_MODE || MODE == PRE_RMS_NORM_MODE) { |
| 168 | DataCopyCustom<T>(xGm[i_idx * numCol + j_idx * ubFactor], x_out, num); | 168 | DataCopyCustom<T>(xGm[i_idx * numCol + j_idx * ubFactor], x_out, num); |
| @@ -209,13 +209,13 @@ private: | |||
| 209 | 209 | ||
| 210 | __aicore__ inline void ComputeRstd(LocalTensor<float> rstdLocal, uint32_t num) | 210 | __aicore__ inline void ComputeRstd(LocalTensor<float> rstdLocal, uint32_t num) |
| 211 | { | 211 | { |
| 212 | - LocalTensor<float> reduce_buf_local = reduceFp32Buf.Get<float>(); | 212 | + LocalTensor<float> splitReduceBufLocal = reduceFp32Buf.Get<float>(); |
| 213 | - Adds(rstdLocal, rstdLocal, epsilon, num); | 213 | + Adds(rstdLocal, rstdLocal, this->epsilon, num); |
| 214 | PipeBarrier<PIPE_V>(); | 214 | PipeBarrier<PIPE_V>(); |
| 215 | Sqrt(rstdLocal, rstdLocal, num); | 215 | Sqrt(rstdLocal, rstdLocal, num); |
| 216 | - Duplicate(reduce_buf_local, ONE, num); | 216 | + Duplicate(splitReduceBufLocal, ONE, num); |
| 217 | PipeBarrier<PIPE_V>(); | 217 | PipeBarrier<PIPE_V>(); |
| 218 | - Div(rstdLocal, reduce_buf_local, rstdLocal, num); | 218 | + Div(rstdLocal, splitReduceBufLocal, rstdLocal, num); |
| 219 | PipeBarrier<PIPE_V>(); | 219 | PipeBarrier<PIPE_V>(); |
| 220 | } | 220 | } |
| 221 | 221 | ||
| @@ -223,13 +223,13 @@ private: | |||
| 223 | uint32_t i_o_idx, uint32_t calc_row_num, uint32_t j_idx, LocalTensor<float>& rstdLocal, uint32_t num) | 223 | uint32_t i_o_idx, uint32_t calc_row_num, uint32_t j_idx, LocalTensor<float>& rstdLocal, uint32_t num) |
| 224 | { | 224 | { |
| 225 | CopyInGamma(j_idx, num); | 225 | CopyInGamma(j_idx, num); |
| 226 | - LocalTensor<T> gammaLocal = inQueueGamma.DeQue<T>(); | 226 | + LocalTensor<T> splitGammaLocal = inQueueGamma.DeQue<T>(); |
| 227 | for (uint32_t i_i = 0; i_i < calc_row_num; i_i++) { | 227 | for (uint32_t i_i = 0; i_i < calc_row_num; i_i++) { |
| 228 | CopyInX(i_o_idx * rowFactor + i_i, j_idx, num); | 228 | CopyInX(i_o_idx * rowFactor + i_i, j_idx, num); |
| 229 | - ComputeY(i_i, gammaLocal, rstdLocal, num); | 229 | + ComputeY(i_i, splitGammaLocal, rstdLocal, num); |
| 230 | CopyOutY(i_o_idx * rowFactor + i_i, j_idx, num); | 230 | CopyOutY(i_o_idx * rowFactor + i_i, j_idx, num); |
| 231 | } | 231 | } |
| 232 | - inQueueGamma.FreeTensor(gammaLocal); | 232 | + inQueueGamma.FreeTensor(splitGammaLocal); |
| 233 | } | 233 | } |
| 234 | 234 | ||
| 235 | __aicore__ inline void CopyInGamma(uint32_t j_idx, uint32_t num) | 235 | __aicore__ inline void CopyInGamma(uint32_t j_idx, uint32_t num) |
| @@ -250,35 +250,35 @@ private: | |||
| 250 | } | 250 | } |
| 251 | inQueueX.EnQue<T>(xLocal); | 251 | inQueueX.EnQue<T>(xLocal); |
| 252 | if constexpr (is_same<T, half>::value || is_same<T, bfloat16_t>::value) { | 252 | if constexpr (is_same<T, half>::value || is_same<T, bfloat16_t>::value) { |
| 253 | - LocalTensor<float> x_fp32 = xFp32Buf.Get<float>(); | 253 | + LocalTensor<float> splitXFp32 = xFp32Buf.Get<float>(); |
| 254 | - LocalTensor<T> xLocal = inQueueX.DeQue<T>(); | 254 | + LocalTensor<T> splitXLocalDeq = inQueueX.DeQue<T>(); |
| 255 | - Cast(x_fp32, xLocal, RoundMode::CAST_NONE, num); | 255 | + Cast(splitXFp32, splitXLocalDeq, RoundMode::CAST_NONE, num); |
| 256 | PipeBarrier<PIPE_V>(); | 256 | PipeBarrier<PIPE_V>(); |
| 257 | - inQueueX.FreeTensor(xLocal); | 257 | + inQueueX.FreeTensor(splitXLocalDeq); |
| 258 | } | 258 | } |
| 259 | } | 259 | } |
| 260 | 260 | ||
| 261 | __aicore__ inline void ComputeY( | 261 | __aicore__ inline void ComputeY( |
| 262 | - uint32_t i_i_idx, LocalTensor<half>& gammaLocal, LocalTensor<float>& rstdLocal, uint32_t num) | 262 | + uint32_t i_i_idx, LocalTensor<half>& splitGammaLocal, LocalTensor<float>& splitRstdLocal, uint32_t num) |
| 263 | { | 263 | { |
| 264 | - LocalTensor<float> x_fp32 = xFp32Buf.Get<float>(); | 264 | + LocalTensor<float> splitXFp32 = xFp32Buf.Get<float>(); |
| 265 | - LocalTensor<float> sqx = sqxBuf.Get<float>(); | 265 | + LocalTensor<float> splitSqx = sqxBuf.Get<float>(); |
| 266 | - event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 266 | + event_t splitEventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 267 | - SetFlag<HardEvent::V_S>(event_v_s); | 267 | + SetFlag<HardEvent::V_S>(splitEventVS); |
| 268 | - WaitFlag<HardEvent::V_S>(event_v_s); | 268 | + WaitFlag<HardEvent::V_S>(splitEventVS); |
| 269 | - float rstdValue = rstdLocal.GetValue(i_i_idx); | 269 | + float splitRstdValue = splitRstdLocal.GetValue(i_i_idx); |
| 270 | - event_t event_s_v = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | 270 | + event_t splitEventSV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); |
| 271 | - SetFlag<HardEvent::S_V>(event_s_v); | 271 | + SetFlag<HardEvent::S_V>(splitEventSV); |
| 272 | - WaitFlag<HardEvent::S_V>(event_s_v); | 272 | + WaitFlag<HardEvent::S_V>(splitEventSV); |
| 273 | PipeBarrier<PIPE_V>(); | 273 | PipeBarrier<PIPE_V>(); |
| 274 | - Muls(x_fp32, x_fp32, rstdValue, num); | 274 | + Muls(splitXFp32, splitXFp32, splitRstdValue, num); |
| 275 | PipeBarrier<PIPE_V>(); | 275 | PipeBarrier<PIPE_V>(); |
| 276 | - LocalTensor<half> yLocal = outQueueY.AllocTensor<half>(); | 276 | + LocalTensor<half> splitYLocal = outQueueY.AllocTensor<half>(); |
| 277 | - Cast(yLocal, x_fp32, RoundMode::CAST_NONE, num); | 277 | + Cast(splitYLocal, splitXFp32, RoundMode::CAST_NONE, num); |
| 278 | PipeBarrier<PIPE_V>(); | 278 | PipeBarrier<PIPE_V>(); |
| 279 | - Mul(yLocal, gammaLocal, yLocal, num); | 279 | + Mul(splitYLocal, splitGammaLocal, splitYLocal, num); |
| 280 | PipeBarrier<PIPE_V>(); | 280 | PipeBarrier<PIPE_V>(); |
| 281 | - outQueueY.EnQue<half>(yLocal); | 281 | + outQueueY.EnQue<half>(splitYLocal); |
| 282 | } | 282 | } |
| 283 | 283 | ||
| 284 | __aicore__ inline void ComputeY( | 284 | __aicore__ inline void ComputeY( |
| @@ -305,54 +305,54 @@ private: | |||
| 305 | __aicore__ inline void ComputeY( | 305 | __aicore__ inline void ComputeY( |
| 306 | uint32_t i_i_idx, LocalTensor<bfloat16_t>& gammaLocal, LocalTensor<float>& rstdLocal, uint32_t num) | 306 | uint32_t i_i_idx, LocalTensor<bfloat16_t>& gammaLocal, LocalTensor<float>& rstdLocal, uint32_t num) |
| 307 | { | 307 | { |
| 308 | - LocalTensor<float> x_fp32 = xFp32Buf.Get<float>(); | 308 | + LocalTensor<float> splitXFp32Bf16 = xFp32Buf.Get<float>(); |
| 309 | - LocalTensor<float> sqx = sqxBuf.Get<float>(); | 309 | + LocalTensor<float> splitSqxBf16 = sqxBuf.Get<float>(); |
| 310 | - event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 310 | + event_t splitEventVSBf16 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 311 | - SetFlag<HardEvent::V_S>(event_v_s); | 311 | + SetFlag<HardEvent::V_S>(splitEventVSBf16); |
| 312 | - WaitFlag<HardEvent::V_S>(event_v_s); | 312 | + WaitFlag<HardEvent::V_S>(splitEventVSBf16); |
| 313 | - float rstdValue = rstdLocal.GetValue(i_i_idx); | 313 | + float splitRstdValueBf16 = rstdLocal.GetValue(i_i_idx); |
| 314 | - event_t event_s_v = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | 314 | + event_t splitEventSVBf16 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); |
| 315 | - SetFlag<HardEvent::S_V>(event_s_v); | 315 | + SetFlag<HardEvent::S_V>(splitEventSVBf16); |
| 316 | - WaitFlag<HardEvent::S_V>(event_s_v); | 316 | + WaitFlag<HardEvent::S_V>(splitEventSVBf16); |
| 317 | PipeBarrier<PIPE_V>(); | 317 | PipeBarrier<PIPE_V>(); |
| 318 | - Muls(x_fp32, x_fp32, rstdValue, num); | 318 | + Muls(splitXFp32Bf16, splitXFp32Bf16, splitRstdValueBf16, num); |
| 319 | PipeBarrier<PIPE_V>(); | 319 | PipeBarrier<PIPE_V>(); |
| 320 | - LocalTensor<bfloat16_t> yLocal = outQueueY.AllocTensor<bfloat16_t>(); | 320 | + LocalTensor<bfloat16_t> splitYLocalBf16 = outQueueY.AllocTensor<bfloat16_t>(); |
| 321 | - Cast(yLocal, x_fp32, RoundMode::CAST_RINT, num); | 321 | + Cast(splitYLocalBf16, splitXFp32Bf16, RoundMode::CAST_RINT, num); |
| 322 | PipeBarrier<PIPE_V>(); | 322 | PipeBarrier<PIPE_V>(); |
| 323 | - Cast(x_fp32, yLocal, RoundMode::CAST_NONE, num); | 323 | + Cast(splitXFp32Bf16, splitYLocalBf16, RoundMode::CAST_NONE, num); |
| 324 | PipeBarrier<PIPE_V>(); | 324 | PipeBarrier<PIPE_V>(); |
| 325 | - Cast(sqx, gammaLocal, RoundMode::CAST_NONE, num); | 325 | + Cast(splitSqxBf16, gammaLocal, RoundMode::CAST_NONE, num); |
| 326 | PipeBarrier<PIPE_V>(); | 326 | PipeBarrier<PIPE_V>(); |
| 327 | - Mul(x_fp32, x_fp32, sqx, num); | 327 | + Mul(splitXFp32Bf16, splitXFp32Bf16, splitSqxBf16, num); |
| 328 | PipeBarrier<PIPE_V>(); | 328 | PipeBarrier<PIPE_V>(); |
| 329 | - Cast(yLocal, x_fp32, RoundMode::CAST_RINT, num); | 329 | + Cast(splitYLocalBf16, splitXFp32Bf16, RoundMode::CAST_RINT, num); |
| 330 | PipeBarrier<PIPE_V>(); | 330 | PipeBarrier<PIPE_V>(); |
| 331 | - outQueueY.EnQue<bfloat16_t>(yLocal); | 331 | + outQueueY.EnQue<bfloat16_t>(splitYLocalBf16); |
| 332 | } | 332 | } |
| 333 | 333 | ||
| 334 | __aicore__ inline void CopyOutY(uint32_t i_idx, uint32_t j_idx, uint32_t num) | 334 | __aicore__ inline void CopyOutY(uint32_t i_idx, uint32_t j_idx, uint32_t num) |
| 335 | { | 335 | { |
| 336 | - LocalTensor<T> yLocal = outQueueY.DeQue<T>(); | 336 | + LocalTensor<T> splitYLocalOut = outQueueY.DeQue<T>(); |
| 337 | - DataCopyCustom<T>(yGm[i_idx * numCol + j_idx * ubFactor], yLocal, num); | 337 | + DataCopyCustom<T>(yGm[i_idx * this->numCol + j_idx * this->ubFactor], splitYLocalOut, num); |
| 338 | - outQueueY.FreeTensor(yLocal); | 338 | + outQueueY.FreeTensor(splitYLocalOut); |
| 339 | } | 339 | } |
| 340 | 340 | ||
| 341 | __aicore__ inline void CopyOutRstd(uint32_t i_o_idx, uint32_t num) | 341 | __aicore__ inline void CopyOutRstd(uint32_t i_o_idx, uint32_t num) |
| 342 | { | 342 | { |
| 343 | - LocalTensor<float> rstdLocal = outQueueRstd.DeQue<float>(); | 343 | + LocalTensor<float> splitRstdLocal = outQueueRstd.DeQue<float>(); |
| 344 | 344 | ||
| 345 | - DataCopyCustom<float>(rstdGm[i_o_idx * rowFactor], rstdLocal, num); | 345 | + DataCopyCustom<float>(rstdGm[i_o_idx * this->rowFactor], splitRstdLocal, num); |
| 346 | 346 | ||
| 347 | - outQueueRstd.FreeTensor(rstdLocal); | 347 | + outQueueRstd.FreeTensor(splitRstdLocal); |
| 348 | } | 348 | } |
| 349 | 349 | ||
| 350 | private: | 350 | private: |
| 351 | TPipe* Ppipe = nullptr; | 351 | TPipe* Ppipe = nullptr; |
| 352 | - // create queues for input, in this case depth is equal to buffer num | 352 | + // create input queues for split_d |
| 353 | TQue<QuePosition::VECIN, BUFFER_NUM> inQueueX; | 353 | TQue<QuePosition::VECIN, BUFFER_NUM> inQueueX; |
| 354 | TQue<QuePosition::VECIN, BUFFER_NUM> inQueueGamma; | 354 | TQue<QuePosition::VECIN, BUFFER_NUM> inQueueGamma; |
| 355 | - // create queues for output, in this case depth is equal to buffer num | 355 | + // create output queues for split_d |
| 356 | TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueY; | 356 | TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueY; |
| 357 | TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueRstd; | 357 | TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueRstd; |
| 358 | TBuf<TPosition::VECCALC> xFp32Buf; | 358 | TBuf<TPosition::VECCALC> xFp32Buf; |
| @@ -370,7 +370,7 @@ private: | |||
| 370 | 370 | ||
| 371 | uint32_t numRow; | 371 | uint32_t numRow; |
| 372 | uint32_t numCol; | 372 | uint32_t numCol; |
| 373 | - uint32_t blockFactor; // number of calculations rows on each core | 373 | + uint32_t blockFactor; |
| 374 | uint32_t rowFactor; | 374 | uint32_t rowFactor; |
| 375 | uint32_t ubFactor; | 375 | uint32_t ubFactor; |
| 376 | float epsilon; | 376 | float epsilon; |
| @@ -37,18 +37,18 @@ TILING_DATA_FIELD_DEF(float, avg_factor); | |||
| 37 | END_TILING_DATA_DEF; | 37 | END_TILING_DATA_DEF; |
| 38 | 38 | ||
| 39 | BEGIN_TILING_DATA_DEF(AddRmsNormCastRegbaseTilingData) | 39 | BEGIN_TILING_DATA_DEF(AddRmsNormCastRegbaseTilingData) |
| 40 | -TILING_DATA_FIELD_DEF(uint64_t, numM); | ||
| 41 | TILING_DATA_FIELD_DEF(uint64_t, numN); | 40 | TILING_DATA_FIELD_DEF(uint64_t, numN); |
| 42 | -TILING_DATA_FIELD_DEF(uint64_t, baseM); | 41 | +TILING_DATA_FIELD_DEF(uint64_t, numM); |
| 43 | TILING_DATA_FIELD_DEF(uint64_t, baseN); | 42 | TILING_DATA_FIELD_DEF(uint64_t, baseN); |
| 44 | -TILING_DATA_FIELD_DEF(uint64_t, baseNDtypeAlign); | 43 | +TILING_DATA_FIELD_DEF(uint64_t, baseM); |
| 45 | TILING_DATA_FIELD_DEF(uint64_t, baseNReduceAlign); | 44 | TILING_DATA_FIELD_DEF(uint64_t, baseNReduceAlign); |
| 46 | -TILING_DATA_FIELD_DEF(uint64_t, powerSplit); | 45 | +TILING_DATA_FIELD_DEF(uint64_t, baseNDtypeAlign); |
| 47 | TILING_DATA_FIELD_DEF(uint64_t, powerLoop); | 46 | TILING_DATA_FIELD_DEF(uint64_t, powerLoop); |
| 48 | -TILING_DATA_FIELD_DEF(uint64_t, mPerCore); | 47 | +TILING_DATA_FIELD_DEF(uint64_t, powerSplit); |
| 49 | TILING_DATA_FIELD_DEF(uint64_t, mLastCore); | 48 | TILING_DATA_FIELD_DEF(uint64_t, mLastCore); |
| 50 | -TILING_DATA_FIELD_DEF(float, epsilon); | 49 | +TILING_DATA_FIELD_DEF(uint64_t, mPerCore); |
| 51 | TILING_DATA_FIELD_DEF(float, avgFactor); | 50 | TILING_DATA_FIELD_DEF(float, avgFactor); |
| 51 | +TILING_DATA_FIELD_DEF(float, epsilon); | ||
| 52 | TILING_DATA_FIELD_DEF(uint32_t, isNddma); | 52 | TILING_DATA_FIELD_DEF(uint32_t, isNddma); |
| 53 | END_TILING_DATA_DEF; | 53 | END_TILING_DATA_DEF; |
| 54 | 54 | ||
| @@ -76,18 +76,18 @@ REGISTER_TILING_DATA_CLASS(AddRmsNormCast_199, AddRmsNormCastRegbaseTilingData) | |||
| 76 | 76 | ||
| 77 | struct AddRmsNormCastRegbaseTilingParams { | 77 | struct AddRmsNormCastRegbaseTilingParams { |
| 78 | // Platform | 78 | // Platform |
| 79 | - uint64_t maxUbSize{0}; | ||
| 80 | uint64_t totalCoreNum{0}; | 79 | uint64_t totalCoreNum{0}; |
| 80 | + uint64_t maxUbSize{0}; | ||
| 81 | uint64_t vecLength{0}; | 81 | uint64_t vecLength{0}; |
| 82 | // Input Info | 82 | // Input Info |
| 83 | - uint64_t numM{0}; | ||
| 84 | uint64_t numN{0}; | 83 | uint64_t numN{0}; |
| 84 | + uint64_t numM{0}; | ||
| 85 | uint64_t xDtypeSize{0}; | 85 | uint64_t xDtypeSize{0}; |
| 86 | uint64_t xDtypeAlignNum{0}; | 86 | uint64_t xDtypeAlignNum{0}; |
| 87 | uint64_t xReduceAlignNum{0}; | 87 | uint64_t xReduceAlignNum{0}; |
| 88 | // Cal params | 88 | // Cal params |
| 89 | - uint64_t baseM{0}; | ||
| 90 | uint64_t baseN{0}; | 89 | uint64_t baseN{0}; |
| 90 | + uint64_t baseM{0}; | ||
| 91 | uint64_t baseNDtypeAlign{0}; | 91 | uint64_t baseNDtypeAlign{0}; |
| 92 | uint64_t baseNReduceAlign{0}; | 92 | uint64_t baseNReduceAlign{0}; |
| 93 | uint64_t powerSplit{0}; | 93 | uint64_t powerSplit{0}; |
| @@ -133,11 +133,11 @@ protected: | |||
| 133 | // Order: GetShapeAttrsInfo->GetPlatformInfo-> | 133 | // Order: GetShapeAttrsInfo->GetPlatformInfo-> |
| 134 | // IsCapable->DoOpTiling->DoLibApiTiling-> | 134 | // IsCapable->DoOpTiling->DoLibApiTiling-> |
| 135 | // GetWorkspaceSize->PostTiling->GetTilingKey | 135 | // GetWorkspaceSize->PostTiling->GetTilingKey |
| 136 | - ge::graphStatus GetShapeAttrsInfo() override; | ||
| 137 | ge::graphStatus GetPlatformInfo() override; | 136 | ge::graphStatus GetPlatformInfo() override; |
| 137 | + ge::graphStatus GetShapeAttrsInfo() override; | ||
| 138 | bool IsCapable() override; | 138 | bool IsCapable() override; |
| 139 | - ge::graphStatus DoOpTiling() override; | ||
| 140 | ge::graphStatus DoLibApiTiling() override; | 139 | ge::graphStatus DoLibApiTiling() override; |
| 140 | + ge::graphStatus DoOpTiling() override; | ||
| 141 | ge::graphStatus GetWorkspaceSize() override; | 141 | ge::graphStatus GetWorkspaceSize() override; |
| 142 | ge::graphStatus PostTiling() override; | 142 | ge::graphStatus PostTiling() override; |
| 143 | uint64_t GetTilingKey() const override; | 143 | uint64_t GetTilingKey() const override; |
| @@ -39,17 +39,17 @@ public: | |||
| 39 | this->rowFactor = tiling->row_factor; | 39 | this->rowFactor = tiling->row_factor; |
| 40 | this->ubFactor = tiling->ub_factor; | 40 | this->ubFactor = tiling->ub_factor; |
| 41 | this->epsilon = tiling->epsilon; | 41 | this->epsilon = tiling->epsilon; |
| 42 | - this->avgFactor = (numCol != 0) ? (float)1.0 / numCol : 0; | 42 | + this->avgFactor = (this->numCol != 0) ? (float)1.0 / this->numCol : 0; |
| 43 | 43 | ||
| 44 | blockIdx_ = GetBlockIdx(); | 44 | blockIdx_ = GetBlockIdx(); |
| 45 | if (blockIdx_ < GetBlockNum() - 1) { | 45 | if (blockIdx_ < GetBlockNum() - 1) { |
| 46 | - this->rowWork = blockFactor; | 46 | + this->rowWork = this->blockFactor; |
| 47 | } else if (blockIdx_ == GetBlockNum() - 1) { | 47 | } else if (blockIdx_ == GetBlockNum() - 1) { |
| 48 | - this->rowWork = numRow - (GetBlockNum() - 1) * blockFactor; | 48 | + this->rowWork = this->numRow - (GetBlockNum() - 1) * this->blockFactor; |
| 49 | } | 49 | } |
| 50 | // get start index for current core, core parallel | 50 | // get start index for current core, core parallel |
| 51 | - uint64_t calcOffset = blockIdx_ * blockFactor * numCol; | 51 | + uint64_t calcOffset = blockIdx_ * this->blockFactor * this->numCol; |
| 52 | - uint64_t calcNum = rowWork * numCol; | 52 | + uint64_t calcNum = this->rowWork * this->numCol; |
| 53 | x1Gm.SetGlobalBuffer((__gm__ T*)x1 + calcOffset, calcNum); | 53 | x1Gm.SetGlobalBuffer((__gm__ T*)x1 + calcOffset, calcNum); |
| 54 | x2Gm.SetGlobalBuffer((__gm__ T*)x2 + calcOffset, calcNum); | 54 | x2Gm.SetGlobalBuffer((__gm__ T*)x2 + calcOffset, calcNum); |
| 55 | gammaGm.SetGlobalBuffer((__gm__ T*)gamma, numCol); | 55 | gammaGm.SetGlobalBuffer((__gm__ T*)gamma, numCol); |
| @@ -195,61 +195,61 @@ private: | |||
| 195 | 195 | ||
| 196 | __aicore__ inline void Computefp16(uint32_t outer_progress, uint32_t inner_progress, uint32_t progress) | 196 | __aicore__ inline void Computefp16(uint32_t outer_progress, uint32_t inner_progress, uint32_t progress) |
| 197 | { | 197 | { |
| 198 | - LocalTensor<float> xFp32 = xFp32Buf.Get<float>(); | 198 | + LocalTensor<float> xFp32Local = xFp32Buf.Get<float>(); |
| 199 | - LocalTensor<float> sqx = sqxBuf.Get<float>(); | 199 | + LocalTensor<float> sqxLocal = sqxBuf.Get<float>(); |
| 200 | - LocalTensor<float> reduceBufLocal = reduceFp32Buf.Get<float>(); | 200 | + LocalTensor<float> reduceBufLocalFp16 = reduceFp32Buf.Get<float>(); |
| 201 | - LocalTensor<float> resFp32 = tmpBuf2.Get<float>(); | 201 | + LocalTensor<float> resFp32Local = tmpBuf2.Get<float>(); |
| 202 | - LocalTensor<T> gammaLocal = resFp32.template ReinterpretCast<T>()[ubFactor * 2]; | 202 | + LocalTensor<T> gammaLocalFp16 = resFp32Local.template ReinterpretCast<T>()[ubFactor * 2]; |
| 203 | if (inner_progress == 0) { | 203 | if (inner_progress == 0) { |
| 204 | - DataCopyCustom<T>(gammaLocal, gammaGm, numCol); | 204 | + DataCopyCustom<T>(gammaLocalFp16, gammaGm, numCol); |
| 205 | } | 205 | } |
| 206 | - event_t eventMte2V = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); | 206 | + event_t eventMte2VFp16 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); |
| 207 | - SetFlag<HardEvent::MTE2_V>(eventMte2V); | 207 | + SetFlag<HardEvent::MTE2_V>(eventMte2VFp16); |
| 208 | 208 | ||
| 209 | - Mul(sqx, xFp32, xFp32, numCol); | 209 | + Mul(sqxLocal, xFp32Local, xFp32Local, numCol); |
| 210 | PipeBarrier<PIPE_V>(); | 210 | PipeBarrier<PIPE_V>(); |
| 211 | 211 | ||
| 212 | - Muls(sqx, sqx, avgFactor, numCol); | 212 | + Muls(sqxLocal, sqxLocal, avgFactor, numCol); |
| 213 | PipeBarrier<PIPE_V>(); | 213 | PipeBarrier<PIPE_V>(); |
| 214 | 214 | ||
| 215 | - ReduceSumCustom(sqx, sqx, reduceBufLocal, numCol); | 215 | + ReduceSumCustom(sqxLocal, sqxLocal, reduceBufLocalFp16, numCol); |
| 216 | PipeBarrier<PIPE_V>(); | 216 | PipeBarrier<PIPE_V>(); |
| 217 | 217 | ||
| 218 | - Adds(sqx, sqx, epsilon, 1); | 218 | + Adds(sqxLocal, sqxLocal, epsilon, 1); |
| 219 | PipeBarrier<PIPE_V>(); | 219 | PipeBarrier<PIPE_V>(); |
| 220 | 220 | ||
| 221 | - Sqrt(sqx, sqx, 1); | 221 | + Sqrt(sqxLocal, sqxLocal, 1); |
| 222 | - Duplicate(reduceBufLocal, ONE, 1); | 222 | + Duplicate(reduceBufLocalFp16, ONE, 1); |
| 223 | PipeBarrier<PIPE_V>(); | 223 | PipeBarrier<PIPE_V>(); |
| 224 | - Div(sqx, reduceBufLocal, sqx, 1); | 224 | + Div(sqxLocal, reduceBufLocalFp16, sqxLocal, 1); |
| 225 | PipeBarrier<PIPE_V>(); | 225 | PipeBarrier<PIPE_V>(); |
| 226 | event_t eventVMte31 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 226 | event_t eventVMte31 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| 227 | SetFlag<HardEvent::V_MTE3>(eventVMte31); | 227 | SetFlag<HardEvent::V_MTE3>(eventVMte31); |
| 228 | WaitFlag<HardEvent::V_MTE3>(eventVMte31); | 228 | WaitFlag<HardEvent::V_MTE3>(eventVMte31); |
| 229 | - Brcb(reduceBufLocal, sqx, 1, {1, 8}); | 229 | + Brcb(reduceBufLocalFp16, sqxLocal, 1, {1, 8}); |
| 230 | - DataCopyCustom<float>(rstdGm[outer_progress * rowFactor + inner_progress], sqx, 1); | 230 | + DataCopyCustom<float>(rstdGm[outer_progress * rowFactor + inner_progress], sqxLocal, 1); |
| 231 | event_t eventMte3V1 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); | 231 | event_t eventMte3V1 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); |
| 232 | SetFlag<HardEvent::MTE3_V>(eventMte3V1); | 232 | SetFlag<HardEvent::MTE3_V>(eventMte3V1); |
| 233 | PipeBarrier<PIPE_V>(); | 233 | PipeBarrier<PIPE_V>(); |
| 234 | uint32_t repeat = (numCol + VL - 1) / VL; | 234 | uint32_t repeat = (numCol + VL - 1) / VL; |
| 235 | uint64_t mask = VL; | 235 | uint64_t mask = VL; |
| 236 | - Mul(xFp32, xFp32, reduceBufLocal, mask, repeat, {1, 1, 0, 8, 8, 0}); | 236 | + Mul(xFp32Local, xFp32Local, reduceBufLocalFp16, mask, repeat, {1, 1, 0, 8, 8, 0}); |
| 237 | PipeBarrier<PIPE_V>(); | 237 | PipeBarrier<PIPE_V>(); |
| 238 | 238 | ||
| 239 | LocalTensor<half> yLocal = outQueueY.AllocTensor<half>(); | 239 | LocalTensor<half> yLocal = outQueueY.AllocTensor<half>(); |
| 240 | - Cast(yLocal, xFp32, RoundMode::CAST_NONE, numCol); | 240 | + Cast(yLocal, xFp32Local, RoundMode::CAST_NONE, numCol); |
| 241 | PipeBarrier<PIPE_V>(); | 241 | PipeBarrier<PIPE_V>(); |
| 242 | - WaitFlag<HardEvent::MTE2_V>(eventMte2V); | 242 | + WaitFlag<HardEvent::MTE2_V>(eventMte2VFp16); |
| 243 | 243 | ||
| 244 | - Mul(yLocal, gammaLocal, yLocal, numCol); | 244 | + Mul(yLocal, gammaLocalFp16, yLocal, numCol); |
| 245 | PipeBarrier<PIPE_V>(); | 245 | PipeBarrier<PIPE_V>(); |
| 246 | outQueueY.EnQue<half>(yLocal); | 246 | outQueueY.EnQue<half>(yLocal); |
| 247 | - Cast(resFp32, yLocal, RoundMode::CAST_NONE, numCol); | 247 | + Cast(resFp32Local, yLocal, RoundMode::CAST_NONE, numCol); |
| 248 | PipeBarrier<PIPE_V>(); | 248 | PipeBarrier<PIPE_V>(); |
| 249 | event_t event_v_mte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 249 | event_t event_v_mte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| 250 | SetFlag<HardEvent::V_MTE3>(event_v_mte3); | 250 | SetFlag<HardEvent::V_MTE3>(event_v_mte3); |
| 251 | WaitFlag<HardEvent::V_MTE3>(event_v_mte3); | 251 | WaitFlag<HardEvent::V_MTE3>(event_v_mte3); |
| 252 | - DataCopyCustom<float>(y1Gm[progress], resFp32, numCol); | 252 | + DataCopyCustom<float>(y1Gm[progress], resFp32Local, numCol); |
| 253 | WaitFlag<HardEvent::MTE3_V>(eventMte3V1); | 253 | WaitFlag<HardEvent::MTE3_V>(eventMte3V1); |
| 254 | } | 254 | } |
| 255 | 255 | ||
| @@ -290,8 +290,8 @@ private: | |||
| 290 | GlobalTensor<float> rstdGm; | 290 | GlobalTensor<float> rstdGm; |
| 291 | GlobalTensor<T> xGm; | 291 | GlobalTensor<T> xGm; |
| 292 | 292 | ||
| 293 | - uint32_t numRow; | ||
| 294 | uint32_t numCol; | 293 | uint32_t numCol; |
| 294 | + uint32_t numRow; | ||
| 295 | uint32_t blockFactor; // number of calculations rows on each core | 295 | uint32_t blockFactor; // number of calculations rows on each core |
| 296 | uint32_t rowFactor; | 296 | uint32_t rowFactor; |
| 297 | uint32_t ubFactor; | 297 | uint32_t ubFactor; |
| @@ -31,8 +31,8 @@ public: | |||
| 31 | const AddRMSNormCastTilingData* tiling) | 31 | const AddRMSNormCastTilingData* tiling) |
| 32 | { | 32 | { |
| 33 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); | 33 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); |
| 34 | - this->numRow = tiling->num_row; | ||
| 35 | this->numCol = tiling->num_col; | 34 | this->numCol = tiling->num_col; |
| 35 | + this->numRow = tiling->num_row; | ||
| 36 | uint32_t numPerBlock = ONE_BLK_SIZE / sizeof(T); | 36 | uint32_t numPerBlock = ONE_BLK_SIZE / sizeof(T); |
| 37 | this->numColAlign = AlignUp(numCol, numPerBlock); | 37 | this->numColAlign = AlignUp(numCol, numPerBlock); |
| 38 | this->blockFactor = tiling->block_factor; | 38 | this->blockFactor = tiling->block_factor; |
| @@ -277,8 +277,8 @@ private: | |||
| 277 | GlobalTensor<float> rstdGm; | 277 | GlobalTensor<float> rstdGm; |
| 278 | GlobalTensor<T> xGm; | 278 | GlobalTensor<T> xGm; |
| 279 | 279 | ||
| 280 | - uint32_t numRow; | ||
| 281 | uint32_t numCol; | 280 | uint32_t numCol; |
| 281 | + uint32_t numRow; | ||
| 282 | uint32_t blockFactor; // number of calculations rows on each core | 282 | uint32_t blockFactor; // number of calculations rows on each core |
| 283 | uint32_t rowFactor; | 283 | uint32_t rowFactor; |
| 284 | uint32_t ubFactor; | 284 | uint32_t ubFactor; |
| @@ -153,99 +153,99 @@ private: | |||
| 153 | 153 | ||
| 154 | __aicore__ inline void ProcessBf16() | 154 | __aicore__ inline void ProcessBf16() |
| 155 | { | 155 | { |
| 156 | - LocalTensor<float> ubLocal = unitBuf.Get<float>(); | 156 | + LocalTensor<float> ubLocalBf16 = unitBuf.Get<float>(); |
| 157 | - LocalTensor<T> xLocal = ubLocal.template ReinterpretCast<T>(); | 157 | + LocalTensor<T> xLocalBf16 = ubLocalBf16.template ReinterpretCast<T>(); |
| 158 | - LocalTensor<T> x1Local = xLocal[0]; | 158 | + LocalTensor<T> x1LocalBf16 = xLocalBf16[0]; |
| 159 | - LocalTensor<T> x2Local = xLocal[ubFactor]; | 159 | + LocalTensor<T> x2LocalBf16 = xLocalBf16[ubFactor]; |
| 160 | - LocalTensor<float> xFp32Local = ubLocal[ubFactor]; | 160 | + LocalTensor<float> xFp32LocalBf16 = ubLocalBf16[ubFactor]; |
| 161 | - LocalTensor<float> sqxLocal = ubLocal[ubFactor * 2]; | 161 | + LocalTensor<float> sqxLocalBf16 = ubLocalBf16[ubFactor * 2]; |
| 162 | - LocalTensor<float> tmpLocal = ubLocal[ubFactor * 3]; | 162 | + LocalTensor<float> tmpLocalBf16 = ubLocalBf16[ubFactor * 3]; |
| 163 | 163 | ||
| 164 | - DataCopyCustom<T>(x1Local, x1Gm, numCol); | 164 | + DataCopyCustom<T>(x1LocalBf16, x1Gm, numCol); |
| 165 | - event_t eventMTE2V1 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); | 165 | + event_t eventMTE2V1Bf16 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); |
| 166 | - SetFlag<HardEvent::MTE2_V>(eventMTE2V1); | 166 | + SetFlag<HardEvent::MTE2_V>(eventMTE2V1Bf16); |
| 167 | - DataCopyCustom<T>(x2Local, x2Gm, numCol); | 167 | + DataCopyCustom<T>(x2LocalBf16, x2Gm, numCol); |
| 168 | - event_t eventMTE2V2 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); | 168 | + event_t eventMTE2V2Bf16 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); |
| 169 | - SetFlag<HardEvent::MTE2_V>(eventMTE2V2); | 169 | + SetFlag<HardEvent::MTE2_V>(eventMTE2V2Bf16); |
| 170 | - WaitFlag<HardEvent::MTE2_V>(eventMTE2V1); | 170 | + WaitFlag<HardEvent::MTE2_V>(eventMTE2V1Bf16); |
| 171 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V1); | 171 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V1Bf16); |
| 172 | - Cast(xFp32Local, x1Local, RoundMode::CAST_NONE, numCol); | 172 | + Cast(xFp32LocalBf16, x1LocalBf16, RoundMode::CAST_NONE, numCol); |
| 173 | - WaitFlag<HardEvent::MTE2_V>(eventMTE2V2); | 173 | + WaitFlag<HardEvent::MTE2_V>(eventMTE2V2Bf16); |
| 174 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V2); | 174 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V2Bf16); |
| 175 | - Cast(sqxLocal, x2Local, RoundMode::CAST_NONE, numCol); | 175 | + Cast(sqxLocalBf16, x2LocalBf16, RoundMode::CAST_NONE, numCol); |
| 176 | PipeBarrier<PIPE_V>(); | 176 | PipeBarrier<PIPE_V>(); |
| 177 | - Add(xFp32Local, xFp32Local, sqxLocal, numCol); | 177 | + Add(xFp32LocalBf16, xFp32LocalBf16, sqxLocalBf16, numCol); |
| 178 | PipeBarrier<PIPE_V>(); | 178 | PipeBarrier<PIPE_V>(); |
| 179 | - Cast(x1Local, xFp32Local, RoundMode::CAST_RINT, numCol); | 179 | + Cast(x1LocalBf16, xFp32LocalBf16, RoundMode::CAST_RINT, numCol); |
| 180 | PipeBarrier<PIPE_V>(); | 180 | PipeBarrier<PIPE_V>(); |
| 181 | // copy gamma | 181 | // copy gamma |
| 182 | - event_t eventVMTE2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE2)); | 182 | + event_t eventVMTE2Bf16 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE2)); |
| 183 | - SetFlag<HardEvent::V_MTE2>(eventVMTE2); | 183 | + SetFlag<HardEvent::V_MTE2>(eventVMTE2Bf16); |
| 184 | - WaitFlag<HardEvent::V_MTE2>(eventVMTE2); | 184 | + WaitFlag<HardEvent::V_MTE2>(eventVMTE2Bf16); |
| 185 | 185 | ||
| 186 | - DataCopyCustom<T>(x2Local, gammaGm, numCol); // gammaLocal use x2Local | 186 | + DataCopyCustom<T>(x2LocalBf16, gammaGm, numCol); // gammaLocal use x2Local |
| 187 | - event_t eventMTE2V3 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); | 187 | + event_t eventMTE2V3Bf16 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); |
| 188 | - SetFlag<HardEvent::MTE2_V>(eventMTE2V3); | 188 | + SetFlag<HardEvent::MTE2_V>(eventMTE2V3Bf16); |
| 189 | 189 | ||
| 190 | // copy x out | 190 | // copy x out |
| 191 | - event_t eventVMTE3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 191 | + event_t eventVMTE3Bf16 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| 192 | - SetFlag<HardEvent::V_MTE3>(eventVMTE3); | 192 | + SetFlag<HardEvent::V_MTE3>(eventVMTE3Bf16); |
| 193 | - WaitFlag<HardEvent::V_MTE3>(eventVMTE3); | 193 | + WaitFlag<HardEvent::V_MTE3>(eventVMTE3Bf16); |
| 194 | - DataCopyCustom<T>(xGm, x1Local, numCol); | 194 | + DataCopyCustom<T>(xGm, x1LocalBf16, numCol); |
| 195 | - event_t eventMTE3V = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE3_V>()); | 195 | + event_t eventMTE3VBf16 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE3_V>()); |
| 196 | - SetFlag<HardEvent::MTE3_V>(eventMTE3V); | 196 | + SetFlag<HardEvent::MTE3_V>(eventMTE3VBf16); |
| 197 | 197 | ||
| 198 | - Mul(sqxLocal, xFp32Local, xFp32Local, numCol); | 198 | + Mul(sqxLocalBf16, xFp32LocalBf16, xFp32LocalBf16, numCol); |
| 199 | PipeBarrier<PIPE_V>(); | 199 | PipeBarrier<PIPE_V>(); |
| 200 | - Muls(sqxLocal, sqxLocal, avgFactor, numCol); | 200 | + Muls(sqxLocalBf16, sqxLocalBf16, avgFactor, numCol); |
| 201 | PipeBarrier<PIPE_V>(); | 201 | PipeBarrier<PIPE_V>(); |
| 202 | - ReduceSumCustom(sqxLocal, sqxLocal, tmpLocal, numCol); | 202 | + ReduceSumCustom(sqxLocalBf16, sqxLocalBf16, tmpLocalBf16, numCol); |
| 203 | PipeBarrier<PIPE_V>(); | 203 | PipeBarrier<PIPE_V>(); |
| 204 | - Adds(sqxLocal, sqxLocal, epsilon, 1); | 204 | + Adds(sqxLocalBf16, sqxLocalBf16, epsilon, 1); |
| 205 | PipeBarrier<PIPE_V>(); | 205 | PipeBarrier<PIPE_V>(); |
| 206 | - Sqrt(sqxLocal, sqxLocal, 1); | 206 | + Sqrt(sqxLocalBf16, sqxLocalBf16, 1); |
| 207 | - Duplicate(tmpLocal, ONE, 1); | 207 | + Duplicate(tmpLocalBf16, ONE, 1); |
| 208 | PipeBarrier<PIPE_V>(); | 208 | PipeBarrier<PIPE_V>(); |
| 209 | - Div(sqxLocal, tmpLocal, sqxLocal, 1); | 209 | + Div(sqxLocalBf16, tmpLocalBf16, sqxLocalBf16, 1); |
| 210 | PipeBarrier<PIPE_V>(); | 210 | PipeBarrier<PIPE_V>(); |
| 211 | - event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 211 | + event_t eventVSBf16 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 212 | - SetFlag<HardEvent::V_S>(eventVS); | 212 | + SetFlag<HardEvent::V_S>(eventVSBf16); |
| 213 | - WaitFlag<HardEvent::V_S>(eventVS); | 213 | + WaitFlag<HardEvent::V_S>(eventVSBf16); |
| 214 | - float rstdValue = sqxLocal.GetValue(0); | 214 | + float rstdValueBf16 = sqxLocalBf16.GetValue(0); |
| 215 | - event_t eventSV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | 215 | + event_t eventSVBf16 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); |
| 216 | - SetFlag<HardEvent::S_V>(eventSV); | 216 | + SetFlag<HardEvent::S_V>(eventSVBf16); |
| 217 | - WaitFlag<HardEvent::S_V>(eventSV); | 217 | + WaitFlag<HardEvent::S_V>(eventSVBf16); |
| 218 | // copyout rstd | 218 | // copyout rstd |
| 219 | 219 | ||
| 220 | - SetFlag<HardEvent::V_MTE3>(eventVMTE3); | 220 | + SetFlag<HardEvent::V_MTE3>(eventVMTE3Bf16); |
| 221 | - WaitFlag<HardEvent::V_MTE3>(eventVMTE3); | 221 | + WaitFlag<HardEvent::V_MTE3>(eventVMTE3Bf16); |
| 222 | - DataCopyCustom<float>(rstdGm, sqxLocal, 1); | 222 | + DataCopyCustom<float>(rstdGm, sqxLocalBf16, 1); |
| 223 | - event_t eventMTE3V2 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE3_V>()); | 223 | + event_t eventMTE3V2Bf16 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE3_V>()); |
| 224 | - SetFlag<HardEvent::MTE3_V>(eventMTE3V2); | 224 | + SetFlag<HardEvent::MTE3_V>(eventMTE3V2Bf16); |
| 225 | 225 | ||
| 226 | 226 | ||
| 227 | - Muls(xFp32Local, xFp32Local, rstdValue, numCol); | 227 | + Muls(xFp32LocalBf16, xFp32LocalBf16, rstdValueBf16, numCol); |
| 228 | PipeBarrier<PIPE_V>(); | 228 | PipeBarrier<PIPE_V>(); |
| 229 | - WaitFlag<HardEvent::MTE3_V>(eventMTE3V); | 229 | + WaitFlag<HardEvent::MTE3_V>(eventMTE3VBf16); |
| 230 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE3_V>(eventMTE3V); | 230 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE3_V>(eventMTE3VBf16); |
| 231 | - WaitFlag<HardEvent::MTE2_V>(eventMTE2V3); | 231 | + WaitFlag<HardEvent::MTE2_V>(eventMTE2V3Bf16); |
| 232 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V3); | 232 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V3Bf16); |
| 233 | 233 | ||
| 234 | - WaitFlag<HardEvent::MTE3_V>(eventMTE3V2); | 234 | + WaitFlag<HardEvent::MTE3_V>(eventMTE3V2Bf16); |
| 235 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE3_V>(eventMTE3V2); | 235 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE3_V>(eventMTE3V2Bf16); |
| 236 | 236 | ||
| 237 | - Cast(sqxLocal, x2Local, RoundMode::CAST_NONE, numCol); | 237 | + Cast(sqxLocalBf16, x2LocalBf16, RoundMode::CAST_NONE, numCol); |
| 238 | PipeBarrier<PIPE_V>(); | 238 | PipeBarrier<PIPE_V>(); |
| 239 | - Mul(xFp32Local, xFp32Local, sqxLocal, numCol); | 239 | + Mul(xFp32LocalBf16, xFp32LocalBf16, sqxLocalBf16, numCol); |
| 240 | PipeBarrier<PIPE_V>(); | 240 | PipeBarrier<PIPE_V>(); |
| 241 | - Cast(x1Local, xFp32Local, RoundMode::CAST_RINT, numCol); | 241 | + Cast(x1LocalBf16, xFp32LocalBf16, RoundMode::CAST_RINT, numCol); |
| 242 | PipeBarrier<PIPE_V>(); | 242 | PipeBarrier<PIPE_V>(); |
| 243 | - SetFlag<HardEvent::V_MTE3>(eventVMTE3); | 243 | + SetFlag<HardEvent::V_MTE3>(eventVMTE3Bf16); |
| 244 | - WaitFlag<HardEvent::V_MTE3>(eventVMTE3); | 244 | + WaitFlag<HardEvent::V_MTE3>(eventVMTE3Bf16); |
| 245 | - DataCopyCustom<T>(y2Gm, x1Local, numCol); | 245 | + DataCopyCustom<T>(y2Gm, x1LocalBf16, numCol); |
| 246 | - SetFlag<HardEvent::V_MTE3>(eventVMTE3); | 246 | + SetFlag<HardEvent::V_MTE3>(eventVMTE3Bf16); |
| 247 | - WaitFlag<HardEvent::V_MTE3>(eventVMTE3); | 247 | + WaitFlag<HardEvent::V_MTE3>(eventVMTE3Bf16); |
| 248 | - DataCopyCustom<float>(y1Gm, xFp32Local, numCol); | 248 | + DataCopyCustom<float>(y1Gm, xFp32LocalBf16, numCol); |
| 249 | } | 249 | } |
| 250 | 250 | ||
| 251 | private: | 251 | private: |
| @@ -22,17 +22,13 @@ using namespace RmsNorm; | |||
| 22 | template <typename T> | 22 | template <typename T> |
| 23 | class KernelAddRmsNormCastSplitD { | 23 | class KernelAddRmsNormCastSplitD { |
| 24 | public: | 24 | public: |
| 25 | - __aicore__ inline KernelAddRmsNormCastSplitD(TPipe* pipe) | ||
| 26 | - { | ||
| 27 | - Ppipe = pipe; | ||
| 28 | - } | ||
| 29 | __aicore__ inline void Init( | 25 | __aicore__ inline void Init( |
| 30 | GM_ADDR x1, GM_ADDR x2, GM_ADDR gamma, GM_ADDR y1, GM_ADDR y2, GM_ADDR rstd, GM_ADDR x, GM_ADDR workspace, | 26 | GM_ADDR x1, GM_ADDR x2, GM_ADDR gamma, GM_ADDR y1, GM_ADDR y2, GM_ADDR rstd, GM_ADDR x, GM_ADDR workspace, |
| 31 | const AddRMSNormCastTilingData* tiling) | 27 | const AddRMSNormCastTilingData* tiling) |
| 32 | { | 28 | { |
| 33 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); | 29 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); |
| 34 | - this->numRow = tiling->num_row; | ||
| 35 | this->numCol = tiling->num_col; | 30 | this->numCol = tiling->num_col; |
| 31 | + this->numRow = tiling->num_row; | ||
| 36 | this->blockFactor = tiling->block_factor; | 32 | this->blockFactor = tiling->block_factor; |
| 37 | this->rowFactor = tiling->row_factor; | 33 | this->rowFactor = tiling->row_factor; |
| 38 | this->ubFactor = tiling->ub_factor; | 34 | this->ubFactor = tiling->ub_factor; |
| @@ -70,14 +66,19 @@ public: | |||
| 70 | Ppipe->InitBuffer(reduceFp32Buf, NUM_PER_REP_FP32 * sizeof(float)); | 66 | Ppipe->InitBuffer(reduceFp32Buf, NUM_PER_REP_FP32 * sizeof(float)); |
| 71 | } | 67 | } |
| 72 | 68 | ||
| 69 | + __aicore__ inline KernelAddRmsNormCastSplitD(TPipe* pipe) | ||
| 70 | + { | ||
| 71 | + Ppipe = pipe; | ||
| 72 | + } | ||
| 73 | + | ||
| 73 | __aicore__ inline void Process() | 74 | __aicore__ inline void Process() |
| 74 | { | 75 | { |
| 75 | uint32_t i_o_max = RmsNorm::CeilDiv(rowWork, rowFactor); | 76 | uint32_t i_o_max = RmsNorm::CeilDiv(rowWork, rowFactor); |
| 76 | uint32_t row_tail = rowWork - (i_o_max - 1) * rowFactor; | 77 | uint32_t row_tail = rowWork - (i_o_max - 1) * rowFactor; |
| 77 | uint32_t j_max = RmsNorm::CeilDiv(numCol, ubFactor); | 78 | uint32_t j_max = RmsNorm::CeilDiv(numCol, ubFactor); |
| 78 | uint32_t col_tail = numCol - (j_max - 1) * ubFactor; | 79 | uint32_t col_tail = numCol - (j_max - 1) * ubFactor; |
| 79 | - for (uint32_t i_o = 0; i_o < i_o_max - 1; i_o++) { | 80 | + for (uint32_t i_o_1 = 0; i_o_1 < i_o_max - 1; i_o_1++) { |
| 80 | - SubProcess(i_o, rowFactor, j_max, col_tail); | 81 | + SubProcess(i_o_1, rowFactor, j_max, col_tail); |
| 81 | } | 82 | } |
| 82 | SubProcess(i_o_max - 1, row_tail, j_max, col_tail); | 83 | SubProcess(i_o_max - 1, row_tail, j_max, col_tail); |
| 83 | } | 84 | } |
| @@ -89,15 +90,15 @@ public: | |||
| 89 | LocalTensor<float> rstdLocal = outQueueRstd.AllocTensor<float>(); | 90 | LocalTensor<float> rstdLocal = outQueueRstd.AllocTensor<float>(); |
| 90 | Duplicate(rstdLocal, (float)0.0, calc_row_num); | 91 | Duplicate(rstdLocal, (float)0.0, calc_row_num); |
| 91 | PipeBarrier<PIPE_V>(); | 92 | PipeBarrier<PIPE_V>(); |
| 92 | - for (uint32_t j = 0; j < j_max - 1; j++) { | 93 | + for (uint32_t j1 = 0; j1 < j_max - 1; j1++) { |
| 93 | - ComputeFormer(i_o, calc_row_num, j, rstdLocal, sumLocal, ubFactor); | 94 | + ComputeFormer(i_o, calc_row_num, j1, rstdLocal, sumLocal, ubFactor); |
| 94 | } | 95 | } |
| 95 | // do tail | 96 | // do tail |
| 96 | ComputeFormer(i_o, calc_row_num, j_max - 1, rstdLocal, sumLocal, col_tail); | 97 | ComputeFormer(i_o, calc_row_num, j_max - 1, rstdLocal, sumLocal, col_tail); |
| 97 | ComputeRstd(rstdLocal, calc_row_num); | 98 | ComputeRstd(rstdLocal, calc_row_num); |
| 98 | 99 | ||
| 99 | - for (uint32_t j = 0; j < j_max - 1; j++) { | 100 | + for (uint32_t j2 = 0; j2 < j_max - 1; j2++) { |
| 100 | - ComputeLatter(i_o, calc_row_num, j, rstdLocal, ubFactor); | 101 | + ComputeLatter(i_o, calc_row_num, j2, rstdLocal, ubFactor); |
| 101 | } | 102 | } |
| 102 | ComputeLatter(i_o, calc_row_num, j_max - 1, rstdLocal, col_tail); | 103 | ComputeLatter(i_o, calc_row_num, j_max - 1, rstdLocal, col_tail); |
| 103 | outQueueRstd.EnQue<float>(rstdLocal); | 104 | outQueueRstd.EnQue<float>(rstdLocal); |
| @@ -153,48 +154,48 @@ private: | |||
| 153 | } | 154 | } |
| 154 | 155 | ||
| 155 | __aicore__ inline void ComputeFormer( | 156 | __aicore__ inline void ComputeFormer( |
| 156 | - uint32_t i_o_idx, uint32_t calc_row_num, uint32_t j_idx, LocalTensor<float>& rstdLocal, | 157 | + uint32_t i_o_idx, uint32_t calc_row_num_1, uint32_t j_idx, LocalTensor<float>& rstdLocal, |
| 157 | LocalTensor<float>& sumLocal, uint32_t num) | 158 | LocalTensor<float>& sumLocal, uint32_t num) |
| 158 | { | 159 | { |
| 159 | - for (uint32_t i_i = 0; i_i < calc_row_num; i_i++) { | 160 | + for (uint32_t i_i = 0; i_i < calc_row_num_1; i_i++) { |
| 160 | CopyInAndAdd(i_o_idx * rowFactor + i_i, j_idx, num); | 161 | CopyInAndAdd(i_o_idx * rowFactor + i_i, j_idx, num); |
| 161 | ComputeSum(i_i, sumLocal, num); | 162 | ComputeSum(i_i, sumLocal, num); |
| 162 | } | 163 | } |
| 163 | - BlockReduceSumFP32(sumLocal, sumLocal, calc_row_num * NUM_PER_BLK_FP32); | 164 | + BlockReduceSumFP32(sumLocal, sumLocal, calc_row_num_1 * NUM_PER_BLK_FP32); |
| 164 | - Add(rstdLocal, rstdLocal, sumLocal, calc_row_num); | 165 | + Add(rstdLocal, rstdLocal, sumLocal, calc_row_num_1); |
| 165 | PipeBarrier<PIPE_V>(); | 166 | PipeBarrier<PIPE_V>(); |
| 166 | } | 167 | } |
| 167 | 168 | ||
| 168 | __aicore__ inline void ComputeSum(uint32_t i_i_idx, LocalTensor<float>& sumLocal, uint32_t num) | 169 | __aicore__ inline void ComputeSum(uint32_t i_i_idx, LocalTensor<float>& sumLocal, uint32_t num) |
| 169 | { | 170 | { |
| 170 | - LocalTensor<float> sqx = sqxBuf.Get<float>(); | 171 | + LocalTensor<float> sqxV1 = sqxBuf.Get<float>(); |
| 171 | LocalTensor<float> reduce_buf_local = reduceFp32Buf.Get<float>(); | 172 | LocalTensor<float> reduce_buf_local = reduceFp32Buf.Get<float>(); |
| 172 | if constexpr (is_same<T, half>::value || is_same<T, bfloat16_t>::value) { | 173 | if constexpr (is_same<T, half>::value || is_same<T, bfloat16_t>::value) { |
| 173 | - LocalTensor<float> x_fp32 = xFp32Buf.Get<float>(); | 174 | + LocalTensor<float> x_fp32_v1 = xFp32Buf.Get<float>(); |
| 174 | PipeBarrier<PIPE_V>(); | 175 | PipeBarrier<PIPE_V>(); |
| 175 | - Mul(sqx, x_fp32, x_fp32, num); | 176 | + Mul(sqxV1, x_fp32_v1, x_fp32_v1, num); |
| 176 | } else { | 177 | } else { |
| 177 | LocalTensor<T> xLocal = inQueueX.AllocTensor<float>(); | 178 | LocalTensor<T> xLocal = inQueueX.AllocTensor<float>(); |
| 178 | PipeBarrier<PIPE_V>(); | 179 | PipeBarrier<PIPE_V>(); |
| 179 | - Mul(sqx, xLocal, xLocal, num); | 180 | + Mul(sqxV1, xLocal, xLocal, num); |
| 180 | inQueueX.FreeTensor(xLocal); | 181 | inQueueX.FreeTensor(xLocal); |
| 181 | } | 182 | } |
| 182 | PipeBarrier<PIPE_V>(); | 183 | PipeBarrier<PIPE_V>(); |
| 183 | - Muls(sqx, sqx, avgFactor, num); | 184 | + Muls(sqxV1, sqxV1, avgFactor, num); |
| 184 | PipeBarrier<PIPE_V>(); | 185 | PipeBarrier<PIPE_V>(); |
| 185 | // 8 means 8 fp32 pre block | 186 | // 8 means 8 fp32 pre block |
| 186 | - ReduceSumFP32ToBlock(sumLocal[i_i_idx * 8], sqx, reduce_buf_local, num); | 187 | + ReduceSumFP32ToBlock(sumLocal[i_i_idx * 8], sqxV1, reduce_buf_local, num); |
| 187 | } | 188 | } |
| 188 | 189 | ||
| 189 | - __aicore__ inline void ComputeRstd(LocalTensor<float> rstdLocal, uint32_t num) | 190 | + __aicore__ inline void ComputeRstd(LocalTensor<float> rstdLocalV1, uint32_t num) |
| 190 | { | 191 | { |
| 191 | LocalTensor<float> reduce_buf_local = reduceFp32Buf.Get<float>(); | 192 | LocalTensor<float> reduce_buf_local = reduceFp32Buf.Get<float>(); |
| 192 | - Adds(rstdLocal, rstdLocal, epsilon, num); | 193 | + Adds(rstdLocalV1, rstdLocalV1, epsilon, num); |
| 193 | PipeBarrier<PIPE_V>(); | 194 | PipeBarrier<PIPE_V>(); |
| 194 | - Sqrt(rstdLocal, rstdLocal, num); | 195 | + Sqrt(rstdLocalV1, rstdLocalV1, num); |
| 195 | Duplicate(reduce_buf_local, ONE, num); | 196 | Duplicate(reduce_buf_local, ONE, num); |
| 196 | PipeBarrier<PIPE_V>(); | 197 | PipeBarrier<PIPE_V>(); |
| 197 | - Div(rstdLocal, reduce_buf_local, rstdLocal, num); | 198 | + Div(rstdLocalV1, reduce_buf_local, rstdLocalV1, num); |
| 198 | PipeBarrier<PIPE_V>(); | 199 | PipeBarrier<PIPE_V>(); |
| 199 | } | 200 | } |
| 200 | 201 | ||
| @@ -203,25 +204,25 @@ private: | |||
| 203 | { | 204 | { |
| 204 | CopyInGamma(j_idx, num); | 205 | CopyInGamma(j_idx, num); |
| 205 | LocalTensor<T> gammaLocal = inQueueGamma.DeQue<T>(); | 206 | LocalTensor<T> gammaLocal = inQueueGamma.DeQue<T>(); |
| 206 | - for (uint32_t i_i = 0; i_i < calc_row_num; i_i++) { | 207 | + for (uint32_t i_i_1 = 0; i_i_1 < calc_row_num; i_i_1++) { |
| 207 | - CopyInX(i_o_idx * rowFactor + i_i, j_idx, num); | 208 | + CopyInX(i_o_idx * rowFactor + i_i_1, j_idx, num); |
| 208 | - ComputeY(i_i, gammaLocal, rstdLocal, num, (i_o_idx * rowFactor + i_i) * numCol + j_idx * ubFactor); | 209 | + ComputeY(i_i_1, gammaLocal, rstdLocal, num, (i_o_idx * rowFactor + i_i_1) * numCol + j_idx * ubFactor); |
| 209 | - CopyOutY(i_o_idx * rowFactor + i_i, j_idx, num); | 210 | + CopyOutY(i_o_idx * rowFactor + i_i_1, j_idx, num); |
| 210 | } | 211 | } |
| 211 | inQueueGamma.FreeTensor(gammaLocal); | 212 | inQueueGamma.FreeTensor(gammaLocal); |
| 212 | } | 213 | } |
| 213 | 214 | ||
| 214 | - __aicore__ inline void CopyInGamma(uint32_t j_idx, uint32_t num) | 215 | + __aicore__ inline void CopyInGamma(uint32_t j_idx_1, uint32_t num) |
| 215 | { | 216 | { |
| 216 | LocalTensor<T> gammaLocal = inQueueGamma.AllocTensor<T>(); | 217 | LocalTensor<T> gammaLocal = inQueueGamma.AllocTensor<T>(); |
| 217 | - DataCopyCustom<T>(gammaLocal, gammaGm[j_idx * ubFactor], num); | 218 | + DataCopyCustom<T>(gammaLocal, gammaGm[j_idx_1 * ubFactor], num); |
| 218 | inQueueGamma.EnQue(gammaLocal); | 219 | inQueueGamma.EnQue(gammaLocal); |
| 219 | } | 220 | } |
| 220 | 221 | ||
| 221 | - __aicore__ inline void CopyInX(uint32_t i_idx, uint32_t j_idx, uint32_t num) | 222 | + __aicore__ inline void CopyInX(uint32_t i_idx, uint32_t j_idx_1, uint32_t num) |
| 222 | { | 223 | { |
| 223 | LocalTensor<T> xLocal = inQueueX.AllocTensor<T>(); | 224 | LocalTensor<T> xLocal = inQueueX.AllocTensor<T>(); |
| 224 | - DataCopyCustom<T>(xLocal, xGm[i_idx * numCol + j_idx * ubFactor], num); | 225 | + DataCopyCustom<T>(xLocal, xGm[i_idx * numCol + j_idx_1 * ubFactor], num); |
| 225 | inQueueX.EnQue<T>(xLocal); | 226 | inQueueX.EnQue<T>(xLocal); |
| 226 | if constexpr (is_same<T, half>::value || is_same<T, bfloat16_t>::value) { | 227 | if constexpr (is_same<T, half>::value || is_same<T, bfloat16_t>::value) { |
| 227 | LocalTensor<float> x_fp32 = xFp32Buf.Get<float>(); | 228 | LocalTensor<float> x_fp32 = xFp32Buf.Get<float>(); |
| @@ -233,14 +234,14 @@ private: | |||
| 233 | } | 234 | } |
| 234 | 235 | ||
| 235 | __aicore__ inline void ComputeY( | 236 | __aicore__ inline void ComputeY( |
| 236 | - uint32_t i_i_idx, LocalTensor<half>& gammaLocal, LocalTensor<float>& rstdLocal, uint32_t num, uint32_t gmOffset) | 237 | + uint32_t i_i_idx_1, LocalTensor<half>& gammaLocal, LocalTensor<float>& rstdLocal, uint32_t num, uint32_t gmOffset) |
| 237 | { | 238 | { |
| 238 | LocalTensor<float> x_fp32 = xFp32Buf.Get<float>(); | 239 | LocalTensor<float> x_fp32 = xFp32Buf.Get<float>(); |
| 239 | LocalTensor<float> sqx = sqxBuf.Get<float>(); | 240 | LocalTensor<float> sqx = sqxBuf.Get<float>(); |
| 240 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 241 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 241 | SetFlag<HardEvent::V_S>(event_v_s); | 242 | SetFlag<HardEvent::V_S>(event_v_s); |
| 242 | WaitFlag<HardEvent::V_S>(event_v_s); | 243 | WaitFlag<HardEvent::V_S>(event_v_s); |
| 243 | - float rstdValue = rstdLocal.GetValue(i_i_idx); | 244 | + float rstdValue = rstdLocal.GetValue(i_i_idx_1); |
| 244 | event_t event_s_v = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | 245 | event_t event_s_v = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); |
| 245 | SetFlag<HardEvent::S_V>(event_s_v); | 246 | SetFlag<HardEvent::S_V>(event_s_v); |
| 246 | WaitFlag<HardEvent::S_V>(event_s_v); | 247 | WaitFlag<HardEvent::S_V>(event_s_v); |
| @@ -280,19 +281,19 @@ private: | |||
| 280 | PipeBarrier<PIPE_V>(); | 281 | PipeBarrier<PIPE_V>(); |
| 281 | Muls(x_fp32, x_fp32, rstdValue, num); | 282 | Muls(x_fp32, x_fp32, rstdValue, num); |
| 282 | PipeBarrier<PIPE_V>(); | 283 | PipeBarrier<PIPE_V>(); |
| 283 | - LocalTensor<bfloat16_t> yLocal = outQueueY.AllocTensor<bfloat16_t>(); | 284 | + LocalTensor<bfloat16_t> yLocalV1 = outQueueY.AllocTensor<bfloat16_t>(); |
| 284 | Cast(sqx, gammaLocal, RoundMode::CAST_NONE, num); | 285 | Cast(sqx, gammaLocal, RoundMode::CAST_NONE, num); |
| 285 | PipeBarrier<PIPE_V>(); | 286 | PipeBarrier<PIPE_V>(); |
| 286 | Mul(x_fp32, x_fp32, sqx, num); | 287 | Mul(x_fp32, x_fp32, sqx, num); |
| 287 | PipeBarrier<PIPE_V>(); | 288 | PipeBarrier<PIPE_V>(); |
| 288 | - Cast(yLocal, x_fp32, RoundMode::CAST_RINT, num); | 289 | + Cast(yLocalV1, x_fp32, RoundMode::CAST_RINT, num); |
| 289 | PipeBarrier<PIPE_V>(); | 290 | PipeBarrier<PIPE_V>(); |
| 290 | - outQueueY.EnQue<bfloat16_t>(yLocal); | 291 | + outQueueY.EnQue<bfloat16_t>(yLocalV1); |
| 291 | - Cast(x_fp32, yLocal, RoundMode::CAST_NONE, num); | 292 | + Cast(x_fp32, yLocalV1, RoundMode::CAST_NONE, num); |
| 292 | PipeBarrier<PIPE_V>(); | 293 | PipeBarrier<PIPE_V>(); |
| 293 | - event_t event_v_mte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 294 | + event_t event_v_mte3_1 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| 294 | - SetFlag<HardEvent::V_MTE3>(event_v_mte3); | 295 | + SetFlag<HardEvent::V_MTE3>(event_v_mte3_1); |
| 295 | - WaitFlag<HardEvent::V_MTE3>(event_v_mte3); | 296 | + WaitFlag<HardEvent::V_MTE3>(event_v_mte3_1); |
| 296 | DataCopyCustom<float>(y1Gm[gmOffset], x_fp32, num); | 297 | DataCopyCustom<float>(y1Gm[gmOffset], x_fp32, num); |
| 297 | event_t event_mte3_v = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); | 298 | event_t event_mte3_v = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); |
| 298 | SetFlag<HardEvent::MTE3_V>(event_mte3_v); | 299 | SetFlag<HardEvent::MTE3_V>(event_mte3_v); |
| @@ -318,23 +319,23 @@ private: | |||
| 318 | private: | 319 | private: |
| 319 | TPipe* Ppipe = nullptr; | 320 | TPipe* Ppipe = nullptr; |
| 320 | // create queues for input, in this case depth is equal to buffer num | 321 | // create queues for input, in this case depth is equal to buffer num |
| 321 | - TQue<QuePosition::VECIN, BUFFER_NUM> inQueueX; | ||
| 322 | TQue<QuePosition::VECIN, BUFFER_NUM> inQueueGamma; | 322 | TQue<QuePosition::VECIN, BUFFER_NUM> inQueueGamma; |
| 323 | + TQue<QuePosition::VECIN, BUFFER_NUM> inQueueX; | ||
| 323 | // create queues for output, in this case depth is equal to buffer num | 324 | // create queues for output, in this case depth is equal to buffer num |
| 324 | - TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueY; | ||
| 325 | TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueRstd; | 325 | TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueRstd; |
| 326 | - TBuf<TPosition::VECCALC> xFp32Buf; | 326 | + TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueY; |
| 327 | TBuf<TPosition::VECCALC> sqxBuf; | 327 | TBuf<TPosition::VECCALC> sqxBuf; |
| 328 | + TBuf<TPosition::VECCALC> xFp32Buf; | ||
| 328 | TBuf<TPosition::VECCALC> sumBuf; | 329 | TBuf<TPosition::VECCALC> sumBuf; |
| 329 | TBuf<TPosition::VECCALC> reduceFp32Buf; | 330 | TBuf<TPosition::VECCALC> reduceFp32Buf; |
| 330 | 331 | ||
| 331 | - GlobalTensor<T> x1Gm; | ||
| 332 | GlobalTensor<T> x2Gm; | 332 | GlobalTensor<T> x2Gm; |
| 333 | - GlobalTensor<T> gammaGm; | 333 | + GlobalTensor<T> x1Gm; |
| 334 | GlobalTensor<float> y1Gm; | 334 | GlobalTensor<float> y1Gm; |
| 335 | + GlobalTensor<T> gammaGm; | ||
| 335 | GlobalTensor<T> y2Gm; | 336 | GlobalTensor<T> y2Gm; |
| 336 | - GlobalTensor<float> rstdGm; | ||
| 337 | GlobalTensor<T> xGm; | 337 | GlobalTensor<T> xGm; |
| 338 | + GlobalTensor<float> rstdGm; | ||
| 338 | 339 | ||
| 339 | uint32_t numRow; | 340 | uint32_t numRow; |
| 340 | uint32_t numCol; | 341 | uint32_t numCol; |
| @@ -244,8 +244,8 @@ protected: | |||
| 244 | // Order: GetShapeAttrsInfo->GetPlatformInfo-> | 244 | // Order: GetShapeAttrsInfo->GetPlatformInfo-> |
| 245 | // IsCapable->DoOpTiling->DoLibApiTiling-> | 245 | // IsCapable->DoOpTiling->DoLibApiTiling-> |
| 246 | // GetWorkspaceSize->PostTiling->GetTilingKey | 246 | // GetWorkspaceSize->PostTiling->GetTilingKey |
| 247 | - ge::graphStatus GetShapeAttrsInfo() override; | ||
| 248 | ge::graphStatus GetPlatformInfo() override; | 247 | ge::graphStatus GetPlatformInfo() override; |
| 248 | + ge::graphStatus GetShapeAttrsInfo() override; | ||
| 249 | bool IsCapable() override; | 249 | bool IsCapable() override; |
| 250 | ge::graphStatus DoOpTiling() override; | 250 | ge::graphStatus DoOpTiling() override; |
| 251 | ge::graphStatus DoLibApiTiling() override; | 251 | ge::graphStatus DoLibApiTiling() override; |
| @@ -15,34 +15,34 @@ | |||
| 15 | 15 | ||
| 16 | using namespace AscendC; | 16 | using namespace AscendC; |
| 17 | 17 | ||
| 18 | -constexpr uint32_t MAX_REP_NUM = 255; | ||
| 19 | constexpr uint32_t ELEM_PER_REP_FP32 = 64; | 18 | constexpr uint32_t ELEM_PER_REP_FP32 = 64; |
| 20 | -constexpr uint32_t ELEM_PER_BLK_FP32 = 8; | 19 | +constexpr uint32_t MAX_REP_NUM = 255; |
| 21 | constexpr float ZERO = 0; | 20 | constexpr float ZERO = 0; |
| 21 | +constexpr uint32_t ELEM_PER_BLK_FP32 = 8; | ||
| 22 | constexpr int32_t HALf_INTERVAL = 2; | 22 | constexpr int32_t HALf_INTERVAL = 2; |
| 23 | constexpr int32_t INDEX_TWO = 2; | 23 | constexpr int32_t INDEX_TWO = 2; |
| 24 | constexpr int32_t INDEX_FOUR = 4; | 24 | constexpr int32_t INDEX_FOUR = 4; |
| 25 | -constexpr int32_t INDEX_EIGHT = 8; | ||
| 26 | constexpr int32_t INDEX_SIXTEEN = 16; | 25 | constexpr int32_t INDEX_SIXTEEN = 16; |
| 26 | +constexpr int32_t INDEX_EIGHT = 8; | ||
| 27 | 27 | ||
| 28 | __aicore__ inline void ReduceSumForSmallReduceDimPreRepeat( | 28 | __aicore__ inline void ReduceSumForSmallReduceDimPreRepeat( |
| 29 | const LocalTensor<float>& dstLocal, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, | 29 | const LocalTensor<float>& dstLocal, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, |
| 30 | - const uint32_t elemNum, const uint32_t numLastDim, const uint32_t tailCount, const uint32_t repeat, | 30 | + const uint32_t elemNum, const uint32_t numLastDim, const uint32_t tailCount, const uint32_t repeat1, |
| 31 | const uint8_t repStride) | 31 | const uint8_t repStride) |
| 32 | { | 32 | { |
| 33 | uint32_t elemIndex = 0; | 33 | uint32_t elemIndex = 0; |
| 34 | for (; elemIndex + ELEM_PER_REP_FP32 <= numLastDim; elemIndex += ELEM_PER_REP_FP32) { | 34 | for (; elemIndex + ELEM_PER_REP_FP32 <= numLastDim; elemIndex += ELEM_PER_REP_FP32) { |
| 35 | - Add(tmpLocal, srcLocal[elemIndex], tmpLocal, elemNum, repeat, | 35 | + Add(tmpLocal, srcLocal[elemIndex], tmpLocal, elemNum, repeat1, |
| 36 | {1, 1, 1, ELEM_PER_BLK_FP32, repStride, ELEM_PER_BLK_FP32}); | 36 | {1, 1, 1, ELEM_PER_BLK_FP32, repStride, ELEM_PER_BLK_FP32}); |
| 37 | PipeBarrier<PIPE_V>(); | 37 | PipeBarrier<PIPE_V>(); |
| 38 | } | 38 | } |
| 39 | if (unlikely(tailCount != 0)) { | 39 | if (unlikely(tailCount != 0)) { |
| 40 | - Add(tmpLocal, srcLocal[elemIndex], tmpLocal, tailCount, repeat, | 40 | + Add(tmpLocal, srcLocal[elemIndex], tmpLocal, tailCount, repeat1, |
| 41 | {1, 1, 1, ELEM_PER_BLK_FP32, repStride, ELEM_PER_BLK_FP32}); | 41 | {1, 1, 1, ELEM_PER_BLK_FP32, repStride, ELEM_PER_BLK_FP32}); |
| 42 | } | 42 | } |
| 43 | PipeBarrier<PIPE_V>(); | 43 | PipeBarrier<PIPE_V>(); |
| 44 | AscendCUtils::SetMask<float>(ELEM_PER_REP_FP32); // set mask = 64 | 44 | AscendCUtils::SetMask<float>(ELEM_PER_REP_FP32); // set mask = 64 |
| 45 | - WholeReduceSum<float, false>(dstLocal, tmpLocal, MASK_PLACEHOLDER, repeat, 1, 1, ELEM_PER_BLK_FP32); | 45 | + WholeReduceSum<float, false>(dstLocal, tmpLocal, MASK_PLACEHOLDER, repeat1, 1, 1, ELEM_PER_BLK_FP32); |
| 46 | } | 46 | } |
| 47 | 47 | ||
| 48 | /* | 48 | /* |
| @@ -54,44 +54,27 @@ __aicore__ inline void ReduceSumForSmallReduceDim( | |||
| 54 | const uint32_t numLastDimAligned, const uint32_t numLastDim, const uint32_t tailCount, const uint32_t repeat, | 54 | const uint32_t numLastDimAligned, const uint32_t numLastDim, const uint32_t tailCount, const uint32_t repeat, |
| 55 | const uint8_t repStride) | 55 | const uint8_t repStride) |
| 56 | { | 56 | { |
| 57 | - uint32_t repeatTimes = repeat / MAX_REP_NUM; | 57 | + uint32_t smallRepeatTimes = repeat / MAX_REP_NUM; |
| 58 | - if (repeatTimes == 0) { | 58 | + if (smallRepeatTimes == 0) { |
| 59 | ReduceSumForSmallReduceDimPreRepeat( | 59 | ReduceSumForSmallReduceDimPreRepeat( |
| 60 | dstLocal, srcLocal, tmpLocal, ELEM_PER_REP_FP32, numLastDim, tailCount, repeat, repStride); | 60 | dstLocal, srcLocal, tmpLocal, ELEM_PER_REP_FP32, numLastDim, tailCount, repeat, repStride); |
| 61 | } else { | 61 | } else { |
| 62 | - uint32_t repTailNum = repeat % MAX_REP_NUM; | 62 | + uint32_t smallRepTailNum = repeat % MAX_REP_NUM; |
| 63 | - uint32_t repIndex = 0; | 63 | + uint32_t smallRepIndex = 0; |
| 64 | - uint32_t repElem; | 64 | + uint32_t smallRepElem; |
| 65 | - for (; repIndex + MAX_REP_NUM <= repeat; repIndex += MAX_REP_NUM) { | 65 | + for (; smallRepIndex + MAX_REP_NUM <= repeat; smallRepIndex += MAX_REP_NUM) { |
| 66 | ReduceSumForSmallReduceDimPreRepeat( | 66 | ReduceSumForSmallReduceDimPreRepeat( |
| 67 | - dstLocal[repIndex], srcLocal[repIndex * numLastDimAligned], tmpLocal[repIndex * ELEM_PER_REP_FP32], | 67 | + dstLocal[smallRepIndex], srcLocal[smallRepIndex * numLastDimAligned], tmpLocal[smallRepIndex * ELEM_PER_REP_FP32], |
| 68 | ELEM_PER_REP_FP32, numLastDim, tailCount, MAX_REP_NUM, repStride); | 68 | ELEM_PER_REP_FP32, numLastDim, tailCount, MAX_REP_NUM, repStride); |
| 69 | } | 69 | } |
| 70 | - if (repTailNum != 0) { | 70 | + if (smallRepTailNum != 0) { |
| 71 | ReduceSumForSmallReduceDimPreRepeat( | 71 | ReduceSumForSmallReduceDimPreRepeat( |
| 72 | - dstLocal[repIndex], srcLocal[repIndex * numLastDimAligned], tmpLocal[repIndex * ELEM_PER_REP_FP32], | 72 | + dstLocal[smallRepIndex], srcLocal[smallRepIndex * numLastDimAligned], tmpLocal[smallRepIndex * ELEM_PER_REP_FP32], |
| 73 | - ELEM_PER_REP_FP32, numLastDim, tailCount, repTailNum, repStride); | 73 | + ELEM_PER_REP_FP32, numLastDim, tailCount, smallRepTailNum, repStride); |
| 74 | } | 74 | } |
| 75 | } | 75 | } |
| 76 | } | 76 | } |
| 77 | 77 | ||
| 78 | -/* | ||
| 79 | - * reduce dim form (N, D) to (N, 1) | ||
| 80 | - * this reduce sum is for small reduce dim, require D < 255 * 8. | ||
| 81 | - * size of tmpLocal: (N, 64) | ||
| 82 | - */ | ||
| 83 | -__aicore__ inline void ReduceSumMultiN( | ||
| 84 | - const LocalTensor<float>& dstLocal, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, | ||
| 85 | - const uint32_t numRow, const uint32_t numCol, const uint32_t numColAlign) | ||
| 86 | -{ | ||
| 87 | - const uint32_t tailCount = numCol % ELEM_PER_REP_FP32; | ||
| 88 | - const uint32_t repeat = numRow; | ||
| 89 | - const uint8_t repStride = numColAlign / ELEM_PER_BLK_FP32; | ||
| 90 | - Duplicate(tmpLocal, ZERO, numRow * ELEM_PER_REP_FP32); | ||
| 91 | - PipeBarrier<PIPE_V>(); | ||
| 92 | - ReduceSumForSmallReduceDim(dstLocal, srcLocal, tmpLocal, numColAlign, numCol, tailCount, repeat, repStride); | ||
| 93 | -} | ||
| 94 | - | ||
| 95 | __aicore__ inline int32_t findPowerTwo(int32_t n) | 78 | __aicore__ inline int32_t findPowerTwo(int32_t n) |
| 96 | { | 79 | { |
| 97 | // find max power of 2 no more than n (32 bit) | 80 | // find max power of 2 no more than n (32 bit) |
| @@ -103,6 +86,23 @@ __aicore__ inline int32_t findPowerTwo(int32_t n) | |||
| 103 | return (n + 1) >> 1; | 86 | return (n + 1) >> 1; |
| 104 | } | 87 | } |
| 105 | 88 | ||
| 89 | +/* | ||
| 90 | + * reduce dim form (N, D) to (N, 1) | ||
| 91 | + * this reduce sum is for small reduce dim, require D < 255 * 8. | ||
| 92 | + * size of tmpLocal: (N, 64) | ||
| 93 | + */ | ||
| 94 | +__aicore__ inline void ReduceSumMultiN( | ||
| 95 | + const LocalTensor<float>& dstLocal2, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, | ||
| 96 | + const uint32_t numRow, const uint32_t numCol, const uint32_t numColAlign) | ||
| 97 | +{ | ||
| 98 | + const uint32_t tailCount = numCol % ELEM_PER_REP_FP32; | ||
| 99 | + const uint32_t repeat = numRow; | ||
| 100 | + const uint8_t repStride = numColAlign / ELEM_PER_BLK_FP32; | ||
| 101 | + Duplicate(tmpLocal, ZERO, numRow * ELEM_PER_REP_FP32); | ||
| 102 | + PipeBarrier<PIPE_V>(); | ||
| 103 | + ReduceSumForSmallReduceDim(dstLocal2, srcLocal, tmpLocal, numColAlign, numCol, tailCount, repeat, repStride); | ||
| 104 | +} | ||
| 105 | + | ||
| 106 | __aicore__ inline void ReduceSumHalfInterval( | 106 | __aicore__ inline void ReduceSumHalfInterval( |
| 107 | const LocalTensor<float>& dst_local, const LocalTensor<float>& src_local, int32_t count) | 107 | const LocalTensor<float>& dst_local, const LocalTensor<float>& src_local, int32_t count) |
| 108 | { | 108 | { |
| @@ -133,18 +133,18 @@ __aicore__ inline void ReduceSumHalfInterval( | |||
| 133 | PipeBarrier<PIPE_V>(); | 133 | PipeBarrier<PIPE_V>(); |
| 134 | } | 134 | } |
| 135 | 135 | ||
| 136 | -__aicore__ inline float ReduceSumHalfInterval(const LocalTensor<float>& src_local, int32_t count) | 136 | +__aicore__ inline float ReduceSumHalfInterval(const LocalTensor<float>& src_local1, int32_t count) |
| 137 | { | 137 | { |
| 138 | if (likely(count > ELEM_PER_REP_FP32)) { | 138 | if (likely(count > ELEM_PER_REP_FP32)) { |
| 139 | int32_t bodyCount = findPowerTwo(count); | 139 | int32_t bodyCount = findPowerTwo(count); |
| 140 | int32_t tailCount = count - bodyCount; | 140 | int32_t tailCount = count - bodyCount; |
| 141 | if (tailCount > 0) { | 141 | if (tailCount > 0) { |
| 142 | - Add(src_local, src_local, src_local[bodyCount], tailCount); | 142 | + Add(src_local1, src_local1, src_local1[bodyCount], tailCount); |
| 143 | PipeBarrier<PIPE_V>(); | 143 | PipeBarrier<PIPE_V>(); |
| 144 | } | 144 | } |
| 145 | while (bodyCount > ELEM_PER_REP_FP32) { | 145 | while (bodyCount > ELEM_PER_REP_FP32) { |
| 146 | bodyCount = bodyCount / HALf_INTERVAL; | 146 | bodyCount = bodyCount / HALf_INTERVAL; |
| 147 | - Add(src_local, src_local, src_local[bodyCount], bodyCount); | 147 | + Add(src_local1, src_local1, src_local1[bodyCount], bodyCount); |
| 148 | PipeBarrier<PIPE_V>(); | 148 | PipeBarrier<PIPE_V>(); |
| 149 | } | 149 | } |
| 150 | 150 | ||
| @@ -154,14 +154,14 @@ __aicore__ inline float ReduceSumHalfInterval(const LocalTensor<float>& src_loca | |||
| 154 | } | 154 | } |
| 155 | 155 | ||
| 156 | if (g_coreType == AIV) { | 156 | if (g_coreType == AIV) { |
| 157 | - WholeReduceSum<float, false>(src_local, src_local, MASK_PLACEHOLDER, 1, 0, 1, 0); | 157 | + WholeReduceSum<float, false>(src_local1, src_local1, MASK_PLACEHOLDER, 1, 0, 1, 0); |
| 158 | } | 158 | } |
| 159 | 159 | ||
| 160 | - WholeReduceSum<float, false>(src_local, src_local, MASK_PLACEHOLDER, 1, 1, 1, DEFAULT_REPEAT_STRIDE); | 160 | + WholeReduceSum<float, false>(src_local1, src_local1, MASK_PLACEHOLDER, 1, 1, 1, DEFAULT_REPEAT_STRIDE); |
| 161 | 161 | ||
| 162 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 162 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 163 | SetFlag<HardEvent::V_S>(event_v_s); | 163 | SetFlag<HardEvent::V_S>(event_v_s); |
| 164 | WaitFlag<HardEvent::V_S>(event_v_s); | 164 | WaitFlag<HardEvent::V_S>(event_v_s); |
| 165 | - return src_local.GetValue(0); | 165 | + return src_local1.GetValue(0); |
| 166 | } | 166 | } |
| 167 | 167 | ||
| @@ -136,7 +136,7 @@ bool AddRmsNormDynamicQuantV2TilingHelper::DoBlockTiling() | |||
| 136 | this->firstDimPerCore_ = Ops::Base::CeilDiv(this->numFirstDim_, this->useCore_); | 136 | this->firstDimPerCore_ = Ops::Base::CeilDiv(this->numFirstDim_, this->useCore_); |
| 137 | this->firstDimPerCoreTail_ = this->numFirstDim_ - this->firstDimPerCore_ * (this->useCore_ - 1); | 137 | this->firstDimPerCoreTail_ = this->numFirstDim_ - this->firstDimPerCore_ * (this->useCore_ - 1); |
| 138 | OP_LOGI( | 138 | OP_LOGI( |
| 139 | - "DoBlockTiling", "BlockTiling Factor: useCore_: %lu, firstDimPerCore_: %lu, firstDimPerCoreTail_: %lu", | 139 | + "DoBlockTiling", "BlockTiling Factor: useCore_ is : %lu, firstDimPerCore_ is : %lu, firstDimPerCoreTail_ is : %lu", |
| 140 | this->useCore_, this->firstDimPerCore_, this->firstDimPerCoreTail_); | 140 | this->useCore_, this->firstDimPerCore_, this->firstDimPerCoreTail_); |
| 141 | return true; | 141 | return true; |
| 142 | } | 142 | } |
| @@ -203,17 +203,17 @@ bool AddRmsNormDynamicQuantV2TilingHelper::GetShapeInfo() | |||
| 203 | (smooth2Exist && smooth2Shape->GetStorageShape() != gammaShape), | 203 | (smooth2Exist && smooth2Shape->GetStorageShape() != gammaShape), |
| 204 | OP_LOGE(context_->GetNodeName(), "GammaShape is not same to smooth2Shape."), return false); | 204 | OP_LOGE(context_->GetNodeName(), "GammaShape is not same to smooth2Shape."), return false); |
| 205 | 205 | ||
| 206 | - uint64_t numRow = 1; | 206 | + uint64_t numRowV2 = 1; |
| 207 | - uint64_t numCol = 1; | 207 | + uint64_t numColV2 = 1; |
| 208 | for (size_t i = 0; i < xDimNum - gammaDimNum; i++) { | 208 | for (size_t i = 0; i < xDimNum - gammaDimNum; i++) { |
| 209 | - numRow *= xShape.GetDim(i); | 209 | + numRowV2 *= xShape.GetDim(i); |
| 210 | } | 210 | } |
| 211 | for (size_t i = 0; i < gammaDimNum; i++) { | 211 | for (size_t i = 0; i < gammaDimNum; i++) { |
| 212 | - numCol *= gammaShape.GetDim(i); | 212 | + numColV2 *= gammaShape.GetDim(i); |
| 213 | } | 213 | } |
| 214 | - this->numFirstDim_ = numRow; | 214 | + this->numFirstDim_ = numRowV2; |
| 215 | - this->numLastDim_ = numCol; | 215 | + this->numLastDim_ = numColV2; |
| 216 | - this->numLastDimAligned_ = Ops::Base::CeilDiv(numCol, static_cast<uint64_t>(BLOCK_SIZE)) * BLOCK_SIZE; | 216 | + this->numLastDimAligned_ = Ops::Base::CeilDiv(numColV2, static_cast<uint64_t>(BLOCK_SIZE)) * BLOCK_SIZE; |
| 217 | this->avgFactor_ = 1.0 / ((float)this->numLastDim_); | 217 | this->avgFactor_ = 1.0 / ((float)this->numLastDim_); |
| 218 | 218 | ||
| 219 | OP_LOGI("GetShapeInfo", "[N, D] = [%lu, %lu]", this->numFirstDim_, this->numLastDim_); | 219 | OP_LOGI("GetShapeInfo", "[N, D] = [%lu, %lu]", this->numFirstDim_, this->numLastDim_); |
| @@ -84,8 +84,8 @@ private: | |||
| 84 | 84 | ||
| 85 | uint64_t useCore_{1}; | 85 | uint64_t useCore_{1}; |
| 86 | uint64_t numFirstDim_{1}; | 86 | uint64_t numFirstDim_{1}; |
| 87 | - uint64_t numLastDim_{1}; | ||
| 88 | uint64_t numLastDimAligned_{1}; | 87 | uint64_t numLastDimAligned_{1}; |
| 88 | + uint64_t numLastDim_{1}; | ||
| 89 | uint64_t firstDimPerCore_{1}; | 89 | uint64_t firstDimPerCore_{1}; |
| 90 | uint64_t firstDimPerCoreTail_{1}; | 90 | uint64_t firstDimPerCoreTail_{1}; |
| 91 | uint64_t firstDimPerLoop_{1}; | 91 | uint64_t firstDimPerLoop_{1}; |
| @@ -85,14 +85,14 @@ public: | |||
| 85 | } | 85 | } |
| 86 | { | 86 | { |
| 87 | scalesLocalOut = scalesQue.template AllocTensor<float>(); | 87 | scalesLocalOut = scalesQue.template AllocTensor<float>(); |
| 88 | - for (int32_t innerIdx = 0; innerIdx < outLoopTail; ++innerIdx) { | 88 | + for (int32_t innerIdxTail = 0; innerIdxTail < outLoopTail; ++innerIdxTail) { |
| 89 | CopyInX1X2(gmOffset); | 89 | CopyInX1X2(gmOffset); |
| 90 | AddSingleRow(gmOffset); | 90 | AddSingleRow(gmOffset); |
| 91 | CopyInGamma(); | 91 | CopyInGamma(); |
| 92 | ComputeRmsNorm(gmOffset); | 92 | ComputeRmsNorm(gmOffset); |
| 93 | CopyOutRmsNormAndCast(gmOffset); | 93 | CopyOutRmsNormAndCast(gmOffset); |
| 94 | CopyInSmooth(); | 94 | CopyInSmooth(); |
| 95 | - ComputeDynamicQuant(innerIdx, scalesLocalOut, gmOffset); | 95 | + ComputeDynamicQuant(innerIdxTail, scalesLocalOut, gmOffset); |
| 96 | CopyOut(gmOffset); | 96 | CopyOut(gmOffset); |
| 97 | gmOffset += this->numLastDim; | 97 | gmOffset += this->numLastDim; |
| 98 | } | 98 | } |
| @@ -47,14 +47,14 @@ TILING_DATA_FIELD_DEF(uint64_t, numM); //A | |||
| 47 | TILING_DATA_FIELD_DEF(uint64_t, numN); //R | 47 | TILING_DATA_FIELD_DEF(uint64_t, numN); //R |
| 48 | TILING_DATA_FIELD_DEF(uint64_t, baseM); //ubfactor ub处理a的大小 | 48 | TILING_DATA_FIELD_DEF(uint64_t, baseM); //ubfactor ub处理a的大小 |
| 49 | TILING_DATA_FIELD_DEF(uint64_t, baseN); //全载时=R | 49 | TILING_DATA_FIELD_DEF(uint64_t, baseN); //全载时=R |
| 50 | -TILING_DATA_FIELD_DEF(uint64_t, baseNDtypeAlign); //R对32B对齐的个数 | ||
| 51 | TILING_DATA_FIELD_DEF(uint64_t, baseNReduceAlign); | 50 | TILING_DATA_FIELD_DEF(uint64_t, baseNReduceAlign); |
| 52 | -TILING_DATA_FIELD_DEF(uint64_t, powerSplit); //binaryAdd 二分折叠点 | 51 | +TILING_DATA_FIELD_DEF(uint64_t, baseNDtypeAlign); //R对32B对齐的个数 |
| 53 | TILING_DATA_FIELD_DEF(uint64_t, powerLoop); | 52 | TILING_DATA_FIELD_DEF(uint64_t, powerLoop); |
| 53 | +TILING_DATA_FIELD_DEF(uint64_t, powerSplit); //binaryAdd 二分折叠点 | ||
| 54 | TILING_DATA_FIELD_DEF(uint64_t, mPerCore); //blockFactor 单核处理a的大小 | 54 | TILING_DATA_FIELD_DEF(uint64_t, mPerCore); //blockFactor 单核处理a的大小 |
| 55 | TILING_DATA_FIELD_DEF(uint64_t, mLastCore); //blockTail 尾核处理a的大小 | 55 | TILING_DATA_FIELD_DEF(uint64_t, mLastCore); //blockTail 尾核处理a的大小 |
| 56 | -TILING_DATA_FIELD_DEF(float, epsilon); | ||
| 57 | TILING_DATA_FIELD_DEF(float, avgFactor); | 56 | TILING_DATA_FIELD_DEF(float, avgFactor); |
| 57 | +TILING_DATA_FIELD_DEF(float, epsilon); | ||
| 58 | END_TILING_DATA_DEF; | 58 | END_TILING_DATA_DEF; |
| 59 | 59 | ||
| 60 | REGISTER_TILING_DATA_CLASS(AddRmsNormQuant, AddRMSNormQuantTilingData) | 60 | REGISTER_TILING_DATA_CLASS(AddRmsNormQuant, AddRMSNormQuantTilingData) |
| @@ -156,8 +156,8 @@ struct AddRmsNormQuantRegbaseTilingParams { | |||
| 156 | uint64_t baseNQuantAlign{0}; | 156 | uint64_t baseNQuantAlign{0}; |
| 157 | uint64_t baseNReduceAlign{0}; | 157 | uint64_t baseNReduceAlign{0}; |
| 158 | uint64_t reduceBufLenAlign{0}; | 158 | uint64_t reduceBufLenAlign{0}; |
| 159 | - uint64_t powerSplit{0}; | ||
| 160 | uint64_t powerLoop{0}; | 159 | uint64_t powerLoop{0}; |
| 160 | + uint64_t powerSplit{0}; | ||
| 161 | uint64_t mPerCore{0}; | 161 | uint64_t mPerCore{0}; |
| 162 | uint64_t mLastCore{0}; | 162 | uint64_t mLastCore{0}; |
| 163 | uint64_t usedCoreNum{0}; | 163 | uint64_t usedCoreNum{0}; |
| @@ -166,8 +166,8 @@ struct AddRmsNormQuantRegbaseTilingParams { | |||
| 166 | // Tiling key parmas | 166 | // Tiling key parmas |
| 167 | uint64_t tilingType{0}; | 167 | uint64_t tilingType{0}; |
| 168 | 168 | ||
| 169 | - float epsilon{0}; | ||
| 170 | float avgFactor{0}; | 169 | float avgFactor{0}; |
| 170 | + float epsilon{0}; | ||
| 171 | uint32_t quantBufCnt{0}; | 171 | uint32_t quantBufCnt{0}; |
| 172 | bool divMode{false}; | 172 | bool divMode{false}; |
| 173 | bool hasScales2{false}; | 173 | bool hasScales2{false}; |
| @@ -31,13 +31,13 @@ using namespace op; | |||
| 31 | extern "C" { | 31 | extern "C" { |
| 32 | 32 | ||
| 33 | 33 | ||
| 34 | -constexpr int IDX_INT8 = 2; | 34 | +constexpr int IDX_INT8_QUANT = 2; |
| 35 | -constexpr int IDX_HIFLOAT8 = 34; | 35 | +constexpr int IDX_HIFLOAT8_QUANT = 34; |
| 36 | -constexpr int IDX_FLOAT8_E5M2 = 35; | 36 | +constexpr int IDX_FLOAT8_E5M2_QUANT = 35; |
| 37 | -constexpr int IDX_FLOAT8_E4M3FN = 36; | 37 | +constexpr int IDX_FLOAT8_E4M3FN_QUANT = 36; |
| 38 | -static std::map<op::DataType, int> dstTypeMap = {{op::DataType::DT_INT8, IDX_INT8}, | 38 | +static std::map<op::DataType, int> dstTypeMapQuant = {{op::DataType::DT_INT8, IDX_INT8_QUANT}, |
| 39 | - {op::DataType::DT_HIFLOAT8, IDX_HIFLOAT8}, {op::DataType::DT_FLOAT8_E5M2, IDX_FLOAT8_E5M2}, | 39 | + {op::DataType::DT_HIFLOAT8, IDX_HIFLOAT8_QUANT}, {op::DataType::DT_FLOAT8_E5M2, IDX_FLOAT8_E5M2_QUANT}, |
| 40 | - {op::DataType::DT_FLOAT8_E4M3FN, IDX_FLOAT8_E4M3FN}}; | 40 | + {op::DataType::DT_FLOAT8_E4M3FN, IDX_FLOAT8_E4M3FN_QUANT}}; |
| 41 | namespace AddRmsNormQuantACLNN { | 41 | namespace AddRmsNormQuantACLNN { |
| 42 | constexpr int IDX_0 = 0; | 42 | constexpr int IDX_0 = 0; |
| 43 | constexpr int IDX_1 = 1; | 43 | constexpr int IDX_1 = 1; |
| @@ -177,7 +177,7 @@ aclnnStatus ComputeAddRmsNormQuant( | |||
| 177 | aclTensor* xComputeOut = nullptr; | 177 | aclTensor* xComputeOut = nullptr; |
| 178 | bool isDual = (nullptr != scales2Optional); | 178 | bool isDual = (nullptr != scales2Optional); |
| 179 | 179 | ||
| 180 | - int dstType = dstTypeMap[y1Out->GetDataType()]; | 180 | + int dstType = dstTypeMapQuant[y1Out->GetDataType()]; |
| 181 | 181 | ||
| 182 | auto addRmsNormQuantOuts = l0op::AddRmsNormQuant( | 182 | auto addRmsNormQuantOuts = l0op::AddRmsNormQuant( |
| 183 | x1, x2, gamma, scales1, scales2Optional, zeroPoints1Optional, zeroPoints2Optional, nullptr, axis, epsilon, | 183 | x1, x2, gamma, scales1, scales2Optional, zeroPoints1Optional, zeroPoints2Optional, nullptr, axis, epsilon, |
| @@ -245,13 +245,13 @@ aclnnStatus aclnnAddRmsNormQuantGetWorkspaceSize( | |||
| 245 | } | 245 | } |
| 246 | 246 | ||
| 247 | // 固定写法，将输入转换成连续的tensor，可选输入不做判空校验 | 247 | // 固定写法，将输入转换成连续的tensor，可选输入不做判空校验 |
| 248 | - auto x1Cont = l0op::Contiguous(x1, uniqueExecutor.get()); | 248 | + auto x1ContQuant = l0op::Contiguous(x1, uniqueExecutor.get()); |
| 249 | - auto x2Cont = l0op::Contiguous(x2, uniqueExecutor.get()); | 249 | + auto x2ContQuant = l0op::Contiguous(x2, uniqueExecutor.get()); |
| 250 | - auto gammaCont = l0op::Contiguous(gamma, uniqueExecutor.get()); | 250 | + auto gammaContQuant = l0op::Contiguous(gamma, uniqueExecutor.get()); |
| 251 | 251 | ||
| 252 | - CHECK_RET(x1Cont != nullptr, ACLNN_ERR_INNER_NULLPTR); | 252 | + CHECK_RET(x1ContQuant != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 253 | - CHECK_RET(x2Cont != nullptr, ACLNN_ERR_INNER_NULLPTR); | 253 | + CHECK_RET(x2ContQuant != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 254 | - CHECK_RET(gammaCont != nullptr, ACLNN_ERR_INNER_NULLPTR); | 254 | + CHECK_RET(gammaContQuant != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 255 | 255 | ||
| 256 | auto s1Cont = GetTensorContiguous(scales1, uniqueExecutor.get()); | 256 | auto s1Cont = GetTensorContiguous(scales1, uniqueExecutor.get()); |
| 257 | auto s2Cont = GetTensorContiguous(scales2Optional, uniqueExecutor.get()); | 257 | auto s2Cont = GetTensorContiguous(scales2Optional, uniqueExecutor.get()); |
| @@ -259,7 +259,7 @@ aclnnStatus aclnnAddRmsNormQuantGetWorkspaceSize( | |||
| 259 | auto z2Cont = GetTensorContiguous(zeroPoints2Optional, uniqueExecutor.get()); | 259 | auto z2Cont = GetTensorContiguous(zeroPoints2Optional, uniqueExecutor.get()); |
| 260 | 260 | ||
| 261 | ret = ComputeAddRmsNormQuant( | 261 | ret = ComputeAddRmsNormQuant( |
| 262 | - x1Cont, x2Cont, gammaCont, s1Cont, s2Cont, z1Cont, z2Cont, axis, epsilon, divMode, y1Out, y2Out, xOut, | 262 | + x1ContQuant, x2ContQuant, gammaContQuant, s1Cont, s2Cont, z1Cont, z2Cont, axis, epsilon, divMode, y1Out, y2Out, xOut, |
| 263 | uniqueExecutor.get()); | 263 | uniqueExecutor.get()); |
| 264 | CHECK_RET(ret == ACLNN_SUCCESS, ret); | 264 | CHECK_RET(ret == ACLNN_SUCCESS, ret); |
| 265 | 265 | ||
| @@ -33,8 +33,8 @@ public: | |||
| 33 | { | 33 | { |
| 34 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); | 34 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); |
| 35 | 35 | ||
| 36 | - this->numRow = tilingData->numRow; | ||
| 37 | this->numCol = tilingData->numCol; | 36 | this->numCol = tilingData->numCol; |
| 37 | + this->numRow = tilingData->numRow; | ||
| 38 | this->blockFactor = tilingData->blockFactor; | 38 | this->blockFactor = tilingData->blockFactor; |
| 39 | this->rowFactor = tilingData->rowFactor; | 39 | this->rowFactor = tilingData->rowFactor; |
| 40 | this->ubFactor = tilingData->ubFactor; | 40 | this->ubFactor = tilingData->ubFactor; |
| @@ -255,174 +255,174 @@ private: | |||
| 255 | 255 | ||
| 256 | __aicore__ inline void ProcessBf16() | 256 | __aicore__ inline void ProcessBf16() |
| 257 | { | 257 | { |
| 258 | - LocalTensor<float> ubLocal = unitBuf.Get<float>(); | 258 | + LocalTensor<float> ubLocalQuant = unitBuf.Get<float>(); |
| 259 | - LocalTensor<TX> xLocal = ubLocal.template ReinterpretCast<TX>(); | 259 | + LocalTensor<TX> xLocalQuant = ubLocalQuant.template ReinterpretCast<TX>(); |
| 260 | - LocalTensor<TX> x1Local = xLocal[0]; | 260 | + LocalTensor<TX> x1LocalQuant = xLocalQuant[0]; |
| 261 | - LocalTensor<TX> x2Local = xLocal[ubFactor]; | 261 | + LocalTensor<TX> x2LocalQuant = xLocalQuant[ubFactor]; |
| 262 | - LocalTensor<float> xFp32Local = ubLocal[ubFactor]; | 262 | + LocalTensor<float> xFp32LocalQuant = ubLocalQuant[ubFactor]; |
| 263 | - LocalTensor<float> sqxLocal = ubLocal[ubFactor * 2]; | 263 | + LocalTensor<float> sqxLocalQuant = ubLocalQuant[ubFactor * 2]; |
| 264 | - LocalTensor<float> tmpLocal = ubLocal[ubFactor * 3]; | 264 | + LocalTensor<float> tmpLocalQuant = ubLocalQuant[ubFactor * 3]; |
| 265 | 265 | ||
| 266 | - DataCopyCustom<TX>(x1Local, x1Gm, numCol); | 266 | + DataCopyCustom<TX>(x1LocalQuant, x1Gm, numCol); |
| 267 | - event_t eventMTE2V1 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); | 267 | + event_t eventMTE2V1Quant = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); |
| 268 | - SetFlag<HardEvent::MTE2_V>(eventMTE2V1); | 268 | + SetFlag<HardEvent::MTE2_V>(eventMTE2V1Quant); |
| 269 | - DataCopyCustom<TX>(x2Local, x2Gm, numCol); | 269 | + DataCopyCustom<TX>(x2LocalQuant, x2Gm, numCol); |
| 270 | - event_t eventMTE2V2 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); | 270 | + event_t eventMTE2V2Quant = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); |
| 271 | - SetFlag<HardEvent::MTE2_V>(eventMTE2V2); | 271 | + SetFlag<HardEvent::MTE2_V>(eventMTE2V2Quant); |
| 272 | - WaitFlag<HardEvent::MTE2_V>(eventMTE2V1); | 272 | + WaitFlag<HardEvent::MTE2_V>(eventMTE2V1Quant); |
| 273 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V1); | 273 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V1Quant); |
| 274 | - Cast(xFp32Local, x1Local, RoundMode::CAST_NONE, numCol); | 274 | + Cast(xFp32LocalQuant, x1LocalQuant, RoundMode::CAST_NONE, numCol); |
| 275 | - WaitFlag<HardEvent::MTE2_V>(eventMTE2V2); | 275 | + WaitFlag<HardEvent::MTE2_V>(eventMTE2V2Quant); |
| 276 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V2); | 276 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V2Quant); |
| 277 | - Cast(sqxLocal, x2Local, RoundMode::CAST_NONE, numCol); | 277 | + Cast(sqxLocalQuant, x2LocalQuant, RoundMode::CAST_NONE, numCol); |
| 278 | PipeBarrier<PIPE_V>(); | 278 | PipeBarrier<PIPE_V>(); |
| 279 | - Add(xFp32Local, xFp32Local, sqxLocal, numCol); | 279 | + Add(xFp32LocalQuant, xFp32LocalQuant, sqxLocalQuant, numCol); |
| 280 | PipeBarrier<PIPE_V>(); | 280 | PipeBarrier<PIPE_V>(); |
| 281 | 281 | ||
| 282 | - Cast(x1Local, xFp32Local, RoundMode::CAST_NONE, numCol); | 282 | + Cast(x1LocalQuant, xFp32LocalQuant, RoundMode::CAST_NONE, numCol); |
| 283 | 283 | ||
| 284 | - Cast(x1Local, xFp32Local, RoundMode::CAST_RINT, numCol); | 284 | + Cast(x1LocalQuant, xFp32LocalQuant, RoundMode::CAST_RINT, numCol); |
| 285 | 285 | ||
| 286 | PipeBarrier<PIPE_V>(); | 286 | PipeBarrier<PIPE_V>(); |
| 287 | // copy gamma | 287 | // copy gamma |
| 288 | - event_t eventVMTE2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE2)); | 288 | + event_t eventVMTE2Quant = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE2)); |
| 289 | - SetFlag<HardEvent::V_MTE2>(eventVMTE2); | 289 | + SetFlag<HardEvent::V_MTE2>(eventVMTE2Quant); |
| 290 | - WaitFlag<HardEvent::V_MTE2>(eventVMTE2); | 290 | + WaitFlag<HardEvent::V_MTE2>(eventVMTE2Quant); |
| 291 | 291 | ||
| 292 | - DataCopyCustom<TX>(x2Local, gammaGm, numCol); // gammaLocal use x2Local | 292 | + DataCopyCustom<TX>(x2LocalQuant, gammaGm, numCol); // gammaLocal use x2Local |
| 293 | - event_t eventMTE2V4 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); | 293 | + event_t eventMTE2V4Quant = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); |
| 294 | - SetFlag<HardEvent::MTE2_V>(eventMTE2V4); | 294 | + SetFlag<HardEvent::MTE2_V>(eventMTE2V4Quant); |
| 295 | 295 | ||
| 296 | // copy x out | 296 | // copy x out |
| 297 | - event_t eventVMTE3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 297 | + event_t eventVMTE3Quant = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| 298 | - SetFlag<HardEvent::V_MTE3>(eventVMTE3); | 298 | + SetFlag<HardEvent::V_MTE3>(eventVMTE3Quant); |
| 299 | - WaitFlag<HardEvent::V_MTE3>(eventVMTE3); | 299 | + WaitFlag<HardEvent::V_MTE3>(eventVMTE3Quant); |
| 300 | if constexpr (A) { | 300 | if constexpr (A) { |
| 301 | - DataCopyCustom<TX>(xGm, x1Local, numCol); | 301 | + DataCopyCustom<TX>(xGm, x1LocalQuant, numCol); |
| 302 | } | 302 | } |
| 303 | - event_t eventMTE3V = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); | 303 | + event_t eventMTE3VQuant = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_V)); |
| 304 | - SetFlag<HardEvent::MTE3_V>(eventMTE3V); | 304 | + SetFlag<HardEvent::MTE3_V>(eventMTE3VQuant); |
| 305 | 305 | ||
| 306 | if constexpr (!PT) { | 306 | if constexpr (!PT) { |
| 307 | - Cast(xFp32Local, x1Local, RoundMode::CAST_NONE, numCol); | 307 | + Cast(xFp32LocalQuant, x1LocalQuant, RoundMode::CAST_NONE, numCol); |
| 308 | PipeBarrier<PIPE_V>(); | 308 | PipeBarrier<PIPE_V>(); |
| 309 | } | 309 | } |
| 310 | - Mul(sqxLocal, xFp32Local, xFp32Local, numCol); | 310 | + Mul(sqxLocalQuant, xFp32LocalQuant, xFp32LocalQuant, numCol); |
| 311 | PipeBarrier<PIPE_V>(); | 311 | PipeBarrier<PIPE_V>(); |
| 312 | - Muls(sqxLocal, sqxLocal, avgFactor, numCol); | 312 | + Muls(sqxLocalQuant, sqxLocalQuant, avgFactor, numCol); |
| 313 | PipeBarrier<PIPE_V>(); | 313 | PipeBarrier<PIPE_V>(); |
| 314 | - ReduceSumCustom(sqxLocal, sqxLocal, tmpLocal, numCol); | 314 | + ReduceSumCustom(sqxLocalQuant, sqxLocalQuant, tmpLocalQuant, numCol); |
| 315 | PipeBarrier<PIPE_V>(); | 315 | PipeBarrier<PIPE_V>(); |
| 316 | - Adds(sqxLocal, sqxLocal, epsilon, 1); | 316 | + Adds(sqxLocalQuant, sqxLocalQuant, epsilon, 1); |
| 317 | PipeBarrier<PIPE_V>(); | 317 | PipeBarrier<PIPE_V>(); |
| 318 | - Sqrt(sqxLocal, sqxLocal, 1); | 318 | + Sqrt(sqxLocalQuant, sqxLocalQuant, 1); |
| 319 | - Duplicate(tmpLocal, ONE, 1); | 319 | + Duplicate(tmpLocalQuant, ONE, 1); |
| 320 | PipeBarrier<PIPE_V>(); | 320 | PipeBarrier<PIPE_V>(); |
| 321 | - Div(sqxLocal, tmpLocal, sqxLocal, 1); | 321 | + Div(sqxLocalQuant, tmpLocalQuant, sqxLocalQuant, 1); |
| 322 | PipeBarrier<PIPE_V>(); | 322 | PipeBarrier<PIPE_V>(); |
| 323 | - event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 323 | + event_t eventVSQuant = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 324 | - SetFlag<HardEvent::V_S>(eventVS); | 324 | + SetFlag<HardEvent::V_S>(eventVSQuant); |
| 325 | - WaitFlag<HardEvent::V_S>(eventVS); | 325 | + WaitFlag<HardEvent::V_S>(eventVSQuant); |
| 326 | - float rstdValue = sqxLocal.GetValue(0); | 326 | + float rstdValueQuant = sqxLocalQuant.GetValue(0); |
| 327 | - event_t eventSV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | 327 | + event_t eventSVQuant = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); |
| 328 | - SetFlag<HardEvent::S_V>(eventSV); | 328 | + SetFlag<HardEvent::S_V>(eventSVQuant); |
| 329 | - WaitFlag<HardEvent::S_V>(eventSV); | 329 | + WaitFlag<HardEvent::S_V>(eventSVQuant); |
| 330 | 330 | ||
| 331 | - SetFlag<HardEvent::V_MTE2>(eventVMTE2); | 331 | + SetFlag<HardEvent::V_MTE2>(eventVMTE2Quant); |
| 332 | - WaitFlag<HardEvent::V_MTE2>(eventVMTE2); | 332 | + WaitFlag<HardEvent::V_MTE2>(eventVMTE2Quant); |
| 333 | // copy in scales | 333 | // copy in scales |
| 334 | if constexpr (is_same<TScale, bfloat16_t>::value) { | 334 | if constexpr (is_same<TScale, bfloat16_t>::value) { |
| 335 | - DataCopyCustom<TScale>(tmpLocal.template ReinterpretCast<TScale>()[ubFactor], scales1Gm, numCol); | 335 | + DataCopyCustom<TScale>(tmpLocalQuant.template ReinterpretCast<TScale>()[ubFactor], scales1Gm, numCol); |
| 336 | } else { // float | 336 | } else { // float |
| 337 | - DataCopyCustom<TScale>(tmpLocal.template ReinterpretCast<TScale>(), scales1Gm, numCol); | 337 | + DataCopyCustom<TScale>(tmpLocalQuant.template ReinterpretCast<TScale>(), scales1Gm, numCol); |
| 338 | } | 338 | } |
| 339 | - event_t eventMTE2V3 = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); | 339 | + event_t eventMTE2V3Quant = static_cast<event_t>(GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>()); |
| 340 | - SetFlag<HardEvent::MTE2_V>(eventMTE2V3); | 340 | + SetFlag<HardEvent::MTE2_V>(eventMTE2V3Quant); |
| 341 | - Muls(xFp32Local, xFp32Local, rstdValue, numCol); | 341 | + Muls(xFp32LocalQuant, xFp32LocalQuant, rstdValueQuant, numCol); |
| 342 | PipeBarrier<PIPE_V>(); | 342 | PipeBarrier<PIPE_V>(); |
| 343 | - WaitFlag<HardEvent::MTE3_V>(eventMTE3V); | 343 | + WaitFlag<HardEvent::MTE3_V>(eventMTE3VQuant); |
| 344 | - WaitFlag<HardEvent::MTE2_V>(eventMTE2V4); | 344 | + WaitFlag<HardEvent::MTE2_V>(eventMTE2V4Quant); |
| 345 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V4); | 345 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V4Quant); |
| 346 | - Cast(sqxLocal, x2Local, RoundMode::CAST_NONE, numCol); | 346 | + Cast(sqxLocalQuant, x2LocalQuant, RoundMode::CAST_NONE, numCol); |
| 347 | PipeBarrier<PIPE_V>(); | 347 | PipeBarrier<PIPE_V>(); |
| 348 | - Mul(xFp32Local, xFp32Local, sqxLocal, numCol); | 348 | + Mul(xFp32LocalQuant, xFp32LocalQuant, sqxLocalQuant, numCol); |
| 349 | PipeBarrier<PIPE_V>(); | 349 | PipeBarrier<PIPE_V>(); |
| 350 | if constexpr (RN) { | 350 | if constexpr (RN) { |
| 351 | 351 | ||
| 352 | - Cast(x1Local, xFp32Local, RoundMode::CAST_NONE, numCol); | 352 | + Cast(x1LocalQuant, xFp32LocalQuant, RoundMode::CAST_NONE, numCol); |
| 353 | 353 | ||
| 354 | - Cast(x1Local, xFp32Local, RoundMode::CAST_RINT, numCol); | 354 | + Cast(x1LocalQuant, xFp32LocalQuant, RoundMode::CAST_RINT, numCol); |
| 355 | 355 | ||
| 356 | - SetFlag<HardEvent::V_MTE3>(eventVMTE3); | 356 | + SetFlag<HardEvent::V_MTE3>(eventVMTE3Quant); |
| 357 | - WaitFlag<HardEvent::V_MTE3>(eventVMTE3); | 357 | + WaitFlag<HardEvent::V_MTE3>(eventVMTE3Quant); |
| 358 | - DataCopyCustom<TX>(resOutGm, x1Local, numCol); | 358 | + DataCopyCustom<TX>(resOutGm, x1LocalQuant, numCol); |
| 359 | } | 359 | } |
| 360 | 360 | ||
| 361 | if (hasBeta) { | 361 | if (hasBeta) { |
| 362 | - LocalTensor<TX> betaLocal = inQueueBeta.DeQue<TX>(); | 362 | + LocalTensor<TX> betaLocalQuant = inQueueBeta.DeQue<TX>(); |
| 363 | PipeBarrier<PIPE_ALL>(); | 363 | PipeBarrier<PIPE_ALL>(); |
| 364 | - Cast(sqxLocal, betaLocal, RoundMode::CAST_NONE, numCol); | 364 | + Cast(sqxLocalQuant, betaLocalQuant, RoundMode::CAST_NONE, numCol); |
| 365 | PipeBarrier<PIPE_V>(); | 365 | PipeBarrier<PIPE_V>(); |
| 366 | - Add(xFp32Local, xFp32Local, sqxLocal, numCol); | 366 | + Add(xFp32LocalQuant, xFp32LocalQuant, sqxLocalQuant, numCol); |
| 367 | PipeBarrier<PIPE_V>(); | 367 | PipeBarrier<PIPE_V>(); |
| 368 | - inQueueBeta.FreeTensor(betaLocal); | 368 | + inQueueBeta.FreeTensor(betaLocalQuant); |
| 369 | } | 369 | } |
| 370 | - SetFlag<HardEvent::V_MTE2>(eventVMTE2); | 370 | + SetFlag<HardEvent::V_MTE2>(eventVMTE2Quant); |
| 371 | - WaitFlag<HardEvent::MTE2_V>(eventMTE2V3); | 371 | + WaitFlag<HardEvent::MTE2_V>(eventMTE2V3Quant); |
| 372 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V3); | 372 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventMTE2V3Quant); |
| 373 | if (hasScales2) { | 373 | if (hasScales2) { |
| 374 | - LocalTensor<float> y2Local = inQueueBeta.AllocTensor<float>(); | 374 | + LocalTensor<float> y2LocalQuant = inQueueBeta.AllocTensor<float>(); |
| 375 | - AddRmsNormQuantBase::doScales(y2Local, xFp32Local, scales2Buf, divMode, numCol); | 375 | + AddRmsNormQuantBase::doScales(y2LocalQuant, xFp32LocalQuant, scales2Buf, divMode, numCol); |
| 376 | - AddRmsNormQuantBase::doZeroPoints(y2Local, zeroPoints2Buf, numCol, hasZeroPoints2); | 376 | + AddRmsNormQuantBase::doZeroPoints(y2LocalQuant, zeroPoints2Buf, numCol, hasZeroPoints2); |
| 377 | - LocalTensor<int8_t> y2Out = scales2Buf.Get<int8_t>(); | 377 | + LocalTensor<int8_t> y2OutQuant = scales2Buf.Get<int8_t>(); |
| 378 | - RoundFloat2Int8(y2Out, y2Local, numCol); | 378 | + RoundFloat2Int8(y2OutQuant, y2LocalQuant, numCol); |
| 379 | - event_t event_V_MTE3_1 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 379 | + event_t event_V_MTE3_1Quant = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| 380 | - SetFlag<HardEvent::V_MTE3>(event_V_MTE3_1); | 380 | + SetFlag<HardEvent::V_MTE3>(event_V_MTE3_1Quant); |
| 381 | - WaitFlag<HardEvent::V_MTE3>(event_V_MTE3_1); | 381 | + WaitFlag<HardEvent::V_MTE3>(event_V_MTE3_1Quant); |
| 382 | - DataCopyCustom<int8_t>(y2Gm, y2Out, numCol); | 382 | + DataCopyCustom<int8_t>(y2Gm, y2OutQuant, numCol); |
| 383 | - inQueueBeta.FreeTensor(y2Local); | 383 | + inQueueBeta.FreeTensor(y2LocalQuant); |
| 384 | } | 384 | } |
| 385 | 385 | ||
| 386 | if constexpr (is_same<TScale, bfloat16_t>::value) { | 386 | if constexpr (is_same<TScale, bfloat16_t>::value) { |
| 387 | - Cast(tmpLocal, tmpLocal.template ReinterpretCast<TScale>()[ubFactor], RoundMode::CAST_NONE, numCol); | 387 | + Cast(tmpLocalQuant, tmpLocalQuant.template ReinterpretCast<TScale>()[ubFactor], RoundMode::CAST_NONE, numCol); |
| 388 | PipeBarrier<PIPE_V>(); | 388 | PipeBarrier<PIPE_V>(); |
| 389 | } | 389 | } |
| 390 | 390 | ||
| 391 | - Quant_Mul(scales1Gm, xFp32Local, tmpLocal, numCol); | 391 | + Quant_Mul(scales1Gm, xFp32LocalQuant, tmpLocalQuant, numCol); |
| 392 | 392 | ||
| 393 | - WaitFlag<HardEvent::V_MTE2>(eventVMTE2); | 393 | + WaitFlag<HardEvent::V_MTE2>(eventVMTE2Quant); |
| 394 | if (hasZeroPoints1) { | 394 | if (hasZeroPoints1) { |
| 395 | if constexpr (is_same<TOffset, bfloat16_t>::value) { | 395 | if constexpr (is_same<TOffset, bfloat16_t>::value) { |
| 396 | - DataCopyCustom<TOffset>(sqxLocal.ReinterpretCast<TOffset>()[ubFactor], zeroPoints1Gm, numCol); | 396 | + DataCopyCustom<TOffset>(sqxLocalQuant.ReinterpretCast<TOffset>()[ubFactor], zeroPoints1Gm, numCol); |
| 397 | } else { // int32 | 397 | } else { // int32 |
| 398 | - DataCopyCustom<TOffset>(sqxLocal.ReinterpretCast<TOffset>(), zeroPoints1Gm, numCol); | 398 | + DataCopyCustom<TOffset>(sqxLocalQuant.ReinterpretCast<TOffset>(), zeroPoints1Gm, numCol); |
| 399 | } | 399 | } |
| 400 | 400 | ||
| 401 | - SetFlag<HardEvent::MTE2_V>(eventMTE2V3); | 401 | + SetFlag<HardEvent::MTE2_V>(eventMTE2V3Quant); |
| 402 | - WaitFlag<HardEvent::MTE2_V>(eventMTE2V3); | 402 | + WaitFlag<HardEvent::MTE2_V>(eventMTE2V3Quant); |
| 403 | if constexpr (is_same<TOffset, bfloat16_t>::value) { | 403 | if constexpr (is_same<TOffset, bfloat16_t>::value) { |
| 404 | - Cast(sqxLocal, sqxLocal.ReinterpretCast<TOffset>()[ubFactor], RoundMode::CAST_NONE, numCol); | 404 | + Cast(sqxLocalQuant, sqxLocalQuant.ReinterpretCast<TOffset>()[ubFactor], RoundMode::CAST_NONE, numCol); |
| 405 | } else { // int32 | 405 | } else { // int32 |
| 406 | - Cast(sqxLocal, sqxLocal.ReinterpretCast<TOffset>(), RoundMode::CAST_NONE, numCol); | 406 | + Cast(sqxLocalQuant, sqxLocalQuant.ReinterpretCast<TOffset>(), RoundMode::CAST_NONE, numCol); |
| 407 | } | 407 | } |
| 408 | PipeBarrier<PIPE_V>(); | 408 | PipeBarrier<PIPE_V>(); |
| 409 | if (PT) { | 409 | if (PT) { |
| 410 | - int32_t eventIDVToS = static_cast<int32_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 410 | + int32_t eventIDVToSQuant = static_cast<int32_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 411 | - SetFlag<HardEvent::V_S>(eventIDVToS); | 411 | + SetFlag<HardEvent::V_S>(eventIDVToSQuant); |
| 412 | - WaitFlag<HardEvent::V_S>(eventIDVToS); | 412 | + WaitFlag<HardEvent::V_S>(eventIDVToSQuant); |
| 413 | - Adds(xFp32Local, xFp32Local, sqxLocal.GetValue(0), numCol); | 413 | + Adds(xFp32LocalQuant, xFp32LocalQuant, sqxLocalQuant.GetValue(0), numCol); |
| 414 | } else { | 414 | } else { |
| 415 | - Add(xFp32Local, xFp32Local, sqxLocal, numCol); | 415 | + Add(xFp32LocalQuant, xFp32LocalQuant, sqxLocalQuant, numCol); |
| 416 | } | 416 | } |
| 417 | PipeBarrier<PIPE_V>(); | 417 | PipeBarrier<PIPE_V>(); |
| 418 | } | 418 | } |
| 419 | 419 | ||
| 420 | - LocalTensor<int8_t> y1Out = tmpLocal.ReinterpretCast<int8_t>(); | 420 | + LocalTensor<int8_t> y1OutQuant = tmpLocalQuant.ReinterpretCast<int8_t>(); |
| 421 | - RoundFloat2Int8(y1Out, xFp32Local, numCol); | 421 | + RoundFloat2Int8(y1OutQuant, xFp32LocalQuant, numCol); |
| 422 | - event_t event_V_MTE3_0 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 422 | + event_t event_V_MTE3_0Quant = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| 423 | - SetFlag<HardEvent::V_MTE3>(event_V_MTE3_0); | 423 | + SetFlag<HardEvent::V_MTE3>(event_V_MTE3_0Quant); |
| 424 | - WaitFlag<HardEvent::V_MTE3>(event_V_MTE3_0); | 424 | + WaitFlag<HardEvent::V_MTE3>(event_V_MTE3_0Quant); |
| 425 | - DataCopyCustom<int8_t>(y1Gm, y1Out, numCol); | 425 | + DataCopyCustom<int8_t>(y1Gm, y1OutQuant, numCol); |
| 426 | } | 426 | } |
| 427 | 427 | ||
| 428 | __aicore__ inline void Quant_Mul( | 428 | __aicore__ inline void Quant_Mul( |
| @@ -34,16 +34,16 @@ public: | |||
| 34 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); | 34 | ASSERT(GetBlockNum() != 0 && "Block dim can not be zero!"); |
| 35 | this->numRow = tilingData->numRow; | 35 | this->numRow = tilingData->numRow; |
| 36 | this->numCol = tilingData->numCol; | 36 | this->numCol = tilingData->numCol; |
| 37 | - this->blockFactor = tilingData->blockFactor; | ||
| 38 | this->rowFactor = tilingData->rowFactor; | 37 | this->rowFactor = tilingData->rowFactor; |
| 39 | - this->ubFactor = tilingData->ubFactor; | 38 | + this->blockFactor = tilingData->blockFactor; |
| 40 | this->epsilon = tilingData->epsilon; | 39 | this->epsilon = tilingData->epsilon; |
| 41 | - this->avgFactor = (float)1.0 / numCol; | 40 | + this->ubFactor = tilingData->ubFactor; |
| 42 | this->hasZeroPoints1 = tilingData->hasZeroPoints1; | 41 | this->hasZeroPoints1 = tilingData->hasZeroPoints1; |
| 43 | - this->hasBeta = tilingData->hasBeta; | 42 | + this->avgFactor = (float)1.0 / numCol; |
| 44 | this->divMode = tilingData->divMode; | 43 | this->divMode = tilingData->divMode; |
| 45 | - this->hasScales2 = tilingData->hasScales2 && !PT; | 44 | + this->hasBeta = tilingData->hasBeta; |
| 46 | this->hasZeroPoints2 = tilingData->hasZeroPoints2 && !PT; | 45 | this->hasZeroPoints2 = tilingData->hasZeroPoints2 && !PT; |
| 46 | + this->hasScales2 = tilingData->hasScales2 && !PT; | ||
| 47 | 47 | ||
| 48 | blockIdx_ = GetBlockIdx(); | 48 | blockIdx_ = GetBlockIdx(); |
| 49 | if (blockIdx_ < GetBlockNum() - 1) { | 49 | if (blockIdx_ < GetBlockNum() - 1) { |
| @@ -82,7 +82,7 @@ public: | |||
| 82 | } | 82 | } |
| 83 | Ppipe->InitBuffer(sqxBuf, ubFactor * sizeof(float)); | 83 | Ppipe->InitBuffer(sqxBuf, ubFactor * sizeof(float)); |
| 84 | 84 | ||
| 85 | - Ppipe->InitBuffer(sumBuf, rowFactor * NUM_PER_BLK_FP32 * sizeof(float)); | 85 | + Ppipe->InitBuffer(sumBufSplit, rowFactor * NUM_PER_BLK_FP32 * sizeof(float)); |
| 86 | Ppipe->InitBuffer(reduceFp32Buf, NUM_PER_REP_FP32 * sizeof(float)); | 86 | Ppipe->InitBuffer(reduceFp32Buf, NUM_PER_REP_FP32 * sizeof(float)); |
| 87 | Ppipe->InitBuffer(rstdBuf, rowFactor * sizeof(float)); | 87 | Ppipe->InitBuffer(rstdBuf, rowFactor * sizeof(float)); |
| 88 | initOptionalParams(scales2, zero_points1, zero_points2, beta); | 88 | initOptionalParams(scales2, zero_points1, zero_points2, beta); |
| @@ -99,7 +99,7 @@ public: | |||
| 99 | scales2Gm.SetGlobalBuffer((__gm__ TScale*)scales2, numCol); | 99 | scales2Gm.SetGlobalBuffer((__gm__ TScale*)scales2, numCol); |
| 100 | Ppipe->InitBuffer(outQueueY2, BUFFER_NUM, ubFactor * sizeof(TX)); | 100 | Ppipe->InitBuffer(outQueueY2, BUFFER_NUM, ubFactor * sizeof(TX)); |
| 101 | Ppipe->InitBuffer(scales2Buf, ubFactor * sizeof(float)); | 101 | Ppipe->InitBuffer(scales2Buf, ubFactor * sizeof(float)); |
| 102 | - Ppipe->InitBuffer(tmpBuf, ubFactor * sizeof(float)); | 102 | + Ppipe->InitBuffer(tmpBufSplit, ubFactor * sizeof(float)); |
| 103 | if (hasZeroPoints2) { | 103 | if (hasZeroPoints2) { |
| 104 | zeroPoints2Gm.SetGlobalBuffer((__gm__ TOffset*)zero_points2, numCol); | 104 | zeroPoints2Gm.SetGlobalBuffer((__gm__ TOffset*)zero_points2, numCol); |
| 105 | Ppipe->InitBuffer(zeroPoints2Buf, ubFactor * sizeof(int32_t)); | 105 | Ppipe->InitBuffer(zeroPoints2Buf, ubFactor * sizeof(int32_t)); |
| @@ -121,7 +121,7 @@ public: | |||
| 121 | 121 | ||
| 122 | __aicore__ inline void SubProcess(uint32_t iO, uint32_t calcRowNum, uint32_t jMax, uint32_t colTail) | 122 | __aicore__ inline void SubProcess(uint32_t iO, uint32_t calcRowNum, uint32_t jMax, uint32_t colTail) |
| 123 | { | 123 | { |
| 124 | - LocalTensor<float> sumLocal = sumBuf.Get<float>(); | 124 | + LocalTensor<float> sumLocal = sumBufSplit.Get<float>(); |
| 125 | 125 | ||
| 126 | LocalTensor<float> rstdLocal = rstdBuf.Get<float>(); | 126 | LocalTensor<float> rstdLocal = rstdBuf.Get<float>(); |
| 127 | Duplicate(rstdLocal, (float)0.0, calcRowNum); | 127 | Duplicate(rstdLocal, (float)0.0, calcRowNum); |
| @@ -359,7 +359,7 @@ private: | |||
| 359 | __aicore__ inline void doQuant(LocalTensor<float> xFp32Local, uint32_t num) | 359 | __aicore__ inline void doQuant(LocalTensor<float> xFp32Local, uint32_t num) |
| 360 | { | 360 | { |
| 361 | if (hasScales2) { | 361 | if (hasScales2) { |
| 362 | - LocalTensor<float> tmpFp32 = tmpBuf.Get<float>(); | 362 | + LocalTensor<float> tmpFp32 = tmpBufSplit.Get<float>(); |
| 363 | AddRmsNormQuantBase::doScales(tmpFp32, xFp32Local, scales2Buf, divMode, num); | 363 | AddRmsNormQuantBase::doScales(tmpFp32, xFp32Local, scales2Buf, divMode, num); |
| 364 | AddRmsNormQuantBase::doZeroPoints(tmpFp32, zeroPoints2Buf, num, hasZeroPoints2); | 364 | AddRmsNormQuantBase::doZeroPoints(tmpFp32, zeroPoints2Buf, num, hasZeroPoints2); |
| 365 | LocalTensor<int8_t> y2Local = outQueueY2.AllocTensor<int8_t>(); | 365 | LocalTensor<int8_t> y2Local = outQueueY2.AllocTensor<int8_t>(); |
| @@ -425,8 +425,8 @@ private: | |||
| 425 | TBuf<TPosition::VECCALC> zeroPoints1Buf; | 425 | TBuf<TPosition::VECCALC> zeroPoints1Buf; |
| 426 | TBuf<TPosition::VECCALC> xFp32Buf; | 426 | TBuf<TPosition::VECCALC> xFp32Buf; |
| 427 | TBuf<TPosition::VECCALC> sqxBuf; | 427 | TBuf<TPosition::VECCALC> sqxBuf; |
| 428 | - TBuf<TPosition::VECCALC> tmpBuf; | 428 | + TBuf<TPosition::VECCALC> tmpBufSplit; |
| 429 | - TBuf<TPosition::VECCALC> sumBuf; | 429 | + TBuf<TPosition::VECCALC> sumBufSplit; |
| 430 | TBuf<TPosition::VECCALC> reduceFp32Buf; | 430 | TBuf<TPosition::VECCALC> reduceFp32Buf; |
| 431 | TBuf<TPosition::VECCALC> rstdBuf; | 431 | TBuf<TPosition::VECCALC> rstdBuf; |
| 432 | TBuf<TPosition::VECCALC> scales2Buf; | 432 | TBuf<TPosition::VECCALC> scales2Buf; |
| @@ -25,8 +25,8 @@ | |||
| 25 | 25 | ||
| 26 | 26 | ||
| 27 | 27 | ||
| 28 | - | ||
| 29 | 28 | ||
| 29 | + | ||
| 30 | 30 | ||
| 31 | using namespace op; | 31 | using namespace op; |
| 32 | 32 | ||
| @@ -716,15 +716,15 @@ ge::graphStatus BatchNormGradV3RARRecomputeTilingBase::GetWorkspaceSize() | |||
| 716 | 716 | ||
| 717 | ge::graphStatus BatchNormGradV3RARRecomputeTilingBase::PostTiling() | 717 | ge::graphStatus BatchNormGradV3RARRecomputeTilingBase::PostTiling() |
| 718 | { | 718 | { |
| 719 | - uint64_t tilingKey = GetTilingKey(); | 719 | + uint64_t curTilingKey = GetTilingKey(); |
| 720 | OP_TILING_CHECK( | 720 | OP_TILING_CHECK( |
| 721 | - tilingKey == BNG_V3_TK_DEFAULT, | 721 | + curTilingKey == BNG_V3_TK_DEFAULT, |
| 722 | VECTOR_INNER_ERR_REPORT_TILIING(context_->GetNodeName(), "failed to get tiling key."), return ge::GRAPH_FAILED); | 722 | VECTOR_INNER_ERR_REPORT_TILIING(context_->GetNodeName(), "failed to get tiling key."), return ge::GRAPH_FAILED); |
| 723 | - context_->SetTilingKey(tilingKey); | 723 | + context_->SetTilingKey(curTilingKey); |
| 724 | context_->SetBlockDim(baseTilingData.get_blockNum()); | 724 | context_->SetBlockDim(baseTilingData.get_blockNum()); |
| 725 | - size_t* workspaces = context_->GetWorkspaceSizes(1); | 725 | + size_t* curWorkspaces = context_->GetWorkspaceSizes(1); |
| 726 | - OP_CHECK_NULL_WITH_CONTEXT(context_, workspaces); | 726 | + OP_CHECK_NULL_WITH_CONTEXT(context_, curWorkspaces); |
| 727 | - workspaces[0] = workspaceSize_; | 727 | + curWorkspaces[0] = workspaceSize_; |
| 728 | tilingData.SaveToBuffer(context_->GetRawTilingData()->GetData(), context_->GetRawTilingData()->GetCapacity()); | 728 | tilingData.SaveToBuffer(context_->GetRawTilingData()->GetData(), context_->GetRawTilingData()->GetCapacity()); |
| 729 | context_->GetRawTilingData()->SetDataSize(tilingData.GetDataSize()); | 729 | context_->GetRawTilingData()->SetDataSize(tilingData.GetDataSize()); |
| 730 | return ge::GRAPH_SUCCESS; | 730 | return ge::GRAPH_SUCCESS; |
| @@ -959,12 +959,12 @@ ge::graphStatus BatchNormGradV3RARecomputeTilingBase::GetWorkspaceSize() | |||
| 959 | 959 | ||
| 960 | ge::graphStatus BatchNormGradV3RARecomputeTilingBase::PostTiling() | 960 | ge::graphStatus BatchNormGradV3RARecomputeTilingBase::PostTiling() |
| 961 | { | 961 | { |
| 962 | - uint64_t tilingKey = GetTilingKey(); | 962 | + uint64_t raTilingKey = GetTilingKey(); |
| 963 | - context_->SetTilingKey(tilingKey); | 963 | + context_->SetTilingKey(raTilingKey); |
| 964 | context_->SetBlockDim(tilingData.get_numBlocks()); | 964 | context_->SetBlockDim(tilingData.get_numBlocks()); |
| 965 | - size_t* workspaces = context_->GetWorkspaceSizes(1); | 965 | + size_t* raWorkspaces = context_->GetWorkspaceSizes(1); |
| 966 | - OP_CHECK_NULL_WITH_CONTEXT(context_, workspaces); | 966 | + OP_CHECK_NULL_WITH_CONTEXT(context_, raWorkspaces); |
| 967 | - workspaces[0] = workspaceSize_; | 967 | + raWorkspaces[0] = workspaceSize_; |
| 968 | tilingData.SaveToBuffer(context_->GetRawTilingData()->GetData(), context_->GetRawTilingData()->GetCapacity()); | 968 | tilingData.SaveToBuffer(context_->GetRawTilingData()->GetData(), context_->GetRawTilingData()->GetCapacity()); |
| 969 | context_->GetRawTilingData()->SetDataSize(tilingData.GetDataSize()); | 969 | context_->GetRawTilingData()->SetDataSize(tilingData.GetDataSize()); |
| 970 | return ge::GRAPH_SUCCESS; | 970 | return ge::GRAPH_SUCCESS; |
| @@ -165,10 +165,10 @@ END_TILING_DATA_DEF; | |||
| 165 | 165 | ||
| 166 | REGISTER_TILING_DATA_CLASS(BatchNormGradV3_50000000, BatchNormGradV3RASplitRTilingData); | 166 | REGISTER_TILING_DATA_CLASS(BatchNormGradV3_50000000, BatchNormGradV3RASplitRTilingData); |
| 167 | 167 | ||
| 168 | -// inference | 168 | +// inference - channel last format tiling data (NHWC format) |
| 169 | BEGIN_TILING_DATA_DEF(BatchNormGradV3InferChannelLastTilingData) | 169 | BEGIN_TILING_DATA_DEF(BatchNormGradV3InferChannelLastTilingData) |
| 170 | -TILING_DATA_FIELD_DEF(int64_t, totalTiles); | 170 | +TILING_DATA_FIELD_DEF(int64_t, totalTiles); // total tile count for grad |
| 171 | -TILING_DATA_FIELD_DEF(int64_t, tilesPerCore); | 171 | +TILING_DATA_FIELD_DEF(int64_t, tilesPerCore); // tiles per core for grad |
| 172 | TILING_DATA_FIELD_DEF(int64_t, usedCoreNums); | 172 | TILING_DATA_FIELD_DEF(int64_t, usedCoreNums); |
| 173 | TILING_DATA_FIELD_DEF(int64_t, totalALen); | 173 | TILING_DATA_FIELD_DEF(int64_t, totalALen); |
| 174 | TILING_DATA_FIELD_DEF(int64_t, aOuter); | 174 | TILING_DATA_FIELD_DEF(int64_t, aOuter); |
| @@ -183,9 +183,10 @@ END_TILING_DATA_DEF; | |||
| 183 | 183 | ||
| 184 | REGISTER_TILING_DATA_CLASS(BatchNormGradV3_900000, BatchNormGradV3InferChannelLastTilingData); | 184 | REGISTER_TILING_DATA_CLASS(BatchNormGradV3_900000, BatchNormGradV3InferChannelLastTilingData); |
| 185 | 185 | ||
| 186 | +// BatchNormGradV3 inference tiling data for NCHW/NCDHW format | ||
| 186 | BEGIN_TILING_DATA_DEF(BatchNormGradV3InferTilingData) | 187 | BEGIN_TILING_DATA_DEF(BatchNormGradV3InferTilingData) |
| 187 | -TILING_DATA_FIELD_DEF(int64_t, totalTiles); | 188 | +TILING_DATA_FIELD_DEF(int64_t, totalTiles); // grad total tile count |
| 188 | -TILING_DATA_FIELD_DEF(int64_t, tilesPerCore); | 189 | +TILING_DATA_FIELD_DEF(int64_t, tilesPerCore); // grad tiles per core |
| 189 | TILING_DATA_FIELD_DEF(int64_t, usedCoreNums); | 190 | TILING_DATA_FIELD_DEF(int64_t, usedCoreNums); |
| 190 | TILING_DATA_FIELD_DEF(int64_t, totalB0Len); | 191 | TILING_DATA_FIELD_DEF(int64_t, totalB0Len); |
| 191 | TILING_DATA_FIELD_DEF(int64_t, totalALen); | 192 | TILING_DATA_FIELD_DEF(int64_t, totalALen); |
| @@ -485,8 +486,9 @@ private: | |||
| 485 | void SetBlockFactors(int64_t aDim_, int64_t dtypeSize); | 486 | void SetBlockFactors(int64_t aDim_, int64_t dtypeSize); |
| 486 | 487 | ||
| 487 | private: | 488 | private: |
| 488 | - int64_t reservUbSizeForAlign{0}; | 489 | + // member variables for RA recompute tiling |
| 489 | int64_t binaryAddUbNeed{0}; | 490 | int64_t binaryAddUbNeed{0}; |
| 491 | + int64_t reservUbSizeForAlign{0}; | ||
| 490 | int64_t binaryAddQuotient{0}; | 492 | int64_t binaryAddQuotient{0}; |
| 491 | BatchNormGradV3RARecomputeTilingData tilingData; | 493 | BatchNormGradV3RARecomputeTilingData tilingData; |
| 492 | }; | 494 | }; |
| @@ -45,7 +45,7 @@ void BatchNormGradV3InferBase::Reset() | |||
| 45 | weightDimLen_ = 0; | 45 | weightDimLen_ = 0; |
| 46 | runningVarDimLen_ = 0; | 46 | runningVarDimLen_ = 0; |
| 47 | 47 | ||
| 48 | - epsilon_ = DEFAULT_EPSILON; | 48 | + epsilon_ = DEFAULT_EPSILON_VAL; |
| 49 | } | 49 | } |
| 50 | 50 | ||
| 51 | void BatchNormGradV3InferBase::CalcBasicInfo() | 51 | void BatchNormGradV3InferBase::CalcBasicInfo() |
| @@ -58,18 +58,18 @@ void BatchNormGradV3InferBase::CalcBasicInfo() | |||
| 58 | aTileBase_ = vlFp16_; | 58 | aTileBase_ = vlFp16_; |
| 59 | } | 59 | } |
| 60 | 60 | ||
| 61 | - if (weightDtype_ == ge::DT_FLOAT) { | ||
| 62 | - bytesPerWeight_ = FLOAT32_BYTES; | ||
| 63 | - } else { | ||
| 64 | - bytesPerWeight_ = FLOAT16_BYTES; | ||
| 65 | - } | ||
| 66 | - | ||
| 67 | if (runningVarDtype_ == ge::DT_FLOAT) { | 61 | if (runningVarDtype_ == ge::DT_FLOAT) { |
| 68 | bytesPerRunningVar_ = FLOAT32_BYTES; | 62 | bytesPerRunningVar_ = FLOAT32_BYTES; |
| 69 | } else { | 63 | } else { |
| 70 | bytesPerRunningVar_ = FLOAT16_BYTES; | 64 | bytesPerRunningVar_ = FLOAT16_BYTES; |
| 71 | } | 65 | } |
| 72 | 66 | ||
| 67 | + if (weightDtype_ == ge::DT_FLOAT) { | ||
| 68 | + bytesPerWeight_ = FLOAT32_BYTES; | ||
| 69 | + } else { | ||
| 70 | + bytesPerWeight_ = FLOAT16_BYTES; | ||
| 71 | + } | ||
| 72 | + | ||
| 73 | OP_LOGD( | 73 | OP_LOGD( |
| 74 | context_->GetNodeName(), "aTileBase_: %ld, bytesPerDy_: %ld, bytesPerWeight_: %ld,bytesPerRunningVar_: %ld.", | 74 | context_->GetNodeName(), "aTileBase_: %ld, bytesPerDy_: %ld, bytesPerWeight_: %ld,bytesPerRunningVar_: %ld.", |
| 75 | aTileBase_, bytesPerDy_, bytesPerWeight_, bytesPerRunningVar_); | 75 | aTileBase_, bytesPerDy_, bytesPerWeight_, bytesPerRunningVar_); |
| @@ -116,11 +116,10 @@ ge::graphStatus BatchNormGradV3InferBase::GetShapeAttrsInfo() | |||
| 116 | return ge::GRAPH_PARAM_INVALID); | 116 | return ge::GRAPH_PARAM_INVALID); |
| 117 | 117 | ||
| 118 | const float* epsilonPtr = attrs->GetFloat(PARAM_ATTRS_EPSILON_INDEX); | 118 | const float* epsilonPtr = attrs->GetFloat(PARAM_ATTRS_EPSILON_INDEX); |
| 119 | - epsilon_ = (epsilonPtr == nullptr) ? DEFAULT_EPSILON : *epsilonPtr; | 119 | + epsilon_ = (epsilonPtr == nullptr) ? DEFAULT_EPSILON_VAL : *epsilonPtr; |
| 120 | 120 | ||
| 121 | auto ret = GetDyInfo(); | 121 | auto ret = GetDyInfo(); |
| 122 | - OP_TILING_CHECK( | 122 | + OP_TILING_CHECK(ret != ge::GRAPH_SUCCESS, OP_LOGW(context_->GetNodeName(), "GetShapeAttrsInfo failed."), return ret); |
| 123 | - ret != ge::GRAPH_SUCCESS, OP_LOGW(context_->GetNodeName(), "GetShapeAttrsInfo failed."), return ret); | ||
| 124 | 123 | ||
| 125 | OP_TILING_CHECK( | 124 | OP_TILING_CHECK( |
| 126 | GetWeightRunningVarDxInfo() != ge::GRAPH_SUCCESS, | 125 | GetWeightRunningVarDxInfo() != ge::GRAPH_SUCCESS, |
| @@ -161,16 +160,14 @@ ge::graphStatus BatchNormGradV3InferBase::GetDyInfo() | |||
| 161 | dyStorageShape.GetDim(DIM_3); | 160 | dyStorageShape.GetDim(DIM_3); |
| 162 | fusedB1Len_ = 1; | 161 | fusedB1Len_ = 1; |
| 163 | } else if (dyFormat_ == FORMAT_NCHW) { | 162 | } else if (dyFormat_ == FORMAT_NCHW) { |
| 164 | - OP_TILING_CHECK( | 163 | + OP_TILING_CHECK(dyDimNum_ != DIM_NUM_4, |
| 165 | - dyDimNum_ != DIM_NUM_4, | ||
| 166 | VECTOR_INNER_ERR_REPORT_TILIING(context_->GetNodeName(), "Dims should be 4 with NCHW format."), | 164 | VECTOR_INNER_ERR_REPORT_TILIING(context_->GetNodeName(), "Dims should be 4 with NCHW format."), |
| 167 | return ge::GRAPH_FAILED); | 165 | return ge::GRAPH_FAILED); |
| 168 | fusedB0Len_ = dyStorageShape.GetDim(DIM_0); | 166 | fusedB0Len_ = dyStorageShape.GetDim(DIM_0); |
| 169 | fusedALen_ = dyStorageShape.GetDim(DIM_1); | 167 | fusedALen_ = dyStorageShape.GetDim(DIM_1); |
| 170 | fusedB1Len_ = dyStorageShape.GetDim(DIM_2) * dyStorageShape.GetDim(DIM_3); | 168 | fusedB1Len_ = dyStorageShape.GetDim(DIM_2) * dyStorageShape.GetDim(DIM_3); |
| 171 | } else if (dyFormat_ == FORMAT_NCDHW) { | 169 | } else if (dyFormat_ == FORMAT_NCDHW) { |
| 172 | - OP_TILING_CHECK( | 170 | + OP_TILING_CHECK(dyDimNum_ != DIM_NUM_5, |
| 173 | - dyDimNum_ != DIM_NUM_5, | ||
| 174 | VECTOR_INNER_ERR_REPORT_TILIING(context_->GetNodeName(), "Dims should be 5 with NCDHW format."), | 171 | VECTOR_INNER_ERR_REPORT_TILIING(context_->GetNodeName(), "Dims should be 5 with NCDHW format."), |
| 175 | return ge::GRAPH_FAILED); | 172 | return ge::GRAPH_FAILED); |
| 176 | fusedB0Len_ = dyStorageShape.GetDim(DIM_0); | 173 | fusedB0Len_ = dyStorageShape.GetDim(DIM_0); |
| @@ -192,31 +189,31 @@ ge::graphStatus BatchNormGradV3InferBase::GetDyInfo() | |||
| 192 | 189 | ||
| 193 | ge::graphStatus BatchNormGradV3InferBase::GetWeightRunningVarDxInfo() | 190 | ge::graphStatus BatchNormGradV3InferBase::GetWeightRunningVarDxInfo() |
| 194 | { | 191 | { |
| 195 | - auto weightDesc = context_->GetInputDesc(PARAM_INPUT_WEIGHT_INDEX); | 192 | + auto weightTensorDesc = context_->GetInputDesc(PARAM_INPUT_WEIGHT_INDEX); |
| 196 | - OP_CHECK_NULL_WITH_CONTEXT(context_, weightDesc); | 193 | + OP_CHECK_NULL_WITH_CONTEXT(context_, weightTensorDesc); |
| 197 | - weightDtype_ = weightDesc->GetDataType(); | 194 | + weightDtype_ = weightTensorDesc->GetDataType(); |
| 198 | - auto weightShape = context_->GetInputShape(PARAM_INPUT_WEIGHT_INDEX); | 195 | + auto weightTensorShape = context_->GetInputShape(PARAM_INPUT_WEIGHT_INDEX); |
| 199 | - OP_CHECK_NULL_WITH_CONTEXT(context_, weightShape); | 196 | + OP_CHECK_NULL_WITH_CONTEXT(context_, weightTensorShape); |
| 200 | - auto weightStorageShape = weightShape->GetStorageShape(); | 197 | + auto weightTensorStorageShape = weightTensorShape->GetStorageShape(); |
| 201 | - weightDimNum_ = weightStorageShape.GetDimNum(); | 198 | + weightDimNum_ = weightTensorStorageShape.GetDimNum(); |
| 202 | - weightDimLen_ = weightStorageShape.GetDim(0); | 199 | + weightDimLen_ = weightTensorStorageShape.GetDim(0); |
| 203 | 200 | ||
| 204 | - auto runningVarDesc = context_->GetInputDesc(PARAM_INPUT_RUNNINGVAR_INDEX); | 201 | + auto runningVarTensorDesc = context_->GetInputDesc(PARAM_INPUT_RUNNINGVAR_INDEX); |
| 205 | - OP_CHECK_NULL_WITH_CONTEXT(context_, runningVarDesc); | 202 | + OP_CHECK_NULL_WITH_CONTEXT(context_, runningVarTensorDesc); |
| 206 | - runningVarDtype_ = runningVarDesc->GetDataType(); | 203 | + runningVarDtype_ = runningVarTensorDesc->GetDataType(); |
| 207 | - auto runningVarShape = context_->GetInputShape(PARAM_INPUT_RUNNINGVAR_INDEX); | 204 | + auto runningVarTensorShape = context_->GetInputShape(PARAM_INPUT_RUNNINGVAR_INDEX); |
| 208 | - OP_CHECK_NULL_WITH_CONTEXT(context_, runningVarShape); | 205 | + OP_CHECK_NULL_WITH_CONTEXT(context_, runningVarTensorShape); |
| 209 | - auto runningVarStorageShape = runningVarShape->GetStorageShape(); | 206 | + auto runningVarTensorStorageShape = runningVarTensorShape->GetStorageShape(); |
| 210 | - runningVarDimNum_ = runningVarStorageShape.GetDimNum(); | 207 | + runningVarDimNum_ = runningVarTensorStorageShape.GetDimNum(); |
| 211 | - runningVarDimLen_ = runningVarStorageShape.GetDim(0); | 208 | + runningVarDimLen_ = runningVarTensorStorageShape.GetDim(0); |
| 212 | 209 | ||
| 213 | - auto dxDesc = context_->GetOutputDesc(PARAM_OUTPUT_DX_INDEX); | 210 | + auto dxTensorDesc = context_->GetOutputDesc(PARAM_OUTPUT_DX_INDEX); |
| 214 | - OP_CHECK_NULL_WITH_CONTEXT(context_, dxDesc); | 211 | + OP_CHECK_NULL_WITH_CONTEXT(context_, dxTensorDesc); |
| 215 | - dxDtype_ = dxDesc->GetDataType(); | 212 | + dxDtype_ = dxTensorDesc->GetDataType(); |
| 216 | - auto dxShape = context_->GetOutputShape(PARAM_OUTPUT_DX_INDEX); | 213 | + auto dxTensorShape = context_->GetOutputShape(PARAM_OUTPUT_DX_INDEX); |
| 217 | - OP_CHECK_NULL_WITH_CONTEXT(context_, dxShape); | 214 | + OP_CHECK_NULL_WITH_CONTEXT(context_, dxTensorShape); |
| 218 | - auto dxStorageShape = dxShape->GetStorageShape(); | 215 | + auto dxTensorStorageShape = dxTensorShape->GetStorageShape(); |
| 219 | - dxDimNum_ = dxStorageShape.GetDimNum(); | 216 | + dxDimNum_ = dxTensorStorageShape.GetDimNum(); |
| 220 | 217 | ||
| 221 | return ge::GRAPH_SUCCESS; | 218 | return ge::GRAPH_SUCCESS; |
| 222 | } | 219 | } |
| @@ -230,29 +227,29 @@ ge::graphStatus BatchNormGradV3InferBase::CheckInputValid() | |||
| 230 | dxDtype_), | 227 | dxDtype_), |
| 231 | return ge::GRAPH_FAILED); | 228 | return ge::GRAPH_FAILED); |
| 232 | 229 | ||
| 233 | - bool dtypeValid = false; | 230 | + bool inputDtypeValid = false; |
| 234 | for (uint32_t i = 0; i < validInputDtypes.size(); i++) { | 231 | for (uint32_t i = 0; i < validInputDtypes.size(); i++) { |
| 235 | if (dyDtype_ == validInputDtypes[i][DTYPE_DX_OFFSET] && | 232 | if (dyDtype_ == validInputDtypes[i][DTYPE_DX_OFFSET] && |
| 236 | weightDtype_ == validInputDtypes[i][DTYPE_WEIGHT_OFFSET] && | 233 | weightDtype_ == validInputDtypes[i][DTYPE_WEIGHT_OFFSET] && |
| 237 | runningVarDtype_ == validInputDtypes[i][DTYPE_RUNNINGVAR_OFFSET]) { | 234 | runningVarDtype_ == validInputDtypes[i][DTYPE_RUNNINGVAR_OFFSET]) { |
| 238 | - dtypeValid = true; | 235 | + inputDtypeValid = true; |
| 239 | break; | 236 | break; |
| 240 | } | 237 | } |
| 241 | } | 238 | } |
| 242 | 239 | ||
| 243 | OP_TILING_CHECK( | 240 | OP_TILING_CHECK( |
| 244 | - !dtypeValid, | 241 | + !inputDtypeValid, |
| 245 | VECTOR_INNER_ERR_REPORT_TILIING( | 242 | VECTOR_INNER_ERR_REPORT_TILIING( |
| 246 | context_->GetNodeName(), | 243 | context_->GetNodeName(), |
| 247 | "input dtypes are not supported, dyDtype_: %d, weightDtype_: %d, runningVarDtype: %d.", dyDtype_, | 244 | "input dtypes are not supported, dyDtype_: %d, weightDtype_: %d, runningVarDtype: %d.", dyDtype_, |
| 248 | weightDtype_, runningVarDtype_), | 245 | weightDtype_, runningVarDtype_), |
| 249 | return ge::GRAPH_FAILED); | 246 | return ge::GRAPH_FAILED); |
| 250 | 247 | ||
| 251 | - bool shapeValid = weightDimNum_ == runningVarDimNum_ && dyDimNum_ == dxDimNum_ && weightDimNum_ == 1 && | 248 | + bool inputShapeValid = weightDimNum_ == runningVarDimNum_ && dyDimNum_ == dxDimNum_ && weightDimNum_ == 1 && |
| 252 | weightDimLen_ == runningVarDimLen_ && weightDimLen_ == fusedALen_; | 249 | weightDimLen_ == runningVarDimLen_ && weightDimLen_ == fusedALen_; |
| 253 | 250 | ||
| 254 | OP_TILING_CHECK( | 251 | OP_TILING_CHECK( |
| 255 | - !shapeValid, | 252 | + !inputShapeValid, |
| 256 | VECTOR_INNER_ERR_REPORT_TILIING( | 253 | VECTOR_INNER_ERR_REPORT_TILIING( |
| 257 | context_->GetNodeName(), | 254 | context_->GetNodeName(), |
| 258 | "input shapes are not supported, dy dims: %ld, weight dims: %ld, runningvar dims: %ld, dx dims: %ld, " | 255 | "input shapes are not supported, dy dims: %ld, weight dims: %ld, runningvar dims: %ld, dx dims: %ld, " |
| @@ -63,7 +63,7 @@ constexpr int64_t PARAM_INPUT_RUNNINGVAR_INDEX = 4; | |||
| 63 | constexpr int64_t PARAM_OUTPUT_DX_INDEX = 0; | 63 | constexpr int64_t PARAM_OUTPUT_DX_INDEX = 0; |
| 64 | constexpr int64_t PARAM_ATTRS_EPSILON_INDEX = 1; | 64 | constexpr int64_t PARAM_ATTRS_EPSILON_INDEX = 1; |
| 65 | 65 | ||
| 66 | -constexpr float DEFAULT_EPSILON = 1e-5; | 66 | +constexpr float DEFAULT_EPSILON_VAL = 1e-5; |
| 67 | 67 | ||
| 68 | // 框架侧占位可以只预留32B（ttk正常），debugTool执行时需要预留16M | 68 | // 框架侧占位可以只预留32B（ttk正常），debugTool执行时需要预留16M |
| 69 | constexpr uint32_t MINIMAL_WORKSPACE = 16 * 1024 * 1024; | 69 | constexpr uint32_t MINIMAL_WORKSPACE = 16 * 1024 * 1024; |
| @@ -117,6 +117,7 @@ protected: | |||
| 117 | ge::graphStatus GetDyInfo(); | 117 | ge::graphStatus GetDyInfo(); |
| 118 | ge::graphStatus GetWeightRunningVarDxInfo(); | 118 | ge::graphStatus GetWeightRunningVarDxInfo(); |
| 119 | ge::graphStatus CheckInputValid(); | 119 | ge::graphStatus CheckInputValid(); |
| 120 | + // infer base class - no GetAlignValue method | ||
| 120 | 121 | ||
| 121 | protected: | 122 | protected: |
| 122 | const char* opName_ = "BatchNormGradV3InferBase"; | 123 | const char* opName_ = "BatchNormGradV3InferBase"; |
| @@ -206,15 +206,15 @@ static bool CheckFormat( | |||
| 206 | static bool CheckShape( | 206 | static bool CheckShape( |
| 207 | const aclTensor* gradOut, const aclTensor* input, const aclTensor* gradInput, const aclBoolArray* outputMask) | 207 | const aclTensor* gradOut, const aclTensor* input, const aclTensor* gradInput, const aclBoolArray* outputMask) |
| 208 | { | 208 | { |
| 209 | - const int max_check_nums = 8; | 209 | + const int max_dim_check = 8; |
| 210 | - OP_CHECK_MAX_DIM(input, max_check_nums, return false); | 210 | + OP_CHECK_MAX_DIM(input, max_dim_check, return false); |
| 211 | OP_CHECK_MIN_DIM(input, MIN_BN_DIMS, return false); | 211 | OP_CHECK_MIN_DIM(input, MIN_BN_DIMS, return false); |
| 212 | - OP_CHECK_MAX_DIM(gradOut, max_check_nums, return false); | 212 | + OP_CHECK_MAX_DIM(gradOut, max_dim_check, return false); |
| 213 | OP_CHECK_MIN_DIM(gradOut, MIN_BN_DIMS, return false); | 213 | OP_CHECK_MIN_DIM(gradOut, MIN_BN_DIMS, return false); |
| 214 | OP_CHECK_SHAPE_NOT_EQUAL(gradOut, input, return false); | 214 | OP_CHECK_SHAPE_NOT_EQUAL(gradOut, input, return false); |
| 215 | 215 | ||
| 216 | if ((*outputMask)[0]) { | 216 | if ((*outputMask)[0]) { |
| 217 | - OP_CHECK_MAX_DIM(gradInput, max_check_nums, return false); | 217 | + OP_CHECK_MAX_DIM(gradInput, max_dim_check, return false); |
| 218 | OP_CHECK_MIN_DIM(gradInput, MIN_BN_DIMS, return false); | 218 | OP_CHECK_MIN_DIM(gradInput, MIN_BN_DIMS, return false); |
| 219 | OP_CHECK_SHAPE_NOT_EQUAL(gradInput, gradOut, return false); | 219 | OP_CHECK_SHAPE_NOT_EQUAL(gradInput, gradOut, return false); |
| 220 | } | 220 | } |
| @@ -225,23 +225,20 @@ static bool CheckOtherShape( | |||
| 225 | int dimC, const aclTensor* weight, const aclTensor* runningMean, const aclTensor* runningVar) | 225 | int dimC, const aclTensor* weight, const aclTensor* runningMean, const aclTensor* runningVar) |
| 226 | { | 226 | { |
| 227 | if (weight != nullptr && (weight->GetViewShape().GetDimNum() != 1 || weight->GetViewShape()[0] != dimC)) { | 227 | if (weight != nullptr && (weight->GetViewShape().GetDimNum() != 1 || weight->GetViewShape()[0] != dimC)) { |
| 228 | - OP_LOGE( | 228 | + OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Dim of weight should be one and shape is channel num of input[%d], but got [%s].", |
| 229 | - ACLNN_ERR_PARAM_INVALID, "Dim of weight should be one and shape is channel num of input[%d], but got [%s].", | ||
| 230 | dimC, op::ToString(weight->GetViewShape()).GetString()); | 229 | dimC, op::ToString(weight->GetViewShape()).GetString()); |
| 231 | return false; | 230 | return false; |
| 232 | } | 231 | } |
| 233 | if (runningMean != nullptr && | 232 | if (runningMean != nullptr && |
| 234 | (runningMean->GetViewShape().GetDimNum() != 1 || runningMean->GetViewShape()[0] != dimC)) { | 233 | (runningMean->GetViewShape().GetDimNum() != 1 || runningMean->GetViewShape()[0] != dimC)) { |
| 235 | - OP_LOGE( | 234 | + OP_LOGE(ACLNN_ERR_PARAM_INVALID, |
| 236 | - ACLNN_ERR_PARAM_INVALID, | ||
| 237 | "Dim of runningMean should be one and shape is channel num of input[%d], but got [%s].", dimC, | 235 | "Dim of runningMean should be one and shape is channel num of input[%d], but got [%s].", dimC, |
| 238 | op::ToString(runningMean->GetViewShape()).GetString()); | 236 | op::ToString(runningMean->GetViewShape()).GetString()); |
| 239 | return false; | 237 | return false; |
| 240 | } | 238 | } |
| 241 | if (runningVar != nullptr && | 239 | if (runningVar != nullptr && |
| 242 | (runningVar->GetViewShape().GetDimNum() != 1 || runningVar->GetViewShape()[0] != dimC)) { | 240 | (runningVar->GetViewShape().GetDimNum() != 1 || runningVar->GetViewShape()[0] != dimC)) { |
| 243 | - OP_LOGE( | 241 | + OP_LOGE(ACLNN_ERR_PARAM_INVALID, |
| 244 | - ACLNN_ERR_PARAM_INVALID, | ||
| 245 | "Dim of runningVar should be one and shape is channel num of input[%d], but got [%s].", dimC, | 242 | "Dim of runningVar should be one and shape is channel num of input[%d], but got [%s].", dimC, |
| 246 | op::ToString(runningVar->GetViewShape()).GetString()); | 243 | op::ToString(runningVar->GetViewShape()).GetString()); |
| 247 | return false; | 244 | return false; |
| @@ -300,6 +297,7 @@ static bool CheckGradWeightGradBiasShape( | |||
| 300 | return true; | 297 | return true; |
| 301 | } | 298 | } |
| 302 | 299 | ||
| 300 | +// batch norm backward: get channel dimension from input tensor | ||
| 303 | static int64_t GetDimC(const aclTensor* input) | 301 | static int64_t GetDimC(const aclTensor* input) |
| 304 | { | 302 | { |
| 305 | auto viewShape = input->GetViewShape(); | 303 | auto viewShape = input->GetViewShape(); |
| @@ -409,12 +407,12 @@ aclnnStatus BatchNormPost(op::Shape& inputShape, aclTensor* bnGradInput, aclTens | |||
| 409 | { | 407 | { |
| 410 | auto inputDims = inputShape.GetDimNum(); | 408 | auto inputDims = inputShape.GetDimNum(); |
| 411 | if (inputDims > MAX_BN_DIMS) { | 409 | if (inputDims > MAX_BN_DIMS) { |
| 412 | - int64_t originShapes[inputDims]; | 410 | + int64_t batchNormOriginShapes[inputDims]; |
| 413 | for (size_t i = 0; i < inputDims; ++i) { | 411 | for (size_t i = 0; i < inputDims; ++i) { |
| 414 | - originShapes[i] = inputShape[i]; | 412 | + batchNormOriginShapes[i] = inputShape[i]; |
| 415 | } | 413 | } |
| 416 | - aclIntArray* originShapeArray = executor->AllocIntArray(originShapes, inputDims); | 414 | + aclIntArray* batchNormOriginShapeArray = executor->AllocIntArray(batchNormOriginShapes, inputDims); |
| 417 | - auto bnGradInputReshape = l0op::Reshape(bnGradInput, originShapeArray, executor); | 415 | + auto bnGradInputReshape = l0op::Reshape(bnGradInput, batchNormOriginShapeArray, executor); |
| 418 | auto bnGradInputReformat = l0op::ReFormat(bnGradInputReshape, Format::FORMAT_ND); | 416 | auto bnGradInputReformat = l0op::ReFormat(bnGradInputReshape, Format::FORMAT_ND); |
| 419 | auto viewCopyResult = l0op::ViewCopy(bnGradInputReformat, gradInput, executor); | 417 | auto viewCopyResult = l0op::ViewCopy(bnGradInputReformat, gradInput, executor); |
| 420 | CHECK_RET(viewCopyResult != nullptr, ACLNN_ERR_INNER_NULLPTR); | 418 | CHECK_RET(viewCopyResult != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| @@ -437,18 +435,18 @@ aclnnStatus BatchNormBackwardProc( | |||
| 437 | 435 | ||
| 438 | std::array<aclTensor*, UPDATE_GRAD_RESULT_CNT> grad; | 436 | std::array<aclTensor*, UPDATE_GRAD_RESULT_CNT> grad; |
| 439 | if (!training) { | 437 | if (!training) { |
| 440 | - auto runningMeanResize = op::ResizeFrom1D(runningMean, input, isSupportNcdhw, executor); | 438 | + auto batchNormRunningMeanResize = op::ResizeFrom1D(runningMean, input, isSupportNcdhw, executor); |
| 441 | - CHECK_RET(runningMeanResize != nullptr, ACLNN_ERR_INNER_NULLPTR); | 439 | + CHECK_RET(batchNormRunningMeanResize != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 442 | 440 | ||
| 443 | - auto runningVarResize = op::ResizeFrom1D(runningVar, input, isSupportNcdhw, executor); | 441 | + auto batchNormRunningVarResize = op::ResizeFrom1D(runningVar, input, isSupportNcdhw, executor); |
| 444 | - CHECK_RET(runningVarResize != nullptr, ACLNN_ERR_INNER_NULLPTR); | 442 | + CHECK_RET(batchNormRunningVarResize != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 445 | 443 | ||
| 446 | - grad = l0op::BNTrainingUpdateGrad(gradOut, input, runningMeanResize, runningVarResize, eps, executor); | 444 | + grad = l0op::BNTrainingUpdateGrad(gradOut, input, batchNormRunningMeanResize, batchNormRunningVarResize, eps, executor); |
| 447 | 445 | ||
| 448 | - auto reduceGrad = l0op::BNInferGrad(gradOut, weightResize, runningVarResize, eps, executor); | 446 | + auto batchNormReduceGrad = l0op::BNInferGrad(gradOut, weightResize, batchNormRunningVarResize, eps, executor); |
| 449 | - CHECK_RET(reduceGrad != nullptr, ACLNN_ERR_INNER_NULLPTR); | 447 | + CHECK_RET(batchNormReduceGrad != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 450 | 448 | ||
| 451 | - *gradInput = const_cast<aclTensor*>(reduceGrad); | 449 | + *gradInput = const_cast<aclTensor*>(batchNormReduceGrad); |
| 452 | } else { | 450 | } else { |
| 453 | auto saveMeanResize = op::ResizeFrom1D(saveMean, input, isSupportNcdhw, executor); | 451 | auto saveMeanResize = op::ResizeFrom1D(saveMean, input, isSupportNcdhw, executor); |
| 454 | CHECK_RET(saveMeanResize != nullptr, ACLNN_ERR_INNER_NULLPTR); | 452 | CHECK_RET(saveMeanResize != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| @@ -472,16 +470,16 @@ aclnnStatus BatchNormBackwardProc( | |||
| 472 | *gradInput = const_cast<aclTensor*>(resultNcdhw); | 470 | *gradInput = const_cast<aclTensor*>(resultNcdhw); |
| 473 | } else if (input->GetViewShape().GetDimNum() == MAX_BN_DIMS) { | 471 | } else if (input->GetViewShape().GetDimNum() == MAX_BN_DIMS) { |
| 474 | grad = l0op::BN3DTrainingUpdateGrad(gradOut, input, saveMeanResize, saveInvstdResize, eps, executor); | 472 | grad = l0op::BN3DTrainingUpdateGrad(gradOut, input, saveMeanResize, saveInvstdResize, eps, executor); |
| 475 | - auto reduceGrad = l0op::BN3DTrainingReduceGrad( | 473 | + auto batchNormReduceGrad = l0op::BN3DTrainingReduceGrad( |
| 476 | gradOut, input, grad[0], grad[1], weightResize, saveMeanResize, saveInvstdResize, eps, executor); | 474 | gradOut, input, grad[0], grad[1], weightResize, saveMeanResize, saveInvstdResize, eps, executor); |
| 477 | - CHECK_RET(reduceGrad != nullptr, ACLNN_ERR_INNER_NULLPTR); | 475 | + CHECK_RET(batchNormReduceGrad != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 478 | - *gradInput = const_cast<aclTensor*>(reduceGrad); | 476 | + *gradInput = const_cast<aclTensor*>(batchNormReduceGrad); |
| 479 | } else { | 477 | } else { |
| 480 | grad = l0op::BNTrainingUpdateGrad(gradOut, input, saveMeanResize, saveInvstdResize, eps, executor); | 478 | grad = l0op::BNTrainingUpdateGrad(gradOut, input, saveMeanResize, saveInvstdResize, eps, executor); |
| 481 | - auto reduceGrad = l0op::BNTrainingReduceGrad( | 479 | + auto batchNormReduceGrad = l0op::BNTrainingReduceGrad( |
| 482 | gradOut, input, grad[0], grad[1], weightResize, saveMeanResize, saveInvstdResize, eps, executor); | 480 | gradOut, input, grad[0], grad[1], weightResize, saveMeanResize, saveInvstdResize, eps, executor); |
| 483 | - CHECK_RET(reduceGrad != nullptr, ACLNN_ERR_INNER_NULLPTR); | 481 | + CHECK_RET(batchNormReduceGrad != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 484 | - *gradInput = const_cast<aclTensor*>(reduceGrad); | 482 | + *gradInput = const_cast<aclTensor*>(batchNormReduceGrad); |
| 485 | } | 483 | } |
| 486 | } | 484 | } |
| 487 | *gradWeight = grad[0]; | 485 | *gradWeight = grad[0]; |
| @@ -494,59 +492,60 @@ aclnnStatus BatchNormBackward( | |||
| 494 | const aclTensor* runningVar, const aclTensor* saveMean, const aclTensor* saveInvstd, bool training, float eps, | 492 | const aclTensor* runningVar, const aclTensor* saveMean, const aclTensor* saveInvstd, bool training, float eps, |
| 495 | aclTensor** gradInput, aclTensor** gradWeight, aclTensor** gradBias, aclOpExecutor* executor) | 493 | aclTensor** gradInput, aclTensor** gradWeight, aclTensor** gradBias, aclOpExecutor* executor) |
| 496 | { | 494 | { |
| 497 | - size_t dimC = input->GetViewShape()[1]; | 495 | + // batch norm backward: prepare default values for optional tensors |
| 496 | + size_t batchNormDimC = input->GetViewShape()[1]; | ||
| 498 | if (runningMean == nullptr) { | 497 | if (runningMean == nullptr) { |
| 499 | - runningMean = op::FillScalar(dimC, 0, executor); | 498 | + runningMean = op::FillScalar(batchNormDimC, 0, executor); |
| 500 | CHECK_RET(runningMean != nullptr, ACLNN_ERR_INNER_NULLPTR); | 499 | CHECK_RET(runningMean != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 501 | } | 500 | } |
| 502 | if (runningVar == nullptr) { | 501 | if (runningVar == nullptr) { |
| 503 | - runningVar = op::FillScalar(dimC, 1, executor); | 502 | + runningVar = op::FillScalar(batchNormDimC, 1, executor); |
| 504 | CHECK_RET(runningVar != nullptr, ACLNN_ERR_INNER_NULLPTR); | 503 | CHECK_RET(runningVar != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 505 | } | 504 | } |
| 506 | if (weight == nullptr) { | 505 | if (weight == nullptr) { |
| 507 | - weight = op::FillScalar(dimC, 1, executor); | 506 | + weight = op::FillScalar(batchNormDimC, 1, executor); |
| 508 | CHECK_RET(weight != nullptr, ACLNN_ERR_INNER_NULLPTR); | 507 | CHECK_RET(weight != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 509 | } | 508 | } |
| 510 | if (saveMean == nullptr) { | 509 | if (saveMean == nullptr) { |
| 511 | - saveMean = op::FillScalar(dimC, 0, executor); | 510 | + saveMean = op::FillScalar(batchNormDimC, 0, executor); |
| 512 | CHECK_RET(saveMean != nullptr, ACLNN_ERR_INNER_NULLPTR); | 511 | CHECK_RET(saveMean != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 513 | } | 512 | } |
| 514 | if (saveInvstd == nullptr) { | 513 | if (saveInvstd == nullptr) { |
| 515 | - saveInvstd = op::FillScalar(dimC, 1, executor); | 514 | + saveInvstd = op::FillScalar(batchNormDimC, 1, executor); |
| 516 | CHECK_RET(saveInvstd != nullptr, ACLNN_ERR_INNER_NULLPTR); | 515 | CHECK_RET(saveInvstd != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 517 | } | 516 | } |
| 518 | 517 | ||
| 519 | size_t dimNum = input->GetViewShape().GetDimNum(); | 518 | size_t dimNum = input->GetViewShape().GetDimNum(); |
| 520 | - auto gradOutPre = gradOut; | 519 | + auto batchNormGradOutPre = gradOut; |
| 521 | - auto inputPre = input; | 520 | + auto batchNormInputPre = input; |
| 522 | if (dimNum < BN2D_INPUT_DIMS) { | 521 | if (dimNum < BN2D_INPUT_DIMS) { |
| 523 | - gradOutPre = op::ResizeFromND(gradOut, executor); | 522 | + batchNormGradOutPre = op::ResizeFromND(gradOut, executor); |
| 524 | - CHECK_RET(gradOutPre != nullptr, ACLNN_ERR_INNER_NULLPTR); | 523 | + CHECK_RET(batchNormGradOutPre != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 525 | 524 | ||
| 526 | - inputPre = op::ResizeFromND(input, executor); | 525 | + batchNormInputPre = op::ResizeFromND(input, executor); |
| 527 | - CHECK_RET(inputPre != nullptr, ACLNN_ERR_INNER_NULLPTR); | 526 | + CHECK_RET(batchNormInputPre != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 528 | } else if (!training && dimNum == MAX_BN_DIMS) { | 527 | } else if (!training && dimNum == MAX_BN_DIMS) { |
| 529 | - gradOutPre = op::ResizeFrom5D(gradOut, executor); | 528 | + batchNormGradOutPre = op::ResizeFrom5D(gradOut, executor); |
| 530 | - CHECK_RET(gradOutPre != nullptr, ACLNN_ERR_INNER_NULLPTR); | 529 | + CHECK_RET(batchNormGradOutPre != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 531 | 530 | ||
| 532 | - inputPre = op::ResizeFrom5D(input, executor); | 531 | + batchNormInputPre = op::ResizeFrom5D(input, executor); |
| 533 | - CHECK_RET(inputPre != nullptr, ACLNN_ERR_INNER_NULLPTR); | 532 | + CHECK_RET(batchNormInputPre != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 534 | } | 533 | } |
| 535 | 534 | ||
| 536 | - aclTensor* result = nullptr; | 535 | + aclTensor* batchNormResult = nullptr; |
| 537 | auto bnResult = BatchNormBackwardProc( | 536 | auto bnResult = BatchNormBackwardProc( |
| 538 | - gradOutPre, inputPre, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, &result, gradWeight, | 537 | + batchNormGradOutPre, batchNormInputPre, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, &batchNormResult, gradWeight, |
| 539 | gradBias, executor); | 538 | gradBias, executor); |
| 540 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); | 539 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); |
| 541 | 540 | ||
| 542 | - *gradInput = result; | 541 | + *gradInput = batchNormResult; |
| 543 | if (dimNum < BN2D_INPUT_DIMS) { | 542 | if (dimNum < BN2D_INPUT_DIMS) { |
| 544 | - auto outputFormat = op::ResizeToND(result, input, executor); | 543 | + auto outputFormat = op::ResizeToND(batchNormResult, input, executor); |
| 545 | CHECK_RET(outputFormat != nullptr, ACLNN_ERR_INNER_NULLPTR); | 544 | CHECK_RET(outputFormat != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 546 | 545 | ||
| 547 | *gradInput = const_cast<aclTensor*>(outputFormat); | 546 | *gradInput = const_cast<aclTensor*>(outputFormat); |
| 548 | } else if (!training && dimNum == MAX_BN_DIMS) { | 547 | } else if (!training && dimNum == MAX_BN_DIMS) { |
| 549 | - auto outputTranspose = op::ResizeTo5D(result, input, executor); | 548 | + auto outputTranspose = op::ResizeTo5D(batchNormResult, input, executor); |
| 550 | CHECK_RET(outputTranspose != nullptr, ACLNN_ERR_INNER_NULLPTR); | 549 | CHECK_RET(outputTranspose != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 551 | 550 | ||
| 552 | *gradInput = const_cast<aclTensor*>(outputTranspose); | 551 | *gradInput = const_cast<aclTensor*>(outputTranspose); |
| @@ -763,18 +762,18 @@ aclnnStatus BatchNormBackwardDavid( | |||
| 763 | } | 762 | } |
| 764 | 763 | ||
| 765 | size_t dimNum = input->GetViewShape().GetDimNum(); | 764 | size_t dimNum = input->GetViewShape().GetDimNum(); |
| 766 | - auto gradOutPre = gradOut; | 765 | + auto batchNormGradOutPre2 = gradOut; |
| 767 | - auto inputPre = input; | 766 | + auto batchNormInputPre2 = input; |
| 768 | if (dimNum < BN2D_INPUT_DIMS) { | 767 | if (dimNum < BN2D_INPUT_DIMS) { |
| 769 | - gradOutPre = op::ResizeFromND(gradOut, executor); | 768 | + batchNormGradOutPre2 = op::ResizeFromND(gradOut, executor); |
| 770 | - CHECK_RET(gradOutPre != nullptr, ACLNN_ERR_INNER_NULLPTR); | 769 | + CHECK_RET(batchNormGradOutPre2 != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 771 | 770 | ||
| 772 | - inputPre = op::ResizeFromND(input, executor); | 771 | + batchNormInputPre2 = op::ResizeFromND(input, executor); |
| 773 | - CHECK_RET(inputPre != nullptr, ACLNN_ERR_INNER_NULLPTR); | 772 | + CHECK_RET(batchNormInputPre2 != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 774 | } | 773 | } |
| 775 | aclTensor* result = nullptr; | 774 | aclTensor* result = nullptr; |
| 776 | auto bnResult = BatchNormBackwardProcDavid( | 775 | auto bnResult = BatchNormBackwardProcDavid( |
| 777 | - gradOutPre, inputPre, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, outputMask, &result, | 776 | + batchNormGradOutPre2, batchNormInputPre2, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, outputMask, &result, |
| 778 | gradWeight, gradBias, executor); | 777 | gradWeight, gradBias, executor); |
| 779 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); | 778 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); |
| 780 | 779 | ||
| @@ -800,15 +799,15 @@ aclnnStatus aclnnBatchNormBackwardGetWorkspaceSize( | |||
| 800 | DFX_IN(gradOut, input, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, outputMask), | 799 | DFX_IN(gradOut, input, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, outputMask), |
| 801 | DFX_OUT(gradInput, gradWeight, gradBias)); | 800 | DFX_OUT(gradInput, gradWeight, gradBias)); |
| 802 | 801 | ||
| 803 | - // 固定写法，创建OpExecutor | 802 | + // batch norm backward: 创建OpExecutor |
| 804 | - auto uniqueExecutor = CREATE_EXECUTOR(); | 803 | + auto batchNormUniqueExecutor = CREATE_EXECUTOR(); |
| 805 | - CHECK_RET(uniqueExecutor.get() != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR); | 804 | + CHECK_RET(batchNormUniqueExecutor.get() != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR); |
| 806 | 805 | ||
| 807 | CHECK_RET(CheckInputNotNull(gradOut, input), ACLNN_ERR_PARAM_NULLPTR); | 806 | CHECK_RET(CheckInputNotNull(gradOut, input), ACLNN_ERR_PARAM_NULLPTR); |
| 808 | 807 | ||
| 809 | if (input->IsEmpty() || gradOut->IsEmpty()) { | 808 | if (input->IsEmpty() || gradOut->IsEmpty()) { |
| 810 | *workspaceSize = 0UL; | 809 | *workspaceSize = 0UL; |
| 811 | - uniqueExecutor.ReleaseTo(executor); | 810 | + batchNormUniqueExecutor.ReleaseTo(executor); |
| 812 | return ACLNN_SUCCESS; | 811 | return ACLNN_SUCCESS; |
| 813 | } | 812 | } |
| 814 | 813 | ||
| @@ -821,23 +820,23 @@ aclnnStatus aclnnBatchNormBackwardGetWorkspaceSize( | |||
| 821 | !(*outputMask)[0] && !(*outputMask)[GRAD_WEIGHT_INDEX] && !(*outputMask)[GRAD_BIAS_INDEX]; | 820 | !(*outputMask)[0] && !(*outputMask)[GRAD_WEIGHT_INDEX] && !(*outputMask)[GRAD_BIAS_INDEX]; |
| 822 | if (canEarlyReturn) { | 821 | if (canEarlyReturn) { |
| 823 | *workspaceSize = 0UL; | 822 | *workspaceSize = 0UL; |
| 824 | - uniqueExecutor.ReleaseTo(executor); | 823 | + batchNormUniqueExecutor.ReleaseTo(executor); |
| 825 | return ACLNN_SUCCESS; | 824 | return ACLNN_SUCCESS; |
| 826 | } | 825 | } |
| 827 | 826 | ||
| 828 | - auto inputContiguous = l0op::Contiguous(input, uniqueExecutor.get()); | 827 | + auto batchNormInputContiguous = l0op::Contiguous(input, batchNormUniqueExecutor.get()); |
| 829 | - CHECK_RET(inputContiguous != nullptr, ACLNN_ERR_INNER_NULLPTR); | 828 | + CHECK_RET(batchNormInputContiguous != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 830 | - auto gradOutContigous = l0op::Contiguous(gradOut, uniqueExecutor.get()); | 829 | + auto batchNormGradOutContigous = l0op::Contiguous(gradOut, batchNormUniqueExecutor.get()); |
拼写错误: 变量名'batchNormGradOutContigous'中的'Contigous'拼写错误，应为'Contiguous'。这个拼写错误出现在多个地方（第830、839行），会影响代码的可读性和专业性。 问题类型: 拼写错误 文件路径: norm/batch_norm_grad_v3/op_host/op_api/aclnn_batch_norm_backward.cpp行号: 829 问题代码: auto batchNormGradOutContigous = l0op::Contiguous(gradOut, batchNormUniqueExecutor.get()); 修改建议: 将变量名更正为'batchNormGradOutContiguous'以保持拼写正确性。同时检查代码中其他类似拼写错误并一并更正。 --- 此评论由代码审查工具自动生成 | |||
| 831 | - CHECK_RET(gradOutContigous != nullptr, ACLNN_ERR_INNER_NULLPTR); | 830 | + CHECK_RET(batchNormGradOutContigous != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 832 | 831 | ||
| 833 | auto inputShape = input->GetViewShape(); | 832 | auto inputShape = input->GetViewShape(); |
| 834 | if (inputShape.GetDimNum() > MAX_BN_DIMS) { | 833 | if (inputShape.GetDimNum() > MAX_BN_DIMS) { |
| 835 | - const int64_t shapes[5] = {inputShape[0], inputShape[1], inputShape[2], inputShape[3], -1}; | 834 | + const int64_t batchNormShapes[5] = {inputShape[0], inputShape[1], inputShape[2], inputShape[3], -1}; |
| 836 | - aclIntArray* shapeArray = uniqueExecutor.get()->AllocIntArray(shapes, 5); | 835 | + aclIntArray* batchNormShapeArray = batchNormUniqueExecutor.get()->AllocIntArray(batchNormShapes, 5); |
| 837 | - inputContiguous = l0op::Reshape(inputContiguous, shapeArray, uniqueExecutor.get()); | 836 | + batchNormInputContiguous = l0op::Reshape(batchNormInputContiguous, batchNormShapeArray, batchNormUniqueExecutor.get()); |
| 838 | - inputContiguous = l0op::ReFormat(inputContiguous, Format::FORMAT_NCDHW); | 837 | + batchNormInputContiguous = l0op::ReFormat(batchNormInputContiguous, Format::FORMAT_NCDHW); |
| 839 | - gradOutContigous = l0op::Reshape(gradOutContigous, shapeArray, uniqueExecutor.get()); | 838 | + batchNormGradOutContigous = l0op::Reshape(batchNormGradOutContigous, batchNormShapeArray, batchNormUniqueExecutor.get()); |
| 840 | - gradOutContigous = l0op::ReFormat(gradOutContigous, Format::FORMAT_NCDHW); | 839 | + batchNormGradOutContigous = l0op::ReFormat(batchNormGradOutContigous, Format::FORMAT_NCDHW); |
| 841 | } | 840 | } |
| 842 | 841 | ||
| 843 | aclTensor* bnGradInput = nullptr; | 842 | aclTensor* bnGradInput = nullptr; |
| @@ -846,56 +845,56 @@ aclnnStatus aclnnBatchNormBackwardGetWorkspaceSize( | |||
| 846 | aclnnStatus bnResult; | 845 | aclnnStatus bnResult; |
| 847 | if (Ops::NN::AclnnUtil::IsRegbase()) { | 846 | if (Ops::NN::AclnnUtil::IsRegbase()) { |
| 848 | bnResult = BatchNormBackwardDavid( | 847 | bnResult = BatchNormBackwardDavid( |
| 849 | - gradOutContigous, inputContiguous, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, | 848 | + batchNormGradOutContigous, batchNormInputContiguous, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, |
| 850 | - outputMask, &bnGradInput, &bnGradWeight, &bnGradBias, uniqueExecutor.get()); | 849 | + outputMask, &bnGradInput, &bnGradWeight, &bnGradBias, batchNormUniqueExecutor.get()); |
| 851 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); | 850 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); |
| 852 | 851 | ||
| 853 | if ((*outputMask)[GRAD_WEIGHT_INDEX]) { | 852 | if ((*outputMask)[GRAD_WEIGHT_INDEX]) { |
| 854 | CHECK_RET(bnGradWeight != nullptr, ACLNN_ERR_INNER_NULLPTR); | 853 | CHECK_RET(bnGradWeight != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 855 | - auto gradWeightCast = l0op::Cast(bnGradWeight, gradWeight->GetDataType(), uniqueExecutor.get()); | 854 | + auto gradWeightCast = l0op::Cast(bnGradWeight, gradWeight->GetDataType(), batchNormUniqueExecutor.get()); |
| 856 | CHECK_RET(gradWeightCast != nullptr, ACLNN_ERR_INNER_NULLPTR); | 855 | CHECK_RET(gradWeightCast != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 857 | - auto gradWeightResult = l0op::ViewCopy(gradWeightCast, gradWeight, uniqueExecutor.get()); | 856 | + auto gradWeightResult = l0op::ViewCopy(gradWeightCast, gradWeight, batchNormUniqueExecutor.get()); |
| 858 | CHECK_RET(gradWeightResult != nullptr, ACLNN_ERR_INNER_NULLPTR); | 857 | CHECK_RET(gradWeightResult != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 859 | } | 858 | } |
| 860 | 859 | ||
| 861 | if ((*outputMask)[GRAD_BIAS_INDEX]) { | 860 | if ((*outputMask)[GRAD_BIAS_INDEX]) { |
| 862 | CHECK_RET(bnGradBias != nullptr, ACLNN_ERR_INNER_NULLPTR); | 861 | CHECK_RET(bnGradBias != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 863 | - auto gradBiasCast = l0op::Cast(bnGradBias, gradBias->GetDataType(), uniqueExecutor.get()); | 862 | + auto gradBiasCast = l0op::Cast(bnGradBias, gradBias->GetDataType(), batchNormUniqueExecutor.get()); |
| 864 | CHECK_RET(gradBiasCast != nullptr, ACLNN_ERR_INNER_NULLPTR); | 863 | CHECK_RET(gradBiasCast != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 865 | - auto gradBiasResult = l0op::ViewCopy(gradBiasCast, gradBias, uniqueExecutor.get()); | 864 | + auto gradBiasResult = l0op::ViewCopy(gradBiasCast, gradBias, batchNormUniqueExecutor.get()); |
| 866 | CHECK_RET(gradBiasResult != nullptr, ACLNN_ERR_INNER_NULLPTR); | 865 | CHECK_RET(gradBiasResult != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 867 | } | 866 | } |
| 868 | 867 | ||
| 869 | if ((*outputMask)[0]) { | 868 | if ((*outputMask)[0]) { |
| 870 | CHECK_RET(bnGradInput != nullptr, ACLNN_ERR_INNER_NULLPTR); | 869 | CHECK_RET(bnGradInput != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 871 | - auto viewCopyInput = BatchNormPost(inputShape, bnGradInput, gradInput, uniqueExecutor.get()); | 870 | + auto viewCopyInput = BatchNormPost(inputShape, bnGradInput, gradInput, batchNormUniqueExecutor.get()); |
| 872 | CHECK_RET(viewCopyInput == ACLNN_SUCCESS, viewCopyInput); | 871 | CHECK_RET(viewCopyInput == ACLNN_SUCCESS, viewCopyInput); |
| 873 | } | 872 | } |
| 874 | } else { | 873 | } else { |
| 875 | bnResult = BatchNormBackward( | 874 | bnResult = BatchNormBackward( |
| 876 | - gradOutContigous, inputContiguous, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, | 875 | + batchNormGradOutContigous, batchNormInputContiguous, weight, runningMean, runningVar, saveMean, saveInvstd, training, eps, |
| 877 | - &bnGradInput, &bnGradWeight, &bnGradBias, uniqueExecutor.get()); | 876 | + &bnGradInput, &bnGradWeight, &bnGradBias, batchNormUniqueExecutor.get()); |
| 878 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); | 877 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); |
| 879 | if ((*outputMask)[GRAD_WEIGHT_INDEX]) { | 878 | if ((*outputMask)[GRAD_WEIGHT_INDEX]) { |
| 880 | auto viewCopyWeight = | 879 | auto viewCopyWeight = |
| 881 | - op::ResizeTo1D(bnGradWeight, gradWeight, isBatchNormSupportNcdhw(), uniqueExecutor.get()); | 880 | + op::ResizeTo1D(bnGradWeight, gradWeight, isBatchNormSupportNcdhw(), batchNormUniqueExecutor.get()); |
| 882 | CHECK_RET(viewCopyWeight != nullptr, ACLNN_ERR_INNER_NULLPTR); | 881 | CHECK_RET(viewCopyWeight != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 883 | } | 882 | } |
| 884 | 883 | ||
| 885 | if ((*outputMask)[GRAD_BIAS_INDEX]) { | 884 | if ((*outputMask)[GRAD_BIAS_INDEX]) { |
| 886 | - auto viewCopyBias = op::ResizeTo1D(bnGradBias, gradBias, isBatchNormSupportNcdhw(), uniqueExecutor.get()); | 885 | + auto viewCopyBias = op::ResizeTo1D(bnGradBias, gradBias, isBatchNormSupportNcdhw(), batchNormUniqueExecutor.get()); |
| 887 | CHECK_RET(viewCopyBias != nullptr, ACLNN_ERR_INNER_NULLPTR); | 886 | CHECK_RET(viewCopyBias != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 888 | } | 887 | } |
| 889 | 888 | ||
| 890 | if ((*outputMask)[0]) { | 889 | if ((*outputMask)[0]) { |
| 891 | - auto viewCopyInput = BatchNormPost(inputShape, bnGradInput, gradInput, uniqueExecutor.get()); | 890 | + auto viewCopyInput = BatchNormPost(inputShape, bnGradInput, gradInput, batchNormUniqueExecutor.get()); |
| 892 | CHECK_RET(viewCopyInput == ACLNN_SUCCESS, viewCopyInput); | 891 | CHECK_RET(viewCopyInput == ACLNN_SUCCESS, viewCopyInput); |
| 893 | } | 892 | } |
| 894 | } | 893 | } |
| 895 | 894 | ||
| 896 | // 固定写法，获取计算过程中需要使用的workspace大小 | 895 | // 固定写法，获取计算过程中需要使用的workspace大小 |
| 897 | - *workspaceSize = uniqueExecutor->GetWorkspaceSize(); | 896 | + *workspaceSize = batchNormUniqueExecutor->GetWorkspaceSize(); |
| 898 | - uniqueExecutor.ReleaseTo(executor); | 897 | + batchNormUniqueExecutor.ReleaseTo(executor); |
| 899 | return ACLNN_SUCCESS; | 898 | return ACLNN_SUCCESS; |
| 900 | } | 899 | } |
| 901 | 900 | ||
| @@ -40,6 +40,7 @@ using namespace op; | |||
| 40 | extern "C" { | 40 | extern "C" { |
| 41 | 41 | ||
| 42 | 42 | ||
| 43 | +// batch norm backward constants | ||
| 43 | constexpr size_t GRAD_WEIGHT_INDEX = 1; | 44 | constexpr size_t GRAD_WEIGHT_INDEX = 1; |
| 44 | constexpr size_t GRAD_BIAS_INDEX = 2; | 45 | constexpr size_t GRAD_BIAS_INDEX = 2; |
| 45 | constexpr size_t MIN_BN_DIMS = 2; | 46 | constexpr size_t MIN_BN_DIMS = 2; |
| @@ -89,6 +90,7 @@ static inline bool isBatchNormSupportAscendC(void) | |||
| 89 | static bool CheckMaskNotNull( | 90 | static bool CheckMaskNotNull( |
| 90 | const aclTensor* gradInput, const aclTensor* gradWeight, const aclTensor* gradBias, const aclBoolArray* outputMask) | 91 | const aclTensor* gradInput, const aclTensor* gradWeight, const aclTensor* gradBias, const aclBoolArray* outputMask) |
| 91 | { | 92 | { |
| 93 | + // fast batch norm backward: check output mask validity | ||
| 92 | OP_CHECK_NULL(outputMask, return false); | 94 | OP_CHECK_NULL(outputMask, return false); |
| 93 | if (outputMask->Size() < GRAD_WEIGHT_INDEX) { | 95 | if (outputMask->Size() < GRAD_WEIGHT_INDEX) { |
| 94 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, "outputMask size should not less than 1."); | 96 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, "outputMask size should not less than 1."); |
| @@ -267,12 +269,12 @@ aclnnStatus FastBatchNormPost( | |||
| 267 | { | 269 | { |
| 268 | auto inputDims = inputShape.GetDimNum(); | 270 | auto inputDims = inputShape.GetDimNum(); |
| 269 | if (inputDims > MAX_BN_DIMS) { | 271 | if (inputDims > MAX_BN_DIMS) { |
| 270 | - int64_t originShapes[inputDims]; | 272 | + int64_t fastOriginShapes[inputDims]; |
| 271 | for (size_t i = 0; i < inputDims; ++i) { | 273 | for (size_t i = 0; i < inputDims; ++i) { |
| 272 | - originShapes[i] = inputShape[i]; | 274 | + fastOriginShapes[i] = inputShape[i]; |
| 273 | } | 275 | } |
| 274 | - aclIntArray* originShapeArray = executor->AllocIntArray(originShapes, inputDims); | 276 | + aclIntArray* fastOriginShapeArray = executor->AllocIntArray(fastOriginShapes, inputDims); |
| 275 | - auto bnGradInputReshape = l0op::Reshape(bnGradInput, originShapeArray, executor); | 277 | + auto bnGradInputReshape = l0op::Reshape(bnGradInput, fastOriginShapeArray, executor); |
| 276 | auto bnGradInputReformat = l0op::ReFormat(bnGradInputReshape, Format::FORMAT_ND); | 278 | auto bnGradInputReformat = l0op::ReFormat(bnGradInputReshape, Format::FORMAT_ND); |
| 277 | auto viewCopyResult = l0op::ViewCopy(bnGradInputReformat, gradInput, executor); | 279 | auto viewCopyResult = l0op::ViewCopy(bnGradInputReformat, gradInput, executor); |
| 278 | CHECK_RET(viewCopyResult != nullptr, ACLNN_ERR_INNER_NULLPTR); | 280 | CHECK_RET(viewCopyResult != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| @@ -315,12 +317,14 @@ aclnnStatus FastBatchNormBackwardProc( | |||
| 315 | CHECK_RET(saveInvstdResize != nullptr, ACLNN_ERR_INNER_NULLPTR); | 317 | CHECK_RET(saveInvstdResize != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 316 | 318 | ||
| 317 | if (input->GetViewShape().GetDimNum() == MAX_BN_DIMS) { | 319 | if (input->GetViewShape().GetDimNum() == MAX_BN_DIMS) { |
| 320 | + // fast batch norm: 3D training update grad | ||
| 318 | grad = l0op::BN3DTrainingUpdateGrad(gradOut, input, saveMeanResize, saveInvstdResize, eps, executor); | 321 | grad = l0op::BN3DTrainingUpdateGrad(gradOut, input, saveMeanResize, saveInvstdResize, eps, executor); |
| 319 | auto reduceGrad = l0op::BN3DTrainingReduceGrad( | 322 | auto reduceGrad = l0op::BN3DTrainingReduceGrad( |
| 320 | gradOut, input, grad[0], grad[1], weightResize, saveMeanResize, saveInvstdResize, eps, executor); | 323 | gradOut, input, grad[0], grad[1], weightResize, saveMeanResize, saveInvstdResize, eps, executor); |
| 321 | CHECK_RET(reduceGrad != nullptr, ACLNN_ERR_INNER_NULLPTR); | 324 | CHECK_RET(reduceGrad != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 322 | *gradInput = const_cast<aclTensor*>(reduceGrad); | 325 | *gradInput = const_cast<aclTensor*>(reduceGrad); |
| 323 | } else { | 326 | } else { |
| 327 | + // fast batch norm: 2D training update grad | ||
| 324 | grad = l0op::BNTrainingUpdateGrad(gradOut, input, saveMeanResize, saveInvstdResize, eps, executor); | 328 | grad = l0op::BNTrainingUpdateGrad(gradOut, input, saveMeanResize, saveInvstdResize, eps, executor); |
| 325 | auto reduceGrad = l0op::BNTrainingReduceGrad( | 329 | auto reduceGrad = l0op::BNTrainingReduceGrad( |
| 326 | gradOut, input, grad[0], grad[1], weightResize, saveMeanResize, saveInvstdResize, eps, executor); | 330 | gradOut, input, grad[0], grad[1], weightResize, saveMeanResize, saveInvstdResize, eps, executor); |
| @@ -485,6 +489,7 @@ aclnnStatus FastBatchNormBackwardPrepare( | |||
| 485 | const aclTensor* input, const aclTensor*& weight, const aclTensor*& runningMean, const aclTensor*& runningVar, | 489 | const aclTensor* input, const aclTensor*& weight, const aclTensor*& runningMean, const aclTensor*& runningVar, |
| 486 | const aclTensor*& saveMean, const aclTensor*& saveInvstd, aclOpExecutor* executor) | 490 | const aclTensor*& saveMean, const aclTensor*& saveInvstd, aclOpExecutor* executor) |
| 487 | { | 491 | { |
| 492 | + // fast batch norm backward: prepare default values for optional tensors | ||
| 488 | size_t dimC = input->GetViewShape()[1]; | 493 | size_t dimC = input->GetViewShape()[1]; |
| 489 | if (runningMean == nullptr) { | 494 | if (runningMean == nullptr) { |
| 490 | runningMean = op::FillScalar(dimC, 0, executor); | 495 | runningMean = op::FillScalar(dimC, 0, executor); |
| @@ -519,6 +524,7 @@ aclnnStatus FastBatchNormBackward( | |||
| 519 | CHECK_RET(bnPrepareResp == ACLNN_SUCCESS, bnPrepareResp); | 524 | CHECK_RET(bnPrepareResp == ACLNN_SUCCESS, bnPrepareResp); |
| 520 | 525 | ||
| 521 | size_t dimNum = input->GetViewShape().GetDimNum(); | 526 | size_t dimNum = input->GetViewShape().GetDimNum(); |
| 527 | + // fast batch norm: prepare input tensors for processing | ||
| 522 | auto gradOutPre = gradOut; | 528 | auto gradOutPre = gradOut; |
| 523 | auto inputPre = input; | 529 | auto inputPre = input; |
| 524 | if (dimNum < BN2D_INPUT_DIMS) { | 530 | if (dimNum < BN2D_INPUT_DIMS) { |
| @@ -529,6 +535,7 @@ aclnnStatus FastBatchNormBackward( | |||
| 529 | CHECK_RET(inputPre != nullptr, ACLNN_ERR_INNER_NULLPTR); | 535 | CHECK_RET(inputPre != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 530 | } | 536 | } |
| 531 | 537 | ||
| 538 | + // fast batch norm: process backward computation | ||
| 532 | aclTensor* result = nullptr; | 539 | aclTensor* result = nullptr; |
| 533 | if (isBatchNormSupportAscendC()) { | 540 | if (isBatchNormSupportAscendC()) { |
| 534 | auto bnResult = FastBatchNormBackwardProcForAscendC( | 541 | auto bnResult = FastBatchNormBackwardProcForAscendC( |
| @@ -97,16 +97,16 @@ bool BatchNormV3TilingBase::CheckInputShape() | |||
| 97 | auto xStorageShape = xShape->GetStorageShape(); | 97 | auto xStorageShape = xShape->GetStorageShape(); |
| 98 | auto weightShape = context_->GetInputShape(WEIGHT_INPUT_IDX); | 98 | auto weightShape = context_->GetInputShape(WEIGHT_INPUT_IDX); |
| 99 | OP_CHECK_IF(weightShape == nullptr, OP_LOGE("BatchNormV3TilingBase", "weightShape is null"), return false); | 99 | OP_CHECK_IF(weightShape == nullptr, OP_LOGE("BatchNormV3TilingBase", "weightShape is null"), return false); |
| 100 | - auto weightStorageShape = weightShape->GetStorageShape(); | 100 | + auto bnWeightStorageShape = weightShape->GetStorageShape(); |
| 101 | - auto biasShape = context_->GetInputShape(BIAS_INPUT_IDX); | 101 | + auto bnBiasShape = context_->GetInputShape(BIAS_INPUT_IDX); |
| 102 | - OP_CHECK_IF(biasShape == nullptr, OP_LOGE("BatchNormV3TilingBase", "biasShape is null"), return false); | 102 | + OP_CHECK_IF(bnBiasShape == nullptr, OP_LOGE("BatchNormV3TilingBase", "biasShape is null"), return false); |
| 103 | - auto biasStorageShape = biasShape->GetStorageShape(); | 103 | + auto bnBiasStorageShape = bnBiasShape->GetStorageShape(); |
| 104 | - auto meanShape = context_->GetInputShape(MEAN_INPUT_IDX); | 104 | + auto bnMeanShape = context_->GetInputShape(MEAN_INPUT_IDX); |
| 105 | - OP_CHECK_IF(meanShape == nullptr, OP_LOGE("BatchNormV3TilingBase", "meanShape is null"), return false); | 105 | + OP_CHECK_IF(bnMeanShape == nullptr, OP_LOGE("BatchNormV3TilingBase", "meanShape is null"), return false); |
| 106 | - auto meanStorageShape = meanShape->GetStorageShape(); | 106 | + auto bnMeanStorageShape = bnMeanShape->GetStorageShape(); |
| 107 | - auto varShape = context_->GetInputShape(VAR_INPUT_IDX); | 107 | + auto bnVarShape = context_->GetInputShape(VAR_INPUT_IDX); |
| 108 | - OP_CHECK_IF(varShape == nullptr, OP_LOGE("BatchNormV3TilingBase", "varShape is null"), return false); | 108 | + OP_CHECK_IF(bnVarShape == nullptr, OP_LOGE("BatchNormV3TilingBase", "varShape is null"), return false); |
| 109 | - auto varStorageShape = varShape->GetStorageShape(); | 109 | + auto bnVarStorageShape = bnVarShape->GetStorageShape(); |
| 110 | auto xDesc = context_->GetInputDesc(X_INPUT_IDX); | 110 | auto xDesc = context_->GetInputDesc(X_INPUT_IDX); |
| 111 | auto format = xDesc->GetFormat().GetStorageFormat(); | 111 | auto format = xDesc->GetFormat().GetStorageFormat(); |
| 112 | if (format == FORMAT_NCHW) { | 112 | if (format == FORMAT_NCHW) { |
| @@ -138,28 +138,28 @@ bool BatchNormV3TilingBase::CheckInputShape() | |||
| 138 | commonParams.patternR0 <= 0, OP_LOGE(commonParams.nodeName, "x shape dim_2 * dim_3 should be more than zero."), | 138 | commonParams.patternR0 <= 0, OP_LOGE(commonParams.nodeName, "x shape dim_2 * dim_3 should be more than zero."), |
| 139 | return false); | 139 | return false); |
| 140 | OP_CHECK_IF( | 140 | OP_CHECK_IF( |
| 141 | - weightStorageShape.GetShapeSize() != commonParams.patternA, | 141 | + bnWeightStorageShape.GetShapeSize() != commonParams.patternA, |
| 142 | OP_LOGE( | 142 | OP_LOGE( |
| 143 | commonParams.nodeName, "weight ShapeSize: %ld should equal x shape C dim: %ld", | 143 | commonParams.nodeName, "weight ShapeSize: %ld should equal x shape C dim: %ld", |
| 144 | - weightStorageShape.GetShapeSize(), commonParams.patternA), | 144 | + bnWeightStorageShape.GetShapeSize(), commonParams.patternA), |
| 145 | return false); | 145 | return false); |
| 146 | OP_CHECK_IF( | 146 | OP_CHECK_IF( |
| 147 | - biasStorageShape.GetShapeSize() != commonParams.patternA, | 147 | + bnBiasStorageShape.GetShapeSize() != commonParams.patternA, |
| 148 | OP_LOGE( | 148 | OP_LOGE( |
| 149 | commonParams.nodeName, "bias ShapeSize: %ld should equal x shape C dim: %ld", | 149 | commonParams.nodeName, "bias ShapeSize: %ld should equal x shape C dim: %ld", |
| 150 | - biasStorageShape.GetShapeSize(), commonParams.patternA), | 150 | + bnBiasStorageShape.GetShapeSize(), commonParams.patternA), |
| 151 | return false); | 151 | return false); |
| 152 | OP_CHECK_IF( | 152 | OP_CHECK_IF( |
| 153 | - meanStorageShape.GetShapeSize() != commonParams.patternA, | 153 | + bnMeanStorageShape.GetShapeSize() != commonParams.patternA, |
| 154 | OP_LOGE( | 154 | OP_LOGE( |
| 155 | commonParams.nodeName, "running_mean ShapeSize: %ld should equal x shape C dim: %ld", | 155 | commonParams.nodeName, "running_mean ShapeSize: %ld should equal x shape C dim: %ld", |
| 156 | - meanStorageShape.GetShapeSize(), commonParams.patternA), | 156 | + bnMeanStorageShape.GetShapeSize(), commonParams.patternA), |
| 157 | return false); | 157 | return false); |
| 158 | OP_CHECK_IF( | 158 | OP_CHECK_IF( |
| 159 | - varStorageShape.GetShapeSize() != commonParams.patternA, | 159 | + bnVarStorageShape.GetShapeSize() != commonParams.patternA, |
| 160 | OP_LOGE( | 160 | OP_LOGE( |
| 161 | commonParams.nodeName, "running_var ShapeSize: %ld should equal x shape C dim: %ld", | 161 | commonParams.nodeName, "running_var ShapeSize: %ld should equal x shape C dim: %ld", |
| 162 | - varStorageShape.GetShapeSize(), commonParams.patternA), | 162 | + bnVarStorageShape.GetShapeSize(), commonParams.patternA), |
| 163 | return false); | 163 | return false); |
| 164 | return true; | 164 | return true; |
| 165 | } | 165 | } |
| @@ -202,10 +202,10 @@ ge::graphStatus BatchNormV3TilingBase::GetPlatformInfo() | |||
| 202 | commonParams.ubSizePlatForm = compileInfoPtr->ubSize; | 202 | commonParams.ubSizePlatForm = compileInfoPtr->ubSize; |
| 203 | } | 203 | } |
| 204 | OP_CHECK_IF( | 204 | OP_CHECK_IF( |
| 205 | - commonParams.coreNum == 0, OP_LOGE(commonParams.nodeName, "numBlocks should not be equal to zero."), | 205 | + commonParams.coreNum == 0, OP_LOGE(commonParams.nodeName, "coreNum should not be equal to zero."), |
| 206 | return ge::GRAPH_FAILED); | 206 | return ge::GRAPH_FAILED); |
| 207 | OP_CHECK_IF( | 207 | OP_CHECK_IF( |
| 208 | - commonParams.ubSizePlatForm == 0, OP_LOGE(commonParams.nodeName, "ubSize should not be equal to zero."), | 208 | + commonParams.ubSizePlatForm == 0, OP_LOGE(commonParams.nodeName, "ubSizePlatForm should not be equal to zero."), |
| 209 | return ge::GRAPH_FAILED); | 209 | return ge::GRAPH_FAILED); |
| 210 | return ge::GRAPH_SUCCESS; | 210 | return ge::GRAPH_SUCCESS; |
| 211 | } | 211 | } |
| @@ -37,13 +37,13 @@ uint32_t BatchNormV3WelfordTiling::FindDichotomizeAddDiffSize(uint32_t parallelN | |||
| 37 | { | 37 | { |
| 38 | // 找到parallelN与小于parallelN的最近二次幂的差值 例如：parallelN = 15，结果为15 - 8 = 7 | 38 | // 找到parallelN与小于parallelN的最近二次幂的差值 例如：parallelN = 15，结果为15 - 8 = 7 |
| 39 | if ((parallelN & (parallelN - 1)) != 0) { | 39 | if ((parallelN & (parallelN - 1)) != 0) { |
| 40 | - uint32_t temp = parallelN - 1; | 40 | + uint32_t welfordTemp = parallelN - 1; |
| 41 | - temp |= temp >> 1; | 41 | + welfordTemp |= welfordTemp >> 1; |
| 42 | - temp |= temp >> TWO_POWER_ONE; | 42 | + welfordTemp |= welfordTemp >> TWO_POWER_ONE; |
| 43 | - temp |= temp >> TWO_POWER_TWO; | 43 | + welfordTemp |= welfordTemp >> TWO_POWER_TWO; |
| 44 | - temp |= temp >> TWO_POWER_THREE; | 44 | + welfordTemp |= welfordTemp >> TWO_POWER_THREE; |
| 45 | - temp |= temp >> TWO_POWER_FOUR; | 45 | + welfordTemp |= welfordTemp >> TWO_POWER_FOUR; |
| 46 | - return (parallelN - ((temp + 1) / TWO_POWER_ONE)); | 46 | + return (parallelN - ((welfordTemp + 1) / TWO_POWER_ONE)); |
| 47 | } else { | 47 | } else { |
| 48 | return 0; | 48 | return 0; |
| 49 | } | 49 | } |
| @@ -83,36 +83,36 @@ ge::graphStatus BatchNormV3WelfordTiling::DoOpTiling() | |||
| 83 | usedCoreNum = Ops::Base::CeilDiv(commonParams.patternA, blockFactor); | 83 | usedCoreNum = Ops::Base::CeilDiv(commonParams.patternA, blockFactor); |
| 84 | td_.set_blockFactor(blockFactor); | 84 | td_.set_blockFactor(blockFactor); |
| 85 | td_.set_tailCoreBlockFactor(commonParams.patternA - (usedCoreNum - 1) * blockFactor); | 85 | td_.set_tailCoreBlockFactor(commonParams.patternA - (usedCoreNum - 1) * blockFactor); |
| 86 | - float batchVarScale = (commonParams.patternR0 * commonParams.patternR1 == 1) ? | 86 | + // Calculate batch variance scale for Welford algorithm |
| 87 | - 1.0 : | 87 | + float welfordBatchVarScale = (commonParams.patternR0 * commonParams.patternR1 == 1) ? |
| 88 | - static_cast<float>( | 88 | + 1.0 : |
| 89 | - static_cast<double>(commonParams.patternR0 * commonParams.patternR1) / | 89 | + static_cast<float>( |
| 90 | - static_cast<double>(commonParams.patternR0 * commonParams.patternR1 - 1)); | 90 | + static_cast<double>(commonParams.patternR0 * commonParams.patternR1) / |
| 91 | - td_.set_batchVarScale(batchVarScale); | 91 | + static_cast<double>(commonParams.patternR0 * commonParams.patternR1 - 1)); |
| 92 | - int64_t aUbFactor = 1; | 92 | + td_.set_batchVarScale(welfordBatchVarScale); |
| 93 | - int64_t r0UbFactor = 1; | 93 | + int64_t bnAUbFactor = 1; |
| 94 | - DoUbTiling(aUbFactor, r0UbFactor); | 94 | + int64_t bnR0UbFactor = 1; |
| 95 | - td_.set_aUbFactor(aUbFactor); | 95 | + DoUbTiling(bnAUbFactor, bnR0UbFactor); |
| 96 | - td_.set_r0UbFactor(r0UbFactor); | 96 | + td_.set_aUbFactor(bnAUbFactor); |
| 97 | - td_.set_aUbLoop(Ops::Base::CeilDiv(blockFactor, aUbFactor)); | 97 | + td_.set_r0UbFactor(bnR0UbFactor); |
| 98 | - td_.set_aUbTail(blockFactor - (td_.get_aUbLoop() - 1) * aUbFactor); | 98 | + td_.set_aUbLoop(Ops::Base::CeilDiv(blockFactor, bnAUbFactor)); |
| 99 | - td_.set_tailCoreAUbLoop(Ops::Base::CeilDiv(td_.get_tailCoreBlockFactor(), aUbFactor)); | 99 | + td_.set_aUbTail(blockFactor - (td_.get_aUbLoop() - 1) * bnAUbFactor); |
| 100 | - td_.set_tailCoreAUbTail(td_.get_tailCoreBlockFactor() - (td_.get_tailCoreAUbLoop() - 1) * aUbFactor); | 100 | + td_.set_tailCoreAUbLoop(Ops::Base::CeilDiv(td_.get_tailCoreBlockFactor(), bnAUbFactor)); |
| 101 | - td_.set_r0UbLoop(Ops::Base::CeilDiv(commonParams.patternR0, r0UbFactor)); | 101 | + td_.set_tailCoreAUbTail(td_.get_tailCoreBlockFactor() - (td_.get_tailCoreAUbLoop() - 1) * bnAUbFactor); |
| 102 | - td_.set_r0UbTail(commonParams.patternR0 - (td_.get_r0UbLoop() - 1) * r0UbFactor); | 102 | + td_.set_r0UbLoop(Ops::Base::CeilDiv(commonParams.patternR0, bnR0UbFactor)); |
| 103 | + td_.set_r0UbTail(commonParams.patternR0 - (td_.get_r0UbLoop() - 1) * bnR0UbFactor); | ||
| 103 | td_.set_procNR0(1); | 104 | td_.set_procNR0(1); |
| 104 | td_.set_nR0Loop(commonParams.patternR1); | 105 | td_.set_nR0Loop(commonParams.patternR1); |
| 105 | td_.set_lastLoopNR0(1); | 106 | td_.set_lastLoopNR0(1); |
| 106 | uint32_t parallelN = | 107 | uint32_t parallelN = |
| 107 | - (td_.get_r0UbLoop() == 1) ? static_cast<uint32_t>(commonParams.patternR0) : static_cast<uint32_t>(r0UbFactor); | 108 | + (td_.get_r0UbLoop() == 1) ? static_cast<uint32_t>(commonParams.patternR0) : static_cast<uint32_t>(bnR0UbFactor); |
| 108 | if ((td_.get_r0UbLoop() == 1) || (td_.get_r0UbFactor() == td_.get_r0UbTail())) { | 109 | if ((td_.get_r0UbLoop() == 1) || (td_.get_r0UbFactor() == td_.get_r0UbTail())) { |
| 109 | welfordTilingkey = BNV3_WELFORD_R0_SPLIT_ALIGN_TILING_KEY; | 110 | welfordTilingkey = BNV3_WELFORD_R0_SPLIT_ALIGN_TILING_KEY; |
| 110 | } else { | 111 | } else { |
| 111 | welfordTilingkey = BNV3_WELFORD_R0_SPLIT_NOT_ALIGN_TILING_KEY; | 112 | welfordTilingkey = BNV3_WELFORD_R0_SPLIT_NOT_ALIGN_TILING_KEY; |
| 112 | } | 113 | } |
| 113 | - // R0不切分，R1补充切分 | 114 | + if ((commonParams.patternR0Align <= (bnR0UbFactor / TWO_NUM)) && commonParams.patternR1 > 1) { |
| 114 | - if ((commonParams.patternR0Align <= (r0UbFactor / TWO_NUM)) && commonParams.patternR1 > 1) { | 115 | + int64_t procNR0 = Ops::Base::FloorDiv(bnR0UbFactor, commonParams.patternR0Align); |
| 115 | - int64_t procNR0 = Ops::Base::FloorDiv(r0UbFactor, commonParams.patternR0Align); | ||
| 116 | int64_t nR0Loop = Ops::Base::CeilDiv(commonParams.patternR1, procNR0); | 116 | int64_t nR0Loop = Ops::Base::CeilDiv(commonParams.patternR1, procNR0); |
| 117 | int64_t lastLoopNR0 = commonParams.patternR1 - (nR0Loop - 1) * procNR0; | 117 | int64_t lastLoopNR0 = commonParams.patternR1 - (nR0Loop - 1) * procNR0; |
| 118 | td_.set_procNR0(procNR0); | 118 | td_.set_procNR0(procNR0); |
| @@ -133,6 +133,7 @@ ge::graphStatus BatchNormV3WelfordTiling::DoOpTiling() | |||
| 133 | 133 | ||
| 134 | ge::graphStatus BatchNormV3WelfordTiling::PostTiling() | 134 | ge::graphStatus BatchNormV3WelfordTiling::PostTiling() |
| 135 | { | 135 | { |
| 136 | + // Set tiling parameters for Welford algorithm | ||
| 136 | td_.set_patternR1(commonParams.patternR1); | 137 | td_.set_patternR1(commonParams.patternR1); |
| 137 | td_.set_patternR0(commonParams.patternR0); | 138 | td_.set_patternR0(commonParams.patternR0); |
| 138 | td_.set_patternA(commonParams.patternA); | 139 | td_.set_patternA(commonParams.patternA); |
| @@ -141,15 +142,17 @@ ge::graphStatus BatchNormV3WelfordTiling::PostTiling() | |||
| 141 | td_.set_momentum(commonParams.momentum); | 142 | td_.set_momentum(commonParams.momentum); |
| 142 | td_.set_momentumReverse(commonParams.momentumReverse); | 143 | td_.set_momentumReverse(commonParams.momentumReverse); |
| 143 | context_->SetBlockDim(usedCoreNum); | 144 | context_->SetBlockDim(usedCoreNum); |
| 144 | - auto rawTilingData = context_->GetRawTilingData(); | 145 | + |
| 146 | + // Save tiling data to context | ||
| 147 | + auto welfordRawTilingData = context_->GetRawTilingData(); | ||
| 145 | OP_CHECK_IF( | 148 | OP_CHECK_IF( |
| 146 | - td_.GetDataSize() > rawTilingData->GetCapacity(), | 149 | + td_.GetDataSize() > welfordRawTilingData->GetCapacity(), |
| 147 | OP_LOGE( | 150 | OP_LOGE( |
| 148 | commonParams.nodeName, "actual tiling data size %zu > context tiling data size %zu", td_.GetDataSize(), | 151 | commonParams.nodeName, "actual tiling data size %zu > context tiling data size %zu", td_.GetDataSize(), |
| 149 | - rawTilingData->GetCapacity()), | 152 | + welfordRawTilingData->GetCapacity()), |
| 150 | return ge::GRAPH_FAILED); | 153 | return ge::GRAPH_FAILED); |
| 151 | - td_.SaveToBuffer(rawTilingData->GetData(), rawTilingData->GetCapacity()); | 154 | + td_.SaveToBuffer(welfordRawTilingData->GetData(), welfordRawTilingData->GetCapacity()); |
| 152 | - rawTilingData->SetDataSize(td_.GetDataSize()); | 155 | + welfordRawTilingData->SetDataSize(td_.GetDataSize()); |
| 153 | 156 | ||
| 154 | return ge::GRAPH_SUCCESS; | 157 | return ge::GRAPH_SUCCESS; |
| 155 | } | 158 | } |
| @@ -158,23 +158,25 @@ static bool CheckOtherDtypeValid( | |||
| 158 | 158 | ||
| 159 | static bool CheckFormat(const aclTensor* input, const aclTensor* out) | 159 | static bool CheckFormat(const aclTensor* input, const aclTensor* out) |
| 160 | { | 160 | { |
| 161 | - if (input->GetStorageFormat() != out->GetStorageFormat()) { | 161 | + auto inputFormat = input->GetStorageFormat(); |
| 162 | + auto outputFormat = out->GetStorageFormat(); | ||
| 163 | + if (inputFormat != outputFormat) { | ||
| 162 | OP_LOGE( | 164 | OP_LOGE( |
| 163 | ACLNN_ERR_PARAM_INVALID, "Format of input and output should be equal, input [%s], output [%s].", | 165 | ACLNN_ERR_PARAM_INVALID, "Format of input and output should be equal, input [%s], output [%s].", |
| 164 | - op::ToString(input->GetStorageFormat()).GetString(), op::ToString(out->GetStorageFormat()).GetString()); | 166 | + op::ToString(inputFormat).GetString(), op::ToString(outputFormat).GetString()); |
| 165 | return false; | 167 | return false; |
| 166 | } | 168 | } |
| 167 | 169 | ||
| 168 | if (Ops::NN::AclnnUtil::IsRegbase()) { | 170 | if (Ops::NN::AclnnUtil::IsRegbase()) { |
| 169 | if ((input->GetViewShape().GetDimNum() == MAX_BN_DIMS) && | 171 | if ((input->GetViewShape().GetDimNum() == MAX_BN_DIMS) && |
| 170 | - ((input->GetStorageFormat() != Format::FORMAT_NCDHW) && | 172 | + ((inputFormat != Format::FORMAT_NCDHW) && |
| 171 | - (input->GetStorageFormat() != Format::FORMAT_NDHWC))) { | 173 | + (inputFormat != Format::FORMAT_NDHWC))) { |
| 172 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Format of input should be NCDWH or NDWHC, when input dim is 5."); | 174 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Format of input should be NCDWH or NDWHC, when input dim is 5."); |
| 173 | return false; | 175 | return false; |
| 174 | } | 176 | } |
| 175 | 177 | ||
| 176 | if ((out->GetViewShape().GetDimNum() == MAX_BN_DIMS) && | 178 | if ((out->GetViewShape().GetDimNum() == MAX_BN_DIMS) && |
| 177 | - ((out->GetStorageFormat() != Format::FORMAT_NCDHW) && (out->GetStorageFormat() != Format::FORMAT_NDHWC))) { | 179 | + ((outputFormat != Format::FORMAT_NCDHW) && (outputFormat != Format::FORMAT_NDHWC))) { |
| 178 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Format of output should be NCDWH or NDWHC, when input dim is 5."); | 180 | OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Format of output should be NCDWH or NDWHC, when input dim is 5."); |
| 179 | return false; | 181 | return false; |
| 180 | } | 182 | } |
| @@ -233,7 +235,7 @@ static bool CheckOtherShape( | |||
| 233 | return true; | 235 | return true; |
| 234 | } | 236 | } |
| 235 | 237 | ||
| 236 | -static int64_t GetDimC(const aclTensor* input) | 238 | +static int64_t GetBatchNormDimC(const aclTensor* input) |
| 237 | { | 239 | { |
| 238 | auto viewShape = input->GetViewShape(); | 240 | auto viewShape = input->GetViewShape(); |
| 239 | if (Ops::NN::AclnnUtil::IsRegbase()) { | 241 | if (Ops::NN::AclnnUtil::IsRegbase()) { |
| @@ -255,7 +257,7 @@ static aclnnStatus CheckParams( | |||
| 255 | CHECK_RET(CheckFormat(input, output), ACLNN_ERR_PARAM_INVALID); | 257 | CHECK_RET(CheckFormat(input, output), ACLNN_ERR_PARAM_INVALID); |
| 256 | 258 | ||
| 257 | CHECK_RET(CheckShape(input, output), ACLNN_ERR_PARAM_INVALID); | 259 | CHECK_RET(CheckShape(input, output), ACLNN_ERR_PARAM_INVALID); |
| 258 | - int64_t dimC = GetDimC(input); | 260 | + int64_t dimC = GetBatchNormDimC(input); |
| 259 | CHECK_RET(CheckOtherShape(dimC, weight, bias, runningMean, runningVar), ACLNN_ERR_PARAM_INVALID); | 261 | CHECK_RET(CheckOtherShape(dimC, weight, bias, runningMean, runningVar), ACLNN_ERR_PARAM_INVALID); |
| 260 | 262 | ||
| 261 | if (training) { | 263 | if (training) { |
| @@ -325,12 +327,12 @@ aclnnStatus aclnnBatchNormGetWorkspaceSize( | |||
| 325 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); | 327 | CHECK_RET(bnResult == ACLNN_SUCCESS, bnResult); |
| 326 | 328 | ||
| 327 | if (inputDims > MAX_BN_DIMS) { | 329 | if (inputDims > MAX_BN_DIMS) { |
| 328 | - int64_t originShapes[inputDims]; | 330 | + int64_t originShapesBn[inputDims]; |
| 329 | - for (size_t i = 0; i < inputDims; ++i) { | 331 | + for (size_t idxBn = 0; idxBn < inputDims; ++idxBn) { |
| 330 | - originShapes[i] = inputShape[i]; | 332 | + originShapesBn[idxBn] = inputShape[idxBn]; |
| 331 | } | 333 | } |
| 332 | - aclIntArray* originShapeArray = uniqueExecutor.get()->AllocIntArray(originShapes, inputDims); | 334 | + aclIntArray* originShapeArrayBn = uniqueExecutor.get()->AllocIntArray(originShapesBn, inputDims); |
| 333 | - auto bnOutputReshape = l0op::Reshape(bnOutput, originShapeArray, uniqueExecutor.get()); | 335 | + auto bnOutputReshape = l0op::Reshape(bnOutput, originShapeArrayBn, uniqueExecutor.get()); |
| 334 | auto bnOutputReformat = l0op::ReFormat(bnOutputReshape, Format::FORMAT_ND); | 336 | auto bnOutputReformat = l0op::ReFormat(bnOutputReshape, Format::FORMAT_ND); |
| 335 | auto viewCopyResult = l0op::ViewCopy(bnOutputReformat, output, uniqueExecutor.get()); | 337 | auto viewCopyResult = l0op::ViewCopy(bnOutputReformat, output, uniqueExecutor.get()); |
| 336 | CHECK_RET(viewCopyResult != nullptr, ACLNN_ERR_INNER_NULLPTR); | 338 | CHECK_RET(viewCopyResult != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| @@ -554,17 +556,17 @@ aclnnStatus BatchNormProcDavid( | |||
| 554 | 556 | ||
| 555 | if (training) { | 557 | if (training) { |
| 556 | if (!runningMean->IsFromWorkspace()) { | 558 | if (!runningMean->IsFromWorkspace()) { |
| 557 | - auto runningMeanResCast = l0op::Cast(runningMeanOut, runningMean->GetDataType(), executor); | 559 | + auto bnRunningMeanCast = l0op::Cast(runningMeanOut, runningMean->GetDataType(), executor); |
| 558 | - CHECK_RET(runningMeanResCast != nullptr, ACLNN_ERR_INNER_NULLPTR); | 560 | + CHECK_RET(bnRunningMeanCast != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 559 | - auto runningMeanResViewCopy = l0op::ViewCopy(runningMeanResCast, runningMean, executor); | 561 | + auto bnRunningMeanViewCopy = l0op::ViewCopy(bnRunningMeanCast, runningMean, executor); |
| 560 | - CHECK_RET(runningMeanResViewCopy != nullptr, ACLNN_ERR_INNER_NULLPTR); | 562 | + CHECK_RET(bnRunningMeanViewCopy != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 561 | } | 563 | } |
| 562 | 564 | ||
| 563 | if (!runningVar->IsFromWorkspace()) { | 565 | if (!runningVar->IsFromWorkspace()) { |
| 564 | - auto runningVarResCast = l0op::Cast(runningVarOut, runningVar->GetDataType(), executor); | 566 | + auto bnRunningVarCast = l0op::Cast(runningVarOut, runningVar->GetDataType(), executor); |
| 565 | - CHECK_RET(runningVarResCast != nullptr, ACLNN_ERR_INNER_NULLPTR); | 567 | + CHECK_RET(bnRunningVarCast != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 566 | - auto runningVarResViewCopy = l0op::ViewCopy(runningVarResCast, runningVar, executor); | 568 | + auto bnRunningVarViewCopy = l0op::ViewCopy(bnRunningVarCast, runningVar, executor); |
| 567 | - CHECK_RET(runningVarResViewCopy != nullptr, ACLNN_ERR_INNER_NULLPTR); | 569 | + CHECK_RET(bnRunningVarViewCopy != nullptr, ACLNN_ERR_INNER_NULLPTR); |
| 568 | } | 570 | } |
| 569 | } | 571 | } |
| 570 | 572 | ||
| @@ -606,7 +608,7 @@ aclnnStatus BatchNorm( | |||
| 606 | aclTensor* runningVar, bool training, float momentum, float eps, aclTensor** output, aclTensor* saveMean, | 608 | aclTensor* runningVar, bool training, float momentum, float eps, aclTensor** output, aclTensor* saveMean, |
| 607 | aclTensor* saveInvstd, aclOpExecutor* executor) | 609 | aclTensor* saveInvstd, aclOpExecutor* executor) |
| 608 | { | 610 | { |
| 609 | - size_t dimC = GetDimC(input); | 611 | + size_t dimC = GetBatchNormDimC(input); |
| 610 | auto curArch = GetCurrentPlatformInfo().GetCurNpuArch(); | 612 | auto curArch = GetCurrentPlatformInfo().GetCurNpuArch(); |
| 611 | if (runningMean == nullptr) { | 613 | if (runningMean == nullptr) { |
| 612 | runningMean = | 614 | runningMean = |
| @@ -155,22 +155,22 @@ const std::array<aclTensor*, 3> BatchNormV3( | |||
| 155 | { | 155 | { |
| 156 | L0_DFX(BatchNormV3, x, weight, bias, running_mean, running_var, momentum, eps, training); | 156 | L0_DFX(BatchNormV3, x, weight, bias, running_mean, running_var, momentum, eps, training); |
| 157 | 157 | ||
| 158 | - auto y = executor->AllocTensor( | 158 | + auto bnY = executor->AllocTensor( |
| 159 | x->GetStorageShape(), x->GetOriginalShape(), x->GetDataType(), x->GetStorageFormat(), x->GetOriginalFormat()); | 159 | x->GetStorageShape(), x->GetOriginalShape(), x->GetDataType(), x->GetStorageFormat(), x->GetOriginalFormat()); |
| 160 | - auto batchMean = executor->AllocTensor( | 160 | + auto bnBatchMean = executor->AllocTensor( |
| 161 | running_mean->GetStorageShape(), running_mean->GetOriginalShape(), DataType::DT_FLOAT, | 161 | running_mean->GetStorageShape(), running_mean->GetOriginalShape(), DataType::DT_FLOAT, |
| 162 | running_mean->GetStorageFormat(), running_mean->GetOriginalFormat()); | 162 | running_mean->GetStorageFormat(), running_mean->GetOriginalFormat()); |
| 163 | - auto batchVar = executor->AllocTensor( | 163 | + auto bnBatchVar = executor->AllocTensor( |
| 164 | running_var->GetStorageShape(), running_var->GetOriginalShape(), DataType::DT_FLOAT, | 164 | running_var->GetStorageShape(), running_var->GetOriginalShape(), DataType::DT_FLOAT, |
| 165 | running_var->GetStorageFormat(), running_var->GetOriginalFormat()); | 165 | running_var->GetStorageFormat(), running_var->GetOriginalFormat()); |
| 166 | 166 | ||
| 167 | auto ret = ADD_TO_LAUNCHER_LIST_AICORE( | 167 | auto ret = ADD_TO_LAUNCHER_LIST_AICORE( |
| 168 | BatchNormV3, OP_INPUT(x, weight, bias, running_mean, running_var), | 168 | BatchNormV3, OP_INPUT(x, weight, bias, running_mean, running_var), |
| 169 | - OP_OUTPUT(y, running_mean, running_var, batchMean, batchVar), OP_ATTR(eps, momentum, training)); | 169 | + OP_OUTPUT(bnY, running_mean, running_var, bnBatchMean, bnBatchVar), OP_ATTR(eps, momentum, training)); |
| 170 | if (ret != ACL_SUCCESS) { | 170 | if (ret != ACL_SUCCESS) { |
| 171 | OP_LOGE(ACLNN_ERR_INNER_NULLPTR, "BatchNormV3 ADD_TO_LAUNCHER_LIST_AICORE failed."); | 171 | OP_LOGE(ACLNN_ERR_INNER_NULLPTR, "BatchNormV3 ADD_TO_LAUNCHER_LIST_AICORE failed."); |
| 172 | return {nullptr, nullptr, nullptr}; | 172 | return {nullptr, nullptr, nullptr}; |
| 173 | } | 173 | } |
| 174 | - return {y, batchMean, batchVar}; | 174 | + return {bnY, bnBatchMean, bnBatchVar}; |
| 175 | } | 175 | } |
| 176 | } // namespace l0op | 176 | } // namespace l0op |
| @@ -57,6 +57,7 @@ public: | |||
| 57 | this->batchVarScale = tilingData->batchVarScale; | 57 | this->batchVarScale = tilingData->batchVarScale; |
| 58 | dichotomizeAddDiffSize = tilingData->dichotomizeAddDiffSize; | 58 | dichotomizeAddDiffSize = tilingData->dichotomizeAddDiffSize; |
| 59 | 59 | ||
| 60 | + // welford algorithm: compute global memory offsets | ||
| 60 | uint64_t aGmBlockOffset = | 61 | uint64_t aGmBlockOffset = |
| 61 | static_cast<uint64_t>(this->blockIdx) * static_cast<uint64_t>(tilingData->blockFactor); | 62 | static_cast<uint64_t>(this->blockIdx) * static_cast<uint64_t>(tilingData->blockFactor); |
| 62 | uint64_t aR0GmBlockOffset = aGmBlockOffset * patternR0; | 63 | uint64_t aR0GmBlockOffset = aGmBlockOffset * patternR0; |
| @@ -635,6 +636,7 @@ private: | |||
| 635 | } | 636 | } |
| 636 | 637 | ||
| 637 | private: | 638 | private: |
| 639 | + // welford algorithm constants | ||
| 638 | constexpr static uint32_t BLOCK_SIZE = 32; | 640 | constexpr static uint32_t BLOCK_SIZE = 32; |
| 639 | constexpr static uint32_t X_NUM_PER_BLOCK = BLOCK_SIZE / sizeof(T1); | 641 | constexpr static uint32_t X_NUM_PER_BLOCK = BLOCK_SIZE / sizeof(T1); |
| 640 | constexpr static uint32_t FLOAT_SIZE = 4; | 642 | constexpr static uint32_t FLOAT_SIZE = 4; |
| @@ -644,6 +646,7 @@ private: | |||
| 644 | constexpr static uint32_t BLOCK_NUM_PER_REP = 8; | 646 | constexpr static uint32_t BLOCK_NUM_PER_REP = 8; |
| 645 | constexpr static uint32_t B32_BLOCK_ALIGN_NUM = 8; | 647 | constexpr static uint32_t B32_BLOCK_ALIGN_NUM = 8; |
| 646 | constexpr static uint32_t UINT8_MAX_NUM = 255; | 648 | constexpr static uint32_t UINT8_MAX_NUM = 255; |
| 649 | + // welford specific modes | ||
| 647 | constexpr static int R0_SPLIT_NOT_ALIGN_MODE = 0; | 650 | constexpr static int R0_SPLIT_NOT_ALIGN_MODE = 0; |
| 648 | constexpr static int R0_SPLIT_ALIGN_MODE = 1; | 651 | constexpr static int R0_SPLIT_ALIGN_MODE = 1; |
| 649 | constexpr static int R1_SPLIT_NOT_ALIGN_MODE = 2; | 652 | constexpr static int R1_SPLIT_NOT_ALIGN_MODE = 2; |
| @@ -681,7 +684,7 @@ private: | |||
| 681 | float finalVar = 0.0; | 684 | float finalVar = 0.0; |
| 682 | float weightValue = 0.0; | 685 | float weightValue = 0.0; |
| 683 | float biasValue = 0.0; | 686 | float biasValue = 0.0; |
| 684 | - /* ascendc variable */ | 687 | + /* welford ascendc queues and buffers */ |
| 685 | TQue<QuePosition::VECIN, 1> xQueue; | 688 | TQue<QuePosition::VECIN, 1> xQueue; |
| 686 | TQue<QuePosition::VECIN, 1> weightQueue; | 689 | TQue<QuePosition::VECIN, 1> weightQueue; |
| 687 | TQue<QuePosition::VECIN, 1> biasQueue; | 690 | TQue<QuePosition::VECIN, 1> biasQueue; |
| @@ -693,6 +696,7 @@ private: | |||
| 693 | TQue<QuePosition::VECOUT, 1> saveVarQueue; | 696 | TQue<QuePosition::VECOUT, 1> saveVarQueue; |
| 694 | TQue<QuePosition::VECOUT, 1> runningMeanOutQueue; | 697 | TQue<QuePosition::VECOUT, 1> runningMeanOutQueue; |
| 695 | TQue<QuePosition::VECOUT, 1> runningVarOutQueue; | 698 | TQue<QuePosition::VECOUT, 1> runningVarOutQueue; |
| 699 | + // welford temp buffers | ||
| 696 | TBuf<TPosition::VECCALC> tmpBuf0; | 700 | TBuf<TPosition::VECCALC> tmpBuf0; |
| 697 | TBuf<TPosition::VECCALC> tmpBuf3; | 701 | TBuf<TPosition::VECCALC> tmpBuf3; |
| 698 | TBuf<TPosition::VECCALC> tmpBuf4; | 702 | TBuf<TPosition::VECCALC> tmpBuf4; |
| @@ -178,7 +178,6 @@ public: | |||
| 178 | alpha_); | 178 | alpha_); |
| 179 | pipe.InitBuffer(z_que, BUFFER_NUM, row_step * ROUND_UP(num_last_dim) * databyte); | 179 | pipe.InitBuffer(z_que, BUFFER_NUM, row_step * ROUND_UP(num_last_dim) * databyte); |
| 180 | pipe.InitBuffer(calc_buf_fp32, ROUND_UP(num_last_dim) * sizeof(float)); | 180 | pipe.InitBuffer(calc_buf_fp32, ROUND_UP(num_last_dim) * sizeof(float)); |
| 181 | - // calc buffer | ||
| 182 | pipe.InitBuffer(x_buf_fp32, sizeof(float) * ROUND_UP(num_last_dim)); | 181 | pipe.InitBuffer(x_buf_fp32, sizeof(float) * ROUND_UP(num_last_dim)); |
| 183 | pipe.InitBuffer(y_buf_fp32, sizeof(float) * ROUND_UP(num_last_dim)); | 182 | pipe.InitBuffer(y_buf_fp32, sizeof(float) * ROUND_UP(num_last_dim)); |
| 184 | pipe.InitBuffer(z_buf_fp32, sizeof(float) * ROUND_UP(num_last_dim)); | 183 | pipe.InitBuffer(z_buf_fp32, sizeof(float) * ROUND_UP(num_last_dim)); |
| @@ -192,15 +191,15 @@ public: | |||
| 192 | __gm__ uint8_t* x, __gm__ uint8_t* gx, __gm__ uint8_t* beta, __gm__ uint8_t* gamma, __gm__ uint8_t* mean, | 191 | __gm__ uint8_t* x, __gm__ uint8_t* gx, __gm__ uint8_t* beta, __gm__ uint8_t* gamma, __gm__ uint8_t* mean, |
| 193 | __gm__ uint8_t* rstd, __gm__ uint8_t* z, uint32_t num_core_, uint32_t num_Last_dim_, uint32_t num_first_dim_, | 192 | __gm__ uint8_t* rstd, __gm__ uint8_t* z, uint32_t num_core_, uint32_t num_Last_dim_, uint32_t num_first_dim_, |
| 194 | uint32_t nl_first_dim_per_core_, uint32_t l_first_dim_per_core_, uint32_t first_dim_per_times_, | 193 | uint32_t nl_first_dim_per_core_, uint32_t l_first_dim_per_core_, uint32_t first_dim_per_times_, |
| 195 | - uint32_t updated_last_dim_, uint32_t updated_last_times_, uint32_t eps_, uint32_t meanNum_, uint32_t alpha_) | 194 | + uint32_t updated_last_dim_, uint32_t updated_last_times_, uint32_t eps_v1, uint32_t meanNum_, uint32_t alpha_) |
| 196 | { | 195 | { |
| 197 | InitBase( | 196 | InitBase( |
| 198 | x, gx, beta, gamma, mean, rstd, z, num_core_, num_Last_dim_, num_first_dim_, nl_first_dim_per_core_, | 197 | x, gx, beta, gamma, mean, rstd, z, num_core_, num_Last_dim_, num_first_dim_, nl_first_dim_per_core_, |
| 199 | - l_first_dim_per_core_, first_dim_per_times_, updated_last_dim_, updated_last_times_, eps_, meanNum_, | 198 | + l_first_dim_per_core_, first_dim_per_times_, updated_last_dim_, updated_last_times_, eps_v1, meanNum_, |
| 200 | alpha_); | 199 | alpha_); |
| 201 | // calc buffer | 200 | // calc buffer |
| 202 | - pipe.InitBuffer(x_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); | ||
| 203 | pipe.InitBuffer(y_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); | 201 | pipe.InitBuffer(y_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); |
| 202 | + pipe.InitBuffer(x_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); | ||
| 204 | pipe.InitBuffer(z_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); | 203 | pipe.InitBuffer(z_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); |
| 205 | pipe.InitBuffer(z_que, BUFFER_NUM, row_step * ROUND_UP(num_last_dim) * databyte); | 204 | pipe.InitBuffer(z_que, BUFFER_NUM, row_step * ROUND_UP(num_last_dim) * databyte); |
| 206 | pipe.InitBuffer(calc_buf_fp32, ROUND_UP(num_last_dim) * sizeof(float)); | 205 | pipe.InitBuffer(calc_buf_fp32, ROUND_UP(num_last_dim) * sizeof(float)); |
| @@ -216,7 +215,6 @@ public: | |||
| 216 | x, gx, beta, gamma, mean, rstd, z, num_core_, num_Last_dim_, num_first_dim_, nl_first_dim_per_core_, | 215 | x, gx, beta, gamma, mean, rstd, z, num_core_, num_Last_dim_, num_first_dim_, nl_first_dim_per_core_, |
| 217 | l_first_dim_per_core_, first_dim_per_times_, updated_last_dim_, updated_last_times_, eps_, meanNum_, | 216 | l_first_dim_per_core_, first_dim_per_times_, updated_last_dim_, updated_last_times_, eps_, meanNum_, |
| 218 | alpha_); | 217 | alpha_); |
| 219 | - // calc buffer | ||
| 220 | pipe.InitBuffer(x_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); | 218 | pipe.InitBuffer(x_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); |
| 221 | pipe.InitBuffer(y_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); | 219 | pipe.InitBuffer(y_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); |
| 222 | pipe.InitBuffer(z_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); | 220 | pipe.InitBuffer(z_buf_fp32, sizeof(float) * ROUND_UP(updated_last_dim_)); |
| @@ -466,27 +464,27 @@ private: | |||
| 466 | LocalTensor<T> gx_local = gx_que.DeQue<T>(); | 464 | LocalTensor<T> gx_local = gx_que.DeQue<T>(); |
| 467 | LocalTensor<float> local_calc_fp32 = calc_buf_fp32.Get<float>(); | 465 | LocalTensor<float> local_calc_fp32 = calc_buf_fp32.Get<float>(); |
| 468 | // output | 466 | // output |
| 467 | + LocalTensor<float> rstd_local = rstd_que_fp32.AllocTensor<float>(); | ||
| 469 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); | 468 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); |
| 470 | LocalTensor<float> mean_local = mean_que_fp32.AllocTensor<float>(); | 469 | LocalTensor<float> mean_local = mean_que_fp32.AllocTensor<float>(); |
| 471 | - LocalTensor<float> rstd_local = rstd_que_fp32.AllocTensor<float>(); | ||
| 472 | 470 | ||
| 473 | // local temp | 471 | // local temp |
| 474 | - LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); | ||
| 475 | LocalTensor<float> y_local_fp32 = y_buf_fp32.Get<float>(); | 472 | LocalTensor<float> y_local_fp32 = y_buf_fp32.Get<float>(); |
| 473 | + LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); | ||
| 476 | LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); | 474 | LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); |
| 477 | 475 | ||
| 478 | uint32_t realLen = ROUND_UP(num_last_dim); | 476 | uint32_t realLen = ROUND_UP(num_last_dim); |
| 479 | uint32_t stepSize = nums * realLen; | 477 | uint32_t stepSize = nums * realLen; |
| 480 | 478 | ||
| 481 | - LocalTensor<float> local_x_fp32 = calc_x_fp32.Get<float>(); | ||
| 482 | LocalTensor<float> local_y_fp32 = calc_y_fp32.Get<float>(); | 479 | LocalTensor<float> local_y_fp32 = calc_y_fp32.Get<float>(); |
| 480 | + LocalTensor<float> local_x_fp32 = calc_x_fp32.Get<float>(); | ||
| 483 | 481 | ||
| 484 | Cast(local_y_fp32, x_local, RoundMode::CAST_NONE, stepSize); | 482 | Cast(local_y_fp32, x_local, RoundMode::CAST_NONE, stepSize); |
| 485 | PipeBarrier<PIPE_V>(); | 483 | PipeBarrier<PIPE_V>(); |
| 486 | Cast(local_x_fp32, gx_local, RoundMode::CAST_NONE, stepSize); | 484 | Cast(local_x_fp32, gx_local, RoundMode::CAST_NONE, stepSize); |
| 487 | PipeBarrier<PIPE_V>(); | 485 | PipeBarrier<PIPE_V>(); |
| 488 | - x_que.FreeTensor(x_local); | ||
| 489 | gx_que.FreeTensor(gx_local); | 486 | gx_que.FreeTensor(gx_local); |
| 487 | + x_que.FreeTensor(x_local); | ||
| 490 | Axpy(local_x_fp32, local_y_fp32, alphaVal, stepSize); | 488 | Axpy(local_x_fp32, local_y_fp32, alphaVal, stepSize); |
| 491 | PipeBarrier<PIPE_V>(); | 489 | PipeBarrier<PIPE_V>(); |
| 492 | Muls(local_y_fp32, local_x_fp32, 1.0f, stepSize); | 490 | Muls(local_y_fp32, local_x_fp32, 1.0f, stepSize); |
| @@ -504,9 +502,9 @@ private: | |||
| 504 | } | 502 | } |
| 505 | SetFlag<HardEvent::S_V>(EVENT_ID0); | 503 | SetFlag<HardEvent::S_V>(EVENT_ID0); |
| 506 | WaitFlag<HardEvent::S_V>(EVENT_ID0); | 504 | WaitFlag<HardEvent::S_V>(EVENT_ID0); |
| 507 | - event_t event_s_mte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE3)); | 505 | + event_t event_s_mte3_1 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE3)); |
| 508 | - SetFlag<HardEvent::S_MTE3>(event_s_mte3); | 506 | + SetFlag<HardEvent::S_MTE3>(event_s_mte3_1); |
| 509 | - WaitFlag<HardEvent::S_MTE3>(event_s_mte3); | 507 | + WaitFlag<HardEvent::S_MTE3>(event_s_mte3_1); |
| 510 | mean_que_fp32.EnQue(mean_local); | 508 | mean_que_fp32.EnQue(mean_local); |
| 511 | 509 | ||
| 512 | Mul(local_x_fp32, local_y_fp32, local_y_fp32, stepSize); | 510 | Mul(local_x_fp32, local_y_fp32, local_y_fp32, stepSize); |
| @@ -517,15 +515,15 @@ private: | |||
| 517 | Cast(z_local_fp32, gamma_local, RoundMode::CAST_NONE, num_last_dim); | 515 | Cast(z_local_fp32, gamma_local, RoundMode::CAST_NONE, num_last_dim); |
| 518 | Cast(y_local_fp32, beta_local, RoundMode::CAST_NONE, num_last_dim); | 516 | Cast(y_local_fp32, beta_local, RoundMode::CAST_NONE, num_last_dim); |
| 519 | PipeBarrier<PIPE_V>(); | 517 | PipeBarrier<PIPE_V>(); |
| 520 | - for (int32_t rid = 0; rid < nums; ++rid) { | 518 | + for (int32_t rid1 = 0; rid1 < nums; ++rid1) { |
| 521 | - uint32_t offset = rid * realLen; | 519 | + uint32_t offset = rid1 * realLen; |
| 522 | 520 | ||
| 523 | float var_local_temp = ReduceSumCustom(local_x_fp32[offset], num_last_dim) * meanNum; | 521 | float var_local_temp = ReduceSumCustom(local_x_fp32[offset], num_last_dim) * meanNum; |
| 524 | float rstd_local_temp = 1 / sqrt(var_local_temp + eps); | 522 | float rstd_local_temp = 1 / sqrt(var_local_temp + eps); |
| 525 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 523 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 526 | SetFlag<HardEvent::V_S>(event_v_s); | 524 | SetFlag<HardEvent::V_S>(event_v_s); |
| 527 | WaitFlag<HardEvent::V_S>(event_v_s); | 525 | WaitFlag<HardEvent::V_S>(event_v_s); |
| 528 | - rstd_local[rid].SetValue(0, rstd_local_temp); | 526 | + rstd_local[rid1].SetValue(0, rstd_local_temp); |
| 529 | 527 | ||
| 530 | Muls(local_y_fp32[offset], local_y_fp32[offset], rstd_local_temp, num_last_dim); | 528 | Muls(local_y_fp32[offset], local_y_fp32[offset], rstd_local_temp, num_last_dim); |
| 531 | PipeBarrier<PIPE_V>(); | 529 | PipeBarrier<PIPE_V>(); |
| @@ -535,8 +533,8 @@ private: | |||
| 535 | } | 533 | } |
| 536 | PipeBarrier<PIPE_V>(); | 534 | PipeBarrier<PIPE_V>(); |
| 537 | Cast(z_local, local_y_fp32, RoundMode::CAST_NONE, stepSize); | 535 | Cast(z_local, local_y_fp32, RoundMode::CAST_NONE, stepSize); |
| 538 | - SetFlag<HardEvent::S_MTE3>(event_s_mte3); | 536 | + SetFlag<HardEvent::S_MTE3>(event_s_mte3_1); |
| 539 | - WaitFlag<HardEvent::S_MTE3>(event_s_mte3); | 537 | + WaitFlag<HardEvent::S_MTE3>(event_s_mte3_1); |
| 540 | rstd_que_fp32.EnQue(rstd_local); | 538 | rstd_que_fp32.EnQue(rstd_local); |
| 541 | z_que.EnQue(z_local); | 539 | z_que.EnQue(z_local); |
| 542 | } | 540 | } |
| @@ -547,12 +545,10 @@ private: | |||
| 547 | LocalTensor<T> x_local = x_que.DeQue<T>(); | 545 | LocalTensor<T> x_local = x_que.DeQue<T>(); |
| 548 | LocalTensor<T> gx_local = gx_que.DeQue<T>(); | 546 | LocalTensor<T> gx_local = gx_que.DeQue<T>(); |
| 549 | LocalTensor<float> local_calc_fp32 = calc_buf_fp32.Get<float>(); | 547 | LocalTensor<float> local_calc_fp32 = calc_buf_fp32.Get<float>(); |
| 550 | - // output | ||
| 551 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); | 548 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); |
| 552 | LocalTensor<float> mean_local = mean_que_fp32.AllocTensor<float>(); | 549 | LocalTensor<float> mean_local = mean_que_fp32.AllocTensor<float>(); |
| 553 | LocalTensor<float> rstd_local = rstd_que_fp32.AllocTensor<float>(); | 550 | LocalTensor<float> rstd_local = rstd_que_fp32.AllocTensor<float>(); |
| 554 | 551 | ||
| 555 | - // local temp | ||
| 556 | LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); | 552 | LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); |
| 557 | LocalTensor<float> y_local_fp32 = y_buf_fp32.Get<float>(); | 553 | LocalTensor<float> y_local_fp32 = y_buf_fp32.Get<float>(); |
| 558 | LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); | 554 | LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); |
| @@ -573,15 +569,15 @@ private: | |||
| 573 | Muls(local_y_fp32, local_x_fp32, 1.0f, stepSize); | 569 | Muls(local_y_fp32, local_x_fp32, 1.0f, stepSize); |
| 574 | SetFlag<HardEvent::V_S>(EVENT_ID0); | 570 | SetFlag<HardEvent::V_S>(EVENT_ID0); |
| 575 | WaitFlag<HardEvent::V_S>(EVENT_ID0); | 571 | WaitFlag<HardEvent::V_S>(EVENT_ID0); |
| 576 | - for (int32_t rid = 0; rid < nums; ++rid) { | 572 | + for (int32_t rid2 = 0; rid2 < nums; ++rid2) { |
| 577 | - uint32_t offset = rid * realLen; | 573 | + uint32_t offset2 = rid2 * realLen; |
| 578 | - float mean_local_temp = ReduceSumCustom(local_y_fp32[offset], num_last_dim); | 574 | + float mean_local_temp2 = ReduceSumCustom(local_y_fp32[offset2], num_last_dim); |
| 579 | - event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 575 | + event_t event_v_s2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 580 | - SetFlag<HardEvent::V_S>(event_v_s); | 576 | + SetFlag<HardEvent::V_S>(event_v_s2); |
| 581 | - WaitFlag<HardEvent::V_S>(event_v_s); | 577 | + WaitFlag<HardEvent::V_S>(event_v_s2); |
| 582 | - mean_local_temp = mean_local_temp * meanNum; | 578 | + mean_local_temp2 = mean_local_temp2 * meanNum; |
| 583 | - mean_local[rid].SetValue(0, mean_local_temp); | 579 | + mean_local[rid2].SetValue(0, mean_local_temp2); |
| 584 | - Adds(local_y_fp32[offset], local_x_fp32[offset], mean_local_temp * (-1), num_last_dim); | 580 | + Adds(local_y_fp32[offset2], local_x_fp32[offset2], mean_local_temp2 * (-1), num_last_dim); |
| 585 | } | 581 | } |
| 586 | SetFlag<HardEvent::S_V>(EVENT_ID0); | 582 | SetFlag<HardEvent::S_V>(EVENT_ID0); |
| 587 | WaitFlag<HardEvent::S_V>(EVENT_ID0); | 583 | WaitFlag<HardEvent::S_V>(EVENT_ID0); |
| @@ -693,38 +689,38 @@ private: | |||
| 693 | __aicore__ inline void ComputeFp16Short(int32_t nums, LocalTensor<T>& beta_local, LocalTensor<T>& gamma_local) | 689 | __aicore__ inline void ComputeFp16Short(int32_t nums, LocalTensor<T>& beta_local, LocalTensor<T>& gamma_local) |
| 694 | { | 690 | { |
| 695 | // input | 691 | // input |
| 696 | - LocalTensor<T> x_local = x_que.DeQue<T>(); | ||
| 697 | LocalTensor<T> gx_local = gx_que.DeQue<T>(); | 692 | LocalTensor<T> gx_local = gx_que.DeQue<T>(); |
| 693 | + LocalTensor<T> x_local = x_que.DeQue<T>(); | ||
| 698 | // output | 694 | // output |
| 699 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); | 695 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); |
| 700 | - LocalTensor<float> mean_local = mean_que_fp32.AllocTensor<float>(); | ||
| 701 | LocalTensor<float> rstd_local = rstd_que_fp32.AllocTensor<float>(); | 696 | LocalTensor<float> rstd_local = rstd_que_fp32.AllocTensor<float>(); |
| 697 | + LocalTensor<float> mean_local = mean_que_fp32.AllocTensor<float>(); | ||
| 702 | 698 | ||
| 703 | // local temp | 699 | // local temp |
| 704 | LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); | 700 | LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); |
| 701 | + LocalTensor<float> z_local_fp32_v1 = z_buf_fp32.Get<float>(); | ||
| 705 | LocalTensor<float> y_local_fp32 = y_buf_fp32.Get<float>(); | 702 | LocalTensor<float> y_local_fp32 = y_buf_fp32.Get<float>(); |
| 706 | - LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); | 703 | + uint32_t realLen1 = ROUND_UP(num_last_dim); |
| 707 | - uint32_t realLen = ROUND_UP(num_last_dim); | 704 | + uint32_t stepSize = nums * realLen1; |
| 708 | - uint32_t stepSize = nums * realLen; | ||
| 709 | Cast(x_local_fp32, x_local, RoundMode::CAST_NONE, stepSize); | 705 | Cast(x_local_fp32, x_local, RoundMode::CAST_NONE, stepSize); |
| 710 | PipeBarrier<PIPE_V>(); | 706 | PipeBarrier<PIPE_V>(); |
| 711 | Cast(y_local_fp32, gx_local, RoundMode::CAST_NONE, stepSize); | 707 | Cast(y_local_fp32, gx_local, RoundMode::CAST_NONE, stepSize); |
| 712 | PipeBarrier<PIPE_V>(); | 708 | PipeBarrier<PIPE_V>(); |
| 713 | - x_que.FreeTensor(x_local); | ||
| 714 | gx_que.FreeTensor(gx_local); | 709 | gx_que.FreeTensor(gx_local); |
| 710 | + x_que.FreeTensor(x_local); | ||
| 715 | 711 | ||
| 716 | - PrecisionComputeMeanShort(nums, z_local_fp32, x_local_fp32, y_local_fp32, mean_local); | 712 | + PrecisionComputeMeanShort(nums, z_local_fp32_v1, x_local_fp32, y_local_fp32, mean_local); |
| 717 | PipeBarrier<PIPE_V>(); | 713 | PipeBarrier<PIPE_V>(); |
| 718 | mean_que_fp32.EnQue(mean_local); | 714 | mean_que_fp32.EnQue(mean_local); |
| 719 | - PrecisionComputeRstdShort(nums, z_local_fp32, x_local_fp32, y_local_fp32, rstd_local); | 715 | + PrecisionComputeRstdShort(nums, z_local_fp32_v1, x_local_fp32, y_local_fp32, rstd_local); |
| 720 | PipeBarrier<PIPE_V>(); | 716 | PipeBarrier<PIPE_V>(); |
| 721 | rstd_que_fp32.EnQue(rstd_local); | 717 | rstd_que_fp32.EnQue(rstd_local); |
| 722 | 718 | ||
| 723 | Cast(x_local_fp32, gamma_local, RoundMode::CAST_NONE, num_last_dim); | 719 | Cast(x_local_fp32, gamma_local, RoundMode::CAST_NONE, num_last_dim); |
| 724 | Cast(y_local_fp32, beta_local, RoundMode::CAST_NONE, num_last_dim); | 720 | Cast(y_local_fp32, beta_local, RoundMode::CAST_NONE, num_last_dim); |
| 725 | - PrecisionComputeResultShort(nums, z_local_fp32, y_local_fp32, x_local_fp32); | 721 | + PrecisionComputeResultShort(nums, z_local_fp32_v1, y_local_fp32, x_local_fp32); |
| 726 | PipeBarrier<PIPE_V>(); | 722 | PipeBarrier<PIPE_V>(); |
| 727 | - Cast(z_local, z_local_fp32, RoundMode::CAST_NONE, stepSize); | 723 | + Cast(z_local, z_local_fp32_v1, RoundMode::CAST_NONE, stepSize); |
| 728 | z_que.EnQue(z_local); | 724 | z_que.EnQue(z_local); |
| 729 | } | 725 | } |
| 730 | 726 | ||
| @@ -733,12 +729,10 @@ private: | |||
| 733 | // input | 729 | // input |
| 734 | LocalTensor<T> x_local = x_que.DeQue<T>(); | 730 | LocalTensor<T> x_local = x_que.DeQue<T>(); |
| 735 | LocalTensor<T> gx_local = gx_que.DeQue<T>(); | 731 | LocalTensor<T> gx_local = gx_que.DeQue<T>(); |
| 736 | - // output | ||
| 737 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); | 732 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); |
| 738 | LocalTensor<float> mean_local = mean_que_fp32.AllocTensor<float>(); | 733 | LocalTensor<float> mean_local = mean_que_fp32.AllocTensor<float>(); |
| 739 | LocalTensor<float> rstd_local = rstd_que_fp32.AllocTensor<float>(); | 734 | LocalTensor<float> rstd_local = rstd_que_fp32.AllocTensor<float>(); |
| 740 | 735 | ||
| 741 | - // local temp | ||
| 742 | LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); | 736 | LocalTensor<float> x_local_fp32 = x_buf_fp32.Get<float>(); |
| 743 | LocalTensor<float> y_local_fp32 = y_buf_fp32.Get<float>(); | 737 | LocalTensor<float> y_local_fp32 = y_buf_fp32.Get<float>(); |
| 744 | LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); | 738 | LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); |
| @@ -984,46 +978,43 @@ private: | |||
| 984 | // Get Mean | 978 | // Get Mean |
| 985 | meanVal = 0; | 979 | meanVal = 0; |
| 986 | 980 | ||
| 987 | - for (int i = 0; i < updated_last_times; i++) { | 981 | + for (int i2 = 0; i2 < updated_last_times; i2++) { |
| 988 | - uint32_t size = (i == updated_last_times - 1) ? lsize : updated_last_dim; | 982 | + uint32_t size2 = (i2 == updated_last_times - 1) ? lsize : updated_last_dim; |
| 989 | - ExtraCopyXGX(offset + i * updated_last_dim, size); | 983 | + ExtraCopyXGX(offset + i2 * updated_last_dim, size2); |
| 990 | - ComputeMeanFp16Bf16(i, size); | 984 | + ComputeMeanFp16Bf16(i2, size2); |
| 991 | sum_local = z_buf_fp32.Get<float>(); | 985 | sum_local = z_buf_fp32.Get<float>(); |
| 992 | - meanVal += ReduceSumCustom(sum_local, size); | 986 | + meanVal += ReduceSumCustom(sum_local, size2); |
| 993 | } | 987 | } |
| 994 | meanVal = meanVal * meanNum; | 988 | meanVal = meanVal * meanNum; |
| 995 | 989 | ||
| 996 | - // Get Var | ||
| 997 | varVal = 0; | 990 | varVal = 0; |
| 998 | - for (int j = 0; j < updated_last_times; j++) { | 991 | + for (int j2 = 0; j2 < updated_last_times; j2++) { |
| 999 | - uint32_t size = (j == updated_last_times - 1) ? lsize : updated_last_dim; | 992 | + uint32_t size2 = (j2 == updated_last_times - 1) ? lsize : updated_last_dim; |
| 1000 | - ComputeVar(j, size); | 993 | + ComputeVar(j2, size2); |
| 1001 | sum_local = z_buf_fp32.Get<float>(); | 994 | sum_local = z_buf_fp32.Get<float>(); |
| 1002 | - varVal += ReduceSumCustom(sum_local, size); | 995 | + varVal += ReduceSumCustom(sum_local, size2); |
| 1003 | } | 996 | } |
| 1004 | varVal = varVal * meanNum; | 997 | varVal = varVal * meanNum; |
| 1005 | - // Get rstd | ||
| 1006 | varVal = 1 / sqrt(varVal + eps); | 998 | varVal = 1 / sqrt(varVal + eps); |
| 1007 | - // Get result | ||
| 1008 | LocalTensor<float> mean = mean_que_fp32.AllocTensor<float>(); | 999 | LocalTensor<float> mean = mean_que_fp32.AllocTensor<float>(); |
| 1009 | LocalTensor<float> rstd = rstd_que_fp32.AllocTensor<float>(); | 1000 | LocalTensor<float> rstd = rstd_que_fp32.AllocTensor<float>(); |
| 1010 | - event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 1001 | + event_t event_v_s_1 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 1011 | - SetFlag<HardEvent::V_S>(event_v_s); | 1002 | + SetFlag<HardEvent::V_S>(event_v_s_1); |
| 1012 | - WaitFlag<HardEvent::V_S>(event_v_s); | 1003 | + WaitFlag<HardEvent::V_S>(event_v_s_1); |
| 1013 | mean.SetValue(0, meanVal); | 1004 | mean.SetValue(0, meanVal); |
| 1014 | rstd.SetValue(0, varVal); | 1005 | rstd.SetValue(0, varVal); |
| 1015 | - event_t event_s_mte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE3)); | 1006 | + event_t event_s_mte3_11 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE3)); |
| 1016 | - SetFlag<HardEvent::S_MTE3>(event_s_mte3); | 1007 | + SetFlag<HardEvent::S_MTE3>(event_s_mte3_11); |
| 1017 | - WaitFlag<HardEvent::S_MTE3>(event_s_mte3); | 1008 | + WaitFlag<HardEvent::S_MTE3>(event_s_mte3_11); |
| 1018 | 1009 | ||
| 1019 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); | 1010 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); |
| 1020 | - for (int k = 0; k < updated_last_times; k++) { | 1011 | + for (int k1 = 0; k1 < updated_last_times; k1++) { |
| 1021 | - uint32_t size = (k == updated_last_times - 1) ? lsize : updated_last_dim; | 1012 | + uint32_t size = (k1 == updated_last_times - 1) ? lsize : updated_last_dim; |
| 1022 | - ExtraCopyBetaGamma(k * updated_last_dim, size); | 1013 | + ExtraCopyBetaGamma(k1 * updated_last_dim, size); |
| 1023 | - ComputeResFp16Bf16(k, size); | 1014 | + ComputeResFp16Bf16(k1, size); |
| 1024 | LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); | 1015 | LocalTensor<float> z_local_fp32 = z_buf_fp32.Get<float>(); |
| 1025 | PipeBarrier<PIPE_V>(); | 1016 | PipeBarrier<PIPE_V>(); |
| 1026 | - Cast(z_local[k * updated_last_dim], z_local_fp32, RoundMode::CAST_NONE, size); | 1017 | + Cast(z_local[k1 * updated_last_dim], z_local_fp32, RoundMode::CAST_NONE, size); |
| 1027 | PipeBarrier<PIPE_V>(); | 1018 | PipeBarrier<PIPE_V>(); |
| 1028 | } | 1019 | } |
| 1029 | 1020 | ||
| @@ -1037,7 +1028,7 @@ private: | |||
| 1037 | { | 1028 | { |
| 1038 | uint32_t offset = iter * row_step * num_last_dim; | 1029 | uint32_t offset = iter * row_step * num_last_dim; |
| 1039 | uint32_t lsize = num_last_dim - (updated_last_times - 1) * updated_last_dim; | 1030 | uint32_t lsize = num_last_dim - (updated_last_times - 1) * updated_last_dim; |
| 1040 | - LocalTensor<float> sum_local; | 1031 | + LocalTensor<float> sum_local_v1; |
| 1041 | // Get Mean | 1032 | // Get Mean |
| 1042 | meanVal = 0; | 1033 | meanVal = 0; |
| 1043 | 1034 | ||
| @@ -1045,8 +1036,8 @@ private: | |||
| 1045 | uint32_t size = (i == updated_last_times - 1) ? lsize : updated_last_dim; | 1036 | uint32_t size = (i == updated_last_times - 1) ? lsize : updated_last_dim; |
| 1046 | ExtraCopyXGX(offset + i * updated_last_dim, size); | 1037 | ExtraCopyXGX(offset + i * updated_last_dim, size); |
| 1047 | ComputeMeanFp16Bf16(i, size); | 1038 | ComputeMeanFp16Bf16(i, size); |
| 1048 | - sum_local = z_buf_fp32.Get<float>(); | 1039 | + sum_local_v1 = z_buf_fp32.Get<float>(); |
| 1049 | - meanVal += ReduceSumCustom(sum_local, size); | 1040 | + meanVal += ReduceSumCustom(sum_local_v1, size); |
| 1050 | } | 1041 | } |
| 1051 | meanVal = meanVal * meanNum; | 1042 | meanVal = meanVal * meanNum; |
| 1052 | 1043 | ||
| @@ -1055,15 +1046,15 @@ private: | |||
| 1055 | for (int j = 0; j < updated_last_times; j++) { | 1046 | for (int j = 0; j < updated_last_times; j++) { |
| 1056 | uint32_t size = (j == updated_last_times - 1) ? lsize : updated_last_dim; | 1047 | uint32_t size = (j == updated_last_times - 1) ? lsize : updated_last_dim; |
| 1057 | ComputeVar(j, size); | 1048 | ComputeVar(j, size); |
| 1058 | - sum_local = z_buf_fp32.Get<float>(); | 1049 | + sum_local_v1 = z_buf_fp32.Get<float>(); |
| 1059 | - varVal += ReduceSumCustom(sum_local, size); | 1050 | + varVal += ReduceSumCustom(sum_local_v1, size); |
| 1060 | } | 1051 | } |
| 1061 | varVal = varVal * meanNum; | 1052 | varVal = varVal * meanNum; |
| 1062 | // Get rstd | 1053 | // Get rstd |
| 1063 | varVal = 1 / sqrt(varVal + eps); | 1054 | varVal = 1 / sqrt(varVal + eps); |
| 1064 | // Get result | 1055 | // Get result |
| 1065 | - LocalTensor<float> mean = mean_que_fp32.AllocTensor<float>(); | ||
| 1066 | LocalTensor<float> rstd = rstd_que_fp32.AllocTensor<float>(); | 1056 | LocalTensor<float> rstd = rstd_que_fp32.AllocTensor<float>(); |
| 1057 | + LocalTensor<float> mean = mean_que_fp32.AllocTensor<float>(); | ||
| 1067 | 1058 | ||
| 1068 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 1059 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 1069 | SetFlag<HardEvent::V_S>(event_v_s); | 1060 | SetFlag<HardEvent::V_S>(event_v_s); |
| @@ -1119,17 +1110,17 @@ private: | |||
| 1119 | // Get rstd | 1110 | // Get rstd |
| 1120 | varVal = 1 / sqrt(varVal + eps); | 1111 | varVal = 1 / sqrt(varVal + eps); |
| 1121 | // Get result | 1112 | // Get result |
| 1122 | - LocalTensor<float> mean = mean_que_fp32.AllocTensor<float>(); | 1113 | + LocalTensor<float> meanV1 = mean_que_fp32.AllocTensor<float>(); |
| 1123 | LocalTensor<float> rstd = rstd_que_fp32.AllocTensor<float>(); | 1114 | LocalTensor<float> rstd = rstd_que_fp32.AllocTensor<float>(); |
| 1124 | 1115 | ||
| 1125 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 1116 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 1126 | SetFlag<HardEvent::V_S>(event_v_s); | 1117 | SetFlag<HardEvent::V_S>(event_v_s); |
| 1127 | WaitFlag<HardEvent::V_S>(event_v_s); | 1118 | WaitFlag<HardEvent::V_S>(event_v_s); |
| 1128 | - mean.SetValue(0, meanVal); | 1119 | + meanV1.SetValue(0, meanVal); |
| 1129 | rstd.SetValue(0, varVal); | 1120 | rstd.SetValue(0, varVal); |
| 1130 | - event_t event_s_mte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE3)); | 1121 | + event_t event_s_mte3_v1 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE3)); |
| 1131 | - SetFlag<HardEvent::S_MTE3>(event_s_mte3); | 1122 | + SetFlag<HardEvent::S_MTE3>(event_s_mte3_v1); |
| 1132 | - WaitFlag<HardEvent::S_MTE3>(event_s_mte3); | 1123 | + WaitFlag<HardEvent::S_MTE3>(event_s_mte3_v1); |
| 1133 | 1124 | ||
| 1134 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); | 1125 | LocalTensor<T> z_local = z_que.AllocTensor<T>(); |
| 1135 | for (int k = 0; k < updated_last_times; k++) { | 1126 | for (int k = 0; k < updated_last_times; k++) { |
| @@ -1143,7 +1134,7 @@ private: | |||
| 1143 | PipeBarrier<PIPE_V>(); | 1134 | PipeBarrier<PIPE_V>(); |
| 1144 | } | 1135 | } |
| 1145 | z_que.EnQue(z_local); | 1136 | z_que.EnQue(z_local); |
| 1146 | - mean_que_fp32.EnQue(mean); | 1137 | + mean_que_fp32.EnQue(meanV1); |
| 1147 | rstd_que_fp32.EnQue(rstd); | 1138 | rstd_que_fp32.EnQue(rstd); |
| 1148 | ExtraCopyOut(iter); | 1139 | ExtraCopyOut(iter); |
| 1149 | } | 1140 | } |
| @@ -112,16 +112,18 @@ inline void CalMergeCountValue( | |||
| 112 | (tensorNDNum * elemWithDInUB + tensorJustNNum * elemWithoutDInUB)); | 112 | (tensorNDNum * elemWithDInUB + tensorJustNNum * elemWithoutDInUB)); |
| 113 | } | 113 | } |
| 114 | 114 | ||
| 115 | +// SetFp32Config: configure tiling parameters for FP32 data type | ||
| 115 | inline void SetFp32Config( | 116 | inline void SetFp32Config( |
| 116 | DeepNormGradTilingData& tiling, uint32_t& ubElemNum, uint32_t& elemWithDInUB, uint32_t& elemWithoutDInUB, | 117 | DeepNormGradTilingData& tiling, uint32_t& ubElemNum, uint32_t& elemWithDInUB, uint32_t& elemWithoutDInUB, |
| 117 | uint32_t& blockElem, uint32_t& dDimNum, uint32_t& cutDKey) | 118 | uint32_t& blockElem, uint32_t& dDimNum, uint32_t& cutDKey) |
| 118 | { | 119 | { |
| 119 | - uint32_t cutStageMergeN; | ||
| 120 | uint32_t mergeCountMergeN; | 120 | uint32_t mergeCountMergeN; |
| 121 | + uint32_t cutStageMergeN; | ||
| 121 | 122 | ||
| 123 | + // fp32 tensor counts for small D stage | ||
| 124 | + uint32_t tensorJustDNum; | ||
| 122 | uint32_t tensorJustNNum; | 125 | uint32_t tensorJustNNum; |
| 123 | uint32_t tensorNDNum; | 126 | uint32_t tensorNDNum; |
| 124 | - uint32_t tensorJustDNum; | ||
| 125 | uint32_t otherTensorJustDNum; | 127 | uint32_t otherTensorJustDNum; |
| 126 | 128 | ||
| 127 | if (dDimNum < SMALL_D_STAGE) { | 129 | if (dDimNum < SMALL_D_STAGE) { |
| @@ -150,9 +152,9 @@ inline void SetFp32Config( | |||
| 150 | // (5*mergeNCount+3)*elemWithDInUB+(2*mergeNCount)*elemWithoutDInUB(8)=ubElemNum | 152 | // (5*mergeNCount+3)*elemWithDInUB+(2*mergeNCount)*elemWithoutDInUB(8)=ubElemNum |
| 151 | // when mergeNCount=1, elemWithDInUB=(ubElemNum-2*elemWithoutDInUB)/8 | 153 | // when mergeNCount=1, elemWithDInUB=(ubElemNum-2*elemWithoutDInUB)/8 |
| 152 | tensorJustNNum = 2; // mean\rstd | 154 | tensorJustNNum = 2; // mean\rstd |
| 155 | + otherTensorJustDNum = 0; // not use brcb | ||
| 153 | tensorNDNum = CONST_5; // dy\x\gx\dx\dgx | 156 | tensorNDNum = CONST_5; // dy\x\gx\dx\dgx |
| 154 | tensorJustDNum = CONST_3; // gamma\dbeta\dgamma | 157 | tensorJustDNum = CONST_3; // gamma\dbeta\dgamma |
| 155 | - otherTensorJustDNum = 0; // not use brcb | ||
| 156 | 158 | ||
| 157 | CalCutStageValue( | 159 | CalCutStageValue( |
| 158 | cutStageMergeN, ubElemNum, tensorJustNNum, elemWithoutDInUB, | 160 | cutStageMergeN, ubElemNum, tensorJustNNum, elemWithoutDInUB, |
| @@ -165,20 +167,20 @@ inline void SetFp32Config( | |||
| 165 | // == merge N == | 167 | // == merge N == |
| 166 | uint32_t mergeNCount; | 168 | uint32_t mergeNCount; |
| 167 | if (dDimNum < SMALL_D_STAGE) { | 169 | if (dDimNum < SMALL_D_STAGE) { |
| 170 | + mergeNCount = mergeCountMergeN; | ||
| 168 | cutDKey = KEY_LARGE_N_SMALL_D; | 171 | cutDKey = KEY_LARGE_N_SMALL_D; |
| 169 | - mergeNCount = mergeCountMergeN; | ||
| 170 | } else if (dDimNum <= cutStageMergeN) { | 172 | } else if (dDimNum <= cutStageMergeN) { |
| 171 | - cutDKey = KEY_MERGE_N; | ||
| 172 | mergeNCount = mergeCountMergeN; | 173 | mergeNCount = mergeCountMergeN; |
| 174 | + cutDKey = KEY_MERGE_N; | ||
| 173 | } else { | 175 | } else { |
| 174 | - cutDKey = KEY_CUT_D; | ||
| 175 | mergeNCount = 1; | 176 | mergeNCount = 1; |
| 177 | + cutDKey = KEY_CUT_D; | ||
| 176 | } | 178 | } |
| 177 | 179 | ||
| 178 | // == cut D == | 180 | // == cut D == |
| 179 | uint32_t cutStageCutD; | 181 | uint32_t cutStageCutD; |
| 180 | - uint32_t cutDTime; | ||
| 181 | uint32_t cutDPerTime; | 182 | uint32_t cutDPerTime; |
| 183 | + uint32_t cutDTime; | ||
| 182 | uint32_t cutDLastTime; | 184 | uint32_t cutDLastTime; |
| 183 | if (cutDKey == KEY_CUT_D) { | 185 | if (cutDKey == KEY_CUT_D) { |
| 184 | // fp32 cutD | 186 | // fp32 cutD |
| @@ -188,39 +190,41 @@ inline void SetFp32Config( | |||
| 188 | // 7*elemWithDInUB+4*elemWithoutDInUB(8)=ubElemNum | 190 | // 7*elemWithDInUB+4*elemWithoutDInUB(8)=ubElemNum |
| 189 | tensorJustNNum = 4; // mean\rstd\tmp_mean_pd_buf\tmp_var_pd_buf | 191 | tensorJustNNum = 4; // mean\rstd\tmp_mean_pd_buf\tmp_var_pd_buf |
| 190 | tensorNDNum = CONST_5; // dy\x\gx\dx\dgx | 192 | tensorNDNum = CONST_5; // dy\x\gx\dx\dgx |
| 191 | - tensorJustDNum = CONST_3; // gamma\dgamma\dbeta | ||
| 192 | otherTensorJustDNum = 0; // not use brcb | 193 | otherTensorJustDNum = 0; // not use brcb |
| 194 | + tensorJustDNum = CONST_3; // gamma\dgamma\dbeta | ||
| 193 | 195 | ||
| 194 | CalCutStageValue( | 196 | CalCutStageValue( |
| 195 | cutStageCutD, ubElemNum, tensorJustNNum, elemWithoutDInUB, | 197 | cutStageCutD, ubElemNum, tensorJustNNum, elemWithoutDInUB, |
| 196 | tensorNDNum + tensorJustDNum + otherTensorJustDNum, blockElem); | 198 | tensorNDNum + tensorJustDNum + otherTensorJustDNum, blockElem); |
| 197 | 199 | ||
| 198 | - cutDTime = Ops::Base::CeilDiv(dDimNum, cutStageCutD); | ||
| 199 | cutDPerTime = cutStageCutD; | 200 | cutDPerTime = cutStageCutD; |
| 201 | + cutDTime = Ops::Base::CeilDiv(dDimNum, cutStageCutD); | ||
| 200 | cutDLastTime = dDimNum - cutStageCutD * (cutDTime - 1); | 202 | cutDLastTime = dDimNum - cutStageCutD * (cutDTime - 1); |
| 201 | } else { | 203 | } else { |
| 202 | - cutDTime = 1; | ||
| 203 | cutDPerTime = dDimNum; | 204 | cutDPerTime = dDimNum; |
| 205 | + cutDTime = 1; | ||
| 204 | cutDLastTime = dDimNum; | 206 | cutDLastTime = dDimNum; |
| 205 | } | 207 | } |
| 206 | 208 | ||
| 207 | - tiling.set_mergeNCount(mergeNCount); | ||
| 208 | tiling.set_cutDTime(cutDTime); | 209 | tiling.set_cutDTime(cutDTime); |
| 210 | + tiling.set_mergeNCount(mergeNCount); | ||
| 209 | tiling.set_cutDPerTime(cutDPerTime); | 211 | tiling.set_cutDPerTime(cutDPerTime); |
| 210 | tiling.set_cutDLastTime(cutDLastTime); | 212 | tiling.set_cutDLastTime(cutDLastTime); |
| 211 | } | 213 | } |
| 212 | 214 | ||
| 215 | +// SetFp16Bf16Config: configure tiling parameters for FP16/BF16 data types | ||
| 213 | inline void SetFp16Bf16Config( | 216 | inline void SetFp16Bf16Config( |
| 214 | DeepNormGradTilingData& tiling, uint32_t& ubElemNum, uint32_t& elemWithDInUB, uint32_t& elemWithoutDInUB, | 217 | DeepNormGradTilingData& tiling, uint32_t& ubElemNum, uint32_t& elemWithDInUB, uint32_t& elemWithoutDInUB, |
| 215 | uint32_t& blockElem, uint32_t& dDimNum, uint32_t& cutDKey) | 218 | uint32_t& blockElem, uint32_t& dDimNum, uint32_t& cutDKey) |
| 216 | { | 219 | { |
| 217 | - uint32_t cutStageMergeN; | ||
| 218 | uint32_t mergeCountMergeN; | 220 | uint32_t mergeCountMergeN; |
| 221 | + uint32_t cutStageMergeN; | ||
| 219 | 222 | ||
| 223 | + // fp16/bf16 tensor counts for small D stage | ||
| 224 | + uint32_t otherTensorJustDNum; | ||
| 220 | uint32_t tensorJustNNum; | 225 | uint32_t tensorJustNNum; |
| 221 | uint32_t tensorNDNum; | 226 | uint32_t tensorNDNum; |
| 222 | uint32_t tensorJustDNum; | 227 | uint32_t tensorJustDNum; |
| 223 | - uint32_t otherTensorJustDNum; | ||
| 224 | 228 | ||
| 225 | if (dDimNum < SMALL_D_STAGE) { | 229 | if (dDimNum < SMALL_D_STAGE) { |
| 226 | // fp16 smallD | 230 | // fp16 smallD |
| @@ -244,6 +248,7 @@ inline void SetFp16Bf16Config( | |||
| 244 | mergeCountMergeN, ubElemNum, tensorJustDNum + otherTensorJustDNum, elemWithDInUB, tensorNDNum, | 248 | mergeCountMergeN, ubElemNum, tensorJustDNum + otherTensorJustDNum, elemWithDInUB, tensorNDNum, |
| 245 | tensorJustNNum, elemWithoutDInUB); | 249 | tensorJustNNum, elemWithoutDInUB); |
| 246 | } else { | 250 | } else { |
| 251 | + // fp32 large D stage | ||
| 247 | tensorJustNNum = CONST_4; // mean\rstd | 252 | tensorJustNNum = CONST_4; // mean\rstd |
| 248 | tensorNDNum = CONST_5; // dy\x\gx\dx\dgx | 253 | tensorNDNum = CONST_5; // dy\x\gx\dx\dgx |
| 249 | tensorJustDNum = CONST_17; // gamma\dbeta(fp32)\dgamma(fp32)\dy_t(fp32)\x_t(fp32)\gx_t(fp32) | 254 | tensorJustDNum = CONST_17; // gamma\dbeta(fp32)\dgamma(fp32)\dy_t(fp32)\x_t(fp32)\gx_t(fp32) |
| @@ -258,7 +263,7 @@ inline void SetFp16Bf16Config( | |||
| 258 | tensorJustNNum, elemWithoutDInUB); | 263 | tensorJustNNum, elemWithoutDInUB); |
| 259 | } | 264 | } |
| 260 | 265 | ||
| 261 | - // == merge N == | 266 | + // == merge N for fp16/bf16 == |
| 262 | uint32_t mergeNCount; | 267 | uint32_t mergeNCount; |
| 263 | if (dDimNum < SMALL_D_STAGE) { | 268 | if (dDimNum < SMALL_D_STAGE) { |
| 264 | cutDKey = KEY_LARGE_N_SMALL_D; | 269 | cutDKey = KEY_LARGE_N_SMALL_D; |
| @@ -271,7 +276,7 @@ inline void SetFp16Bf16Config( | |||
| 271 | mergeNCount = 1; | 276 | mergeNCount = 1; |
| 272 | } | 277 | } |
| 273 | 278 | ||
| 274 | - // == cut D == | 279 | + // == cut D for fp16/bf16 == |
| 275 | uint32_t cutStageCutD; | 280 | uint32_t cutStageCutD; |
| 276 | uint32_t cutDTime; | 281 | uint32_t cutDTime; |
| 277 | uint32_t cutDPerTime; | 282 | uint32_t cutDPerTime; |
| @@ -283,6 +288,7 @@ inline void SetFp16Bf16Config( | |||
| 283 | // dgamma(fp32)\dbeta(fp32) gamma_t(fp32)\dx_t(fp32)\dgx_t(fp32) | 288 | // dgamma(fp32)\dbeta(fp32) gamma_t(fp32)\dx_t(fp32)\dgx_t(fp32) |
| 284 | // tensor(without D): mean(fp32)\rstd(fp32) tmp_mean_pd_buf(fp32)\tmp_var_pd_buf(fp32) | 289 | // tensor(without D): mean(fp32)\rstd(fp32) tmp_mean_pd_buf(fp32)\tmp_var_pd_buf(fp32) |
| 285 | // 20*elemWithDInUB+8*elemWithoutDInUB(16)=ubElemNum | 290 | // 20*elemWithDInUB+8*elemWithoutDInUB(16)=ubElemNum |
| 291 | + // fp16/bf16 cutD tensor counts | ||
| 286 | tensorJustNNum = 8; // mean(fp32)\rstd(fp32)\tmp_mean_pd_buf(fp32)\tmp_var_pd_buf(fp32) | 292 | tensorJustNNum = 8; // mean(fp32)\rstd(fp32)\tmp_mean_pd_buf(fp32)\tmp_var_pd_buf(fp32) |
| 287 | tensorNDNum = CONST_17; // dy\x\gx\dx\dgx\dy_t(fp32)\x_t(fp32)\gx_t(fp32) | 293 | tensorNDNum = CONST_17; // dy\x\gx\dx\dgx\dy_t(fp32)\x_t(fp32)\gx_t(fp32) |
| 288 | // gamma_t(fp32)\dx_t(fp32)\dgx_t(fp32) | 294 | // gamma_t(fp32)\dx_t(fp32)\dgx_t(fp32) |
| @@ -297,6 +303,7 @@ inline void SetFp16Bf16Config( | |||
| 297 | cutDPerTime = cutStageCutD; | 303 | cutDPerTime = cutStageCutD; |
| 298 | cutDLastTime = dDimNum - cutStageCutD * (cutDTime - 1); | 304 | cutDLastTime = dDimNum - cutStageCutD * (cutDTime - 1); |
| 299 | } else { | 305 | } else { |
| 306 | + // fp16/bf16 no cutD | ||
| 300 | cutDTime = 1; | 307 | cutDTime = 1; |
| 301 | cutDPerTime = dDimNum; | 308 | cutDPerTime = dDimNum; |
| 302 | cutDLastTime = dDimNum; | 309 | cutDLastTime = dDimNum; |
| @@ -310,16 +317,16 @@ inline void SetFp16Bf16Config( | |||
| 310 | 317 | ||
| 311 | static ge::graphStatus CheckInputOutputShapeNull(const gert::TilingContext* context) | 318 | static ge::graphStatus CheckInputOutputShapeNull(const gert::TilingContext* context) |
| 312 | { | 319 | { |
| 313 | - const gert::StorageShape* dyShape = context->GetInputShape(INPUT_DY_INDEX); | ||
| 314 | const gert::StorageShape* xShape = context->GetInputShape(INPUT_X_INDEX); | 320 | const gert::StorageShape* xShape = context->GetInputShape(INPUT_X_INDEX); |
| 315 | - const gert::StorageShape* gxShape = context->GetInputShape(INPUT_GX_INDEX); | 321 | + const gert::StorageShape* dyShape = context->GetInputShape(INPUT_DY_INDEX); |
| 316 | const gert::StorageShape* gammaShape = context->GetInputShape(INPUT_GAMMA_INDEX); | 322 | const gert::StorageShape* gammaShape = context->GetInputShape(INPUT_GAMMA_INDEX); |
| 317 | - const gert::StorageShape* meanShape = context->GetInputShape(INPUT_MEAN_INDEX); | 323 | + const gert::StorageShape* gxShape = context->GetInputShape(INPUT_GX_INDEX); |
| 318 | const gert::StorageShape* rstdShape = context->GetInputShape(INPUT_RSTD_INDEX); | 324 | const gert::StorageShape* rstdShape = context->GetInputShape(INPUT_RSTD_INDEX); |
| 319 | - const gert::StorageShape* dxShape = context->GetOutputShape(OUTPUT_DX_INDEX); | 325 | + const gert::StorageShape* meanShape = context->GetInputShape(INPUT_MEAN_INDEX); |
| 320 | const gert::StorageShape* dgxShape = context->GetOutputShape(OUTPUT_DGX_INDEX); | 326 | const gert::StorageShape* dgxShape = context->GetOutputShape(OUTPUT_DGX_INDEX); |
| 321 | - const gert::StorageShape* dbetaShape = context->GetOutputShape(OUTPUT_DBETA_INDEX); | 327 | + const gert::StorageShape* dxShape = context->GetOutputShape(OUTPUT_DX_INDEX); |
| 322 | const gert::StorageShape* dgammaShape = context->GetOutputShape(OUTPUT_DGAMMA_INDEX); | 328 | const gert::StorageShape* dgammaShape = context->GetOutputShape(OUTPUT_DGAMMA_INDEX); |
| 329 | + const gert::StorageShape* dbetaShape = context->GetOutputShape(OUTPUT_DBETA_INDEX); | ||
| 323 | OP_CHECK_NULL_WITH_CONTEXT(context, dyShape); | 330 | OP_CHECK_NULL_WITH_CONTEXT(context, dyShape); |
| 324 | OP_CHECK_NULL_WITH_CONTEXT(context, xShape); | 331 | OP_CHECK_NULL_WITH_CONTEXT(context, xShape); |
| 325 | OP_CHECK_NULL_WITH_CONTEXT(context, gxShape); | 332 | OP_CHECK_NULL_WITH_CONTEXT(context, gxShape); |
| @@ -335,6 +342,7 @@ static ge::graphStatus CheckInputOutputShapeNull(const gert::TilingContext* cont | |||
| 335 | 342 | ||
| 336 | static bool CheckInputOutputShapeDim(const gert::TilingContext* context) | 343 | static bool CheckInputOutputShapeDim(const gert::TilingContext* context) |
| 337 | { | 344 | { |
| 345 | + // check input/output shape dimensions | ||
| 338 | const gert::StorageShape* dyShape = context->GetInputShape(INPUT_DY_INDEX); | 346 | const gert::StorageShape* dyShape = context->GetInputShape(INPUT_DY_INDEX); |
| 339 | const gert::StorageShape* xShape = context->GetInputShape(INPUT_X_INDEX); | 347 | const gert::StorageShape* xShape = context->GetInputShape(INPUT_X_INDEX); |
| 340 | const gert::StorageShape* gxShape = context->GetInputShape(INPUT_GX_INDEX); | 348 | const gert::StorageShape* gxShape = context->GetInputShape(INPUT_GX_INDEX); |
| @@ -21,17 +21,17 @@ | |||
| 21 | 21 | ||
| 22 | using namespace AscendC; | 22 | using namespace AscendC; |
| 23 | 23 | ||
| 24 | -template <typename Tp, Tp v> | 24 | +template <typename Tp1, Tp1 v> |
| 25 | struct integral_constant { | 25 | struct integral_constant { |
| 26 | - static constexpr Tp value = v; | 26 | + static constexpr Tp1 value = v; |
| 27 | }; | 27 | }; |
| 28 | using true_type = integral_constant<bool, true>; | 28 | using true_type = integral_constant<bool, true>; |
| 29 | using false_type = integral_constant<bool, false>; | 29 | using false_type = integral_constant<bool, false>; |
| 30 | template <typename, typename> | 30 | template <typename, typename> |
| 31 | struct is_same : public false_type { | 31 | struct is_same : public false_type { |
| 32 | }; | 32 | }; |
| 33 | -template <typename Tp> | 33 | +template <typename Tp2> |
| 34 | -struct is_same<Tp, Tp> : public true_type { | 34 | +struct is_same<Tp2, Tp2> : public true_type { |
| 35 | }; | 35 | }; |
| 36 | 36 | ||
| 37 | constexpr uint32_t BUFFER_NUM = 1; | 37 | constexpr uint32_t BUFFER_NUM = 1; |
| @@ -40,14 +40,14 @@ constexpr uint32_t REDUCE_REP_STRIDE = 8; | |||
| 40 | constexpr uint32_t FLOAT_BLOCK_ELEM = 8; | 40 | constexpr uint32_t FLOAT_BLOCK_ELEM = 8; |
| 41 | constexpr uint32_t BRCB_ONCE_ELEM = 8; | 41 | constexpr uint32_t BRCB_ONCE_ELEM = 8; |
| 42 | constexpr uint32_t MAX_REP_NUM = 255; | 42 | constexpr uint32_t MAX_REP_NUM = 255; |
| 43 | -constexpr uint32_t MAX_COPY_LENTH = 2000; | ||
| 44 | constexpr uint32_t USE_INT_TOW = 2; | 43 | constexpr uint32_t USE_INT_TOW = 2; |
| 44 | +constexpr uint32_t MAX_COPY_LENTH = 2000; | ||
| 45 | 45 | ||
| 46 | inline volatile __gm__ uint32_t g_FixedOutputSync[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; | 46 | inline volatile __gm__ uint32_t g_FixedOutputSync[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; |
| 47 | 47 | ||
| 48 | __aicore__ inline constexpr bool IsDataCopyPadSupport() | 48 | __aicore__ inline constexpr bool IsDataCopyPadSupport() |
| 49 | { | 49 | { |
| 50 | -#if __CCE_AICORE__ == 220 || (defined(__NPU_ARCH__) && __NPU_ARCH__ == 3003) | 50 | +#if (defined(__NPU_ARCH__) && __NPU_ARCH__ == 3003) || __CCE_AICORE__ == 220 |
| 51 | return true; | 51 | return true; |
| 52 | 52 | ||
| 53 | return false; | 53 | return false; |
| @@ -76,9 +76,9 @@ __aicore__ inline void SafeDataCopy( | |||
| 76 | return; // �˴���Ȼ���ڴ��̤ | 76 | return; // �˴���Ȼ���ڴ��̤ |
| 77 | } | 77 | } |
| 78 | DataCopy(dstGlobal, srcLocal, numAlignedBlocks); | 78 | DataCopy(dstGlobal, srcLocal, numAlignedBlocks); |
| 79 | - event_t eventID = static_cast<event_t>(GetTPipePtr()->FetchEventID(AscendC::HardEvent::MTE3_S)); | 79 | + event_t eventId = static_cast<event_t>(GetTPipePtr()->FetchEventID(AscendC::HardEvent::MTE3_S)); |
| 80 | - AscendC::SetFlag<AscendC::HardEvent::MTE3_S>(eventID); | 80 | + AscendC::SetFlag<AscendC::HardEvent::MTE3_S>(eventId); |
| 81 | - AscendC::WaitFlag<AscendC::HardEvent::MTE3_S>(eventID); | 81 | + AscendC::WaitFlag<AscendC::HardEvent::MTE3_S>(eventId); |
| 82 | const int rollbackEleCount = calCount - numAlignedBlocks; // ������Ҫ���˴���byte�� | 82 | const int rollbackEleCount = calCount - numAlignedBlocks; // ������Ҫ���˴���byte�� |
| 83 | const size_t rollbackDstIdx = numAlignedBlocks - numElemsPerBlock; // �������˵�blockԪ������ | 83 | const size_t rollbackDstIdx = numAlignedBlocks - numElemsPerBlock; // �������˵�blockԪ������ |
| 84 | const size_t rollbackSrcIdx = rollbackDstIdx + rollbackEleCount; // ������ԴԪ������ | 84 | const size_t rollbackSrcIdx = rollbackDstIdx + rollbackEleCount; // ������ԴԪ������ |
| @@ -99,9 +99,9 @@ __aicore__ inline void SafeDataCopy( | |||
| 99 | } | 99 | } |
| 100 | DataCopy(dstGlobal[calCount - numElemsPerBlock], srcLocal[rollbackDstIdx], numElemsPerBlock); | 100 | DataCopy(dstGlobal[calCount - numElemsPerBlock], srcLocal[rollbackDstIdx], numElemsPerBlock); |
| 101 | if (recoverUbTailFormat) { // ��ԭ�ع��ֳ� | 101 | if (recoverUbTailFormat) { // ��ԭ�ع��ֳ� |
| 102 | - event_t eventID = static_cast<event_t>(GetTPipePtr()->FetchEventID(AscendC::HardEvent::MTE3_MTE2)); | 102 | + event_t eventId = static_cast<event_t>(GetTPipePtr()->FetchEventID(AscendC::HardEvent::MTE3_MTE2)); |
| 103 | - AscendC::SetFlag<AscendC::HardEvent::MTE3_MTE2>(eventID); | 103 | + AscendC::SetFlag<AscendC::HardEvent::MTE3_MTE2>(eventId); |
| 104 | - AscendC::WaitFlag<AscendC::HardEvent::MTE3_MTE2>(eventID); | 104 | + AscendC::WaitFlag<AscendC::HardEvent::MTE3_MTE2>(eventId); |
| 105 | DataCopy( | 105 | DataCopy( |
| 106 | srcLocal[rollbackDstIdx], dstGlobal[rollbackDstIdx], | 106 | srcLocal[rollbackDstIdx], dstGlobal[rollbackDstIdx], |
| 107 | numElemsPerBlock); // ��ԭ���ڻ��˵�block���� | 107 | numElemsPerBlock); // ��ԭ���ڻ��˵�block���� |
| @@ -131,9 +131,9 @@ public: | |||
| 131 | __aicore__ inline float ReduceSumCustom(const LocalTensor<float>& src_local, int32_t count) | 131 | __aicore__ inline float ReduceSumCustom(const LocalTensor<float>& src_local, int32_t count) |
| 132 | { | 132 | { |
| 133 | ReduceSum(src_local, src_local, src_local, count); | 133 | ReduceSum(src_local, src_local, src_local, count); |
| 134 | - event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 134 | + event_t eventVs = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 135 | - SetFlag<HardEvent::V_S>(event_v_s); | 135 | + SetFlag<HardEvent::V_S>(eventVs); |
| 136 | - WaitFlag<HardEvent::V_S>(event_v_s); | 136 | + WaitFlag<HardEvent::V_S>(eventVs); |
| 137 | float rstd_value = src_local.GetValue(0); | 137 | float rstd_value = src_local.GetValue(0); |
| 138 | return rstd_value; | 138 | return rstd_value; |
| 139 | } | 139 | } |
| @@ -26,8 +26,8 @@ | |||
| 26 | 26 | ||
| 27 | 27 | ||
| 28 | 28 | ||
| 29 | - | ||
| 30 | 29 | ||
| 30 | + | ||
| 31 | 31 | ||
| 32 | namespace optiling { | 32 | namespace optiling { |
| 33 | BEGIN_TILING_DATA_DEF(GroupNormGradTilingData) | 33 | BEGIN_TILING_DATA_DEF(GroupNormGradTilingData) |
| @@ -23,14 +23,14 @@ extern "C" __global__ __aicore__ void inplace_add_layer_norm( | |||
| 23 | GM_ADDR workspace, GM_ADDR tiling) | 23 | GM_ADDR workspace, GM_ADDR tiling) |
| 24 | { | 24 | { |
| 25 | TPipe pipe; | 25 | TPipe pipe; |
| 26 | - GET_TILING_DATA(tiling_data, tiling); | 26 | + GET_TILING_DATA(inplaceTilingData, tiling); |
| 27 | 27 | ||
| 28 | -#define INIT_AND_PROCESS \ | 28 | +#define INIT_AND_PROCESS \ |
| 29 | - op.Init( \ | 29 | + op.Init( \ |
| 30 | - x1, x2, gamma, beta, bias, y, mean, rstd, x, workspace, tiling_data.numCore, tiling_data.numLastDim, \ | 30 | + x1, x2, gamma, beta, bias, y, mean, rstd, x, workspace, inplaceTilingData.numCore, inplaceTilingData.numLastDim, \ |
| 31 | - tiling_data.numFirstDim, tiling_data.firstDimPerCore, tiling_data.firstDimPerCoreTail, \ | 31 | + inplaceTilingData.numFirstDim, inplaceTilingData.firstDimPerCore, inplaceTilingData.firstDimPerCoreTail, \ |
| 32 | - tiling_data.firstDimPerTime, tiling_data.lastDimPerTime, tiling_data.eps, tiling_data.aveFactor, \ | 32 | + inplaceTilingData.firstDimPerTime, inplaceTilingData.lastDimPerTime, inplaceTilingData.eps, inplaceTilingData.aveFactor, \ |
| 33 | - tiling_data.colMoveCnt, tiling_data.colTail, tiling_data.workspaceSize); \ | 33 | + inplaceTilingData.colMoveCnt, inplaceTilingData.colTail, inplaceTilingData.workspaceSize); \ |
| 34 | op.Process() | 34 | op.Process() |
| 35 | if (TILING_KEY_IS(0)) { | 35 | if (TILING_KEY_IS(0)) { |
| 36 | KernelAddLayerNorm<DTYPE_X1, DTYPE_X2, DTYPE_GAMMA, DTYPE_X1, 0> op(&pipe); | 36 | KernelAddLayerNorm<DTYPE_X1, DTYPE_X2, DTYPE_GAMMA, DTYPE_X1, 0> op(&pipe); |
| @@ -140,36 +140,36 @@ extern "C" __global__ __aicore__ void inplace_add_layer_norm( | |||
| 140 | } else if (TILING_KEY_IS(152)) { | 140 | } else if (TILING_KEY_IS(152)) { |
| 141 | KernelAddLayerNorm<DTYPE_X1, DTYPE_X2, DTYPE_GAMMA, DTYPE_X1, 152> op(&pipe); | 141 | KernelAddLayerNorm<DTYPE_X1, DTYPE_X2, DTYPE_GAMMA, DTYPE_X1, 152> op(&pipe); |
| 142 | INIT_AND_PROCESS; | 142 | INIT_AND_PROCESS; |
| 143 | - } else if (TILING_KEY_IS(62)) { // Better UB begin | ||
| 144 | - KernelAddLayerNormBetterUB<half, half, half, half, 62> op(&pipe); | ||
| 145 | - INIT_AND_PROCESS; | ||
| 146 | } else if (TILING_KEY_IS(162)) { | 143 | } else if (TILING_KEY_IS(162)) { |
| 147 | KernelAddLayerNormBetterUB<half, half, half, half, 162> op(&pipe); | 144 | KernelAddLayerNormBetterUB<half, half, half, half, 162> op(&pipe); |
| 148 | INIT_AND_PROCESS; | 145 | INIT_AND_PROCESS; |
| 149 | - } else if (TILING_KEY_IS(70)) { // Normal Special Reduce begin | 146 | + } else if (TILING_KEY_IS(62)) { // Better UB begin |
| 150 | - KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 70> op(&pipe); | 147 | + KernelAddLayerNormBetterUB<half, half, half, half, 62> op(&pipe); |
| 151 | INIT_AND_PROCESS; | 148 | INIT_AND_PROCESS; |
| 152 | } else if (TILING_KEY_IS(170)) { | 149 | } else if (TILING_KEY_IS(170)) { |
| 153 | KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 170> op(&pipe); | 150 | KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 170> op(&pipe); |
| 154 | INIT_AND_PROCESS; | 151 | INIT_AND_PROCESS; |
| 155 | - } else if (TILING_KEY_IS(80)) { | 152 | + } else if (TILING_KEY_IS(70)) { // Normal Special Reduce begin |
| 156 | - KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 80> op(&pipe); | 153 | + KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 70> op(&pipe); |
| 157 | INIT_AND_PROCESS; | 154 | INIT_AND_PROCESS; |
| 158 | } else if (TILING_KEY_IS(180)) { | 155 | } else if (TILING_KEY_IS(180)) { |
| 159 | KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 180> op(&pipe); | 156 | KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 180> op(&pipe); |
| 160 | INIT_AND_PROCESS; | 157 | INIT_AND_PROCESS; |
| 161 | - } else if (TILING_KEY_IS(72)) { | 158 | + } else if (TILING_KEY_IS(80)) { |
| 162 | - KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 72> op(&pipe); | 159 | + KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 80> op(&pipe); |
| 163 | INIT_AND_PROCESS; | 160 | INIT_AND_PROCESS; |
| 164 | } else if (TILING_KEY_IS(172)) { | 161 | } else if (TILING_KEY_IS(172)) { |
| 165 | KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 172> op(&pipe); | 162 | KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 172> op(&pipe); |
| 166 | INIT_AND_PROCESS; | 163 | INIT_AND_PROCESS; |
| 167 | - } else if (TILING_KEY_IS(82)) { | 164 | + } else if (TILING_KEY_IS(72)) { |
| 168 | - KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 82> op(&pipe); | 165 | + KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 72> op(&pipe); |
| 169 | INIT_AND_PROCESS; | 166 | INIT_AND_PROCESS; |
| 170 | } else if (TILING_KEY_IS(182)) { | 167 | } else if (TILING_KEY_IS(182)) { |
| 171 | KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 182> op(&pipe); | 168 | KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 182> op(&pipe); |
| 172 | INIT_AND_PROCESS; | 169 | INIT_AND_PROCESS; |
| 170 | + } else if (TILING_KEY_IS(82)) { | ||
| 171 | + KernelAddLayerNormNormalSpecialReduce<half, half, half, half, 82> op(&pipe); | ||
| 172 | + INIT_AND_PROCESS; | ||
| 173 | } else if (TILING_KEY_IS(190)) { // Single Row Less Tensor begin | 173 | } else if (TILING_KEY_IS(190)) { // Single Row Less Tensor begin |
| 174 | KernelAddLayerNormSingleRowLessTensor<DTYPE_X1, DTYPE_X2, float, DTYPE_X1, 190> op(&pipe); | 174 | KernelAddLayerNormSingleRowLessTensor<DTYPE_X1, DTYPE_X2, float, DTYPE_X1, 190> op(&pipe); |
| 175 | INIT_AND_PROCESS; | 175 | INIT_AND_PROCESS; |
| @@ -22,16 +22,16 @@ using namespace AscendC; | |||
| 22 | 22 | ||
| 23 | 23 | ||
| 24 | do { \ | 24 | do { \ |
| 25 | - templateClass<__VA_ARGS__> op(&pipe); \ | 25 | + templateClass<__VA_ARGS__> op(&inplacePipe); \ |
| 26 | - op.Init(x1, x2, gamma, y, rstd, x, workspace, &tilingData); \ | 26 | + op.Init(x1, x2, gamma, y, rstd, x, workspace, &inplaceTilingData); \ |
| 27 | op.Process(); \ | 27 | op.Process(); \ |
| 28 | } while (0) | 28 | } while (0) |
| 29 | 29 | ||
| 30 | extern "C" __global__ __aicore__ void inplace_add_rms_norm( | 30 | extern "C" __global__ __aicore__ void inplace_add_rms_norm( |
| 31 | GM_ADDR x1, GM_ADDR x2, GM_ADDR gamma, GM_ADDR y, GM_ADDR rstd, GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) | 31 | GM_ADDR x1, GM_ADDR x2, GM_ADDR gamma, GM_ADDR y, GM_ADDR rstd, GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) |
| 32 | { | 32 | { |
| 33 | - TPipe pipe; | 33 | + TPipe inplacePipe; |
| 34 | - GET_TILING_DATA(tilingData, tiling); | 34 | + GET_TILING_DATA(inplaceTilingData, tiling); |
| 35 | if (TILING_KEY_IS(10)) { | 35 | if (TILING_KEY_IS(10)) { |
| 36 | GENERAL_OP_IMPL(KernelAddRmsNorm, half, 1); | 36 | GENERAL_OP_IMPL(KernelAddRmsNorm, half, 1); |
| 37 | } else if (TILING_KEY_IS(20)) { | 37 | } else if (TILING_KEY_IS(20)) { |
| @@ -211,10 +211,10 @@ ge::graphStatus LayerNormGradV3RecomputeTiling::DoOpTiling() | |||
| 211 | td_.set_pdxIsRequire(static_cast<int32_t>(commonParams.pdxIsRequire)); | 211 | td_.set_pdxIsRequire(static_cast<int32_t>(commonParams.pdxIsRequire)); |
| 212 | td_.set_pdgammaIsRequire(static_cast<int32_t>(commonParams.pdgammaIsRequire)); | 212 | td_.set_pdgammaIsRequire(static_cast<int32_t>(commonParams.pdgammaIsRequire)); |
| 213 | td_.set_pdbetaIsRequire(static_cast<int32_t>(commonParams.pdbetaIsRequire)); | 213 | td_.set_pdbetaIsRequire(static_cast<int32_t>(commonParams.pdbetaIsRequire)); |
| 214 | - ge::graphStatus statusGammaBeta = GammaBetaKernelTiling(); | 214 | + ge::graphStatus recomputeStatusGammaBeta = GammaBetaKernelTiling(); |
代码可维护性: 变量名从 `statusGammaBeta` 改为 `recomputeStatusGammaBeta`，从 `statusBackward` 改为 `recomputeStatusBackward`。虽然这增加了变量名的描述性（表明是recompute模板的状态），但改动本身是重命名，不涉及功能变更。在PR中，此类重命名应确保所有引用该变量的地方都已同步更新（在本代码片段中已更新）。 问题类型: 代码可维护性 文件路径: norm/layer_norm_grad_v3/op_host/layer_norm_grad_v3_recompute_tiling.cpp行号: 214 问题代码: ge::graphStatus recomputeStatusGammaBeta = GammaBetaKernelTiling(); 修改建议: 确保重命名在整个文件中保持一致。如果这是PR的唯一改动，且已同步更新所有引用点，则没有问题。 --- 此评论由代码审查工具自动生成 | |||
| 215 | - OP_TILING_CHECK(statusGammaBeta != ge::GRAPH_SUCCESS, , return statusGammaBeta); | 215 | + OP_TILING_CHECK(recomputeStatusGammaBeta != ge::GRAPH_SUCCESS, , return recomputeStatusGammaBeta); |
| 216 | - ge::graphStatus statusBackward = BackwardKernelTiling(); | 216 | + ge::graphStatus recomputeStatusBackward = BackwardKernelTiling(); |
| 217 | - OP_TILING_CHECK(statusBackward != ge::GRAPH_SUCCESS, , return statusBackward); | 217 | + OP_TILING_CHECK(recomputeStatusBackward != ge::GRAPH_SUCCESS, , return recomputeStatusBackward); |
| 218 | return ge::GRAPH_SUCCESS; | 218 | return ge::GRAPH_SUCCESS; |
| 219 | } | 219 | } |
| 220 | 220 | ||
| @@ -151,33 +151,33 @@ public: | |||
| 151 | 151 | ||
| 152 | __aicore__ inline void copyOut(int64_t colIndex, int64_t colSize) | 152 | __aicore__ inline void copyOut(int64_t colIndex, int64_t colSize) |
| 153 | { | 153 | { |
| 154 | - DataCopyParams intriParams; | 154 | + DataCopyParams deterministicIntriParams; |
| 155 | - intriParams.blockCount = 1; | 155 | + deterministicIntriParams.blockCount = 1; |
| 156 | - intriParams.blockLen = colSize * sizeof(float); | 156 | + deterministicIntriParams.blockLen = colSize * sizeof(float); |
| 157 | - intriParams.srcStride = 0; | 157 | + deterministicIntriParams.srcStride = 0; |
| 158 | - intriParams.dstStride = 0; | 158 | + deterministicIntriParams.dstStride = 0; |
| 159 | TEventID eventID = GetTPipePtr()->AllocEventID<HardEvent::V_MTE3>(); | 159 | TEventID eventID = GetTPipePtr()->AllocEventID<HardEvent::V_MTE3>(); |
| 160 | SetFlag<HardEvent::V_MTE3>(eventID); | 160 | SetFlag<HardEvent::V_MTE3>(eventID); |
| 161 | WaitFlag<HardEvent::V_MTE3>(eventID); | 161 | WaitFlag<HardEvent::V_MTE3>(eventID); |
| 162 | GetTPipePtr()->ReleaseEventID<HardEvent::V_MTE3>(eventID); | 162 | GetTPipePtr()->ReleaseEventID<HardEvent::V_MTE3>(eventID); |
| 163 | int64_t offset = colIndex * COL_TEMPLATE; | 163 | int64_t offset = colIndex * COL_TEMPLATE; |
| 164 | - DataCopyPad(pdGammaOutTensorGM_[offset], buffer2_, intriParams); | 164 | + DataCopyPad(pdGammaOutTensorGM_[offset], buffer2_, deterministicIntriParams); |
| 165 | - DataCopyPad(pdBetaOutTensorGM_[offset], buffer4_, intriParams); | 165 | + DataCopyPad(pdBetaOutTensorGM_[offset], buffer4_, deterministicIntriParams); |
| 166 | } | 166 | } |
| 167 | 167 | ||
| 168 | private: | 168 | private: |
| 169 | TPipe pipe_; | 169 | TPipe pipe_; |
| 170 | TQue<QuePosition::VECOUT, DOUBLE_BUFFER> queueGammaOut_; | 170 | TQue<QuePosition::VECOUT, DOUBLE_BUFFER> queueGammaOut_; |
| 171 | TQue<QuePosition::VECOUT, DOUBLE_BUFFER> queueBetaOut_; | 171 | TQue<QuePosition::VECOUT, DOUBLE_BUFFER> queueBetaOut_; |
| 172 | - TQue<QuePosition::VECIN, DOUBLE_BUFFER> queueGammaIn_; | ||
| 173 | TQue<QuePosition::VECIN, DOUBLE_BUFFER> queueBetaIn_; | 172 | TQue<QuePosition::VECIN, DOUBLE_BUFFER> queueBetaIn_; |
| 174 | - LocalTensor<float> buffer1_; | 173 | + TQue<QuePosition::VECIN, DOUBLE_BUFFER> queueGammaIn_; |
| 175 | LocalTensor<float> buffer2_; | 174 | LocalTensor<float> buffer2_; |
| 176 | LocalTensor<float> buffer3_; | 175 | LocalTensor<float> buffer3_; |
| 177 | LocalTensor<float> buffer4_; | 176 | LocalTensor<float> buffer4_; |
| 177 | + LocalTensor<float> buffer1_; | ||
| 178 | 178 | ||
| 179 | - GlobalTensor<float> pdGammaOutTensorGM_; | ||
| 180 | GlobalTensor<float> pdBetaOutTensorGM_; | 179 | GlobalTensor<float> pdBetaOutTensorGM_; |
| 180 | + GlobalTensor<float> pdGammaOutTensorGM_; | ||
| 181 | GlobalTensor<float> workspaceGM_; | 181 | GlobalTensor<float> workspaceGM_; |
| 182 | 182 | ||
| 183 | int64_t workspaceNum_; | 183 | int64_t workspaceNum_; |
| @@ -138,27 +138,27 @@ private: | |||
| 138 | { | 138 | { |
| 139 | // copy_in dy to buffer1 | 139 | // copy_in dy to buffer1 |
| 140 | buffer1 = queue1.AllocTensor<float>(); | 140 | buffer1 = queue1.AllocTensor<float>(); |
| 141 | - DataCopyPadParams padParams{false, 0, 0, 0}; | 141 | + DataCopyPadParams singleReadPadParams{false, 0, 0, 0}; |
| 142 | - DataCopyParams intriParams; | 142 | + DataCopyParams singleReadIntriParams; |
| 143 | if (likely(tilingData->colAlignV == tilingData->col)) { | 143 | if (likely(tilingData->colAlignV == tilingData->col)) { |
| 144 | - intriParams.blockCount = 1; | 144 | + singleReadIntriParams.blockCount = 1; |
| 145 | - intriParams.blockLen = curRowsNum * tilingData->col * sizeof(T); | 145 | + singleReadIntriParams.blockLen = curRowsNum * tilingData->col * sizeof(T); |
| 146 | } else { | 146 | } else { |
| 147 | - intriParams.blockCount = curRowsNum; | 147 | + singleReadIntriParams.blockCount = curRowsNum; |
| 148 | - intriParams.blockLen = tilingData->col * sizeof(T); | 148 | + singleReadIntriParams.blockLen = tilingData->col * sizeof(T); |
| 149 | - padParams.isPad = true; | 149 | + singleReadPadParams.isPad = true; |
| 150 | - padParams.rightPadding = tilingData->colAlignM - tilingData->col; | 150 | + singleReadPadParams.rightPadding = tilingData->colAlignM - tilingData->col; |
| 151 | } | 151 | } |
| 152 | - intriParams.srcStride = 0; | 152 | + singleReadIntriParams.srcStride = 0; |
| 153 | - intriParams.dstStride = 0; | 153 | + singleReadIntriParams.dstStride = 0; |
| 154 | if constexpr (IsSameType<T, float>::value) { | 154 | if constexpr (IsSameType<T, float>::value) { |
| 155 | DataCopyPad( | 155 | DataCopyPad( |
| 156 | buffer1.ReinterpretCast<T>(), dyInTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], | 156 | buffer1.ReinterpretCast<T>(), dyInTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], |
| 157 | - intriParams, padParams); | 157 | + singleReadIntriParams, singleReadPadParams); |
| 158 | } else { | 158 | } else { |
| 159 | DataCopyPad( | 159 | DataCopyPad( |
| 160 | buffer1.ReinterpretCast<T>()[tilingData->bufferElemNums], | 160 | buffer1.ReinterpretCast<T>()[tilingData->bufferElemNums], |
| 161 | - dyInTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], intriParams, padParams); | 161 | + dyInTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], singleReadIntriParams, singleReadPadParams); |
| 162 | } | 162 | } |
| 163 | queue1.EnQue(buffer1); | 163 | queue1.EnQue(buffer1); |
| 164 | } | 164 | } |
| @@ -183,25 +183,25 @@ private: | |||
| 183 | { | 183 | { |
| 184 | // copy_in x to buffer0 | 184 | // copy_in x to buffer0 |
| 185 | buffer0 = queue0.AllocTensor<float>(); | 185 | buffer0 = queue0.AllocTensor<float>(); |
| 186 | - DataCopyPadParams padParams{false, 0, 0, 0}; | 186 | + DataCopyPadParams singleReadPadParams2{false, 0, 0, 0}; |
| 187 | - DataCopyParams intriParams; | 187 | + DataCopyParams singleReadIntriParams2; |
| 188 | if (likely(tilingData->colAlignV == tilingData->col)) { | 188 | if (likely(tilingData->colAlignV == tilingData->col)) { |
| 189 | - intriParams.blockCount = 1; | 189 | + singleReadIntriParams2.blockCount = 1; |
| 190 | - intriParams.blockLen = curRowsNum * tilingData->col * sizeof(T); | 190 | + singleReadIntriParams2.blockLen = curRowsNum * tilingData->col * sizeof(T); |
| 191 | } else { | 191 | } else { |
| 192 | - intriParams.blockCount = curRowsNum; | 192 | + singleReadIntriParams2.blockCount = curRowsNum; |
| 193 | - intriParams.blockLen = tilingData->col * sizeof(T); | 193 | + singleReadIntriParams2.blockLen = tilingData->col * sizeof(T); |
| 194 | } | 194 | } |
| 195 | - intriParams.srcStride = 0; | 195 | + singleReadIntriParams2.srcStride = 0; |
| 196 | - intriParams.dstStride = 0; | 196 | + singleReadIntriParams2.dstStride = 0; |
| 197 | if constexpr (IsSameType<T, float>::value) { | 197 | if constexpr (IsSameType<T, float>::value) { |
| 198 | DataCopyPad( | 198 | DataCopyPad( |
| 199 | buffer0.ReinterpretCast<T>(), xInTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], | 199 | buffer0.ReinterpretCast<T>(), xInTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], |
| 200 | - intriParams, padParams); | 200 | + singleReadIntriParams2, singleReadPadParams2); |
| 201 | } else { | 201 | } else { |
| 202 | DataCopyPad( | 202 | DataCopyPad( |
| 203 | buffer0.ReinterpretCast<T>()[tilingData->bufferElemNums], | 203 | buffer0.ReinterpretCast<T>()[tilingData->bufferElemNums], |
| 204 | - xInTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], intriParams, padParams); | 204 | + xInTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], singleReadIntriParams2, singleReadPadParams2); |
| 205 | } | 205 | } |
| 206 | queue0.EnQue(buffer0); | 206 | queue0.EnQue(buffer0); |
| 207 | } | 207 | } |
| @@ -375,17 +375,17 @@ private: | |||
| 375 | { | 375 | { |
| 376 | // copy_out mul_2 from buffer2 and do atomic | 376 | // copy_out mul_2 from buffer2 and do atomic |
| 377 | buffer2 = queue2.DeQue<float>(); | 377 | buffer2 = queue2.DeQue<float>(); |
| 378 | - DataCopyParams intriParams; | 378 | + DataCopyParams intriParamsPhase1; |
| 379 | - intriParams.blockCount = 1; | 379 | + intriParamsPhase1.blockCount = 1; |
| 380 | - intriParams.blockLen = tilingData->colAlignV * sizeof(float); | 380 | + intriParamsPhase1.blockLen = tilingData->colAlignV * sizeof(float); |
| 381 | - intriParams.srcStride = 0; | 381 | + intriParamsPhase1.srcStride = 0; |
| 382 | - intriParams.dstStride = 0; | 382 | + intriParamsPhase1.dstStride = 0; |
| 383 | SetAtomicAdd<float>(); | 383 | SetAtomicAdd<float>(); |
| 384 | for (int64_t i = 0; i < curRowsNum; i++) { | 384 | for (int64_t i = 0; i < curRowsNum; i++) { |
| 385 | if constexpr (isDeterministic) { | 385 | if constexpr (isDeterministic) { |
| 386 | PipeBarrier<PIPE_MTE3>(); | 386 | PipeBarrier<PIPE_MTE3>(); |
| 387 | } | 387 | } |
| 388 | - DataCopyPad(workspaceGM, buffer2[tilingData->colAlignV * i], intriParams); | 388 | + DataCopyPad(workspaceGM, buffer2[tilingData->colAlignV * i], intriParamsPhase1); |
| 389 | } | 389 | } |
| 390 | queue2.FreeTensor(buffer2); | 390 | queue2.FreeTensor(buffer2); |
| 391 | SetAtomicNone(); | 391 | SetAtomicNone(); |
| @@ -396,19 +396,19 @@ private: | |||
| 396 | { | 396 | { |
| 397 | // copy_out mul_10 to pdX | 397 | // copy_out mul_10 to pdX |
| 398 | buffer3 = queue3.DeQue<float>(); | 398 | buffer3 = queue3.DeQue<float>(); |
| 399 | - DataCopyParams intriParams; | 399 | + DataCopyParams singleReadIntriParams3; |
| 400 | if (likely(tilingData->colAlignV == tilingData->col)) { | 400 | if (likely(tilingData->colAlignV == tilingData->col)) { |
| 401 | - intriParams.blockCount = 1; | 401 | + singleReadIntriParams3.blockCount = 1; |
| 402 | - intriParams.blockLen = curRowsNum * tilingData->col * sizeof(T); | 402 | + singleReadIntriParams3.blockLen = curRowsNum * tilingData->col * sizeof(T); |
| 403 | } else { | 403 | } else { |
| 404 | - intriParams.blockCount = curRowsNum; | 404 | + singleReadIntriParams3.blockCount = curRowsNum; |
| 405 | - intriParams.blockLen = tilingData->col * sizeof(T); | 405 | + singleReadIntriParams3.blockLen = tilingData->col * sizeof(T); |
| 406 | } | 406 | } |
| 407 | - intriParams.srcStride = 0; | 407 | + singleReadIntriParams3.srcStride = 0; |
| 408 | - intriParams.dstStride = 0; | 408 | + singleReadIntriParams3.dstStride = 0; |
| 409 | DataCopyPad( | 409 | DataCopyPad( |
| 410 | pdXOutTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], buffer3.ReinterpretCast<T>(), | 410 | pdXOutTensorGM[tilingData->ubFormer * tilingData->col * outerIdx], buffer3.ReinterpretCast<T>(), |
| 411 | - intriParams); | 411 | + singleReadIntriParams3); |
| 412 | queue3.FreeTensor(buffer3); | 412 | queue3.FreeTensor(buffer3); |
| 413 | } | 413 | } |
| 414 | 414 | ||
| @@ -319,19 +319,19 @@ private: | |||
| 319 | dstLocalList[i] = dstAddr + B16_BLOCK_ALIGN_NUM * i; | 319 | dstLocalList[i] = dstAddr + B16_BLOCK_ALIGN_NUM * i; |
| 320 | } | 320 | } |
| 321 | } | 321 | } |
| 322 | - struct TransDataTo5HDParams transDataParams; | 322 | + struct TransDataTo5HDParams gradTransDataParams; |
| 323 | if constexpr (std::is_same<T_TRANS, float>::value) { | 323 | if constexpr (std::is_same<T_TRANS, float>::value) { |
| 324 | - transDataParams.repeatTimes = lineAlignSize / B32_BLOCK_ALIGN_NUM; | 324 | + gradTransDataParams.repeatTimes = lineAlignSize / B32_BLOCK_ALIGN_NUM; |
| 325 | } else { | 325 | } else { |
| 326 | - transDataParams.repeatTimes = lineAlignSize / B16_BLOCK_ALIGN_NUM; | 326 | + gradTransDataParams.repeatTimes = lineAlignSize / B16_BLOCK_ALIGN_NUM; |
| 327 | } | 327 | } |
| 328 | - transDataParams.srcRepStride = 1; | 328 | + gradTransDataParams.srcRepStride = 1; |
| 329 | - transDataParams.dstRepStride = TRANSPOSE_C0_SIZE; | 329 | + gradTransDataParams.dstRepStride = TRANSPOSE_C0_SIZE; |
| 330 | - if (transDataParams.repeatTimes == 1) { | 330 | + if (gradTransDataParams.repeatTimes == 1) { |
| 331 | - transDataParams.srcRepStride = 0; | 331 | + gradTransDataParams.srcRepStride = 0; |
| 332 | - transDataParams.dstRepStride = 0; | 332 | + gradTransDataParams.dstRepStride = 0; |
| 333 | } | 333 | } |
| 334 | - TransDataTo5HDImpl(dstLocalList, srcLocalList, transDataParams); | 334 | + TransDataTo5HDImpl(dstLocalList, srcLocalList, gradTransDataParams); |
| 335 | } | 335 | } |
| 336 | 336 | ||
| 337 | template <typename T_RESHAPE> | 337 | template <typename T_RESHAPE> |
| @@ -344,39 +344,39 @@ private: | |||
| 344 | // 一个repeat处理（128 / IN_NUM_PER_BLOCK）行数据 | 344 | // 一个repeat处理（128 / IN_NUM_PER_BLOCK）行数据 |
| 345 | uint32_t repeatTimes = lastAxisSize / BLOCK_NUM_PER_REP; | 345 | uint32_t repeatTimes = lastAxisSize / BLOCK_NUM_PER_REP; |
| 346 | uint32_t remainRepeat = lastAxisSize % BLOCK_NUM_PER_REP; | 346 | uint32_t remainRepeat = lastAxisSize % BLOCK_NUM_PER_REP; |
| 347 | - uint32_t mask = 0; | 347 | + uint32_t gradMask = 0; |
| 348 | - uint32_t lineBlockNum = 0; | 348 | + uint32_t gradLineBlockNum = 0; |
| 349 | if constexpr (std::is_same<T_RESHAPE, float>::value) { | 349 | if constexpr (std::is_same<T_RESHAPE, float>::value) { |
| 350 | - mask = B32_BLOCK_ALIGN_NUM * BLOCK_NUM_PER_REP; | 350 | + gradMask = B32_BLOCK_ALIGN_NUM * BLOCK_NUM_PER_REP; |
| 351 | - lineBlockNum = TRANSPOSE_C0_SIZE / B32_BLOCK_ALIGN_NUM; | 351 | + gradLineBlockNum = TRANSPOSE_C0_SIZE / B32_BLOCK_ALIGN_NUM; |
| 352 | } else { | 352 | } else { |
| 353 | - mask = B16_BLOCK_ALIGN_NUM * BLOCK_NUM_PER_REP; | 353 | + gradMask = B16_BLOCK_ALIGN_NUM * BLOCK_NUM_PER_REP; |
| 354 | - lineBlockNum = TRANSPOSE_C0_SIZE / B16_BLOCK_ALIGN_NUM; | 354 | + gradLineBlockNum = TRANSPOSE_C0_SIZE / B16_BLOCK_ALIGN_NUM; |
| 355 | } | 355 | } |
| 356 | - if ((bFormerFactor * BLOCK_NUM_PER_REP * lineBlockNum) < MAX_REP_NUM) { | 356 | + if ((bFormerFactor * BLOCK_NUM_PER_REP * gradLineBlockNum) < MAX_REP_NUM) { |
| 357 | if (repeatTimes) { | 357 | if (repeatTimes) { |
| 358 | for (uint32_t i = 0; i < bFormerFactor; i++) { | 358 | for (uint32_t i = 0; i < bFormerFactor; i++) { |
| 359 | Copy( | 359 | Copy( |
| 360 | - dstTensor[i * TRANSPOSE_C0_SIZE], srcTensor[i * TRANSPOSE_C0_SIZE * lastAxisSize], mask, | 360 | + dstTensor[i * TRANSPOSE_C0_SIZE], srcTensor[i * TRANSPOSE_C0_SIZE * lastAxisSize], gradMask, |
| 361 | repeatTimes, | 361 | repeatTimes, |
| 362 | - {(uint16_t)(bFormerFactor * lineBlockNum), (uint16_t)lineBlockNum, | 362 | + {(uint16_t)(bFormerFactor * gradLineBlockNum), (uint16_t)gradLineBlockNum, |
| 363 | - (uint8_t)(BLOCK_NUM_PER_REP * bFormerFactor * lineBlockNum), | 363 | + (uint8_t)(BLOCK_NUM_PER_REP * bFormerFactor * gradLineBlockNum), |
| 364 | - (uint8_t)(BLOCK_NUM_PER_REP * lineBlockNum)}); | 364 | + (uint8_t)(BLOCK_NUM_PER_REP * gradLineBlockNum)}); |
| 365 | if constexpr (std::is_same<T_RESHAPE, float>::value) { | 365 | if constexpr (std::is_same<T_RESHAPE, float>::value) { |
| 366 | Copy( | 366 | Copy( |
| 367 | dstTensor[i * TRANSPOSE_C0_SIZE + B32_BLOCK_ALIGN_NUM], | 367 | dstTensor[i * TRANSPOSE_C0_SIZE + B32_BLOCK_ALIGN_NUM], |
| 368 | - srcTensor[i * TRANSPOSE_C0_SIZE * lastAxisSize + B32_BLOCK_ALIGN_NUM], mask, repeatTimes, | 368 | + srcTensor[i * TRANSPOSE_C0_SIZE * lastAxisSize + B32_BLOCK_ALIGN_NUM], gradMask, repeatTimes, |
| 369 | - {(uint16_t)(bFormerFactor * lineBlockNum), (uint16_t)lineBlockNum, | 369 | + {(uint16_t)(bFormerFactor * gradLineBlockNum), (uint16_t)gradLineBlockNum, |
| 370 | - (uint8_t)(BLOCK_NUM_PER_REP * bFormerFactor * lineBlockNum), | 370 | + (uint8_t)(BLOCK_NUM_PER_REP * bFormerFactor * gradLineBlockNum), |
| 371 | - (uint8_t)(BLOCK_NUM_PER_REP * lineBlockNum)}); | 371 | + (uint8_t)(BLOCK_NUM_PER_REP * gradLineBlockNum)}); |
| 372 | } | 372 | } |
| 373 | } | 373 | } |
| 374 | } | 374 | } |
| 375 | if (remainRepeat) { | 375 | if (remainRepeat) { |
| 376 | if constexpr (std::is_same<T_RESHAPE, float>::value) { | 376 | if constexpr (std::is_same<T_RESHAPE, float>::value) { |
| 377 | - mask = remainRepeat * B32_BLOCK_ALIGN_NUM; | 377 | + gradMask = remainRepeat * B32_BLOCK_ALIGN_NUM; |
| 378 | } else { | 378 | } else { |
| 379 | - mask = remainRepeat * B16_BLOCK_ALIGN_NUM; | 379 | + gradMask = remainRepeat * B16_BLOCK_ALIGN_NUM; |
| 380 | } | 380 | } |
| 381 | for (uint32_t i = 0; i < bFormerFactor; i++) { | 381 | for (uint32_t i = 0; i < bFormerFactor; i++) { |
| 382 | Copy( | 382 | Copy( |
| @@ -386,7 +386,7 @@ private: | |||
| 386 | srcTensor | 386 | srcTensor |
| 387 | [i * TRANSPOSE_C0_SIZE * lastAxisSize + | 387 | [i * TRANSPOSE_C0_SIZE * lastAxisSize + |
| 388 | repeatTimes * TRANSPOSE_C0_SIZE * BLOCK_NUM_PER_REP], | 388 | repeatTimes * TRANSPOSE_C0_SIZE * BLOCK_NUM_PER_REP], |
| 389 | - mask, 1, {(uint16_t)(bFormerFactor * lineBlockNum), (uint16_t)lineBlockNum, 0, 0}); | 389 | + gradMask, 1, {(uint16_t)(bFormerFactor * gradLineBlockNum), (uint16_t)gradLineBlockNum, 0, 0}); |
| 390 | if constexpr (std::is_same<T_RESHAPE, float>::value) { | 390 | if constexpr (std::is_same<T_RESHAPE, float>::value) { |
| 391 | Copy( | 391 | Copy( |
| 392 | dstTensor | 392 | dstTensor |
| @@ -395,14 +395,14 @@ private: | |||
| 395 | srcTensor | 395 | srcTensor |
| 396 | [i * TRANSPOSE_C0_SIZE * lastAxisSize + B32_BLOCK_ALIGN_NUM + | 396 | [i * TRANSPOSE_C0_SIZE * lastAxisSize + B32_BLOCK_ALIGN_NUM + |
| 397 | repeatTimes * TRANSPOSE_C0_SIZE * BLOCK_NUM_PER_REP], | 397 | repeatTimes * TRANSPOSE_C0_SIZE * BLOCK_NUM_PER_REP], |
| 398 | - mask, 1, {(uint16_t)(bFormerFactor * lineBlockNum), (uint16_t)lineBlockNum, 0, 0}); | 398 | + gradMask, 1, {(uint16_t)(bFormerFactor * gradLineBlockNum), (uint16_t)gradLineBlockNum, 0, 0}); |
| 399 | } | 399 | } |
| 400 | } | 400 | } |
| 401 | } | 401 | } |
| 402 | } else { | 402 | } else { |
| 403 | DataCopyParams copyParams; | 403 | DataCopyParams copyParams; |
| 404 | copyParams.blockCount = bFormerFactor; | 404 | copyParams.blockCount = bFormerFactor; |
| 405 | - copyParams.blockLen = lineBlockNum; | 405 | + copyParams.blockLen = gradLineBlockNum; |
| 406 | copyParams.srcStride = (lastAxisSize - 1) * copyParams.blockLen; | 406 | copyParams.srcStride = (lastAxisSize - 1) * copyParams.blockLen; |
| 407 | copyParams.dstStride = 0; | 407 | copyParams.dstStride = 0; |
| 408 | for (uint32_t i = 0; i < lastAxisSize; i++) { | 408 | for (uint32_t i = 0; i < lastAxisSize; i++) { |
| @@ -545,38 +545,38 @@ private: | |||
| 545 | // 一个repeat处理（128 / IN_NUM_PER_BLOCK）行数据 | 545 | // 一个repeat处理（128 / IN_NUM_PER_BLOCK）行数据 |
| 546 | uint32_t repeatTimes = col / BLOCK_NUM_PER_REP; | 546 | uint32_t repeatTimes = col / BLOCK_NUM_PER_REP; |
| 547 | uint32_t remainRepeat = col % BLOCK_NUM_PER_REP; | 547 | uint32_t remainRepeat = col % BLOCK_NUM_PER_REP; |
| 548 | - uint32_t mask = 0; | 548 | + uint32_t postMask = 0; |
| 549 | - uint32_t lineBlockNum = 0; | 549 | + uint32_t postLineBlockNum = 0; |
| 550 | if constexpr (std::is_same<T_POST_RESHAPE, float>::value) { | 550 | if constexpr (std::is_same<T_POST_RESHAPE, float>::value) { |
| 551 | - mask = B32_BLOCK_ALIGN_NUM * BLOCK_NUM_PER_REP; | 551 | + postMask = B32_BLOCK_ALIGN_NUM * BLOCK_NUM_PER_REP; |
| 552 | - lineBlockNum = TRANSPOSE_C0_SIZE / B32_BLOCK_ALIGN_NUM; | 552 | + postLineBlockNum = TRANSPOSE_C0_SIZE / B32_BLOCK_ALIGN_NUM; |
| 553 | } else { | 553 | } else { |
| 554 | - mask = B16_BLOCK_ALIGN_NUM * BLOCK_NUM_PER_REP; | 554 | + postMask = B16_BLOCK_ALIGN_NUM * BLOCK_NUM_PER_REP; |
| 555 | - lineBlockNum = TRANSPOSE_C0_SIZE / B16_BLOCK_ALIGN_NUM; | 555 | + postLineBlockNum = TRANSPOSE_C0_SIZE / B16_BLOCK_ALIGN_NUM; |
| 556 | } | 556 | } |
| 557 | - if ((bFormerFactor * BLOCK_NUM_PER_REP * lineBlockNum) < MAX_REP_NUM) { | 557 | + if ((bFormerFactor * BLOCK_NUM_PER_REP * postLineBlockNum) < MAX_REP_NUM) { |
| 558 | if (repeatTimes) { | 558 | if (repeatTimes) { |
| 559 | for (uint32_t i = 0; i < bFormerFactor; i++) { | 559 | for (uint32_t i = 0; i < bFormerFactor; i++) { |
| 560 | Copy( | 560 | Copy( |
| 561 | - dstTensor[i * TRANSPOSE_C0_SIZE * col], srcTensor[i * TRANSPOSE_C0_SIZE], mask, repeatTimes, | 561 | + dstTensor[i * TRANSPOSE_C0_SIZE * col], srcTensor[i * TRANSPOSE_C0_SIZE], postMask, repeatTimes, |
| 562 | - {(uint16_t)lineBlockNum, (uint16_t)(bFormerFactor * lineBlockNum), | 562 | + {(uint16_t)postLineBlockNum, (uint16_t)(bFormerFactor * postLineBlockNum), |
| 563 | - (uint8_t)(BLOCK_NUM_PER_REP * lineBlockNum), | 563 | + (uint8_t)(BLOCK_NUM_PER_REP * postLineBlockNum), |
| 564 | - (uint8_t)(BLOCK_NUM_PER_REP * bFormerFactor * lineBlockNum)}); | 564 | + (uint8_t)(BLOCK_NUM_PER_REP * bFormerFactor * postLineBlockNum)}); |
| 565 | if constexpr (std::is_same<T_POST_RESHAPE, float>::value) { | 565 | if constexpr (std::is_same<T_POST_RESHAPE, float>::value) { |
| 566 | Copy( | 566 | Copy( |
| 567 | dstTensor[i * TRANSPOSE_C0_SIZE * col + B32_BLOCK_ALIGN_NUM], | 567 | dstTensor[i * TRANSPOSE_C0_SIZE * col + B32_BLOCK_ALIGN_NUM], |
| 568 | - srcTensor[i * TRANSPOSE_C0_SIZE + B32_BLOCK_ALIGN_NUM], mask, repeatTimes, | 568 | + srcTensor[i * TRANSPOSE_C0_SIZE + B32_BLOCK_ALIGN_NUM], postMask, repeatTimes, |
| 569 | - {(uint16_t)lineBlockNum, (uint16_t)(bFormerFactor * lineBlockNum), | 569 | + {(uint16_t)postLineBlockNum, (uint16_t)(bFormerFactor * postLineBlockNum), |
| 570 | - (uint8_t)(BLOCK_NUM_PER_REP * lineBlockNum), | 570 | + (uint8_t)(BLOCK_NUM_PER_REP * postLineBlockNum), |
| 571 | - (uint8_t)(BLOCK_NUM_PER_REP * bFormerFactor * lineBlockNum)}); | 571 | + (uint8_t)(BLOCK_NUM_PER_REP * bFormerFactor * postLineBlockNum)}); |
| 572 | } | 572 | } |
| 573 | } | 573 | } |
| 574 | } | 574 | } |
| 575 | if (remainRepeat) { | 575 | if (remainRepeat) { |
| 576 | if constexpr (std::is_same<T_POST_RESHAPE, float>::value) { | 576 | if constexpr (std::is_same<T_POST_RESHAPE, float>::value) { |
| 577 | - mask = remainRepeat * B32_BLOCK_ALIGN_NUM; | 577 | + postMask = remainRepeat * B32_BLOCK_ALIGN_NUM; |
| 578 | } else { | 578 | } else { |
| 579 | - mask = remainRepeat * B16_BLOCK_ALIGN_NUM; | 579 | + postMask = remainRepeat * B16_BLOCK_ALIGN_NUM; |
| 580 | } | 580 | } |
| 581 | for (uint32_t i = 0; i < bFormerFactor; i++) { | 581 | for (uint32_t i = 0; i < bFormerFactor; i++) { |
| 582 | Copy( | 582 | Copy( |
| @@ -584,7 +584,7 @@ private: | |||
| 584 | srcTensor | 584 | srcTensor |
| 585 | [i * TRANSPOSE_C0_SIZE + | 585 | [i * TRANSPOSE_C0_SIZE + |
| 586 | repeatTimes * TRANSPOSE_C0_SIZE * BLOCK_NUM_PER_REP * bFormerFactor], | 586 | repeatTimes * TRANSPOSE_C0_SIZE * BLOCK_NUM_PER_REP * bFormerFactor], |
| 587 | - mask, 1, {(uint16_t)lineBlockNum, (uint16_t)(bFormerFactor * lineBlockNum), 0, 0}); | 587 | + postMask, 1, {(uint16_t)postLineBlockNum, (uint16_t)(bFormerFactor * postLineBlockNum), 0, 0}); |
| 588 | if constexpr (std::is_same<T_POST_RESHAPE, float>::value) { | 588 | if constexpr (std::is_same<T_POST_RESHAPE, float>::value) { |
| 589 | Copy( | 589 | Copy( |
| 590 | dstTensor | 590 | dstTensor |
| @@ -593,19 +593,19 @@ private: | |||
| 593 | srcTensor | 593 | srcTensor |
| 594 | [i * TRANSPOSE_C0_SIZE + B32_BLOCK_ALIGN_NUM + | 594 | [i * TRANSPOSE_C0_SIZE + B32_BLOCK_ALIGN_NUM + |
| 595 | repeatTimes * TRANSPOSE_C0_SIZE * BLOCK_NUM_PER_REP * bFormerFactor], | 595 | repeatTimes * TRANSPOSE_C0_SIZE * BLOCK_NUM_PER_REP * bFormerFactor], |
| 596 | - mask, 1, {(uint16_t)lineBlockNum, (uint16_t)(bFormerFactor * lineBlockNum), 0, 0}); | 596 | + postMask, 1, {(uint16_t)postLineBlockNum, (uint16_t)(bFormerFactor * postLineBlockNum), 0, 0}); |
| 597 | } | 597 | } |
| 598 | } | 598 | } |
| 599 | } | 599 | } |
| 600 | } else { | 600 | } else { |
| 601 | - DataCopyParams copyParams; | 601 | + DataCopyParams postCopyParams; |
| 602 | - copyParams.blockCount = bFormerFactor; | 602 | + postCopyParams.blockCount = bFormerFactor; |
| 603 | - copyParams.blockLen = lineBlockNum; | 603 | + postCopyParams.blockLen = postLineBlockNum; |
| 604 | - copyParams.srcStride = 0; | 604 | + postCopyParams.srcStride = 0; |
| 605 | - copyParams.dstStride = (col - 1) * copyParams.blockLen; | 605 | + postCopyParams.dstStride = (col - 1) * postCopyParams.blockLen; |
| 606 | for (uint32_t i = 0; i < col; i++) { | 606 | for (uint32_t i = 0; i < col; i++) { |
| 607 | DataCopy( | 607 | DataCopy( |
| 608 | - dstTensor[i * TRANSPOSE_C0_SIZE], srcTensor[i * bFormerFactor * TRANSPOSE_C0_SIZE], copyParams); | 608 | + dstTensor[i * TRANSPOSE_C0_SIZE], srcTensor[i * bFormerFactor * TRANSPOSE_C0_SIZE], postCopyParams); |
| 609 | } | 609 | } |
| 610 | } | 610 | } |
| 611 | } | 611 | } |
| @@ -621,40 +621,40 @@ private: | |||
| 621 | // 每行数据对齐后的size | 621 | // 每行数据对齐后的size |
| 622 | uint32_t lineAlignSize = 0; | 622 | uint32_t lineAlignSize = 0; |
| 623 | lineAlignSize = (bFormerFactor * col + TRANSPOSE_C0_SIZE - 1) / TRANSPOSE_C0_SIZE * TRANSPOSE_C0_SIZE; | 623 | lineAlignSize = (bFormerFactor * col + TRANSPOSE_C0_SIZE - 1) / TRANSPOSE_C0_SIZE * TRANSPOSE_C0_SIZE; |
| 624 | - __ubuf__ T_POST_TRANS* srcAddr = (__ubuf__ T_POST_TRANS*)srcTensor.GetPhyAddr(); | 624 | + __ubuf__ T_POST_TRANS* postSrcAddr = (__ubuf__ T_POST_TRANS*)srcTensor.GetPhyAddr(); |
| 625 | - __ubuf__ T_POST_TRANS* dstAddr = (__ubuf__ T_POST_TRANS*)dstTensor.GetPhyAddr(); | 625 | + __ubuf__ T_POST_TRANS* postDstAddr = (__ubuf__ T_POST_TRANS*)dstTensor.GetPhyAddr(); |
| 626 | - __ubuf__ T_POST_TRANS* srcLocalList[TRANSPOSE_C0_SIZE]; | 626 | + __ubuf__ T_POST_TRANS* postSrcLocalList[TRANSPOSE_C0_SIZE]; |
| 627 | - __ubuf__ T_POST_TRANS* dstLocalList[TRANSPOSE_C0_SIZE]; | 627 | + __ubuf__ T_POST_TRANS* postDstLocalList[TRANSPOSE_C0_SIZE]; |
| 628 | for (uint32_t i = 0; i < TRANSPOSE_C0_SIZE; i++) { | 628 | for (uint32_t i = 0; i < TRANSPOSE_C0_SIZE; i++) { |
| 629 | - srcLocalList[i] = srcAddr + TRANSPOSE_C0_SIZE * i; | 629 | + postSrcLocalList[i] = postSrcAddr + TRANSPOSE_C0_SIZE * i; |
| 630 | if constexpr (std::is_same<T_POST_TRANS, float>::value) { | 630 | if constexpr (std::is_same<T_POST_TRANS, float>::value) { |
| 631 | - dstLocalList[i] = dstAddr + lineAlignSize * (i / TWO_NUM) + B32_BLOCK_ALIGN_NUM * (i % TWO_NUM); | 631 | + postDstLocalList[i] = postDstAddr + lineAlignSize * (i / TWO_NUM) + B32_BLOCK_ALIGN_NUM * (i % TWO_NUM); |
| 632 | } else { | 632 | } else { |
| 633 | - dstLocalList[i] = dstAddr + lineAlignSize * i; | 633 | + postDstLocalList[i] = postDstAddr + lineAlignSize * i; |
| 634 | } | 634 | } |
| 635 | } | 635 | } |
| 636 | - struct TransDataTo5HDParams transDataParams; | 636 | + struct TransDataTo5HDParams postTransDataParams; |
| 637 | if constexpr (std::is_same<T_POST_TRANS, float>::value) { | 637 | if constexpr (std::is_same<T_POST_TRANS, float>::value) { |
| 638 | - transDataParams.repeatTimes = lineAlignSize / B32_BLOCK_ALIGN_NUM / TWO_NUM; | 638 | + postTransDataParams.repeatTimes = lineAlignSize / B32_BLOCK_ALIGN_NUM / TWO_NUM; |
| 639 | - transDataParams.srcRepStride = TRANSPOSE_C0_SIZE * TWO_NUM; | 639 | + postTransDataParams.srcRepStride = TRANSPOSE_C0_SIZE * TWO_NUM; |
| 640 | - transDataParams.dstRepStride = TWO_NUM; | 640 | + postTransDataParams.dstRepStride = TWO_NUM; |
| 641 | } else { | 641 | } else { |
| 642 | - transDataParams.repeatTimes = lineAlignSize / B16_BLOCK_ALIGN_NUM; | 642 | + postTransDataParams.repeatTimes = lineAlignSize / B16_BLOCK_ALIGN_NUM; |
| 643 | - transDataParams.srcRepStride = TRANSPOSE_C0_SIZE; | 643 | + postTransDataParams.srcRepStride = TRANSPOSE_C0_SIZE; |
| 644 | - transDataParams.dstRepStride = 1; | 644 | + postTransDataParams.dstRepStride = 1; |
| 645 | } | 645 | } |
| 646 | - if (transDataParams.repeatTimes == 1) { | 646 | + if (postTransDataParams.repeatTimes == 1) { |
| 647 | - transDataParams.srcRepStride = 0; | 647 | + postTransDataParams.srcRepStride = 0; |
| 648 | - transDataParams.dstRepStride = 0; | 648 | + postTransDataParams.dstRepStride = 0; |
| 649 | } | 649 | } |
| 650 | - TransDataTo5HDImpl(dstLocalList, srcLocalList, transDataParams); | 650 | + TransDataTo5HDImpl(postDstLocalList, postSrcLocalList, postTransDataParams); |
| 651 | if constexpr (std::is_same<T_POST_TRANS, float>::value) { | 651 | if constexpr (std::is_same<T_POST_TRANS, float>::value) { |
| 652 | for (uint32_t i = 0; i < TRANSPOSE_C0_SIZE; i++) { | 652 | for (uint32_t i = 0; i < TRANSPOSE_C0_SIZE; i++) { |
| 653 | - srcLocalList[i] = srcAddr + TRANSPOSE_C0_SIZE * i + B32_BLOCK_ALIGN_NUM; | 653 | + postSrcLocalList[i] = postSrcAddr + TRANSPOSE_C0_SIZE * i + B32_BLOCK_ALIGN_NUM; |
| 654 | - dstLocalList[i] = dstAddr + B32_BLOCK_ALIGN_NUM * lineAlignSize + lineAlignSize * (i / TWO_NUM) + | 654 | + postDstLocalList[i] = postDstAddr + B32_BLOCK_ALIGN_NUM * lineAlignSize + lineAlignSize * (i / TWO_NUM) + |
| 655 | B32_BLOCK_ALIGN_NUM * (i % TWO_NUM); | 655 | B32_BLOCK_ALIGN_NUM * (i % TWO_NUM); |
| 656 | } | 656 | } |
| 657 | - TransDataTo5HDImpl(dstLocalList, srcLocalList, transDataParams); | 657 | + TransDataTo5HDImpl(postDstLocalList, postSrcLocalList, postTransDataParams); |
| 658 | } | 658 | } |
| 659 | } | 659 | } |
| 660 | 660 | ||
| @@ -943,9 +943,8 @@ private: | |||
| 943 | LocalTensor<float> xLocalFp32; | 943 | LocalTensor<float> xLocalFp32; |
| 944 | LocalTensor<float> pdGammaOut; | 944 | LocalTensor<float> pdGammaOut; |
| 945 | LocalTensor<float> pdBetaOut; | 945 | LocalTensor<float> pdBetaOut; |
| 946 | - // tilingData | ||
| 947 | - uint64_t col; | ||
| 948 | uint64_t row; | 946 | uint64_t row; |
| 947 | + uint64_t col; | ||
| 949 | uint64_t numBlocks; | 948 | uint64_t numBlocks; |
| 950 | uint64_t blockFormer; | 949 | uint64_t blockFormer; |
| 951 | uint64_t blockTail; | 950 | uint64_t blockTail; |
| @@ -107,15 +107,15 @@ __aicore__ inline void ReduceSumHalfInterval( | |||
| 107 | const LocalTensor<float>& dst_local, const LocalTensor<float>& src_local, int32_t count) | 107 | const LocalTensor<float>& dst_local, const LocalTensor<float>& src_local, int32_t count) |
| 108 | { | 108 | { |
| 109 | if (likely(count > ELEM_PER_REP_FP32)) { | 109 | if (likely(count > ELEM_PER_REP_FP32)) { |
| 110 | - int32_t bodyCount = findPowerTwo(count); | 110 | + int32_t bodyCountReduce = findPowerTwo(count); |
| 111 | - int32_t tailCount = count - bodyCount; | 111 | + int32_t tailCountReduce = count - bodyCountReduce; |
| 112 | - if (tailCount > 0) { | 112 | + if (tailCountReduce > 0) { |
| 113 | - Add(src_local, src_local, src_local[bodyCount], tailCount); | 113 | + Add(src_local, src_local, src_local[bodyCountReduce], tailCountReduce); |
| 114 | PipeBarrier<PIPE_V>(); | 114 | PipeBarrier<PIPE_V>(); |
| 115 | } | 115 | } |
| 116 | - while (bodyCount > ELEM_PER_REP_FP32) { | 116 | + while (bodyCountReduce > ELEM_PER_REP_FP32) { |
| 117 | - bodyCount = bodyCount / HALf_INTERVAL; | 117 | + bodyCountReduce = bodyCountReduce / HALf_INTERVAL; |
| 118 | - Add(src_local, src_local, src_local[bodyCount], bodyCount); | 118 | + Add(src_local, src_local, src_local[bodyCountReduce], bodyCountReduce); |
| 119 | PipeBarrier<PIPE_V>(); | 119 | PipeBarrier<PIPE_V>(); |
| 120 | } | 120 | } |
| 121 | 121 | ||
| @@ -30,17 +30,17 @@ static constexpr float ZERO = 0; | |||
| 30 | 30 | ||
| 31 | 31 | ||
| 32 | 32 | ||
| 33 | -template <typename Tp, Tp v> | 33 | +template <typename TypeParam, TypeParam val> |
| 34 | struct integral_constant { | 34 | struct integral_constant { |
| 35 | - static constexpr Tp value = v; | 35 | + static constexpr TypeParam value = val; |
| 36 | }; | 36 | }; |
| 37 | using true_type = integral_constant<bool, true>; | 37 | using true_type = integral_constant<bool, true>; |
| 38 | using false_type = integral_constant<bool, false>; | 38 | using false_type = integral_constant<bool, false>; |
| 39 | template <typename, typename> | 39 | template <typename, typename> |
| 40 | struct is_same : public false_type { | 40 | struct is_same : public false_type { |
| 41 | }; | 41 | }; |
| 42 | -template <typename Tp> | 42 | +template <typename TypeParam> |
| 43 | -struct is_same<Tp, Tp> : public true_type { | 43 | +struct is_same<TypeParam, TypeParam> : public true_type { |
| 44 | }; | 44 | }; |
| 45 | 45 | ||
| 46 | template <typename T, template <typename U> typename R, template <typename U> typename S> | 46 | template <typename T, template <typename U> typename R, template <typename U> typename S> |
| @@ -129,15 +129,17 @@ ge::graphStatus QuantizedBatchNormWelfordTiling::PostTiling() | |||
| 129 | td_.set_patternR0Align(commonParams.patternR0Align); | 129 | td_.set_patternR0Align(commonParams.patternR0Align); |
| 130 | td_.set_epsilon(commonParams.epsilon); | 130 | td_.set_epsilon(commonParams.epsilon); |
| 131 | context_->SetBlockDim(usedCoreNum); | 131 | context_->SetBlockDim(usedCoreNum); |
| 132 | - auto rawTilingData = context_->GetRawTilingData(); | 132 | + |
| 133 | + // Save tiling data for quantized batch norm | ||
| 134 | + auto quantRawTilingData = context_->GetRawTilingData(); | ||
| 133 | OP_CHECK_IF( | 135 | OP_CHECK_IF( |
| 134 | - td_.GetDataSize() > rawTilingData->GetCapacity(), | 136 | + td_.GetDataSize() > quantRawTilingData->GetCapacity(), |
| 135 | OP_LOGE( | 137 | OP_LOGE( |
| 136 | commonParams.nodeName, "actual tiling data size %zu > context tiling data size %zu", td_.GetDataSize(), | 138 | commonParams.nodeName, "actual tiling data size %zu > context tiling data size %zu", td_.GetDataSize(), |
| 137 | - rawTilingData->GetCapacity()), | 139 | + quantRawTilingData->GetCapacity()), |
| 138 | return ge::GRAPH_FAILED); | 140 | return ge::GRAPH_FAILED); |
| 139 | - td_.SaveToBuffer(rawTilingData->GetData(), rawTilingData->GetCapacity()); | 141 | + td_.SaveToBuffer(quantRawTilingData->GetData(), quantRawTilingData->GetCapacity()); |
| 140 | - rawTilingData->SetDataSize(td_.GetDataSize()); | 142 | + quantRawTilingData->SetDataSize(td_.GetDataSize()); |
| 141 | 143 | ||
| 142 | return ge::GRAPH_SUCCESS; | 144 | return ge::GRAPH_SUCCESS; |
| 143 | } | 145 | } |
| @@ -41,10 +41,10 @@ __aicore__ inline constexpr uint32_t GetVRegSize() | |||
| 41 | 41 | ||
| 42 | 42 | ||
| 43 | constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue | 43 | constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue |
| 44 | -constexpr int32_t DOUBLE_BUFFER_NUM = 2; | ||
| 45 | -constexpr int32_t UNROLL_NUM = 2; | ||
| 46 | constexpr int32_t NUM_PER_REP_FP32 = 64; // ONE_REPEAT_BYTE_SIZE / sizeof(float); | 44 | constexpr int32_t NUM_PER_REP_FP32 = 64; // ONE_REPEAT_BYTE_SIZE / sizeof(float); |
| 47 | constexpr int32_t NUM_PER_BLK_FP32 = 8; | 45 | constexpr int32_t NUM_PER_BLK_FP32 = 8; |
| 46 | +constexpr int32_t DOUBLE_BUFFER_NUM = 2; | ||
| 47 | +constexpr int32_t UNROLL_NUM = 2; | ||
| 48 | constexpr int32_t FLOAT_BTYPE_SIZE = 4; | 48 | constexpr int32_t FLOAT_BTYPE_SIZE = 4; |
| 49 | constexpr int32_t NUM_PER_BLK_FP16 = 16; | 49 | constexpr int32_t NUM_PER_BLK_FP16 = 16; |
| 50 | constexpr int32_t CONTINUE_STRIDE = 8; | 50 | constexpr int32_t CONTINUE_STRIDE = 8; |
| @@ -15,39 +15,39 @@ | |||
| 15 | 15 | ||
| 16 | using namespace AscendC; | 16 | using namespace AscendC; |
| 17 | 17 | ||
| 18 | -constexpr uint32_t MAX_REP_NUM = 255; | ||
| 19 | -constexpr uint32_t ELEM_PER_REP_FP32 = 64; | ||
| 20 | constexpr uint32_t ELEM_PER_BLK_FP32 = 8; | 18 | constexpr uint32_t ELEM_PER_BLK_FP32 = 8; |
| 19 | +constexpr uint32_t ELEM_PER_REP_FP32 = 64; | ||
| 20 | +constexpr uint32_t MAX_REP_NUM = 255; | ||
| 21 | +constexpr int32_t INDEX_EIGHT = 8; | ||
| 21 | constexpr float ZERO = 0; | 22 | constexpr float ZERO = 0; |
| 22 | constexpr int32_t HALf_INTERVAL = 2; | 23 | constexpr int32_t HALf_INTERVAL = 2; |
| 24 | +constexpr int32_t INDEX_SIXTEEN = 16; | ||
| 23 | constexpr int32_t INDEX_TWO = 2; | 25 | constexpr int32_t INDEX_TWO = 2; |
| 24 | constexpr int32_t INDEX_FOUR = 4; | 26 | constexpr int32_t INDEX_FOUR = 4; |
| 25 | -constexpr int32_t INDEX_EIGHT = 8; | ||
| 26 | -constexpr int32_t INDEX_SIXTEEN = 16; | ||
| 27 | 27 | ||
| 28 | __aicore__ inline void ReduceSumForSmallReduceDimPreRepeat( | 28 | __aicore__ inline void ReduceSumForSmallReduceDimPreRepeat( |
| 29 | const LocalTensor<float>& dstLocal, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, | 29 | const LocalTensor<float>& dstLocal, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, |
| 30 | - const uint32_t elemNum, const uint32_t numLastDim, const uint32_t tailCount, const uint32_t repeat, | 30 | + const uint32_t elemNum, const uint32_t numLastDim, const uint32_t tailCount, const uint32_t repeat2, |
| 31 | const uint8_t repStride) | 31 | const uint8_t repStride) |
| 32 | { | 32 | { |
| 33 | uint32_t elemIndex = 0; | 33 | uint32_t elemIndex = 0; |
| 34 | for (; elemIndex + ELEM_PER_REP_FP32 <= numLastDim; elemIndex += ELEM_PER_REP_FP32) { | 34 | for (; elemIndex + ELEM_PER_REP_FP32 <= numLastDim; elemIndex += ELEM_PER_REP_FP32) { |
| 35 | - Add(tmpLocal, srcLocal[elemIndex], tmpLocal, elemNum, repeat, | 35 | + Add(tmpLocal, srcLocal[elemIndex], tmpLocal, elemNum, repeat2, |
| 36 | {1, 1, 1, ELEM_PER_BLK_FP32, repStride, ELEM_PER_BLK_FP32}); | 36 | {1, 1, 1, ELEM_PER_BLK_FP32, repStride, ELEM_PER_BLK_FP32}); |
| 37 | PipeBarrier<PIPE_V>(); | 37 | PipeBarrier<PIPE_V>(); |
| 38 | } | 38 | } |
| 39 | if (unlikely(tailCount != 0)) { | 39 | if (unlikely(tailCount != 0)) { |
| 40 | - Add(tmpLocal, srcLocal[elemIndex], tmpLocal, tailCount, repeat, | 40 | + Add(tmpLocal, srcLocal[elemIndex], tmpLocal, tailCount, repeat2, |
| 41 | {1, 1, 1, ELEM_PER_BLK_FP32, repStride, ELEM_PER_BLK_FP32}); | 41 | {1, 1, 1, ELEM_PER_BLK_FP32, repStride, ELEM_PER_BLK_FP32}); |
| 42 | } | 42 | } |
| 43 | PipeBarrier<PIPE_V>(); | 43 | PipeBarrier<PIPE_V>(); |
| 44 | AscendCUtils::SetMask<float>(ELEM_PER_REP_FP32); // set mask = 64 | 44 | AscendCUtils::SetMask<float>(ELEM_PER_REP_FP32); // set mask = 64 |
| 45 | 45 | ||
| 46 | if ASCEND_IS_AIV { | 46 | if ASCEND_IS_AIV { |
| 47 | - WholeReduceSum<float, false>(dstLocal, tmpLocal, MASK_PLACEHOLDER, repeat, 1, 1, ELEM_PER_BLK_FP32); | 47 | + WholeReduceSum<float, false>(dstLocal, tmpLocal, MASK_PLACEHOLDER, repeat2, 1, 1, ELEM_PER_BLK_FP32); |
| 48 | } | 48 | } |
| 49 | 49 | ||
| 50 | - WholeReduceSum<float, false>(dstLocal, tmpLocal, MASK_PLACEHOLDER, repeat, 1, 1, ELEM_PER_BLK_FP32); | 50 | + WholeReduceSum<float, false>(dstLocal, tmpLocal, MASK_PLACEHOLDER, repeat2, 1, 1, ELEM_PER_BLK_FP32); |
| 51 | 51 | ||
| 52 | } | 52 | } |
| 53 | 53 | ||
| @@ -58,12 +58,12 @@ __aicore__ inline void ReduceSumForSmallReduceDimPreRepeat( | |||
| 58 | __aicore__ inline void ReduceSumForSmallReduceDim( | 58 | __aicore__ inline void ReduceSumForSmallReduceDim( |
| 59 | const LocalTensor<float>& dstLocal, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, | 59 | const LocalTensor<float>& dstLocal, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, |
| 60 | const uint32_t numLastDimAligned, const uint32_t numLastDim, const uint32_t tailCount, const uint32_t repeat, | 60 | const uint32_t numLastDimAligned, const uint32_t numLastDim, const uint32_t tailCount, const uint32_t repeat, |
| 61 | - const uint8_t repStride) | 61 | + const uint8_t repStride1) |
| 62 | { | 62 | { |
| 63 | uint32_t repeatTimes = repeat / MAX_REP_NUM; | 63 | uint32_t repeatTimes = repeat / MAX_REP_NUM; |
| 64 | if (repeatTimes == 0) { | 64 | if (repeatTimes == 0) { |
| 65 | ReduceSumForSmallReduceDimPreRepeat( | 65 | ReduceSumForSmallReduceDimPreRepeat( |
| 66 | - dstLocal, srcLocal, tmpLocal, ELEM_PER_REP_FP32, numLastDim, tailCount, repeat, repStride); | 66 | + dstLocal, srcLocal, tmpLocal, ELEM_PER_REP_FP32, numLastDim, tailCount, repeat, repStride1); |
| 67 | } else { | 67 | } else { |
| 68 | uint32_t repTailNum = repeat % MAX_REP_NUM; | 68 | uint32_t repTailNum = repeat % MAX_REP_NUM; |
| 69 | uint32_t repIndex = 0; | 69 | uint32_t repIndex = 0; |
| @@ -71,12 +71,12 @@ __aicore__ inline void ReduceSumForSmallReduceDim( | |||
| 71 | for (; repIndex + MAX_REP_NUM <= repeat; repIndex += MAX_REP_NUM) { | 71 | for (; repIndex + MAX_REP_NUM <= repeat; repIndex += MAX_REP_NUM) { |
| 72 | ReduceSumForSmallReduceDimPreRepeat( | 72 | ReduceSumForSmallReduceDimPreRepeat( |
| 73 | dstLocal[repIndex], srcLocal[repIndex * numLastDimAligned], tmpLocal[repIndex * ELEM_PER_REP_FP32], | 73 | dstLocal[repIndex], srcLocal[repIndex * numLastDimAligned], tmpLocal[repIndex * ELEM_PER_REP_FP32], |
| 74 | - ELEM_PER_REP_FP32, numLastDim, tailCount, MAX_REP_NUM, repStride); | 74 | + ELEM_PER_REP_FP32, numLastDim, tailCount, MAX_REP_NUM, repStride1); |
| 75 | } | 75 | } |
| 76 | if (repTailNum != 0) { | 76 | if (repTailNum != 0) { |
| 77 | ReduceSumForSmallReduceDimPreRepeat( | 77 | ReduceSumForSmallReduceDimPreRepeat( |
| 78 | dstLocal[repIndex], srcLocal[repIndex * numLastDimAligned], tmpLocal[repIndex * ELEM_PER_REP_FP32], | 78 | dstLocal[repIndex], srcLocal[repIndex * numLastDimAligned], tmpLocal[repIndex * ELEM_PER_REP_FP32], |
| 79 | - ELEM_PER_REP_FP32, numLastDim, tailCount, repTailNum, repStride); | 79 | + ELEM_PER_REP_FP32, numLastDim, tailCount, repTailNum, repStride1); |
| 80 | } | 80 | } |
| 81 | } | 81 | } |
| 82 | } | 82 | } |
| @@ -87,7 +87,7 @@ __aicore__ inline void ReduceSumForSmallReduceDim( | |||
| 87 | * size of tmpLocal: (N, 64) | 87 | * size of tmpLocal: (N, 64) |
| 88 | */ | 88 | */ |
| 89 | __aicore__ inline void ReduceSumMultiN( | 89 | __aicore__ inline void ReduceSumMultiN( |
| 90 | - const LocalTensor<float>& dstLocal, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, | 90 | + const LocalTensor<float>& dstLocal1, const LocalTensor<float>& srcLocal, const LocalTensor<float>& tmpLocal, |
| 91 | const uint32_t numRow, const uint32_t numCol, const uint32_t numColAlign) | 91 | const uint32_t numRow, const uint32_t numCol, const uint32_t numColAlign) |
| 92 | { | 92 | { |
| 93 | const uint32_t tailCount = numCol % ELEM_PER_REP_FP32; | 93 | const uint32_t tailCount = numCol % ELEM_PER_REP_FP32; |
| @@ -95,33 +95,22 @@ __aicore__ inline void ReduceSumMultiN( | |||
| 95 | const uint8_t repStride = numColAlign / ELEM_PER_BLK_FP32; | 95 | const uint8_t repStride = numColAlign / ELEM_PER_BLK_FP32; |
| 96 | Duplicate(tmpLocal, ZERO, numRow * ELEM_PER_REP_FP32); | 96 | Duplicate(tmpLocal, ZERO, numRow * ELEM_PER_REP_FP32); |
| 97 | PipeBarrier<PIPE_V>(); | 97 | PipeBarrier<PIPE_V>(); |
| 98 | - ReduceSumForSmallReduceDim(dstLocal, srcLocal, tmpLocal, numColAlign, numCol, tailCount, repeat, repStride); | 98 | + ReduceSumForSmallReduceDim(dstLocal1, srcLocal, tmpLocal, numColAlign, numCol, tailCount, repeat, repStride); |
| 99 | -} | ||
| 100 | - | ||
| 101 | -__aicore__ inline int32_t findPowerTwo(int32_t n) | ||
| 102 | -{ | ||
| 103 | - // find max power of 2 no more than n (32 bit) | ||
| 104 | - n |= n >> 1; // Set the first digit of n's binary to 1 | ||
| 105 | - n |= n >> INDEX_TWO; | ||
| 106 | - n |= n >> INDEX_FOUR; | ||
| 107 | - n |= n >> INDEX_EIGHT; | ||
| 108 | - n |= n >> INDEX_SIXTEEN; | ||
| 109 | - return (n + 1) >> 1; | ||
| 110 | } | 99 | } |
| 111 | 100 | ||
| 112 | __aicore__ inline void ReduceSumHalfInterval( | 101 | __aicore__ inline void ReduceSumHalfInterval( |
| 113 | const LocalTensor<float>& dst_local, const LocalTensor<float>& src_local, int32_t count) | 102 | const LocalTensor<float>& dst_local, const LocalTensor<float>& src_local, int32_t count) |
| 114 | { | 103 | { |
| 115 | if (likely(count > ELEM_PER_REP_FP32)) { | 104 | if (likely(count > ELEM_PER_REP_FP32)) { |
| 116 | - int32_t bodyCount = findPowerTwo(count); | 105 | + int32_t bodyCountHalf = findPowerTwo(count); |
| 117 | - int32_t tailCount = count - bodyCount; | 106 | + int32_t tailCountHalf = count - bodyCountHalf; |
| 118 | - if (tailCount > 0) { | 107 | + if (tailCountHalf > 0) { |
| 119 | - Add(src_local, src_local, src_local[bodyCount], tailCount); | 108 | + Add(src_local, src_local, src_local[bodyCountHalf], tailCountHalf); |
| 120 | PipeBarrier<PIPE_V>(); | 109 | PipeBarrier<PIPE_V>(); |
| 121 | } | 110 | } |
| 122 | - while (bodyCount > ELEM_PER_REP_FP32) { | 111 | + while (bodyCountHalf > ELEM_PER_REP_FP32) { |
| 123 | - bodyCount = bodyCount / HALf_INTERVAL; | 112 | + bodyCountHalf = bodyCountHalf / HALf_INTERVAL; |
| 124 | - Add(src_local, src_local, src_local[bodyCount], bodyCount); | 113 | + Add(src_local, src_local, src_local[bodyCountHalf], bodyCountHalf); |
| 125 | PipeBarrier<PIPE_V>(); | 114 | PipeBarrier<PIPE_V>(); |
| 126 | } | 115 | } |
| 127 | 116 | ||
| @@ -139,18 +128,29 @@ __aicore__ inline void ReduceSumHalfInterval( | |||
| 139 | PipeBarrier<PIPE_V>(); | 128 | PipeBarrier<PIPE_V>(); |
| 140 | } | 129 | } |
| 141 | 130 | ||
| 142 | -__aicore__ inline float ReduceSumHalfInterval(const LocalTensor<float>& src_local, int32_t count) | 131 | +__aicore__ inline int32_t findPowerTwo(int32_t n) |
| 132 | +{ | ||
| 133 | + // find max power of 2 no more than n (32 bit) | ||
| 134 | + n |= n >> 1; // Set the first digit of n's binary to 1 | ||
| 135 | + n |= n >> INDEX_TWO; | ||
| 136 | + n |= n >> INDEX_FOUR; | ||
| 137 | + n |= n >> INDEX_EIGHT; | ||
| 138 | + n |= n >> INDEX_SIXTEEN; | ||
| 139 | + return (n + 1) >> 1; | ||
| 140 | +} | ||
| 141 | + | ||
| 142 | +__aicore__ inline float ReduceSumHalfInterval(const LocalTensor<float>& src_local2, int32_t count) | ||
| 143 | { | 143 | { |
| 144 | if (likely(count > ELEM_PER_REP_FP32)) { | 144 | if (likely(count > ELEM_PER_REP_FP32)) { |
| 145 | int32_t bodyCount = findPowerTwo(count); | 145 | int32_t bodyCount = findPowerTwo(count); |
| 146 | int32_t tailCount = count - bodyCount; | 146 | int32_t tailCount = count - bodyCount; |
| 147 | if (tailCount > 0) { | 147 | if (tailCount > 0) { |
| 148 | - Add(src_local, src_local, src_local[bodyCount], tailCount); | 148 | + Add(src_local2, src_local2, src_local2[bodyCount], tailCount); |
| 149 | PipeBarrier<PIPE_V>(); | 149 | PipeBarrier<PIPE_V>(); |
| 150 | } | 150 | } |
| 151 | while (bodyCount > ELEM_PER_REP_FP32) { | 151 | while (bodyCount > ELEM_PER_REP_FP32) { |
| 152 | bodyCount = bodyCount / HALf_INTERVAL; | 152 | bodyCount = bodyCount / HALf_INTERVAL; |
| 153 | - Add(src_local, src_local, src_local[bodyCount], bodyCount); | 153 | + Add(src_local2, src_local2, src_local2[bodyCount], bodyCount); |
| 154 | PipeBarrier<PIPE_V>(); | 154 | PipeBarrier<PIPE_V>(); |
| 155 | } | 155 | } |
| 156 | 156 | ||
| @@ -160,14 +160,14 @@ __aicore__ inline float ReduceSumHalfInterval(const LocalTensor<float>& src_loca | |||
| 160 | } | 160 | } |
| 161 | 161 | ||
| 162 | if (g_coreType == AIV) { | 162 | if (g_coreType == AIV) { |
| 163 | - WholeReduceSum<float, false>(src_local, src_local, MASK_PLACEHOLDER, 1, 0, 1, 0); | 163 | + WholeReduceSum<float, false>(src_local2, src_local2, MASK_PLACEHOLDER, 1, 0, 1, 0); |
| 164 | } | 164 | } |
| 165 | 165 | ||
| 166 | - WholeReduceSum<float, false>(src_local, src_local, MASK_PLACEHOLDER, 1, 1, 1, DEFAULT_REPEAT_STRIDE); | 166 | + WholeReduceSum<float, false>(src_local2, src_local2, MASK_PLACEHOLDER, 1, 1, 1, DEFAULT_REPEAT_STRIDE); |
| 167 | 167 | ||
| 168 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 168 | event_t event_v_s = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 169 | SetFlag<HardEvent::V_S>(event_v_s); | 169 | SetFlag<HardEvent::V_S>(event_v_s); |
| 170 | WaitFlag<HardEvent::V_S>(event_v_s); | 170 | WaitFlag<HardEvent::V_S>(event_v_s); |
| 171 | - return src_local.GetValue(0); | 171 | + return src_local2.GetValue(0); |
| 172 | } | 172 | } |
| 173 | 173 | ||
| @@ -245,17 +245,17 @@ __aicore__ inline void dCompute(int64_t colIndex, int64_t rowIndex, int64_t colS | |||
| 245 | if (colSizeMod != 0) { | 245 | if (colSizeMod != 0) { |
| 246 | colSizeAlign += FLOAT_ALIGN - colSizeMod; | 246 | colSizeAlign += FLOAT_ALIGN - colSizeMod; |
| 247 | } | 247 | } |
| 248 | - TEventID eventID = GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>(); | 248 | + TEventID eventId = GetTPipePtr()->AllocEventID<HardEvent::MTE2_V>(); |
| 249 | - SetFlag<HardEvent::MTE2_V>(eventID); | 249 | + SetFlag<HardEvent::MTE2_V>(eventId); |
| 250 | - WaitFlag<HardEvent::MTE2_V>(eventID); | 250 | + WaitFlag<HardEvent::MTE2_V>(eventId); |
| 251 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventID); | 251 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE2_V>(eventId); |
| 252 | - TEventID eventID1 = GetTPipePtr()->AllocEventID<HardEvent::MTE3_V>(); | 252 | + TEventID eventId1 = GetTPipePtr()->AllocEventID<HardEvent::MTE3_V>(); |
| 253 | - SetFlag<HardEvent::MTE3_V>(eventID1); | 253 | + SetFlag<HardEvent::MTE3_V>(eventId1); |
| 254 | - WaitFlag<HardEvent::MTE3_V>(eventID1); | 254 | + WaitFlag<HardEvent::MTE3_V>(eventId1); |
| 255 | - GetTPipePtr()->ReleaseEventID<HardEvent::MTE3_V>(eventID1); | 255 | + GetTPipePtr()->ReleaseEventID<HardEvent::MTE3_V>(eventId1); |
| 256 | Duplicate(deterministicStruct.buffer2_, static_cast<float>(0.0), COL_TEMPLATE); | 256 | Duplicate(deterministicStruct.buffer2_, static_cast<float>(0.0), COL_TEMPLATE); |
| 257 | PipeBarrier<PIPE_V>(); | 257 | PipeBarrier<PIPE_V>(); |
| 258 | - uint64_t mask = colSize; | 258 | + uint64_t maskVal = colSize; |
| 259 | uint8_t repeatTimes = MAX_REPEAT_TIMES; | 259 | uint8_t repeatTimes = MAX_REPEAT_TIMES; |
| 260 | BinaryRepeatParams binaryRepeatParams; | 260 | BinaryRepeatParams binaryRepeatParams; |
| 261 | binaryRepeatParams.dstBlkStride = 1; | 261 | binaryRepeatParams.dstBlkStride = 1; |
| @@ -269,7 +269,7 @@ __aicore__ inline void dCompute(int64_t colIndex, int64_t rowIndex, int64_t colS | |||
| 269 | if (i == rowRepeatTimes - 1) { | 269 | if (i == rowRepeatTimes - 1) { |
| 270 | repeatTimes = rowSize - (rowRepeatTimes - 1) * MAX_REPEAT_TIMES; | 270 | repeatTimes = rowSize - (rowRepeatTimes - 1) * MAX_REPEAT_TIMES; |
| 271 | } | 271 | } |
| 272 | - Add(deterministicStruct.buffer2_,deterministicStruct.buffer1_[i * MAX_REPEAT_TIMES],deterministicStruct.buffer2_,mask,repeatTimes,binaryRepeatParams); | 272 | + Add(deterministicStruct.buffer2_,deterministicStruct.buffer1_[i * MAX_REPEAT_TIMES],deterministicStruct.buffer2_,maskVal,repeatTimes,binaryRepeatParams); |
| 273 | PipeBarrier<PIPE_V>(); | 273 | PipeBarrier<PIPE_V>(); |
| 274 | } | 274 | } |
| 275 | } | 275 | } |
| @@ -32,7 +32,8 @@ OP_TYPE_REGISTER(RmsNormQuant); | |||
| 32 | OP_TYPE_REGISTER(RmsNormQuantV2); | 32 | OP_TYPE_REGISTER(RmsNormQuantV2); |
| 33 | 33 | ||
| 34 | const aclTensor* RmsNormQuant( | 34 | const aclTensor* RmsNormQuant( |
| 35 | - const aclTensor* x, const aclTensor* gamma, const aclTensor* beta, const aclTensor* scale, const aclTensor* offset, | 35 | + const aclTensor* x, const aclTensor* gamma, const aclTensor* beta, const aclTensor* scale, |
| 36 | + const aclTensor* offset, | ||
| 36 | double epsilon, int32_t dstType, aclOpExecutor* executor) | 37 | double epsilon, int32_t dstType, aclOpExecutor* executor) |
| 37 | { | 38 | { |
| 38 | L0_DFX(RmsNormQuant, x, gamma, beta, scale, offset, epsilon, dstType); | 39 | L0_DFX(RmsNormQuant, x, gamma, beta, scale, offset, epsilon, dstType); |
norm/add_layer_norm/op_graph/fusion_pass/add_layer_norm_v4_fusion_pass.cppNodeIo add1OutNode; NodeIo add2OutNode;此评论由代码审查工具自动生成