Star695
1005
代码
代码
Issues90
Pull Requests187
Actions
项目讨论
Wiki
分析
Star695
1005
已合并
AllReduce lastSlice calculate #749
chengyutao3创建于 3月5日
AllReduce lastSlice calculate #749
已合并
chengyutao3创建于 3月5日
从已删除 :master合入到cann/asc-devkitmaster
共 3 个文件变更+52-9
Mimpl/adv_api/detail/hccl/ccu/hccl_ccu_v0_prepare.h+7-2
@@ -161,7 +161,7 @@ HcclImpl<HcclServerType::HCCL_SERVER_TYPE_CCU, config>::CcuPrepareForAllReduceM2
161 xnData_[2] = (uint64_t)commParam->recvBuf + offset; // ccu xn2161 xnData_[2] = (uint64_t)commParam->recvBuf + offset; // ccu xn2
162 162
163 uint64_t loopCount = CCU_LOOP_COUNT_M2M_RE;163 uint64_t loopCount = CCU_LOOP_COUNT_M2M_RE;
164- uint64_t tmpCount = commParam->count / ccuParam_.rankNum;164+ uint64_t tmpCount = (commParam->count + ccuParam_.rankNum - 1) / ccuParam_.rankNum;
165 uint64_t sliceCount = (ccuParam_.rankId == ccuParam_.rankNum - 1) ?165 uint64_t sliceCount = (ccuParam_.rankId == ccuParam_.rankNum - 1) ?
166 (commParam->count - (ccuParam_.rankNum - 1) * tmpCount) : tmpCount;166 (commParam->count - (ccuParam_.rankNum - 1) * tmpCount) : tmpCount;
167 uint64_t sliceSize = sliceCount * DATA_TYPE_MAP[commParam->dataType];167 uint64_t sliceSize = sliceCount * DATA_TYPE_MAP[commParam->dataType];
@@ -177,7 +177,12 @@ HcclImpl<HcclServerType::HCCL_SERVER_TYPE_CCU, config>::CcuPrepareForAllReduceM2
177 uint64_t normalSliceCount = (commParam->count + ccuParam_.rankNum - 1) / ccuParam_.rankNum; // count/rankNum 向上取整177 uint64_t normalSliceCount = (commParam->count + ccuParam_.rankNum - 1) / ccuParam_.rankNum; // count/rankNum 向上取整
178 uint64_t normalSliceSize = normalSliceCount * dataSize;178 uint64_t normalSliceSize = normalSliceCount * dataSize;
179 uint64_t normalRankCount = commParam->count / normalSliceCount;179 uint64_t normalRankCount = commParam->count / normalSliceCount;
180- uint64_t lastSliceSize = (commParam->count * dataSize) - (normalRankCount * normalSliceSize);180+ uint64_t lastSliceSize = 0;
181+ if (normalRankCount == ccuParam_.rankNum) {
182+ lastSliceSize = (commParam->count * dataSize) - ((normalRankCount - 1) * normalSliceSize);
183+ } else {
184+ lastSliceSize = (commParam->count * dataSize) - (normalRankCount * normalSliceSize);
185+ }
181 KERNEL_LOG(KERNEL_INFO, "ApiClient CcuPrepareForAllReduceM2M normalSliceSize:%d, lastSliceSize:%d", normalSliceSize, lastSliceSize);186 KERNEL_LOG(KERNEL_INFO, "ApiClient CcuPrepareForAllReduceM2M normalSliceSize:%d, lastSliceSize:%d", normalSliceSize, lastSliceSize);
182 187
183 xnData_[6] = normalSliceSize;188 xnData_[6] = normalSliceSize;
Mimpl/adv_api/detail/hccl/impl/platform_v310/hccl_ccu_v0.h+5-6
@@ -161,8 +161,7 @@ HcclImpl<HcclServerType::HCCL_SERVER_TYPE_CCU, config>::InitInner(GM_ADDR contex
161 161 
162 isInited_ = true;162 isInited_ = true;
163 KERNEL_LOG(KERNEL_INFO, "ApiClient InitInner rankId:%d, rankNum:%d, xnAddr:0x%llx, ckeAddr:0x%llx, ccuMsgExt:0x%llx",163 KERNEL_LOG(KERNEL_INFO, "ApiClient InitInner rankId:%d, rankNum:%d, xnAddr:0x%llx, ckeAddr:0x%llx, ccuMsgExt:0x%llx",
164- hcclContext_->rankId, hcclContext_->rankNum, hcclContext_->xnOffset, hcclContext_->ckeOffset,164+ hcclContext_->rankId, hcclContext_->rankNum, hcclContext_->xnOffset, hcclContext_->ckeOffset, reinterpret_cast<uint64_t>(ccuParam_.ccuMsgExt));
165- reinterpret_cast<uint64_t>(ccuParam_.ccuMsgExt));
166}165}
167 166 
168template<const auto &config>167template<const auto &config>
@@ -451,8 +450,10 @@ __aicore__ inline void HcclImpl<HcclServerType::HCCL_SERVER_TYPE_CCU, config>::C
451 ASCENDC_HCCL_API_ASSERT(isInited_, { return; },450 ASCENDC_HCCL_API_ASSERT(isInited_, { return; },
452 "Call Commit failed, please ensure Hccl::Init func has been called successfully already!");451 "Call Commit failed, please ensure Hccl::Init func has been called successfully already!");
453 452 
454- ASCENDC_HCCL_API_ASSERT(handleId > INVALID_HANDLE_ID && handleId < HCCL_MAX_HANDLE_ID, { return; },453+ if (unlikely(handleId >= HCCL_MAX_HANDLE_ID || handleId <= INVALID_HANDLE_ID)) {
455- "Call Commit failed, handleId is[%d], expected in range of [0, %d).", handleId, HCCL_MAX_HANDLE_ID);454+ KERNEL_LOG(KERNEL_ERROR, "Call Commit failed, handleId[%u] is invalid.", handleId);
455+ return;
456+ }
456 457 
457 handleNeedCommitCnt_[handleId]++;458 handleNeedCommitCnt_[handleId]++;
458 if (msgQueueIsAvailable_[globalCurResId_] == false) {459 if (msgQueueIsAvailable_[globalCurResId_] == false) {
@@ -470,8 +471,6 @@ __aicore__ inline int32_t HcclImpl<HcclServerType::HCCL_SERVER_TYPE_CCU, config>
470 "Call Wait failed, please ensure Hccl::Init func has been called successfully already!");471 "Call Wait failed, please ensure Hccl::Init func has been called successfully already!");
471 ASCENDC_HCCL_API_ASSERT(handleCommitCnt_[handleId] > 0, { return HCCL_FAILED; },472 ASCENDC_HCCL_API_ASSERT(handleCommitCnt_[handleId] > 0, { return HCCL_FAILED; },
472 "Call Wait failed, commitCnt [%u] is invalid.", handleCommitCnt_[handleId]);473 "Call Wait failed, commitCnt [%u] is invalid.", handleCommitCnt_[handleId]);
473- ASCENDC_HCCL_API_ASSERT(handleId >INVALID_HANDLE_ID && handleId < HCCL_MAX_HANDLE_ID, { return HCCL_FAILED; },
474- "Call Wait failed, chandleId is[%d], expected in range of [0, %d).", handleId, HCCL_MAX_HANDLE_ID);
475 ASCENDC_HCCL_API_ASSERT(handleId < curHandleId_, { return HCCL_FAILED; },474 ASCENDC_HCCL_API_ASSERT(handleId < curHandleId_, { return HCCL_FAILED; },
476 "Call Wait failed, handleId = %u is invalid please call Preapre Interface before Wait.", handleId);475 "Call Wait failed, handleId = %u is invalid please call Preapre Interface before Wait.", handleId);
477 476 
Mtests/api/adv_api/hccl/test_hccl_950.cpp+40-1
@@ -384,7 +384,7 @@ TEST_F(HcclSuiteAIC, AllReduce_CcuAllReduceMeshMem2Mem1D)
384 Hccl<HcclServerType::HCCL_SERVER_TYPE_CCU> hccl;384 Hccl<HcclServerType::HCCL_SERVER_TYPE_CCU> hccl;
385 hccl.Init(reinterpret_cast<GM_ADDR>(&hcclCombineOpParam));385 hccl.Init(reinterpret_cast<GM_ADDR>(&hcclCombineOpParam));
386 HcclHandle handleId = hccl.AllReduce(reinterpret_cast<__gm__ uint8_t*>(0x11),386 HcclHandle handleId = hccl.AllReduce(reinterpret_cast<__gm__ uint8_t*>(0x11),
387- reinterpret_cast<__gm__ uint8_t*>(0x11), 100,387+ reinterpret_cast<__gm__ uint8_t*>(0x11), 64,
388 HcclDataType::HCCL_DATA_TYPE_INT8,388 HcclDataType::HCCL_DATA_TYPE_INT8,
389 HcclReduceOp::HCCL_REDUCE_SUM, 1);389 HcclReduceOp::HCCL_REDUCE_SUM, 1);
390 hccl.Commit(handleId);390 hccl.Commit(handleId);
@@ -392,6 +392,45 @@ TEST_F(HcclSuiteAIC, AllReduce_CcuAllReduceMeshMem2Mem1D)
392 EXPECT_EQ(hccl.Wait(handleId), HCCL_SUCCESS);392 EXPECT_EQ(hccl.Wait(handleId), HCCL_SUCCESS);
393}393}
394 394 
395+TEST_F(HcclSuiteAIC, AllReduce_CcuAllReduceMeshMem2Mem1D_Cout_not_dived)
396+{
397+ std::vector<uint8_t> workSpace(workSpaceSize + 1024 * 14);
398+ HcclMsgArea* hcclMsgArea = GetHcclMsgArea(workSpace.data());
399+ HcclCombineOpParam hcclCombineOpParam = GetHcclCombineOpParam(workSpace);
400+ hcclCombineOpParam.opType[0] = static_cast<uint32_t>(HcclCMDType::HCCL_CMD_ALLREDUCE);
401+ hcclCombineOpParam.algorithmType[0] = static_cast<uint8_t>(AlgorithmType::CcuAllReduceMeshMem2Mem1D);
402+ 
403+ Hccl<HcclServerType::HCCL_SERVER_TYPE_CCU> hccl;
404+ hccl.Init(reinterpret_cast<GM_ADDR>(&hcclCombineOpParam));
405+ HcclHandle handleId = hccl.AllReduce(reinterpret_cast<__gm__ uint8_t*>(0x11),
406+ reinterpret_cast<__gm__ uint8_t*>(0x11), 9,
407+ HcclDataType::HCCL_DATA_TYPE_INT8,
408+ HcclReduceOp::HCCL_REDUCE_SUM, 1);
409+ hccl.Commit(handleId);
410+ EXPECT_EQ(handleId, 0);
411+ EXPECT_EQ(hccl.Wait(handleId), HCCL_SUCCESS);
412+}
413+ 
414+TEST_F(HcclSuiteAIC, AllReduce_CcuAllReduceMeshMem2Mem1D_Cout_zero)
415+{
416+ std::vector<uint8_t> workSpace(workSpaceSize + 1024 * 14);
417+ HcclMsgArea* hcclMsgArea = GetHcclMsgArea(workSpace.data());
418+ HcclCombineOpParam hcclCombineOpParam = GetHcclCombineOpParam(workSpace);
419+ hcclCombineOpParam.opType[0] = static_cast<uint32_t>(HcclCMDType::HCCL_CMD_ALLREDUCE);
420+ hcclCombineOpParam.algorithmType[0] = static_cast<uint8_t>(AlgorithmType::CcuAllReduceMeshMem2Mem1D);
421+ 
422+ Hccl<HcclServerType::HCCL_SERVER_TYPE_CCU> hccl;
423+ hccl.Init(reinterpret_cast<GM_ADDR>(&hcclCombineOpParam));
424+ HcclHandle handleId = hccl.AllReduce(reinterpret_cast<__gm__ uint8_t*>(0x11),
425+ reinterpret_cast<__gm__ uint8_t*>(0x11), 0,
426+ HcclDataType::HCCL_DATA_TYPE_INT8,
427+ HcclReduceOp::HCCL_REDUCE_SUM, 0);
428+ 
429+ EXPECT_EQ(handleId, -1);
430+ hccl.Commit(handleId);
431+ EXPECT_EQ(hccl.Wait(handleId), HCCL_FAILED);
432+}
433+ 
395TEST_F(HcclSuiteAIC, ReduceScatter_CcuReduceScatterMeshMem2Mem1D)434TEST_F(HcclSuiteAIC, ReduceScatter_CcuReduceScatterMeshMem2Mem1D)
396{435{
397 std::vector<uint8_t> workSpace(workSpaceSize + 1024 * 100 * 1024);436 std::vector<uint8_t> workSpace(workSpaceSize + 1024 * 100 * 1024);