* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
#include "fusion_c.hpp"
#include "stream_c.hpp"
#include "context.hpp"
#include "error_message_manage.hpp"
#include "task_david.hpp"
#include "thread_local_container.hpp"
#include "inner_thread_local.hpp"
#include "stream.hpp"
#include "task_submit.hpp"
#include "profiler_c.hpp"
#include "aix_c.hpp"
#include "task_execute_time.h"
#include "fusion_task.h"
#include "enum_desc.hpp"
namespace cce {
namespace runtime {
TIMESTAMP_EXTERN(rtFusionLaunch_ArgLoadAll);
TIMESTAMP_EXTERN(rtLaunchKernel_SubMit);
constexpr uint8_t RT_FUSION_CCU_DIE0_BIT_MOVE = 3U;
constexpr uint8_t RT_FUSION_CCU_DIE1_BIT_MOVE = 4U;
constexpr uint8_t RT_MISSION_INDEX_ONE = 1U;
constexpr uint8_t RT_MISSION_INDEX_TWO = 2U;
constexpr uint8_t RT_MISSION_INDEX_THREE = 3U;
constexpr uint8_t RT_MISSION_INDEX_FOUR = 4U;
void AixKernelTaskInitForFusion(
TaskInfo* const taskInfo, const rtAicAivFusionInfo_t* const aicAivInfo,
const LaunchTaskCfgInfo_t* const launchTaskCfg)
{
FusionTaskInfo* fusionTaskInfo = &(taskInfo->u.fusionKernelTask);
FusionTaskInfoAicPart* aicPart = &(fusionTaskInfo->aicPart);
GetAicAivTypeForFusion(aicAivInfo->mixType, aicAivInfo->kernelAttrType, fusionTaskInfo->aicAivType);
aicPart->dim = aicAivInfo->dimNum;
aicPart->kernel = aicAivInfo->kernel;
aicPart->kernelFlag = static_cast<uint8_t>(launchTaskCfg->dumpflag & 0xFFU);
aicPart->qos = launchTaskCfg->qos;
aicPart->partId = launchTaskCfg->partId;
aicPart->schemMode = launchTaskCfg->schemMode;
if (aicPart->schemMode == static_cast<uint8_t>(RT_SCHEM_MODE_END)) {
aicPart->schemMode =
(aicPart->kernel == nullptr) ? static_cast<uint8_t>(RT_SCHEM_MODE_NORMAL) : aicPart->kernel->GetSchedMode();
}
aicPart->dynamicShareMemSize = 0U;
aicPart->simtDcuSmSize = RT_SIMT_UB_SIZE;
aicPart->groupDim = 0U;
aicPart->groupBlockDim = 0U;
aicPart->funcAddr = aicAivInfo->funcAddr;
aicPart->funcAddr1 = aicAivInfo->funcAddr1;
rtArgsSizeInfo_t& argsSize = ThreadLocalContainer::GetArgsSizeInfo();
if (argsSize.infoAddr != nullptr) {
aicPart->inputArgsSize.infoAddr = argsSize.infoAddr;
aicPart->inputArgsSize.atomicIndex = argsSize.atomicIndex;
argsSize.infoAddr = nullptr;
argsSize.atomicIndex = 0U;
RT_LOG(
RT_LOG_INFO, "infoAddr=%p, atomicIndex=%u.", aicPart->inputArgsSize.infoAddr,
aicPart->inputArgsSize.atomicIndex);
} else {
aicPart->inputArgsSize.infoAddr = nullptr;
aicPart->inputArgsSize.atomicIndex = 0U;
}
const uint32_t taskRation = (aicPart->kernel != nullptr) ? aicPart->kernel->GetTaskRation() : 0U;
RT_LOG(
RT_LOG_INFO,
"aicAivType=%hhu, taskRation=%u, kernelFlag=0x%x, qos=%u,"
" partId=%u, schemMode=%u.",
fusionTaskInfo->aicAivType, taskRation, aicPart->kernelFlag, aicPart->qos, aicPart->partId, aicPart->schemMode);
}
void FusionKernelTaskInit(TaskInfo* taskInfo)
{
TaskCommonInfoInit(taskInfo);
FusionTaskInfo* fusionTaskInfo = &(taskInfo->u.fusionKernelTask);
taskInfo->type = TS_TASK_TYPE_FUSION_KERNEL;
taskInfo->typeName = const_cast<char_t*>("FUSION_KERNEL");
fusionTaskInfo->argHandle = nullptr;
fusionTaskInfo->argsInfo = nullptr;
return;
}
static rtError_t CheckFusionDynSizeValid(TaskInfo* taskInfo, const rtFunsionTaskInfo_t* const fusionKernel)
{
if ((taskInfo == nullptr) || (fusionKernel == nullptr)) {
return RT_ERROR_NONE;
}
const uint32_t subKernelNum = fusionKernel->subTaskNum;
for (uint32_t idx = 0; idx < subKernelNum; idx++) {
const uint32_t subKernelType = fusionKernel->subTask[idx].type;
if (subKernelType == RT_FUSION_AICORE) {
FusionTaskInfoAicPart* aicPart = &(taskInfo->u.fusionKernelTask.aicPart);
Kernel* kernel = aicPart->kernel;
if (kernel == nullptr) {
continue;
}
const rtError_t ret = CheckAndGetTotalShareMemorySize(
kernel, taskInfo->stream, aicPart->dynamicShareMemSize, aicPart->simtDcuSmSize);
if (ret != RT_ERROR_NONE) {
return ret;
}
}
}
return RT_ERROR_NONE;
}
static rtError_t PreProcAicAivTaskForFusion(
rtFusionSubTaskInfo_t* const fusionTask, const Stream* const stm, Program*& prog, rtAicAivFusionInfo_t* aicAivInfo,
LaunchTaskCfgInfo_t* launchTaskCfg)
{
rtError_t error = RT_ERROR_NONE;
rtAicoreFusionInfo_t* const aicInfo = &(fusionTask->task.aicoreInfo);
void* const stubFunc = aicInfo->stubFunc;
void* const progHandle = aicInfo->hdl;
Program* realProgram = nullptr;
COND_RETURN_AND_MSG_OUTER(
(stubFunc == nullptr && progHandle == nullptr) || (stubFunc != nullptr && progHandle != nullptr),
RT_ERROR_INVALID_VALUE, ErrorCode::EE1017, "AIC/AIV subtask preprocessing", "stubFunc/progHandle",
"exactly one of stubFunc or progHandle must be non-null");
rtLaunchConfig_t* cfgInfo = aicInfo->config;
NULL_PTR_RETURN_MSG_OUTER_WITH_FUNC_DESC(cfgInfo, RT_ERROR_INVALID_VALUE, "AIC/AIV subtask preprocessing");
NULL_PTR_RETURN_MSG_OUTER_WITH_FUNC_DESC(cfgInfo->attrs, RT_ERROR_INVALID_VALUE, "AIC/AIV subtask preprocessing");
error = GetLaunchConfigInfo(cfgInfo, launchTaskCfg);
COND_RETURN_ERROR_MSG_INNER(
error != RT_ERROR_NONE, error, "Kernel launch GetLaunchConfigInfo failed, stream_id=%d.", stm->Id_());
uint64_t tilingKey = 0ULL;
if (progHandle != nullptr) {
const rtError_t handleRet = GetValidatedObject<Program>(progHandle, realProgram);
ERROR_RETURN(
handleRet, "Failed to validate fusion program handle, retCode=%#x.", static_cast<uint32_t>(handleRet));
tilingKey = aicInfo->tilingKey;
}
const uint32_t coreDim = launchTaskCfg->blockDim;
rtKernelAttrType kernelAttrType = RT_KERNEL_ATTR_TYPE_AICORE;
uint8_t mixType = NO_MIX;
uint64_t addr1 = 0ULL;
uint64_t addr2 = 0ULL;
Module* launchMdl = nullptr;
Kernel* registeredKernel = nullptr;
error = StreamLaunchKernelPrepare(
stm, registeredKernel, prog, kernelAttrType, launchMdl, stubFunc, addr1, addr2, realProgram, tilingKey);
ERROR_RETURN(
error, "Failed to prepare kernel launch, stream_id=%d, retCode=%#x.", stm->Id_(), static_cast<uint32_t>(error));
mixType = registeredKernel->GetMixType();
aicAivInfo->kernelAttrType = kernelAttrType;
aicAivInfo->dimNum = coreDim;
aicAivInfo->mixType = mixType;
aicAivInfo->funcAddr = addr1;
aicAivInfo->funcAddr1 = addr2;
aicAivInfo->kernel = registeredKernel;
aicAivInfo->program = (prog != nullptr) ? prog : realProgram;
RT_LOG(
RT_LOG_INFO,
"kernel info: stream_id=%d, kernelAttrType=%s, kernel_name=%s,"
"mixType=%u, taskRation=%u, tilingKey=%llu.",
stm->Id_(), KernelAttrTypeToString(kernelAttrType).c_str(), registeredKernel->Name_().c_str(), mixType,
registeredKernel->GetTaskRation(), tilingKey);
return RT_ERROR_NONE;
}
static bool CheckCcuMissionIdSingleDie(rtCcuTaskInfo_t* ccuTaskInfo, uint32_t num)
{
if (num == 1) {
return true;
}
return (ccuTaskInfo[num - 1].missionId > ccuTaskInfo[num - 2].missionId) &&
CheckCcuMissionIdSingleDie(ccuTaskInfo, num - 1);
}
static bool CheckCcuMissionIdDoubleDie(rtCcuTaskInfo_t* ccuTaskInfo, uint32_t num)
{
if (num == RT_MISSION_INDEX_TWO) {
return true;
}
return (ccuTaskInfo[num - RT_MISSION_INDEX_ONE].missionId > ccuTaskInfo[num - RT_MISSION_INDEX_THREE].missionId) &&
(ccuTaskInfo[num - RT_MISSION_INDEX_TWO].missionId > ccuTaskInfo[num - RT_MISSION_INDEX_FOUR].missionId) &&
CheckCcuMissionIdDoubleDie(ccuTaskInfo, num - RT_MISSION_INDEX_TWO);
}
static bool CheckCcuMissionId(rtCcuTaskInfo_t* ccuTaskInfo, uint32_t num, bool isDoubleDie)
{
if (isDoubleDie) {
return CheckCcuMissionIdDoubleDie(ccuTaskInfo, num);
}
return CheckCcuMissionIdSingleDie(ccuTaskInfo, num);
}
static rtError_t PreProcCcuTaskForFusion(
rtFusionSubTaskInfo_t* const fusionTask, uint8_t& sqeSubType, uint32_t& sqeLen, uint8_t& ccuArgSize,
Stream* const stm)
{
rtCcuTaskGroup_t* const ccuInfo = &(fusionTask->task.ccuInfo);
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_AND_FUNC_DESC(
ccuInfo->taskNum == 0U || ccuInfo->taskNum > FUSION_SUB_TASK_MAX_CCU_NUM, RT_ERROR_INVALID_VALUE,
"CCU subtask preprocessing", ccuInfo->taskNum, "(0, " + std::to_string(FUSION_SUB_TASK_MAX_CCU_NUM) + "]");
NULL_PTR_RETURN_MSG_OUTER_WITH_FUNC_DESC(ccuInfo->ccuTaskInfo, RT_ERROR_INVALID_VALUE, "CCU subtask preprocessing");
uint8_t isHas128B = 0U;
uint8_t isHas32B = 0U;
uint32_t dieIdArrange = 0U;
uint32_t checkArrange = 0U;
const uint32_t devDieNum = stm->Device_()->GetDevProperties().ioDieNum;
const uint32_t dieNum = (devDieNum != 0U) ? devDieNum : 2U;
for (uint32_t i = 0U; i < ccuInfo->taskNum; i++) {
NULL_PTR_RETURN_MSG_OUTER_WITH_FUNC_DESC(
ccuInfo->ccuTaskInfo[i].args, RT_ERROR_INVALID_VALUE, "CCU subtask preprocessing");
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_AND_FUNC_DESC(
ccuInfo->ccuTaskInfo[i].argSize != RT_CCU_SQE128B_ARGS_SIZE &&
ccuInfo->ccuTaskInfo[i].argSize != RT_CCU_SQE32B_ARGS_SIZE,
RT_ERROR_INVALID_VALUE, "CCU subtask preprocessing", ccuInfo->ccuTaskInfo[i].argSize,
std::to_string(RT_CCU_SQE32B_ARGS_SIZE) + " or " + std::to_string(RT_CCU_SQE128B_ARGS_SIZE));
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_AND_FUNC_DESC(
ccuInfo->ccuTaskInfo[i].dieId >= dieNum, RT_ERROR_INVALID_VALUE, "CCU subtask preprocessing",
static_cast<uint32_t>(ccuInfo->ccuTaskInfo[i].dieId), "[0, " + std::to_string(dieNum) + ")");
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_AND_FUNC_DESC(
ccuInfo->ccuTaskInfo[i].missionId > RT_CCU_MISSION_ID_MAX, RT_ERROR_INVALID_VALUE,
"CCU subtask preprocessing", ccuInfo->ccuTaskInfo[i].missionId,
"[0, " + std::to_string(RT_CCU_MISSION_ID_MAX) + "]");
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_AND_FUNC_DESC(
ccuInfo->ccuTaskInfo[i].instCnt == RT_CCU_INST_CNT_INVALID, RT_ERROR_INVALID_VALUE,
"CCU subtask preprocessing", ccuInfo->ccuTaskInfo[i].instCnt, "not equal to 0");
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_AND_FUNC_DESC(
ccuInfo->ccuTaskInfo[i].instStartId >= RT_CCU_INST_START_MAX, RT_ERROR_INVALID_VALUE,
"CCU subtask preprocessing", ccuInfo->ccuTaskInfo[i].instStartId,
"[0, " + std::to_string(RT_CCU_INST_START_MAX) + ")");
sqeSubType |= (ccuInfo->ccuTaskInfo[i].dieId == 0U) ? (1U << RT_FUSION_CCU_DIE0_BIT_MOVE) :
(1U << RT_FUSION_CCU_DIE1_BIT_MOVE);
dieIdArrange |= static_cast<uint32_t>(ccuInfo->ccuTaskInfo[i].dieId) << i;
checkArrange = ((i & 1U) == 1U) ? checkArrange : ((checkArrange << 2U) | 0x2U);
isHas32B |= ccuInfo->ccuTaskInfo[i].argSize == RT_CCU_SQE32B_ARGS_SIZE ? 1U : 0U;
isHas128B |= ccuInfo->ccuTaskInfo[i].argSize == RT_CCU_SQE128B_ARGS_SIZE ? 1U : 0U;
}
COND_RETURN_AND_MSG_OUTER(
(isHas32B == 1U) && (isHas128B == 1U), RT_ERROR_INVALID_VALUE, ErrorCode::EE1017, "CCU subtask preprocessing",
"argSize", "all CCU subtasks must use the same argSize, either 32B or 128B");
const bool isDoubleDie = (sqeSubType & 0x18U) == 0x18U ? true : false;
uint32_t taskNum = ccuInfo->taskNum;
if (isDoubleDie) {
COND_RETURN_AND_MSG_OUTER(
((taskNum & 1U) == 1U), RT_ERROR_INVALID_VALUE, ErrorCode::EE1011, "CCU subtask preprocessing",
static_cast<uint32_t>(taskNum), "taskNum", "taskNum must be an even number when double die is used");
COND_RETURN_AND_MSG_OUTER(
(dieIdArrange != checkArrange), RT_ERROR_INVALID_VALUE, ErrorCode::EE1017, "CCU subtask preprocessing",
"dieIdArrange", "die IDs must alternate between die 0 and die 1 when double die is used");
taskNum = taskNum / 2U;
}
COND_RETURN_AND_MSG_OUTER(
(isHas32B == 1U) && (taskNum <= 1U), RT_ERROR_INVALID_VALUE, ErrorCode::EE1011, "CCU subtask preprocessing",
static_cast<uint32_t>(taskNum), "taskNum", "taskNum must be greater than 1 when argSize is 32B per die");
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_AND_FUNC_DESC(
(isHas128B == 1U) && (taskNum > 1U), RT_ERROR_INVALID_VALUE, "CCU subtask preprocessing", taskNum, "1");
ccuArgSize = (isHas32B == 1U) ? RT_CCU_SQE32B_ARGS_SIZE : RT_CCU_SQE128B_ARGS_SIZE;
COND_RETURN_AND_MSG_OUTER(
!CheckCcuMissionId(ccuInfo->ccuTaskInfo, ccuInfo->taskNum, isDoubleDie), RT_ERROR_INVALID_VALUE,
ErrorCode::EE1017, "CCU subtask preprocessing", "missionId",
"CCU subtask mission IDs must be strictly increasing");
sqeLen += 4U;
RT_LOG(RT_LOG_INFO, "ccu task info: sqeSubType=%hhu, taskNum=%u.", sqeSubType, ccuInfo->taskNum);
return RT_ERROR_NONE;
}
static void FusionKernelTaskProc(
const rtFunsionTaskInfo_t* const fusionKernel, TaskInfo* const taskInfo,
const rtAicAivFusionInfo_t* const aicAivInfo, const LaunchTaskCfgInfo_t* const launchTaskCfg, uint8_t ccuArgSize)
{
const uint32_t subKernelNum = fusionKernel->subTaskNum;
FusionKernelTaskInit(taskInfo);
for (uint32_t idx = 0; idx < subKernelNum; idx++) {
const rtFusionType_t subKernelType = fusionKernel->subTask[idx].type;
const rtFusionSubTaskInfo_t* const subKernelInfo = &(fusionKernel->subTask[idx]);
switch (subKernelType) {
case RT_FUSION_AICORE:
AixKernelTaskInitForFusion(taskInfo, aicAivInfo, launchTaskCfg);
break;
case RT_FUSION_CCU:
taskInfo->u.fusionKernelTask.ccuSqeNum = subKernelInfo->task.ccuInfo.taskNum;
taskInfo->u.fusionKernelTask.ccuArgSize = ccuArgSize;
break;
default:
RT_LOG(RT_LOG_INFO, "sub task [%u]'s type=%u.", idx, static_cast<uint32_t>(subKernelType));
break;
}
}
}
static rtError_t FusionKernelTaskPreProc(
rtFunsionTaskInfo_t* const fusionKernel, Stream* const stm, Program*& prog, uint32_t& sqeLen, uint8_t& sqeSubType,
uint8_t& ccuArgSize, rtAicAivFusionInfo_t* aicAivInfo, LaunchTaskCfgInfo_t* launchTaskCfg)
{
rtError_t error = RT_ERROR_NONE;
const uint32_t subKernelNum = fusionKernel->subTaskNum;
std::array<uint32_t, RT_FUSION_CCU> fusionSubTaskMove = {
RT_FUSION_HCOMCPU_BIT_MOVE, RT_FUSION_AICPU_BIT_MOVE, RT_FUSION_AIC_BIT_MOVE};
for (uint32_t idx = 0; idx < subKernelNum; idx++) {
uint32_t subKernelType = fusionKernel->subTask[idx].type;
rtFusionSubTaskInfo_t* const subKernelInfo = &(fusionKernel->subTask[idx]);
switch (subKernelType) {
case RT_FUSION_AICORE:
error = PreProcAicAivTaskForFusion(subKernelInfo, stm, prog, aicAivInfo, launchTaskCfg);
ERROR_RETURN(
error, "stream_id=%d failed to proc aic/aiv sub task, retCode=%#x.", stm->Id_(),
static_cast<uint32_t>(error));
sqeSubType |= (1U << fusionSubTaskMove[subKernelType]);
sqeLen += 1U;
break;
case RT_FUSION_HCOM_CPU:
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_AND_FUNC_DESC(
fusionKernel->subTask[idx].task.aicpuInfo.blockDim != 1U, RT_ERROR_INVALID_VALUE,
"Fusion kernel task preprocessing", fusionKernel->subTask[idx].task.aicpuInfo.blockDim, "1");
subKernelType += (stm->Device_()->IsSupportHcomcpu() == 1U) ? 0U : 1U;
sqeSubType |= (1U << fusionSubTaskMove[subKernelType]);
sqeLen += 1U;
break;
case RT_FUSION_AICPU:
sqeSubType |= (1U << FUSION_SUBTASK_MOVE[subKernelType]);
sqeLen += 1U;
break;
case RT_FUSION_CCU:
error = PreProcCcuTaskForFusion(subKernelInfo, sqeSubType, sqeLen, ccuArgSize, stm);
ERROR_RETURN(
error, "stream_id=%d failed to proc ccu sub task, retCode=%#x", stm->Id_(),
static_cast<uint32_t>(error));
COND_RETURN_AND_MSG_OUTER(
(subKernelNum == 1U) && ((sqeSubType != 0x18U) || (ccuArgSize != RT_CCU_SQE128B_ARGS_SIZE)),
RT_ERROR_INVALID_VALUE, ErrorCode::EE1017, "Fusion kernel task preprocessing", "ccuArgSize",
"CCU-only fusion task requires double die with 128B argSize");
break;
default:
RT_LOG_OUTER_MSG_WITH_FUNC_DESC(
ErrorCode::EE1003, "Fusion kernel task preprocessing",
RtFmtMsg("%s(%u)", (subKernelType == RT_FUSION_END) ? "FUSION_END" : "UNKNOWN", subKernelType),
"subKernelType", "[0, 3]");
return RT_ERROR_INVALID_VALUE;
}
}
RT_LOG(RT_LOG_INFO, "sqeLen=%hhu, sqeSubType=%hhu, stream_id=%d.", sqeLen, sqeSubType, stm->Id_());
return error;
}
static void SetArgsFusionKernel(
const rtFusionArgsEx_t* const argsInfo, TaskInfo* const taskInfo, StarsArgLoaderResult* const result)
{
FusionTaskInfo* const fusionKernelTask = &(taskInfo->u.fusionKernelTask);
fusionKernelTask->argsInfo = const_cast<rtFusionArgsEx_t*>(argsInfo);
fusionKernelTask->argsSize = argsInfo->argsSize;
fusionKernelTask->args = result->kerArgs;
const uint8_t aicpuNum = argsInfo->aicpuNum;
for (uint8_t i = 0U; i < aicpuNum; ++i) {
fusionKernelTask->aicpuArgsDesc[i].soName =
static_cast<void*>(RtPtrToPtr<char_t*>(fusionKernelTask->args) + argsInfo->aicpuArgs[i].soNameAddrOffset);
fusionKernelTask->aicpuArgsDesc[i].funcName = static_cast<void*>(
RtPtrToPtr<char_t*>(fusionKernelTask->args) + argsInfo->aicpuArgs[i].kernelNameAddrOffset);
}
fusionKernelTask->oldArgHandle = fusionKernelTask->argHandle;
if (result->handle != nullptr) {
fusionKernelTask->argHandle = result->handle;
taskInfo->needPostProc = true;
}
taskInfo->stmArgPos = static_cast<DavidStream*>(taskInfo->stream)->GetArgPos();
result->stmArgPos = UINT32_MAX;
result->handle = nullptr;
}
static rtError_t CheckUpdateFusionTaskInfo(
const TaskInfo* const updateTask, uint32_t sqeLen, uint8_t sqeSubType, const Kernel* const kernel,
uint8_t kernelType, const Stream* const stm)
{
COND_RETURN_AND_MSG_OUTER(
updateTask->stream->Model_() == nullptr, RT_ERROR_MODEL_NULL, ErrorCode::EE1017,
"Checking and updating fusion task information", "task", "The update task must be a sinked task");
COND_RETURN_AND_MSG_OUTER(
stm->Model_() != nullptr, RT_ERROR_STREAM_MODEL, ErrorCode::EE1017,
"Checking and updating fusion task information", "stream",
RtFmtMsg(
"The update stream (stream_id=%d) must be a single operator stream, not bound to a model (model_id=%u)",
stm->Id_(), stm->Model_()->Id_()));
COND_RETURN_AND_MSG_OUTER(
updateTask->u.fusionKernelTask.sqeLen != sqeLen, RT_ERROR_INVALID_VALUE, ErrorCode::EE1011,
"Checking and updating fusion task information", static_cast<uint32_t>(sqeLen), "sqeLen",
"sqeLen must match the original fusion task's sqeLen (" +
std::to_string(updateTask->u.fusionKernelTask.sqeLen) + ")");
COND_RETURN_AND_MSG_OUTER(
(sqeSubType & (1U << RT_FUSION_AIC_BIT_MOVE)) == 0U, RT_ERROR_INVALID_VALUE, ErrorCode::EE1017,
"Checking and updating fusion task information", "sqeSubType",
"The update subtask must include the AI Core task");
COND_RETURN_AND_MSG_OUTER(
(updateTask->u.fusionKernelTask.aicPart.kernel->GetMixType() != kernel->GetMixType()) ||
(updateTask->u.fusionKernelTask.aicPart.kernel->GetFuncType() != kernel->GetFuncType()) ||
(updateTask->u.fusionKernelTask.aicAivType != kernelType),
RT_ERROR_KERNEL_TYPE, ErrorCode::EE1017, "Checking and updating fusion task information", "kernelType",
"kernel type (mixType, funcType, machine) must match the original fusion task");
return RT_ERROR_NONE;
}
void GetAicAivTypeForFusion(uint8_t mixType, rtKernelAttrType kernelAttrType, uint8_t& aicAivType)
{
if (mixType != static_cast<uint8_t>(NO_MIX)) {
if ((mixType == static_cast<uint8_t>(MIX_AIC)) || (mixType == static_cast<uint8_t>(MIX_AIC_AIV_MAIN_AIC))) {
aicAivType = 0U;
} else {
aicAivType = 1U;
}
} else {
if (kernelAttrType == RT_KERNEL_ATTR_TYPE_VECTOR) {
aicAivType = 1U;
} else {
aicAivType = 0U;
}
}
}
rtError_t LaunchFusionKernel(Stream* stm, void* const fusionKernelInfo, rtFusionArgsEx_t* argsInfo)
{
StarsArgLoaderResult result = {nullptr, nullptr, nullptr, UINT32_MAX, nullptr, nullptr};
Program* prog = nullptr;
Runtime* const rt = Runtime::Instance();
rtFunsionTaskInfo_t* fusionKernel = static_cast<rtFunsionTaskInfo_t*>(fusionKernelInfo);
uint32_t sqeLen = 0U;
uint8_t sqeSubType = 0U;
uint8_t ccuArgSize = 0U;
rtAicAivFusionInfo_t aicAivInfo = {};
LaunchTaskCfgInfo_t launchTaskCfg = {};
rtError_t error =
FusionKernelTaskPreProc(fusionKernel, stm, prog, sqeLen, sqeSubType, ccuArgSize, &aicAivInfo, &launchTaskCfg);
ERROR_PROC_RETURN_MSG_INNER(error, rt->PutProgram(prog);
, "Failed to pre proc fusion kernel task, retCode=%#x.", static_cast<uint32_t>(error));
if (sqeLen > SQE_NUM_PER_DAVID_TASK_MAX) {
rt->PutProgram(prog);
RT_LOG_OUTER_MSG_INVALID_PARAM_WITH_DESC(
"Delivering a kernel fusion task", sqeLen, "(0, " + std::to_string(SQE_NUM_PER_DAVID_TASK_MAX) + "]");
return RT_ERROR_INVALID_VALUE;
}
error = CheckTaskCanSend(stm);
ERROR_PROC_RETURN_MSG_INNER(error, rt->PutProgram(prog);
, "stream_id=%d check failed, retCode=%#x.", stm->Id_(), static_cast<uint32_t>(error));
DavidStream* davidStm = static_cast<DavidStream*>(stm);
const bool useArgPool = UseArgsPool(davidStm, argsInfo, stm->IsTaskGroupUpdate());
uint32_t pos = 0xFFFFU;
Stream* dstStm = stm;
TaskInfo* taskInfo = nullptr;
std::function<void()> const errRecycle = [&result, &taskInfo, &stm, &pos, &prog, &dstStm]() {
StreamLaunchKernelRecycle(result, taskInfo, prog, stm);
TaskRollBack(dstStm, pos);
stm->StreamUnLock();
};
stm->StreamLock();
taskInfo = stm->AllocTask(nullptr, TS_TASK_TYPE_FUSION_KERNEL, error, sqeLen, UpdateTaskFlag::SUPPORT);
COND_PROC_RETURN_ERROR_MSG_INNER(taskInfo == nullptr, error, stm->StreamUnLock(); rt->PutProgram(prog);
, "stream_id=%d alloc task failed, retCode=%#x.", stm->Id_(),
static_cast<uint32_t>(error));
ScopeGuard tskErrRecycle(errRecycle);
if (taskInfo->isUpdateSinkSqe == 1U) {
uint8_t aicAivType = 0U;
GetAicAivTypeForFusion(aicAivInfo.mixType, aicAivInfo.kernelAttrType, aicAivType);
error = CheckUpdateFusionTaskInfo(taskInfo, sqeLen, sqeSubType, aicAivInfo.kernel, aicAivType, stm);
ERROR_RETURN(
error, "Failed to check update fusion task info, stream_id=%d, kernelType=%u, retCode=%#x.", stm->Id_(),
aicAivType, static_cast<uint32_t>(error));
} else {
pos = taskInfo->id;
dstStm = taskInfo->stream;
}
FusionKernelTaskProc(fusionKernel, taskInfo, &aicAivInfo, &launchTaskCfg, ccuArgSize);
error = CheckFusionDynSizeValid(taskInfo, fusionKernel);
ERROR_RETURN_MSG_INNER(
error, "Failed to check SimtSmSize, stream_id=%d, retCode=%#x.", stm->Id_(), static_cast<uint32_t>(error));
if (argsInfo != nullptr) {
error = static_cast<DavidStream*>(dstStm)->LoadArgsInfo(argsInfo, useArgPool, &result);
ERROR_RETURN_MSG_INNER(
error, "Failed to load args with stm pool, stream_id=%d, useArgPool=%u, retCode=%#x.", stm->Id_(),
useArgPool, static_cast<uint32_t>(error));
SetArgsFusionKernel(argsInfo, taskInfo, &result);
}
taskInfo->stmArgPos = static_cast<DavidStream*>(dstStm)->GetArgPos();
taskInfo->u.fusionKernelTask.fusionKernelInfo = const_cast<void*>(fusionKernelInfo);
taskInfo->u.fusionKernelTask.sqeLen = sqeLen;
taskInfo->u.fusionKernelTask.sqeSubType = sqeSubType;
RT_LOG(RT_LOG_INFO, "sqeLen=%hhu, sqeSubType=%hhu, stream_id=%d.", sqeLen, sqeSubType, stm->Id_());
if (taskInfo->isUpdateSinkSqe == 1U) {
error = UpdateDavidKernelTaskSubmit(taskInfo, stm, sqeLen);
if ((error != RT_ERROR_NONE) && (taskInfo->u.fusionKernelTask.oldArgHandle != nullptr)) {
if (davidStm->ArgManagePtr() != nullptr) {
davidStm->ArgManagePtr()->RecycleDevLoader(taskInfo->u.fusionKernelTask.oldArgHandle);
}
}
taskInfo->u.fusionKernelTask.oldArgHandle = nullptr;
taskInfo->isUpdateSinkSqe = 0U;
} else {
error = DavidSendTask(taskInfo, dstStm);
}
ERROR_RETURN_MSG_INNER(
error, "Submit task failed, stream_id=%d, retCode=%#x.", stm->Id_(), static_cast<uint32_t>(error));
tskErrRecycle.ReleaseGuard();
stm->StreamUnLock();
rt->PutProgram(prog);
SET_THREAD_TASKID_AND_STREAMID(dstStm->GetExposedStreamId(), taskInfo->taskSn);
error = SubmitTaskPostProc(dstStm, pos);
ERROR_RETURN_MSG_INNER(
error, "Post-processing of task submission failed, stream_id=%d, retCode=%#x.", stm->Id_(),
static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
}
}
