* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
#include "api_impl_david.hpp"
#include "memory_task.h"
#include "event_task.h"
#include "ccu_stream.hpp"
#include "runtime_handle_guard.h"
#include "context.hpp"
#include "context_manage.hpp"
#include "capture_func.hpp"
#include "stream_c.hpp"
#include "aix_c.hpp"
#include "aicpu_c.hpp"
#include "fusion_c.hpp"
#include "dvpp_c.hpp"
#include "event_c.hpp"
#include "ipc_event.hpp"
#include "memcpy_c.hpp"
#include "memory_c.hpp"
#include "notify_c.hpp"
#include "count_notify.hpp"
#include "event_david.hpp"
#include "event_state_callback_manager.hpp"
#include "model_c.hpp"
#include "cond_c.hpp"
#include "cond_enum_desc.hpp"
#include "label_c.hpp"
#include "label.hpp"
#include "cmo_barrier_c.hpp"
#include "profiler_c.hpp"
#include "device_debug_c.hpp"
#include "thread_local_container.hpp"
#include "inner_thread_local.hpp"
#include "device_msg_handler.hpp"
#include "task_david.hpp"
#include "task_recycle.hpp"
#include "fast_recover.hpp"
#include "device/device_error_info.hpp"
#include "capture_model_utils.hpp"
#include "capture_adapt.hpp"
#include "base_david.hpp"
#include "common_task.h"
#include "args_handle_allocator.hpp"
#include "para_convertor.hpp"
#include "runtime/kernel.h"
#include "starsv2_base.hpp"
#include "utils.h"
#include "api_handle_guard.h"
#include "error_message_manage.hpp"
#include "stream_launch_blocking.hpp"
#include "capability.hpp"
#include "notify_enum_desc.hpp"
#include "task.hpp"
#include "context_data_manage.h"
#include "enum_desc.hpp"
namespace cce {
namespace runtime {
rtError_t ApiImplDavid::KernelLaunch(
const void* const stubFunc, const uint32_t coreDim, const rtArgsEx_t* const argsInfo, Stream* const stm,
const rtTaskCfgInfo_t* const cfgInfo, const bool isLaunchVec)
{
COND_RETURN_WARN(
isLaunchVec, RT_ERROR_FEATURE_NOT_SUPPORT, "chip type(%d) does not support.",
static_cast<int32_t>(Runtime::Instance()->GetChipType()));
RT_LOG(RT_LOG_DEBUG, "Launch kernel, stubFunc=%p, blockDim=%u.", stubFunc, coreDim);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Starting the compute task of the corresponding operator");
if ((cfgInfo != nullptr) && ((cfgInfo->dumpflag & RT_KERNEL_DUMPFLAG) != 0U)) {
ERROR_RETURN_MSG_INNER(
Runtime::Instance()->StartAicpuSd(curCtx->Device_()),
"kernel launch with kernel dump flag failed, check and start tsd open aicpu sd error.");
}
TaskCfg taskCfg = {};
(void)ConvertTaskCfgInfoToTaskCfg(taskCfg, cfgInfo);
return StreamLaunchKernelV1(stubFunc, coreDim, argsInfo, curStm, &taskCfg, isLaunchVec);
}
rtError_t ApiImplDavid::KernelLaunchWithHandle(
void* const hdl, const uint64_t tilingKey, const uint32_t coreDim, const rtArgsEx_t* const argsInfo,
Stream* const stm, const rtTaskCfgInfo_t* const cfgInfo, const bool isLaunchVec)
{
COND_RETURN_WARN(
isLaunchVec, RT_ERROR_FEATURE_NOT_SUPPORT, "chip type(%d) does not support.",
static_cast<int32_t>(Runtime::Instance()->GetChipType()));
RT_LOG(RT_LOG_DEBUG, "Launch kernel with hdl, blockDim=%u, tilingKey=%" PRIu64, coreDim, tilingKey);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Starting the compute task of the corresponding operator");
TaskCfg taskCfg = {};
(void)ConvertTaskCfgInfoToTaskCfg(taskCfg, cfgInfo);
return StreamLaunchKernelWithHandle(hdl, tilingKey, coreDim, argsInfo, curStm, &taskCfg, isLaunchVec);
}
rtError_t ApiImplDavid::LaunchKernel(
Kernel* const kernel, uint32_t blockDim, const rtArgsEx_t* const argsInfo, Stream* const stm,
const rtTaskCfgInfo_t* const cfgInfo)
{
RT_LOG(RT_LOG_DEBUG, "Launch kernel, blockDim=%u", blockDim);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Starting the compute task of the corresponding operator");
if (!kernel->Program_()->IsDeviceSoAndNameValid(curCtx->Device_()->Id_())) {
RT_LOG(RT_LOG_WARNING, "kernel is invalid, device_id=%d", curCtx->Device_()->Id_());
return RT_ERROR_KERNEL_INVALID;
}
TaskCfg taskCfg = {};
(void)ConvertTaskCfgInfoToTaskCfg(taskCfg, cfgInfo);
rtStreamLaunchKernelV2ExtendArgs_t launchKernelExtendArgs = {};
launchKernelExtendArgs.argsInfo = argsInfo;
launchKernelExtendArgs.taskCfg = &taskCfg;
const rtError_t error = StreamLaunchKernelV2(kernel, blockDim, curStm, &launchKernelExtendArgs);
if ((error == RT_ERROR_NONE) && StreamLaunchBlocking::ShouldLaunchBlock(curStm)) {
return curStm->Synchronize(false);
}
return error;
}
rtError_t ApiImplDavid::CpuKernelLaunchExAll(
const Kernel* const kernel, const uint32_t coreDim, rtCpuKernelArgs_t* argsInfo, Stream* const stm,
const TaskCfg* const taskCfg)
{
uint32_t flag = RT_KERNEL_DEFAULT;
if (taskCfg->isBaseValid == 1U) {
flag |= taskCfg->base.dumpflag;
}
const uint32_t kernelType = kernel->GetAicpuKernelType_();
COND_PROC_RETURN_AND_MSG_OUTER(
(kernelType != KERNEL_TYPE_FWK) && (kernelType != KERNEL_TYPE_AICPU) &&
(kernelType != KERNEL_TYPE_AICPU_CUSTOM) && (kernelType != KERNEL_TYPE_AICPU_KFC),
RT_ERROR_KERNEL_TYPE, ErrorCode::EE1011,
RT_LOG(RT_LOG_ERROR, "kernel type mismatch kernelType=UNKNOWN(%u).", kernelType),
"Starting the compute task of an AI CPU operator", RtFmtMsg("%u", kernelType), "kernelType",
RtFmtMsg(
"kernelType must be KERNEL_TYPE_FWK(%u), KERNEL_TYPE_AICPU(%u), KERNEL_TYPE_AICPU_CUSTOM(%u), or "
"KERNEL_TYPE_AICPU_KFC(%u)",
static_cast<uint32_t>(KERNEL_TYPE_FWK), static_cast<uint32_t>(KERNEL_TYPE_AICPU),
static_cast<uint32_t>(KERNEL_TYPE_AICPU_CUSTOM), static_cast<uint32_t>(KERNEL_TYPE_AICPU_KFC)));
const rtError_t error = StreamLaunchCpuKernelExWithArgs(
coreDim, static_cast<const rtAicpuArgsEx_t*>(&argsInfo->baseArgs), taskCfg, stm, flag, kernelType, kernel,
argsInfo->cpuParamHeadOffset);
ERROR_RETURN_MSG_INNER(
error, "Cpu kernel launch ex with args failed, check and start tsd open aicpu sd error=%#x.", error);
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::LaunchKernelByHandle(
Kernel* const kernel, uint32_t blockDim, const RtArgsHandle* const argHandle, Stream* const curStm,
const TaskCfg& taskCfg)
{
COND_RETURN_ERROR(argHandle == nullptr, RT_ERROR_INVALID_VALUE, "args handle is nullptr");
rtError_t error = RT_ERROR_NONE;
const KernelRegisterType regType = kernel->GetKernelRegisterType();
const uint8_t phNum = argHandle->placeHolderNum;
if (regType == RT_KERNEL_REG_TYPE_CPU) {
rtCpuKernelArgs_t cpuKernelArgs = {};
if (phNum <= SPECIAL_ARGS_MAX_CNT) {
rtHostInputInfo_t hostArgsInfos[SPECIAL_ARGS_MAX_CNT] = {};
error = ConvertCpuArgsByArgsHandle(cpuKernelArgs, argHandle, hostArgsInfos, SPECIAL_ARGS_MAX_CNT);
ERROR_RETURN_MSG_INNER(error, "convert args failed, error=%#x", static_cast<uint32_t>(error));
return CpuKernelLaunchExAll(kernel, blockDim, &cpuKernelArgs, curStm, &taskCfg);
}
rtHostInputInfo_t* hostArgsInfos = new (std::nothrow) rtHostInputInfo_t[phNum];
COND_RETURN_AND_MSG_OUTER(
hostArgsInfos == nullptr, RT_ERROR_MEMORY_ALLOCATION, ErrorCode::EE1013,
std::to_string(sizeof(rtHostInputInfo_t) * phNum), "new");
error = ConvertCpuArgsByArgsHandle(cpuKernelArgs, argHandle, hostArgsInfos, phNum);
COND_PROC_RETURN_ERROR(error != RT_ERROR_NONE, error, DELETE_A(hostArgsInfos), "convert args failed.");
error = CpuKernelLaunchExAll(kernel, blockDim, &cpuKernelArgs, curStm, &taskCfg);
DELETE_A(hostArgsInfos);
ERROR_RETURN_MSG_INNER(error, "launch kernel failed, error=%#x", static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
rtArgsEx_t argsInfo = {};
if (phNum <= SPECIAL_ARGS_MAX_CNT) {
rtHostInputInfo_t specialArgsInfos[SPECIAL_ARGS_MAX_CNT];
error = ConvertArgsByArgsHandle(argsInfo, argHandle, specialArgsInfos, SPECIAL_ARGS_MAX_CNT);
ERROR_RETURN_MSG_INNER(error, "convert args failed, error=%#x", error);
rtStreamLaunchKernelV2ExtendArgs_t launchKernelExtendArgs = {};
launchKernelExtendArgs.argsInfo = &argsInfo;
launchKernelExtendArgs.taskCfg = &taskCfg;
return StreamLaunchKernelV2(kernel, blockDim, curStm, &launchKernelExtendArgs);
}
rtHostInputInfo_t* hostArgsInfos = new (std::nothrow) rtHostInputInfo_t[phNum];
COND_RETURN_AND_MSG_OUTER(
hostArgsInfos == nullptr, RT_ERROR_MEMORY_ALLOCATION, ErrorCode::EE1013,
std::to_string(sizeof(rtHostInputInfo_t) * phNum), "new");
error = ConvertArgsByArgsHandle(argsInfo, argHandle, hostArgsInfos, phNum);
COND_PROC_RETURN_ERROR(error != RT_ERROR_NONE, error, DELETE_A(hostArgsInfos), "convert args failed.");
rtStreamLaunchKernelV2ExtendArgs_t launchKernelExtendArgs = {};
launchKernelExtendArgs.argsInfo = &argsInfo;
launchKernelExtendArgs.taskCfg = &taskCfg;
error = StreamLaunchKernelV2(kernel, blockDim, curStm, &launchKernelExtendArgs);
DELETE_A(hostArgsInfos);
ERROR_RETURN_MSG_INNER(error, "launch kernel failed, error=%#x", error);
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::KernelLaunchEx(
const char_t* const opName, const void* const args, const uint32_t argsSize, const uint32_t flags,
Stream* const stm)
{
RT_LOG(RT_LOG_DEBUG, "Launch kernel ex, opName=%s, argsSize=%u, flags=%u.", opName, argsSize, flags);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Starting the compute task of the corresponding operator");
Runtime* const rtInstance = Runtime::Instance();
COND_RETURN_ERROR(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
ERROR_RETURN_MSG_INNER(
rtInstance->StartAicpuSd(curCtx->Device_()),
"Cpu kernel launch ex with args failed, check and start tsd open aicpu sd error.");
return StreamLaunchKernelEx(args, argsSize, flags, curStm);
}
rtError_t ApiImplDavid::CpuKernelLaunch(
const rtKernelLaunchNames_t* const launchNames, const uint32_t coreDim, const rtArgsEx_t* const argsInfo,
Stream* const stm, const uint32_t flag)
{
RT_LOG(
RT_LOG_DEBUG,
"Launch cpu kernel, soName=%s, kernelName=%s, opName=%s, blockDim=%u, argsSize=%u, "
"hostInputInfoNum=%hu, flag=%u.",
launchNames->soName, launchNames->kernelName, launchNames->opName, coreDim, argsInfo->argsSize,
argsInfo->hostInputInfoNum, flag);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Starting the compute task of an AI CPU operator");
Runtime* const rtInstance = Runtime::Instance();
COND_RETURN_ERROR_MSG_INNER(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
ERROR_RETURN_MSG_INNER(
rtInstance->StartAicpuSd(curCtx->Device_()),
"Cpu kernel launch failed, check and start tsd open aicpu sd error.");
return StreamLaunchCpuKernel(launchNames, coreDim, argsInfo, curStm, flag);
}
rtError_t ApiImplDavid::CpuKernelLaunchExWithArgs(
const char_t* const opName, const uint32_t coreDim, const rtAicpuArgsEx_t* const argsInfo, Stream* const stm,
const uint32_t flag, const uint32_t kernelType)
{
RT_LOG(
RT_LOG_DEBUG,
"Launch cpu kernel ex, opName=%s, blockDim=%u, argsSize=%u, hostInputInfoNum=%hu, "
"flag=%u, kernelType=%u, isNoNeedH2DCopy=%u.",
opName, coreDim, argsInfo->argsSize, argsInfo->hostInputInfoNum, flag, kernelType, argsInfo->isNoNeedH2DCopy);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Starting the compute task of an AI CPU operator");
Runtime* const rtInstance = Runtime::Instance();
COND_RETURN_ERROR_MSG_INNER(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
ERROR_RETURN_MSG_INNER(
rtInstance->StartAicpuSd(curCtx->Device_()),
"Cpu kernel launch failed, check and start tsd open aicpu sd error.");
return StreamLaunchCpuKernelExWithArgs(coreDim, argsInfo, nullptr, curStm, flag, kernelType, nullptr);
}
rtError_t ApiImplDavid::FusionLaunch(void* const fusionInfo, Stream* const stm, rtFusionArgsEx_t* argsInfo)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Fused operator task delivery");
return LaunchFusionKernel(curStm, fusionInfo, argsInfo);
}
rtError_t ApiImplDavid::CCULaunch(rtCcuTaskInfo_t* taskInfo, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "CCU task delivery");
return StreamCCULaunch(curStm, taskInfo);
}
rtError_t ApiImplDavid::UbDevQueryInfo(rtUbDevQueryCmd cmd, void* devInfo)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
NULL_PTR_RETURN_MSG(dev, RT_ERROR_DEVICE_NULL);
return NpuDriver::QueryUbInfo(dev->Id_(), cmd, devInfo);
}
rtError_t ApiImplDavid::GetDevResAddress(const rtDevResInfo* const resInfo, rtDevResAddrInfo* const addrInfo)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
NULL_PTR_RETURN_MSG(dev, RT_ERROR_DEVICE_NULL);
uint64_t resAddr = 0U;
uint32_t len = 0U;
const rtError_t error = NpuDriver::GetDevResAddress(dev->Id_(), resInfo, &resAddr, &len);
if (error == RT_ERROR_NONE) {
*(addrInfo->resAddress) = resAddr;
*(addrInfo->len) = len;
}
return error;
}
rtError_t ApiImplDavid::ReleaseDevResAddress(rtDevResInfo* const resInfo)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
NULL_PTR_RETURN_MSG(dev, RT_ERROR_DEVICE_NULL);
return NpuDriver::ReleaseDevResAddress(dev->Id_(), resInfo);
}
rtError_t ApiImplDavid::CmoTaskLaunch(const rtCmoTaskInfo_t* const taskInfo, Stream* const stm, const uint32_t flag)
{
RT_LOG(RT_LOG_DEBUG, "Cmo task launch, opCode=%hu, lengthInner=%u", taskInfo->opCode, taskInfo->lengthInner);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Operating the cache memory on the device");
return cce::runtime::CmoTaskLaunch(taskInfo, curStm, flag);
}
rtError_t ApiImplDavid::CmoAddrTaskLaunch(
void* cmoAddrInfo, const uint64_t destMax, const rtCmoOpCode_t cmoOpCode, Stream* const stm, const uint32_t flag)
{
RT_LOG(RT_LOG_DEBUG, "Cmo addr task launch, opCode=%s.", CmoOpCodeToString(cmoOpCode).c_str());
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT,
"Using the memory descriptor to operate the cache memory on the device");
return StreamCmoAddrTaskLaunch(cmoAddrInfo, destMax, cmoOpCode, curStm, flag);
}
rtError_t ApiImplDavid::EventCreate(Event** const evt, const uint64_t flag)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
*evt = new (std::nothrow) DavidEvent(dev, flag, curCtx);
COND_RETURN_AND_MSG_OUTER(*evt == nullptr, RT_ERROR_EVENT_NEW, ErrorCode::EE1013, sizeof(DavidEvent), "new");
if (flag != RT_EVENT_DEFAULT) {
const rtError_t error = (*evt)->GenEventId();
COND_PROC_RETURN_ERROR(error != RT_ERROR_NONE, error, DELETE_O(*evt);
, "Gen event id failed, device_id=%u, tsId=%u, retCode=%#x", dev->Id_(),
dev->DevGetTsId(), static_cast<uint32_t>(error));
}
InitEmbeddedInnerHandle<Event>(*evt);
dev->PushEvent(*evt);
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::EventCreateEx(Event** const evt, const uint64_t flag)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
if (flag == RT_EVENT_IPC) {
*evt = new (std::nothrow) IpcEvent(dev, flag, curCtx);
COND_RETURN_AND_MSG_OUTER(
*evt == nullptr, RT_ERROR_EVENT_NEW, ErrorCode::EE1013, std::to_string(sizeof(IpcEvent)), "new");
const rtError_t error = (*evt)->Setup();
COND_PROC_RETURN_ERROR(error != RT_ERROR_NONE, error, DELETE_O(*evt);, "setup failed, retCode=%#x", error);
} else {
*evt = new (std::nothrow) DavidEvent(dev, flag, curCtx, true);
COND_RETURN_AND_MSG_OUTER(
*evt == nullptr, RT_ERROR_EVENT_NEW, ErrorCode::EE1013, std::to_string(sizeof(DavidEvent)), "new");
}
InitEmbeddedInnerHandle<Event>(*evt);
dev->PushEvent(*evt);
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::EventDestroy(Event* evt)
{
EventStateCallbackManager::Instance().Notify(nullptr, evt, EventStatePeriod::EVENT_STATE_PERIOD_DESTROY);
ResetEmbeddedInnerHandle<Event>(evt);
if (evt->GetEventFlag() == RT_EVENT_IPC) {
IpcEvent* eventIpc = dynamic_cast<IpcEvent*>(evt);
IpcEventDestroy(&eventIpc, MAX_INT32_NUM, true);
} else {
RT_LOG(RT_LOG_INFO, "event destroy event_id=%d.", evt->EventId_());
TryToFreeEventIdAndDestroyEvent(&evt, evt->EventId_(), true);
}
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::EventRecord(Event* const evt, Stream* const stm, const uint32_t flag)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* const curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(curStm);
const rtError_t checkRet = CheckEventAndStreamDevice(evt, curStm, "Event recording");
if (checkRet != RT_ERROR_NONE) {
return checkRet;
}
const bool supportFlag = (evt->IsNewMode() || (evt->GetEventFlag() == RT_EVENT_DEFAULT)) && curStm->IsModelStream();
COND_RETURN_WARN(
supportFlag, RT_ERROR_FEATURE_NOT_SUPPORT,
"Event record is not supported on a model-bound stream when the event was created in Ex mode "
"(for example, via aclrtCreateEventExWithFlag or rtEventCreateExWithFlag) or "
"eventFlag is RT_EVENT_DEFAULT, isNewMode=%d, eventFlag=%#" PRIx64 ", isModelStream=%d.",
evt->IsNewMode(), evt->GetEventFlag(), curStm->IsModelStream());
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Event recording");
if (flag == RT_EVENT_RECORD_EXTERNAL) {
COND_RETURN_AND_MSG_OUTER(
(!curStm->IsCapturing()), RT_ERROR_STREAM_NOT_CAPTURED, ErrorCode::EE1016, "Event recording",
RtFmtMsg("Stream %d is not in the capture stage", curStm->Id_()));
const rtError_t supportRet = CheckCaptureModelSupportExternalEvent(curStm->Device_(), true);
if (supportRet != RT_ERROR_NONE) {
return supportRet;
}
return Starsv2CaptureExternalEventRecord(evt, curStm);
}
if (evt->ToBeCaptured(curStm)) {
COND_RETURN_WARN(
!evt->IsNewMode(), RT_ERROR_FEATURE_NOT_SUPPORT,
"Not support call rtEventCreate or rtEventCreateWithFlag without external flag, mode=%d", evt->IsNewMode());
const char_t* const unsupportedFlag = GetUnsupportedCaptureStreamFlag(curStm->Flags());
COND_RETURN_AND_MSG_OUTER(
unsupportedFlag != nullptr, RT_ERROR_STREAM_INVALID, ErrorCode::EE1006, "Event recording",
RtFmtMsg("Stream flag value %#x", curStm->Flags()),
RtFmtMsg(
"Stream (stream_id=%d) with the flag %s cannot be used for ACL Graph", curStm->Id_(), unsupportedFlag));
COND_RETURN_AND_MSG_OUTER(
curStm == curCtx->DefaultStream_(), RT_ERROR_STREAM_CAPTURE_IMPLICIT, ErrorCode::EE1016, "Event recording",
RtFmtMsg("The default stream (stream_id=%d) cannot be used in the ACL Graph", curStm->Id_()));
COND_RETURN_WARN(
evt->IsEventWithoutWaitTask(), RT_ERROR_NONE,
"The event flag %" PRIu64 " is not supported in capture mode.", evt->GetEventFlag());
const std::lock_guard<std::mutex> lk(curCtx->GetCaptureLock());
if (evt->ToBeCaptured(curStm)) {
const rtError_t retCode = CaptureRecordEvent(curCtx, evt, curStm);
ERROR_PROC_RETURN_MSG_INNER(retCode, TerminateCapture(evt, curStm), "Capture event record failed.");
return RT_ERROR_NONE;
}
}
if (evt->GetEventFlag() == RT_EVENT_IPC) {
return (dynamic_cast<IpcEvent*>(evt))->IpcEventRecordStarsV2(curStm);
} else if (!evt->IsHardwareMode()) {
return EvtRecordSoftwareMode(evt, curStm);
} else {
return EvtRecord(evt, curStm);
}
}
rtError_t ApiImplDavid::EventReset(Event* const evt, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* const curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(curStm);
const rtError_t checkRet = CheckEventAndStreamDevice(evt, curStm, "Event reset");
if (checkRet != RT_ERROR_NONE) {
return checkRet;
}
const bool supportFlag =
(evt->IsNewMode()) || ((evt->GetEventFlag() == RT_EVENT_DEFAULT) && curStm->IsModelStream());
COND_RETURN_WARN(
supportFlag, RT_ERROR_FEATURE_NOT_SUPPORT,
"Not support current mode bind stm, mode=%d, flag=%" PRIu64 ", isModel=%d.", evt->IsNewMode(),
evt->GetEventFlag(), curStm->IsModelStream());
if ((evt->GetEventFlag() == RT_EVENT_DEFAULT) && evt->IsHardwareMode() && !curStm->IsModelStream()) {
return RT_ERROR_NONE;
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Event reset");
if (evt->IsCapturing()) {
const char_t* const unsupportedFlag = GetUnsupportedCaptureStreamFlag(curStm->Flags());
COND_RETURN_AND_MSG_OUTER(
unsupportedFlag != nullptr, RT_ERROR_STREAM_INVALID, ErrorCode::EE1006, "Event reset",
RtFmtMsg("Stream flag value %#x", curStm->Flags()),
RtFmtMsg(
"Stream (stream_id=%d) with the flag %s cannot be used for ACL Graph", curStm->Id_(), unsupportedFlag));
COND_RETURN_AND_MSG_OUTER(
curStm == curCtx->DefaultStream_(), RT_ERROR_STREAM_CAPTURE_IMPLICIT, ErrorCode::EE1016, "Event reset",
RtFmtMsg("The default stream (stream_id=%d) cannot be used in the ACL Graph", curStm->Id_()));
COND_RETURN_ERROR(
evt->IsEventWithoutWaitTask(), RT_ERROR_INVALID_VALUE,
"Resetting an Event (event_id=%d) with flag value %" PRIu64 " is not supported in capture mode.",
evt->EventId_(), evt->GetEventFlag());
const std::lock_guard<std::mutex> lk(curCtx->GetCaptureLock());
if (evt->IsCapturing()) {
const rtError_t retCode = CaptureResetEvent(evt, curStm);
ERROR_PROC_RETURN_MSG_INNER(retCode, TerminateCapture(evt, curStm), "Capture event reset failed.");
return RT_ERROR_NONE;
}
} else {
if ((curStm != curCtx->DefaultStream_()) && evt->IsHardwareMode() && (evt->ToBeCaptured(curStm))) {
RT_LOG(RT_LOG_WARNING, "Not support call rtEventCreate or rtEventCreateWithFlag without external flag");
return RT_ERROR_FEATURE_NOT_SUPPORT;
}
}
return evt->IsHardwareMode() ? EvtReset(evt, curStm) : EvtResetSoftwareMode(evt, curStm);
}
rtError_t ApiImplDavid::LaunchKernelByArgsWithType(
Kernel* const kernel, const uint32_t coreDim, Stream* stm, const RtArgsWithType* const argsWithType,
TaskCfg& taskCfg)
{
rtError_t error = RT_ERROR_NONE;
RT_LOG(
RT_LOG_DEBUG, "LaunchKernelByArgsWithType, device_id=%u, stream_id=%d, blockDim=%u, argsType=%u.",
stm->Device_()->Id_(), stm->Id_(), coreDim, static_cast<uint32_t>(argsWithType->type));
switch (argsWithType->type) {
case RT_ARGS_NON_CPU_EX: {
rtStreamLaunchKernelV2ExtendArgs_t launchKernelExtendArgs = {};
launchKernelExtendArgs.argsInfo = argsWithType->args.nonCpuArgsInfo;
launchKernelExtendArgs.taskCfg = &taskCfg;
error = StreamLaunchKernelV2(kernel, coreDim, stm, &launchKernelExtendArgs);
break;
}
case RT_ARGS_CPU_EX: {
error = CpuKernelLaunchExAll(kernel, coreDim, argsWithType->args.cpuArgsInfo, stm, &taskCfg);
break;
}
case RT_ARGS_HANDLE: {
error = LaunchKernelByHandle(kernel, coreDim, argsWithType->args.argHandle, stm, taskCfg);
break;
}
case RT_ARGS_ARRAY: {
error = StreamLaunchArgsArray(kernel, coreDim, stm, argsWithType->args.argsArrayInfo, taskCfg);
break;
}
case RT_SIMT_ARGS_ARRAY: {
error = StreamLaunchSimtArgsArray(kernel, coreDim, stm, argsWithType->args.simtArgsArray, taskCfg);
break;
}
case RT_SIMT_ARGS_HOST: {
error = StreamLaunchSimtArgsHost(kernel, coreDim, stm, argsWithType->args.simtArgsHost, taskCfg);
break;
}
default:
error = RT_ERROR_INVALID_VALUE;
RT_LOG_OUTER_MSG_WITH_FUNC_DESC(
ErrorCode::EE1003, "Operator task delivery",
RtFmtMsg(
"%s(%d)", (argsWithType->type == RT_ARGS_MAX) ? "ARGS_MAX" : "UNKNOWN",
static_cast<int32_t>(argsWithType->type)),
"argsWithType->type",
"[" + std::to_string(RT_ARGS_NON_CPU_EX) + ", " + std::to_string(RT_ARGS_MAX) + ")");
break;
}
return error;
}
rtError_t ApiImplDavid::StreamWaitEvent(
Stream* const stm, Event* const evt, const uint32_t timeout, const uint32_t flag)
{
RT_LOG(RT_LOG_DEBUG, "Stream wait event, timeout=%us.", timeout);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* const curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(curStm);
const bool supFlag = ((evt->IsNewMode()) || (evt->GetEventFlag() == RT_EVENT_DEFAULT)) && curStm->IsModelStream();
COND_RETURN_WARN(
supFlag, RT_ERROR_FEATURE_NOT_SUPPORT,
"Not support current mode bind stream, mode=%d, flag=%" PRIu64 ", isModel=%d.", evt->IsNewMode(),
evt->GetEventFlag(), curStm->IsModelStream());
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Triggering stream event waiting");
if (flag == RT_EVENT_WAIT_EXTERNAL) {
COND_RETURN_AND_MSG_OUTER(
(!curStm->IsCapturing()), RT_ERROR_STREAM_NOT_CAPTURED, ErrorCode::EE1016, "Triggering event waiting",
RtFmtMsg("Stream %d is not in the capture stage", curStm->Id_()));
const rtError_t supportRet = CheckCaptureModelSupportExternalEvent(curStm->Device_(), false);
if (supportRet != RT_ERROR_NONE) {
return supportRet;
}
return Starsv2CaptureExternalEventWait(evt, curStm);
}
if (evt->IsCapturing()) {
const char_t* const unsupportedFlag = GetUnsupportedCaptureStreamFlag(curStm->Flags());
COND_RETURN_AND_MSG_OUTER(
unsupportedFlag != nullptr, RT_ERROR_STREAM_INVALID, ErrorCode::EE1006, "Triggering event waiting",
RtFmtMsg("Stream flag value %#x", curStm->Flags()),
RtFmtMsg(
"Stream (stream_id=%d) with the flag %s cannot be used for ACL Graph", curStm->Id_(), unsupportedFlag));
COND_RETURN_AND_MSG_OUTER(
curStm == curCtx->DefaultStream_(), RT_ERROR_STREAM_CAPTURE_IMPLICIT, ErrorCode::EE1016,
"Triggering event waiting",
RtFmtMsg("The default stream (stream_id=%d) cannot be used in the ACL Graph", curStm->Id_()));
COND_RETURN_ERROR(
evt->IsEventWithoutWaitTask(), RT_ERROR_INVALID_VALUE,
"Waiting for an Event (event_id=%d) with flag value %" PRIu64 " is not supported in capture mode.",
evt->EventId_(), evt->GetEventFlag());
const std::lock_guard<std::mutex> lk(curCtx->GetCaptureLock());
if (evt->IsCapturing()) {
const rtError_t retCode = CaptureWaitEvent(curCtx, curStm, evt, timeout);
ERROR_PROC_RETURN_MSG_INNER(
retCode, TerminateCapture(evt, curStm), "Capture wait event failed, stream_id=%d.", curStm->Id_());
return RT_ERROR_NONE;
}
} else {
if (curStm->IsCapturing()) {
if ((!(evt->IsNewMode())) && (evt->GetEventFlag() != RT_EVENT_EXTERNAL)) {
RT_LOG(
RT_LOG_WARNING,
"Events created via the APIs rtEventCreate and rtEventCreateWithFlag are not"
" supported, except for the RT_EVENT_EXTERNAL type, mode=%d, flag=%" PRIu64 "",
evt->IsNewMode(), evt->GetEventFlag());
return RT_ERROR_FEATURE_NOT_SUPPORT;
}
if ((evt->IsNewMode()) && (evt->HasRecord())) {
RT_LOG_OUTER_MSG_IMPL(
ErrorCode::EE1016, "Triggering event waiting",
"The event wait task is not supported during model capture because the corresponding event record "
"task is not "
"in the current model. To wait for an event outside the model, see the usage of "
"ACL_EVENT_WAIT_EXTERNAL of "
"the aclrtStreamWaitEventWithFlag API in the API reference");
return RT_ERROR_STREAM_CAPTURE_ISOLATION;
}
}
}
rtError_t error = RT_ERROR_NONE;
if (evt->GetEventFlag() == RT_EVENT_IPC) {
error = (dynamic_cast<IpcEvent*>(evt))->IpcEventWaitStarsV2(curStm);
} else if ((!evt->IsHardwareMode()) && evt->HasRecord() && (evt->GetEventAddr() != nullptr)) {
error = EvtWaitSoftwareMode(evt, curStm);
} else {
error = EvtWait(evt, curStm, timeout);
}
ERROR_RETURN(error, "Stream wait event failed.");
return error;
}
rtError_t ApiImplDavid::SetMemcpyDesc(
rtMemcpyDesc_t desc, const void* const srcAddr, const void* const dstAddr, const size_t count,
const rtMemcpyKind kind, rtMemcpyConfig_t* const config)
{
RT_LOG(
RT_LOG_INFO, "SetMemcpyDesc called, desc=%p, srcAddr=%p, dstAddr=%p, count=%zu, kind=%s, config=%p", desc,
srcAddr, dstAddr, count, MemcpyNewKindToString(kind).c_str(), config);
UNUSED(kind);
UNUSED(config);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
const Device* const dev = curCtx->Device_();
return cce::runtime::SetMemcpyDesc(desc, srcAddr, dstAddr, count, dev);
}
static rtError_t GetUbMemcpyFlag(
const void* const dst, const void* const src, const rtMemcpyKind_t kind, const Stream* const stm, bool& isUbMemcpy)
{
if (kind != RT_MEMCPY_DEVICE_TO_DEVICE) {
isUbMemcpy = Runtime::Instance()->GetConnectUbFlag();
return RT_ERROR_NONE;
}
uint8_t transType = 0U;
const rtError_t error = stm->Device_()->Driver_()->GetTransWayByAddr(
RtPtrToUnConstPtr<void*>(src), RtPtrToUnConstPtr<void*>(dst), &transType);
COND_RETURN_WITH_NOLOG(error != RT_ERROR_NONE, error);
isUbMemcpy = (transType == RT_MEMCPY_CHANNEL_TYPE_UB);
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::MemCopy2DAsync(
void* const dst, const uint64_t dstPitch, const void* const src, const uint64_t srcPitch, const uint64_t width,
const uint64_t height, Stream* const stm, const rtMemcpyKind_t kind, const rtMemcpyKind newKind)
{
UNUSED(newKind);
RT_LOG(
RT_LOG_DEBUG,
"Sync memcpy2d, dstPitch=%" PRIu64 ", srcPitch=%" PRIu64 ", width=%" PRIu64 ", height=%" PRIu64 ", kind=%s.",
dstPitch, srcPitch, width, height, MemcpyKindToStr(kind));
rtError_t error = RT_ERROR_NONE;
uint64_t remainSize = width * height;
const uint64_t totalSize = remainSize;
uint64_t realSize = 0UL;
uint64_t fixedSize = 0UL;
uint64_t srcoffset = 0UL;
uint64_t dstoffset = 0UL;
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Asynchronous 2D memory copy");
bool isUbMemcpy = false;
error = GetUbMemcpyFlag(dst, src, kind, curStm, isUbMemcpy);
COND_RETURN_WITH_NOLOG(error != RT_ERROR_NONE, error);
const bool needHandlePitch = (kind == RT_MEMCPY_DEVICE_TO_DEVICE) && (!isUbMemcpy);
while (remainSize > 0UL) {
if (needHandlePitch) {
error = Memcpy2DAsync(
(static_cast<char_t*>(dst)) + dstoffset, dstPitch, (static_cast<const char_t*>(src)) + srcoffset,
srcPitch, width, height, kind, &realSize, curStm, fixedSize);
dstoffset += dstPitch;
srcoffset += srcPitch;
} else {
error = Memcpy2DAsync(dst, dstPitch, src, srcPitch, width, height, kind, &realSize, curStm, fixedSize);
}
COND_RETURN_WITH_NOLOG((error != RT_ERROR_NONE), error);
if (isUbMemcpy) {
COND_RETURN_ERROR(
realSize > totalSize, RT_ERROR_DRV_ERR,
"Cumulative memcpy2d size exceeds total size, realSize=%" PRIu64 ", totalSize=%" PRIu64 ".", realSize,
totalSize);
fixedSize = realSize;
remainSize = totalSize - fixedSize;
if (remainSize > 0UL && (!curStm->IsCapturing()) && (!curStm->GetBindFlag())) {
error = curStm->Synchronize();
ERROR_RETURN_MSG_INNER(
error, "Failed to synchronize stream, retCode=%#x.", static_cast<uint32_t>(error));
}
} else {
COND_RETURN_ERROR(
realSize > remainSize, RT_ERROR_DRV_ERR,
"Current memcpy2d size exceeds remaining size, realSize=%" PRIu64 ", remainSize=%" PRIu64 ".", realSize,
remainSize);
fixedSize += realSize;
remainSize -= realSize;
}
}
return error;
}
rtError_t ApiImplDavid::BatchMemcpyAsync(
void** const dsts, const size_t* const destMaxs, void** const srcs, const size_t* const sizes, const size_t count,
const rtMemcpyBatchAttr* const attrs, const size_t* const attrsIdxs, const size_t numAttrs, size_t* const failIdx,
Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Asynchronous batch memory copy");
rtError_t error = RT_ERROR_NONE;
rtMemcpyBatchAttr memAttr = attrs[0];
size_t attrIdx = 0U;
rtPtrAttributes_t dstAttr = {};
rtPtrAttributes_t srcAttr = {};
uint64_t realCnt = 0UL;
uint64_t realSize = 0UL;
uint64_t remainCnt = count;
uint64_t fixedCnt = 0UL;
uint64_t fixedSize = 0UL;
bool isD2HorH2DInvolvePageableMemory = false;
for (size_t i = 0U; i < count; i++) {
if (((attrIdx + 1U) < numAttrs) && (i >= attrsIdxs[attrIdx + 1U])) {
attrIdx = attrIdx + 1U;
memAttr = attrs[attrIdx];
}
error = ValidateMemCpyParamsAndAttributes(dsts[i], destMaxs[i], srcs[i], sizes[i], memAttr, dstAttr, srcAttr);
COND_PROC_RETURN_ERROR(
error != RT_ERROR_NONE, error, SetFailIndex(failIdx, i), "ValidateMemCpyParamsAndAttributes %u failed.", i);
if (dstAttr.location.type == RT_MEMORY_LOC_UNREGISTERED ||
srcAttr.location.type == RT_MEMORY_LOC_UNREGISTERED) {
isD2HorH2DInvolvePageableMemory = true;
}
}
if (isD2HorH2DInvolvePageableMemory) {
COND_RETURN_AND_MSG_OUTER(
curStm->IsCapturing(), RT_ERROR_INVALID_VALUE, ErrorCode::EE1016, "Asynchronous batch copy task",
"The pageable memory copy task does not support graph capture");
error = StreamSynchronize(curStm, -1);
ERROR_RETURN(error, "StreamSynchronize failed, stream_id=%d.", curStm->Id_());
RT_LOG(RT_LOG_DEBUG, "Stream Synchronize success, stream_id=%d.", curStm->Id_());
return MemcpyBatch(
dsts, srcs, const_cast<size_t*>(sizes), count, const_cast<rtMemcpyBatchAttr*>(attrs),
const_cast<size_t*>(attrsIdxs), numAttrs, failIdx);
}
std::vector<void*> localDsts(dsts, dsts + count);
std::vector<void*> localSrcs(srcs, srcs + count);
std::vector<uint64_t> localSizes(sizes, sizes + count);
while (remainCnt > 0UL) {
AsyncDmaBatchInfo batchInfo = {localDsts.data(), localSrcs.data(), localSizes.data(),
remainCnt, fixedCnt, fixedSize};
error = MemcopyBatchAsync(batchInfo, &realCnt, &realSize, curStm);
COND_RETURN_WITH_NOLOG((error != RT_ERROR_NONE), error);
COND_RETURN_ERROR(
realCnt > remainCnt, RT_ERROR_DRV_ERR,
"Invalid memcpy batch progress, realCnt=%" PRIu64 ", remainCnt=%" PRIu64 ".", realCnt, remainCnt);
fixedCnt = realCnt;
fixedSize = realSize;
remainCnt -= fixedCnt;
if (remainCnt > 0UL && (!curStm->IsCapturing()) && (!curStm->GetBindFlag())) {
error = curStm->Synchronize();
ERROR_RETURN_MSG_INNER(error, "Failed to synchronize stream, retCode=%#x.", static_cast<uint32_t>(error));
}
}
return error;
}
rtError_t ApiImplDavid::MemcpyBatchAsync(
void** const dsts, const size_t* const destMaxs, void** const srcs, const size_t* const sizes, const size_t count,
const rtMemcpyBatchAttr* const attrs, const size_t* const attrsIdxs, const size_t numAttrs, size_t* const failIdx,
Stream* const stm)
{
Context* curCtx = Runtime::Instance()->CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
NULL_PTR_RETURN_MSG(curCtx->Device_(), RT_ERROR_DEVICE_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
if (!NpuDriver::CheckIsSupportFeature(curCtx->Device_()->Id_(), FEATURE_MEMCPY_BATCH_ASYNC)) {
if (Runtime::Instance()->GetConnectUbFlag()) {
return BatchMemcpyAsync(dsts, destMaxs, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, curStm);
} else {
return LoopMemcpyAsync(dsts, destMaxs, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stm);
}
}
return RT_ERROR_DRV_NOT_SUPPORT;
}
rtError_t ApiImplDavid::MemcpyAsync(
void* const dst, const uint64_t destMax, const void* const src, const uint64_t cnt, const rtMemcpyKind_t kind,
Stream* const stm, const rtTaskCfgInfo_t* const cfgInfo, const rtD2DAddrCfgInfo_t* const addrCfg, bool checkKind,
const rtMemcpyConfig_t* const memcpyConfig)
{
UNUSED(checkKind);
UNUSED(memcpyConfig);
RT_LOG(RT_LOG_DEBUG, "Async memcpy, count=%" PRIu64 ", kind=%s", cnt, MemcpyKindToStr(kind));
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Asynchronous memory copy");
uint32_t transType = UINT32_MAX;
rtError_t error = RT_ERROR_NONE;
if (kind == RT_MEMCPY_DEVICE_TO_DEVICE) {
error = ConvertD2DCpyType(curStm, transType, src, dst);
if (error != RT_ERROR_NONE) {
RT_LOG(RT_LOG_ERROR, "ConvertD2DCpyType failed, retCode=%#x.", static_cast<uint32_t>(error));
return error;
}
}
const uint64_t sqSize = CalculateMemcpyAsyncSingleMaxSize(kind, transType);
uint64_t realSize = cnt;
uint64_t remainSize = cnt;
uint64_t doneSize = 0U;
while (remainSize > 0U) {
const uint64_t doingSize = (remainSize >= sqSize) ? sqSize : remainSize;
realSize = doingSize;
error = MemcopyAsync(
(static_cast<char_t*>(dst)) + doneSize, destMax - doneSize, (static_cast<const char_t*>(src)) + doneSize,
doingSize, kind, curStm, &realSize, nullptr, cfgInfo, addrCfg);
if (error != RT_ERROR_NONE) {
RT_LOG(
RT_LOG_ERROR,
"MemcpyAsync failed, retCode=%#x, count=%" PRIu64 ", doingSize=%" PRIu64 ", realSize=%" PRIu64 ".",
static_cast<uint32_t>(error), cnt, doingSize, realSize);
return error;
}
doneSize += realSize;
remainSize -= realSize;
}
return error;
}
rtError_t ApiImplDavid::ReduceAsync(
void* const dst, const void* const src, const uint64_t cnt, const rtRecudeKind_t kind, const rtDataType_t type,
Stream* const stm, const rtTaskCfgInfo_t* const cfgInfo)
{
RT_LOG(RT_LOG_INFO, "ReduceAsync, count=%" PRIu64 ", kind=%s.", cnt, ReduceKindToString(kind).c_str());
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Asynchronously performing the Reduce operation");
return cce::runtime::ReduceAsync(dst, src, cnt, kind, type, curStm, cfgInfo);
}
rtError_t ApiImplDavid::ModelExit(Model* const mdl, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Model exiting");
COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL_WITH_FUNC_DESC(mdl, curCtx, RT_ERROR_MODEL_CONTEXT, "Model exiting");
const uint32_t modelExitNum = mdl->ModelExitNum_();
COND_RETURN_AND_MSG_OUTER(
modelExitNum >= 1U, RT_ERROR_MODEL_EXIT, ErrorCode::EE1011, "Model exiting", modelExitNum, "modelExitNum",
RtFmtMsg("Model (model_id=%u) must exit only once", mdl->Id_()));
COND_RETURN_AND_MSG_OUTER(
stm->Model_() == nullptr, RT_ERROR_MODEL_EXIT_STREAM_UNBIND, ErrorCode::EE1017, "Model exiting", "stm",
RtFmtMsg("Stream (stream_id=%d) is not bound to any model", stm->Id_()));
COND_RETURN_AND_MSG_OUTER(
stm->Model_()->Id_() != mdl->Id_(), RT_ERROR_MODEL_EXIT_ID, ErrorCode::EE1017, "Model exiting", "stm",
RtFmtMsg(
"The current stream (stream_id=%d) has been bound to another model (model_id=%u) which is different from "
"the input model (model_id=%u). "
"The input model must be the same as the model bound to the input stream",
stm->Id_(), stm->Model_()->Id_(), mdl->Id_()));
return cce::runtime::ModelExit(mdl, stm);
}
rtError_t ApiImplDavid::MemsetAsync(
void* const ptr, const uint64_t destMax, const uint32_t val, const uint64_t cnt, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Asynchronously setting the memory content to a specified value");
RT_LOG(
RT_LOG_DEBUG, "fillVal=%u, fillCount=%" PRIu64 ", destMax=%" PRIu64 ", stream_id=%d.", val, cnt, destMax,
curStm->Id_());
return MemSetAsync(curStm, ptr, destMax, val, cnt);
}
rtError_t ApiImplDavid::CntNotifyCreate(const int32_t deviceId, CountNotify** const cntNotify, const uint32_t flag)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
const uint32_t mc2FeatureFlag = dev->GetDevProperties().mc2FeatureFlag;
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_NAME_AND_FUNC_DESC(
((mc2FeatureFlag == 0U) && (flag == RT_NOTIFY_FLAG_DOWNLOAD_TO_DEV)), RT_ERROR_INVALID_VALUE,
"CntNotify creation", NotifyFlagToString(flag), "flag", "RT_NOTIFY_FLAG_DEFAULT(0)");
*cntNotify = new (std::nothrow) CountNotify(static_cast<uint32_t>(deviceId), dev->DevGetTsId());
COND_RETURN_AND_MSG_OUTER(
*cntNotify == nullptr, RT_ERROR_NOTIFY_NEW, ErrorCode::EE1013, std::to_string(sizeof(CountNotify)), "new");
(*cntNotify)->SetNotifyFlag(flag);
rtError_t error = (*cntNotify)->Setup();
ERROR_PROC_RETURN_MSG_INNER(error, DELETE_O(*cntNotify);
, "Count Notify create failed, setup failed, user device_id=%d, retCode=%#x", deviceId,
static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::CntNotifyDestroy(CountNotify* const inCntNotify)
{
delete inCntNotify;
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::CntNotifyRecord(
CountNotify* const inCntNotify, Stream* const stm, const rtCntNtyRecordInfo_t* const info)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
Stream* targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(targetStm);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
targetStm, curCtx, RT_ERROR_STREAM_CONTEXT, "CntNotify recording");
const uint32_t countNotifyId = inCntNotify->GetCntNotifyId();
const rtError_t error = inCntNotify->Record(targetStm, info);
ERROR_RETURN(
error,
"Count Notify record failed, device_id=%u, stream_id=%d, count notify_id=%u,"
" retCode=%#x",
dev->Id_(), targetStm->Id_(), countNotifyId, static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::CntNotifyReset(CountNotify* const inCntNotify, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
Stream* targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(targetStm);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
targetStm, curCtx, RT_ERROR_STREAM_CONTEXT, "CntNotify reset");
rtCntNtyRecordInfo_t info = {RECORD_STORE_MODE, 0U};
const uint32_t countNotifyId = inCntNotify->GetCntNotifyId();
const rtError_t error = inCntNotify->Record(targetStm, &info);
ERROR_RETURN(
error,
"Count Notify record failed, device_id=%u, stream_id=%d, count notify_id=%u,"
" retCode=%#x",
dev->Id_(), targetStm->Id_(), countNotifyId, static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::CntNotifyWaitWithTimeout(
CountNotify* const inCntNotify, Stream* const stm, const rtCntNtyWaitInfo_t* const info)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
Stream* targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(targetStm);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
targetStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Waiting for CntNotify");
const rtError_t error = inCntNotify->Wait(targetStm, info);
const uint32_t notifyId = inCntNotify->GetCntNotifyId();
ERROR_RETURN(
error,
"count notify wait failed, device_id=%u, stream_id=%d, count notify_id=%u,"
" time_out = %u, retCode=%#x",
dev->Id_(), targetStm->Id_(), notifyId, info->timeout, static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::GetCntNotifyId(CountNotify* const inCntNotify, uint32_t* const notifyId)
{
*notifyId = inCntNotify->GetCntNotifyId();
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::GetCntNotifyAddress(
CountNotify* const inCntNotify, uint64_t* const cntNotifyAddress, rtNotifyType_t const regType)
{
uint64_t addr;
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Device* const dev = curCtx->Device_();
COND_RETURN_ERROR(dev == nullptr, RT_ERROR_INVALID_VALUE, "device is NULL.");
if (!dev->IsSupportFeature(RtOptionalFeatureType::RT_FEATURE_DEVICE_AICPUSD_LATER_PROCEDURE)) {
COND_RETURN_AND_MSG_OUTER_WITH_PARAM_NAME_AND_FUNC_DESC(
(regType != NOTIFY_CNT_ST_SLICE), RT_ERROR_INVALID_VALUE,
"Obtaining the on-device address of a CntNotify object",
RtFmtMsg("UNKNOWN(%d)", static_cast<int32_t>(regType)), "regType", "NOTIFY_CNT_ST_SLICE(1)");
} else {
COND_RETURN_ERROR(
regType == NOTIFY_TABLE_SLICE, RT_ERROR_INVALID_VALUE,
"CntNotify does not support getting notify table address.");
}
const rtError_t error = inCntNotify->GetCntNotifyAddress(addr, regType);
ERROR_RETURN_MSG_INNER(
error, "GetCntNotifyAddress failed, device_id=%d, retCode=%#x", dev->Id_(), static_cast<uint32_t>(error));
RT_LOG(RT_LOG_INFO, "GetCntNotifyAddress ok, device_id=%d, addr=%#" PRIx64, dev->Id_(), addr);
*cntNotifyAddress = addr;
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::NotifyWait(Notify* const notify, Stream* const stm, const uint32_t timeOut)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Waiting for a Notify");
COND_RETURN_ERROR(
notify->CheckIpcNotifyDevId() != RT_ERROR_NONE, RT_ERROR_INVALID_VALUE,
"Failed to check the IPC Notify device for waiting, notify_id=%u, stream_id=%d.", notify->GetNotifyId(),
curStm->Id_());
const uint32_t timeOutTmp = timeOut;
const rtError_t error = NtyWait(notify, curStm, timeOutTmp);
const uint32_t notifyId = notify->GetNotifyId();
ERROR_RETURN_MSG_INNER(
error, "notify wait failed, notify_id=%u, time_out = %u, retCode=%#x", notifyId, timeOutTmp,
static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::NotifyRecord(Notify* const notify, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Notify recording");
const uint32_t notifyId = notify->GetNotifyId();
const rtError_t error = NtyRecord(notify, curStm);
ERROR_RETURN_MSG_INNER(
error, "Notify record failed, notify_id=%u, retCode=%#x", notifyId, static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::NotifyReset(Notify* const notify)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = curCtx->GetCtrlSQStream();
COND_RETURN_ERROR(curStm == nullptr, RT_ERROR_STREAM_NULL, "The internal CtrlSQ stream is null.");
const uint32_t notifyId = notify->GetNotifyId();
const rtError_t error = NtyReset(notify, curStm);
ERROR_RETURN(
error, "Notify reset failed, device_id=%u, notify_id=%u, is_ipc_notify=%d, retCode=%#x",
curStm->Device_()->Id_(), notifyId, notify->IsIpcNotify(), static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::DatadumpInfoLoad(const void* const dumpInfo, const uint32_t length, const uint32_t flag)
{
RT_LOG(RT_LOG_DEBUG, "length=%u, flag=%u.", length, flag);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Runtime* const rtInstance = Runtime::Instance();
COND_RETURN_ERROR_MSG_INNER(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
ERROR_RETURN_MSG_INNER(
rtInstance->StartAicpuSd(curCtx->Device_()),
"Data dump info load failed, check and start tsd open aicpu sd error.");
return StreamDatadumpInfoLoad(dumpInfo, length, flag, curCtx->DefaultStream_());
}
rtError_t ApiImplDavid::DebugRegister(
Model* const mdl, const uint32_t flag, const void* const addr, uint32_t* const streamId, uint32_t* const taskId)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL_WITH_FUNC_DESC(
mdl, curCtx, RT_ERROR_MODEL_CONTEXT, "Registering a debugging callback for a stream");
return ModelDebugRegister(mdl, flag, addr, streamId, taskId, curCtx->DefaultStream_());
}
rtError_t ApiImplDavid::DebugUnRegister(Model* const mdl)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL_WITH_FUNC_DESC(
mdl, curCtx, RT_ERROR_MODEL_CONTEXT, "Registering a debugging callback for a model");
return ModelDebugUnRegister(mdl, curCtx->DefaultStream_());
}
rtError_t ApiImplDavid::DebugRegisterForStream(
Stream* const stm, const uint32_t flag, const void* const addr, uint32_t* const streamId, uint32_t* const taskId)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Registering a debugging callback for a stream");
return StreamDebugRegister(stm, flag, addr, streamId, taskId, nullptr);
}
rtError_t ApiImplDavid::DebugUnRegisterForStream(Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Deregistering the debugging callback of a stream");
return StreamDebugUnRegister(stm, nullptr);
}
rtError_t ApiImplDavid::GetDevRunningStreamSnapshotMsg(const rtGetMsgCallback callback)
{
const std::function<rtError_t(Device* const dev)> getDevHangMsgForDev = [callback](Device* const dev) -> rtError_t {
DeviceStreamSnapshotHandler devStreamSnapshotHandler(dev, callback);
rtError_t error = devStreamSnapshotHandler.Init();
ERROR_RETURN(
error, "Init device stream snapshot msg handler failed, retCode=%#x.", static_cast<uint32_t>(error));
error = SyncGetDeviceMsg(
dev, devStreamSnapshotHandler.GetDevMemAddr(), devStreamSnapshotHandler.GetDevMemSize(),
RT_GET_DEV_RUNNING_STREAM_SNAPSHOT_MSG);
ERROR_RETURN(error, "Sync get device msg failed, retCode=%#x.", static_cast<uint32_t>(error));
error = devStreamSnapshotHandler.HandleMsg();
ERROR_RETURN_MSG_INNER(
error, "Failed to handle get stream snapshot msg, retCode=%#x.", static_cast<uint32_t>(error));
return RT_ERROR_NONE;
};
return Runtime::Instance()->ProcessForAllOpenDevice(getDevHangMsgForDev, false);
}
rtError_t ApiImplDavid::NpuClearFloatStatus(const uint32_t checkMode, Stream* const stm)
{
NULL_STREAM_PTR_RETURN_MSG(stm);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Clearing the Float exception status flag of the NPU");
return StreamNpuClearFloatStatus(checkMode, stm, false);
}
rtError_t ApiImplDavid::NpuGetFloatStatus(
void* const outputAddrPtr, const uint64_t outputSize, const uint32_t checkMode, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* const targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(targetStm);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
targetStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Obtaining the Float exception status of the NPU");
return StreamNpuGetFloatStatus(outputAddrPtr, outputSize, checkMode, targetStm, false);
}
rtError_t ApiImplDavid::NpuClearFloatDebugStatus(const uint32_t checkMode, Stream* const stm)
{
NULL_STREAM_PTR_RETURN_MSG(stm);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Clearing the Float debugging status flag of the NPU");
return StreamNpuClearFloatStatus(checkMode, stm, true);
}
rtError_t ApiImplDavid::NpuGetFloatDebugStatus(
void* const outputAddrPtr, const uint64_t outputSize, const uint32_t checkMode, Stream* const stm)
{
NULL_STREAM_PTR_RETURN_MSG(stm);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Obtaining the Float exception debugging status of the NPU");
return StreamNpuGetFloatStatus(outputAddrPtr, outputSize, checkMode, stm, true);
}
rtError_t ApiImplDavid::GetDeviceSatStatus(void* const outputAddrPtr, const uint64_t outputSize, Stream* const stm)
{
RT_LOG(RT_LOG_DEBUG, "Start to get sat status.");
uint64_t realSize = 0U;
rtError_t error = RT_ERROR_NONE;
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT,
"Obtaining the overflow status of tasks in all streams on the current device");
error = StreamGetSatStatus(outputSize, curStm);
COND_RETURN_WITH_NOLOG((error != RT_ERROR_NONE), error);
error = MemcopyAsync(
outputAddrPtr, outputSize, curCtx->CtxGetOverflowAddr(), outputSize, RT_MEMCPY_DEVICE_TO_DEVICE, curStm,
&realSize, nullptr, nullptr);
if (error != RT_ERROR_NONE) {
RT_LOG(RT_LOG_ERROR, "MemcpyAsync failed destMax=%llu.", outputSize);
}
return error;
}
static rtError_t SetStreamOverflowSwitchInternal(Stream* const stm, const uint32_t flags)
{
Context* const curCtx = Runtime::Instance()->CurrentContext(true, DEFAULT_DEVICE_ID);
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* const targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(targetStm);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
targetStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Setting the stream overflow/underflow detection switch");
return StreamSetOverflowSwitch(targetStm, flags, nullptr);
}
static rtError_t SetStreamTagInternal(Stream* const stm, const uint32_t geOpTag)
{
RT_LOG(RT_LOG_DEBUG, "geOpTag=%#x.", geOpTag);
Context* const curCtx = Runtime::Instance()->CurrentContext(true, DEFAULT_DEVICE_ID);
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* const targetStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(targetStm);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
targetStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Setting the stream tag");
return SetTagOnStream(targetStm, geOpTag);
}
rtError_t ApiImplDavid::StreamSetAttribute(
Stream* const stm, const rtStreamAttr stmAttrId, const rtStreamAttrValue_t* const attrValue)
{
NULL_PTR_RETURN_MSG(attrValue, RT_ERROR_INVALID_VALUE);
rtError_t error;
switch (stmAttrId) {
case RT_STREAM_ATTR_FLOAT_OVERFLOW_CHECK: {
error = SetStreamOverflowSwitchInternal(stm, attrValue->overflowSwitch);
break;
}
case RT_STREAM_ATTR_USER_CUSTOM_TAG: {
error = SetStreamTagInternal(stm, attrValue->userCustomTag);
break;
}
default: {
error = ApiImpl::StreamSetAttribute(stm, stmAttrId, attrValue);
break;
}
}
return error;
}
rtError_t ApiImplDavid::SetStreamOverflowSwitch(Stream* const stm, const uint32_t flags)
{
return SetStreamOverflowSwitchInternal(stm, flags);
}
rtError_t ApiImplDavid::SetStreamTag(Stream* const stm, const uint32_t geOpTag)
{
return SetStreamTagInternal(stm, geOpTag);
}
rtError_t ApiImplDavid::UbDbSend(rtUbDbInfo_t* const dbInfo, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_STREAM_PTR_RETURN_MSG(curStm);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Delivering a UB doorbell task");
return StreamUbDbSend(dbInfo, curStm, static_cast<uint16_t>(UbDmaSqeSource::RT_UBDMA_SOURCE_API));
}
rtError_t ApiImplDavid::UbDirectSend(rtUbWqeInfo_t* const wqeInfo, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_STREAM_PTR_RETURN_MSG(curStm);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Delivering a UB Direct task");
COND_RETURN_ERROR_MSG_INNER(
curStm->GetBindFlag(), RT_ERROR_STREAM_INVALID, "UbDirectSend does not support model stream, stream_id=%d.",
curStm->Id_());
COND_RETURN_WARN(
curStm->IsCapturing(), RT_ERROR_FEATURE_NOT_SUPPORT, "Ub direct tasks cannot be delivered in capture mode.");
return StreamUbDirectSend(wqeInfo, curStm);
}
rtError_t ApiImplDavid::StreamClear(Stream* const stm, rtClearStep_t step)
{
UNUSED(stm);
UNUSED(step);
RT_LOG(RT_LOG_WARNING, "Chip type(%d) does not support.", Runtime::Instance()->GetChipType());
return RT_ERROR_FEATURE_NOT_SUPPORT;
}
rtError_t ApiImplDavid::NopTask(Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
NULL_STREAM_PTR_RETURN_MSG(stm);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Executing a No-Operation (NOP) task");
return StreamNopTask(stm);
}
rtError_t ApiImplDavid::AicpuInfoLoad(const void* const aicpuInfo, const uint32_t length)
{
RT_LOG(RT_LOG_DEBUG, "length=%u.", length);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Runtime* const rtInstance = Runtime::Instance();
COND_RETURN_ERROR_MSG_INNER(rtInstance == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
ERROR_RETURN_MSG_INNER(
rtInstance->StartAicpuSd(curCtx->Device_()),
"aicpu info load failed, check and start tsd open aicpu sd error.");
return StreamAicpuInfoLoad(curCtx->DefaultStream_(), aicpuInfo, length, curCtx->Device_());
}
rtError_t ApiImplDavid::SubscribeReport(const uint64_t threadId, Stream* const stm)
{
rtError_t ret = RT_ERROR_NONE;
Stream* curStm = stm;
if (curStm == nullptr) {
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
curStm = curCtx->DefaultStream_();
NULL_STREAM_PTR_RETURN_MSG(curStm);
}
Notify* curNotify = nullptr;
ret = NotifyCreate(static_cast<int32_t>(curStm->Device_()->Id_()), &curNotify, RT_NOTIFY_DEFAULT);
ERROR_RETURN(ret, "Call NotifyCreate failed, ret=%#x.", ret);
ret = Runtime::Instance()->SubscribeReport(threadId, curStm, static_cast<void*>(curNotify));
if (ret != RT_ERROR_NONE) {
(void)NotifyDestroy(curNotify);
}
return ret;
}
static rtError_t ProcessReportForBlockCqe(
Device* const dev, rtHostFuncCqReport_t* const report, const uint32_t deviceId, uint32_t tsId)
{
if (report->isBlock != 0) {
Runtime* const rt = Runtime::Instance();
Driver* const curDrv = dev->Driver_();
Notify* curNotify = nullptr;
rtError_t ret = rt->GetNotifyByStreamId(deviceId, static_cast<int32_t>(report->streamId), &curNotify);
ERROR_RETURN_MSG_INNER(ret, "Call GetNotifyByStreamId failed for block callback, ret=%#x.", ret);
ret = curDrv->WriteNotifyRecord(deviceId, tsId, curNotify->GetNotifyId());
ERROR_RETURN_MSG_INNER(ret, "Call WriteNotifyRecord failed for block callback, ret=%#x.", ret);
}
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::ProcessReport(const int32_t timeout, const bool noLog)
{
uint64_t cqidBit[HOST_CALLBACK_SQCQ_BIT / 64U] = {0ULL};
uint32_t deviceId = 0U;
uint32_t tsId = 0U;
uint32_t groupId = 0U;
uint64_t bit = 0U;
Runtime* const rt = Runtime::Instance();
const uint64_t threadId = PidTidFetcher::GetCurrentUserTid();
rtError_t ret = rt->GetGroupIdByThreadId(threadId, &deviceId, &tsId, &groupId, noLog);
COND_RETURN_WARN_WITH_NOLOG_SWITCH(
ret != RT_ERROR_NONE, noLog, ret, "get groupId fail, threadIdentifier=%" PRIu64 ", retCode=%#x", threadId, ret);
rt->LockGroupId(groupId);
std::function<void()> const func = [rt, groupId]() { rt->UnlockGroupId(groupId); };
const ScopeGuard groupIdGuarder(func);
Context* priCtx = rt->GetPriCtxByDeviceId(deviceId, tsId);
if (priCtx == nullptr) {
priCtx = CurrentContext();
}
if (noLog) {
CHECK_CONTEXT_VALID_WITH_PROC_RETURN(priCtx, RT_ERROR_CONTEXT_NULL, );
} else {
CHECK_CONTEXT_VALID_WITH_RETURN(priCtx, RT_ERROR_CONTEXT_NULL);
}
Device* const dev = priCtx->Device_();
Driver* const curDrv = dev->Driver_();
ret = curDrv->CqReportIrqWait(deviceId, tsId, groupId, timeout, &cqidBit[0], HOST_CALLBACK_SQCQ_BIT / 64U);
COND_RETURN_WARN_WITH_NOLOG_SWITCH(ret != RT_ERROR_NONE, noLog, ret, "CqReportIrqWait, retCode=%#x", ret);
RT_LOG(RT_LOG_DEBUG, "IrqWait groupId=%u, threadIdentifier=%" PRIu64, groupId, threadId);
for (uint32_t index = 0U; index < (HOST_CALLBACK_SQCQ_BIT / 64U); index++) {
for (;; RT_BITMAP_CLR(cqidBit[index], bit)) {
bit = BitScan(cqidBit[index]);
if (bit >= 64U) {
break;
}
const uint32_t cqidValue = (index << 6U) + static_cast<uint32_t>(bit);
uint32_t cnt = 0U;
rtHostFuncCqReport_t* report = nullptr;
ret = curDrv->CqReportGet(deviceId, tsId, cqidValue, &report, &cnt);
if (unlikely((report == nullptr) || (cnt == 0U))) {
continue;
}
RT_LOG(RT_LOG_DEBUG, "get report info num=%u from cqid = %u.", cnt, cqidValue);
COND_RETURN_WARN_WITH_NOLOG_SWITCH(
ret != RT_ERROR_NONE, noLog, ret, "CqReportGet failed, retCode=%#x", ret);
for (uint32_t idx = 0U; idx < cnt; idx++) {
const rtCallback_t hostFunc = RtValueToPtr<rtCallback_t>(report[idx].hostFuncCbPtr);
NULL_PTR_RETURN_MSG(hostFunc, RT_ERROR_DRV_REPORT);
RT_LOG(
RT_LOG_INFO, "report[%u], streamId=%hu, taskId=%hu, eventNotifyId=%hu, isBlock=%hhu", idx,
report[idx].streamId, report[idx].taskId, report[idx].eventId, report[idx].isBlock);
ProcessHostFunc(report[idx].hostFuncCbPtr, report[idx].fnDataPtr, dev, report[idx].streamId);
ret = ProcessReportForBlockCqe(dev, &report[idx], deviceId, tsId);
ERROR_RETURN(ret, "process block cqe fail, ret=%#x.", ret);
ret = curDrv->CqReportRelease(&report[idx], deviceId, cqidValue, tsId, noLog);
}
}
}
return ret;
}
rtError_t ApiImplDavid::ModelTaskUpdate(
Stream* desStm, uint32_t desTaskId, Stream* sinkStm, rtMdlTaskUpdateInfo_t* para)
{
RT_LOG(
RT_LOG_INFO, "ModelTaskUpdate, desStm=%d, desTaskId=%u, sinkStm=%d", desStm->Id_(), desTaskId, sinkStm->Id_());
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
return MdlTaskUpdate(desStm, desTaskId, sinkStm, para);
}
rtError_t ApiImplDavid::CallbackLaunch(
const rtCallback_t callBackFunc, void* const fnData, Stream* const stm, const bool isBlock)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_STREAM_PTR_RETURN_MSG(curStm);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Delivering an on-host callback function task in a stream");
COND_RETURN_AND_MSG_OUTER(
!curStm->IsHostFuncCbReg(), RT_ERROR_STREAM_NO_CB_REG, ErrorCode::EE1018,
"Delivering an on-host callback function task in a stream",
RtFmtMsg(
"Stream (stream_id=%d) is not bound to any thread. Call the aclrtSubscribeReport API to bind a thread to "
"the stream",
curStm->Id_()));
if (isBlock) {
const rtError_t ret = CallbackLaunchForDavidWithBlock(callBackFunc, fnData, curStm, MAX_UINT64_NUM);
ERROR_RETURN(ret, "Call CallbackLaunch failed for block callback, ret=%#x.", ret);
return ret;
}
return CallbackLaunchForDavidNoBlock(callBackFunc, fnData, curStm, MAX_UINT64_NUM);
}
rtError_t ApiImplDavid::ModelAbort(Model* const mdl)
{
NULL_PTR_RETURN_MSG_OUTER_WITH_FUNC_DESC(mdl, RT_ERROR_MODEL_NULL, "Aborting the model running instance");
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL_WITH_FUNC_DESC(
mdl, curCtx, RT_ERROR_MODEL_CONTEXT, "Aborting the model running instance");
return MdlAbort(mdl);
}
rtError_t ApiImplDavid::ModelEndGraph(Model* const mdl, Stream* const stm, const uint32_t flags)
{
RT_LOG(RT_LOG_DEBUG, "model add end graph task model_id=%u, stream_id=%d, flags=%u", mdl->Id_(), stm->Id_(), flags);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Delivering the EndGraph flag to the stream of a model");
COND_RETURN_AND_MSG_INVALID_CONTEXT_MODEL_WITH_FUNC_DESC(
mdl, curCtx, RT_ERROR_MODEL_CONTEXT, "Delivering the EndGraph flag to the stream of a model");
if ((flags & RT_KERNEL_DUMPFLAG) != 0U) {
ERROR_RETURN_MSG_INNER(
Runtime::Instance()->StartAicpuSd(curCtx->Device_()), "Failed to start the AI CPU service.");
}
return cce::runtime::ModelAddEndGraph(mdl, stm, flags);
}
rtError_t ApiImplDavid::StreamSwitchEx(
void* const ptr, const rtCondition_t condition, void* const valuePtr, Stream* const trueStream, Stream* const stm,
const rtSwitchDataType_t dataType)
{
RT_LOG(
RT_LOG_DEBUG, "Stream switch, condition=%s, dataType=%s.", ConditionToString(condition).c_str(),
SwitchDataTypeToString(dataType).c_str());
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Switching between streams based on conditions");
COND_RETURN_AND_MSG_OUTER(
!stm->IsModelStream(), RT_ERROR_STREAM_MODEL, ErrorCode::EE1011,
"Switching between streams based on conditions", 0, "stm->modelNum",
RtFmtMsg("The stream (stream_id=%d) is not bound to a model", stm->Id_()));
COND_RETURN_AND_MSG_OUTER(
!trueStream->IsModelStream(), RT_ERROR_STREAM_MODEL, ErrorCode::EE1011,
"Switching between streams based on conditions", 0, "trueStream->modelNum",
RtFmtMsg("The stream (stream_id=%d) is not bound to a model", trueStream->Id_()));
return CondStreamSwitchEx(ptr, condition, valuePtr, trueStream, stm, dataType, curCtx);
}
rtError_t ApiImplDavid::LabelSet(Label* const lbl, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_LABEL_WITH_FUNC_DESC(lbl, curCtx, RT_ERROR_LABEL_CONTEXT, "Label setting");
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Label setting");
return CondLabelSet(lbl, stm);
}
rtError_t ApiImplDavid::ProfilerTrace(const uint64_t id, const bool notifyFlag, const uint32_t flags, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_STREAM_PTR_RETURN_MSG(curStm);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Delivering a profiling task with tracepoint");
return cce::runtime::ProfilerTrace(id, notifyFlag, flags, curStm);
}
rtError_t ApiImplDavid::ProfilerTraceEx(const uint64_t id, const uint64_t modelId, const uint16_t tagId, Stream* stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
if (stm == nullptr) {
stm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(stm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Delivering a profiling task with tracepoint");
return ProfTraceEx(id, modelId, tagId, stm, curCtx);
}
rtError_t ApiImplDavid::WriteValue(rtWriteValueInfo_t* const info, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Writing data to the specified memory");
return StreamWriteValue(info, curStm);
}
rtError_t ApiImplDavid::WriteValuePtr(void* const writeValueInfo, Stream* const stm, void* const pointedAddr)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Writing data to the specified memory");
return StreamWriteValuePtr(static_cast<rtWriteValueInfo_t*>(writeValueInfo), curStm, pointedAddr);
}
rtError_t ApiImplDavid::StarsTaskLaunch(
const void* const sqe, const uint32_t sqeLen, Stream* const stm, const uint32_t flag)
{
RT_LOG(RT_LOG_DEBUG, "Stars launch, sqeLen=%u, flag=%u.", sqeLen, flag);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Delivering a Stars task to the device for execution");
COND_RETURN_WARN(
curStm->IsCapturing(), RT_ERROR_FEATURE_NOT_SUPPORT, "stars task cannot be delivered in capture mode.");
return StarsLaunch(sqe, sqeLen, curStm, flag);
}
rtError_t ApiImplDavid::LaunchDvppTask(const void* sqe, uint32_t sqeLen, Stream* stm, rtDvppCfg_t* cfg)
{
RT_LOG(RT_LOG_INFO, "Start to launch dvpp task.");
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(curStm);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "DVPP task delivery");
bool isCmdListNotFree = false;
rtError_t error = GetIsCmdListNotFreeValByDvppCfg(cfg, isCmdListNotFree);
ERROR_RETURN_MSG_INNER(
error, "Failed to get dvpp cmdlist not free flag, streamId=%u, retCode=%#x", curStm->Id_(), error);
const uint32_t flag = isCmdListNotFree ? RT_KERNEL_CMDLIST_NOT_FREE : RT_KERNEL_DEFAULT;
COND_RETURN_WARN(
curStm->IsCapturing(), RT_ERROR_FEATURE_NOT_SUPPORT, "DVPP tasks cannot be delivered in capture mode.");
error = StarsTaskLaunch(sqe, sqeLen, curStm, flag);
ERROR_RETURN(error, "Failed to launch Dvpp task");
return error;
}
rtError_t ApiImplDavid::MultipleTaskInfoLaunch(
const rtMultipleTaskInfo_t* const taskInfo, Stream* const stm, const uint32_t flag)
{
for (size_t idx = 0U; idx < taskInfo->taskNum; idx++) {
if (taskInfo->taskDesc[idx].type == RT_MULTIPLE_TASK_TYPE_DVPP) {
RT_LOG(
RT_LOG_DEBUG, "Launch dvpp task, dvppSqeType=%hhu, pos=%hu",
taskInfo->taskDesc[idx].u.dvppTaskDesc.sqe.sqeHeader.type,
taskInfo->taskDesc[idx].u.dvppTaskDesc.aicpuTaskPos);
} else if (taskInfo->taskDesc[idx].type == RT_MULTIPLE_TASK_TYPE_AICPU) {
RT_LOG(
RT_LOG_DEBUG,
"Launch aicpu task, soName=%s, kernelName=%s, opName=%s, blockDim=%hu, isUnderstudyOp=%hu,"
" argsSize=%u, hostInputInfoNum=%hu",
taskInfo->taskDesc[idx].u.aicpuTaskDesc.kernelLaunchNames.soName,
taskInfo->taskDesc[idx].u.aicpuTaskDesc.kernelLaunchNames.kernelName,
taskInfo->taskDesc[idx].u.aicpuTaskDesc.kernelLaunchNames.opName,
taskInfo->taskDesc[idx].u.aicpuTaskDesc.blockDim,
taskInfo->taskDesc[idx].u.aicpuTaskDesc.isUnderstudyOp,
taskInfo->taskDesc[idx].u.aicpuTaskDesc.argsInfo.argsSize,
taskInfo->taskDesc[idx].u.aicpuTaskDesc.argsInfo.hostInputInfoNum);
} else {
Kernel* hdl = RtPtrToPtr<Kernel*>(taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.funcHdl);
RT_LOG(
RT_LOG_DEBUG,
"launch aicpu task by handle, soName=%s, funcName=%s, opName=%s, blockDim=%hu, isUnderstudyOp=%hu,"
" argsSize=%u, hostInputInfoNum=%hu,",
hdl->GetCpuKernelSo().c_str(), hdl->GetCpuFuncName().c_str(), hdl->GetCpuOpType().c_str(),
taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.blockDim,
taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.isUnderstudyOp,
taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.argsInfo.argsSize,
taskInfo->taskDesc[idx].u.aicpuTaskDescByHandle.argsInfo.hostInputInfoNum);
}
}
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_PTR_RETURN_MSG(curStm, RT_ERROR_STREAM_NULL);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Delivering a DVPP Multiple task");
COND_RETURN_WARN(
curStm->IsCapturing(), RT_ERROR_FEATURE_NOT_SUPPORT, "DVPP tasks cannot be delivered in capture mode.");
return LaunchMultipleTaskInfo(taskInfo, curStm, flag);
}
rtError_t ApiImplDavid::DvppWaitGroupReport(
DvppGrp* const grp, rtDvppGrpCallback const callBackFunc, const int32_t timeout)
{
Context* const curCtx = grp->getContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
return DvppWaitGroup(curCtx->Device_(), grp, callBackFunc, timeout);
}
static rtError_t SetLimitSizeByType(const rtLimitType_t type, const uint32_t val)
{
Runtime* rt = Runtime::Instance();
uint32_t alignVal = (val + STACK_PHY_BASE_ALIGN_LEN - 1U) / STACK_PHY_BASE_ALIGN_LEN * STACK_PHY_BASE_ALIGN_LEN;
switch (type) {
case RT_LIMIT_TYPE_SIMT_STACK_SIZE:
rt->SetSimtWarpStkSize(alignVal * RT_MAX_THREAD_NUM_PER_WARP);
break;
case RT_LIMIT_TYPE_SIMT_DVG_WARP_STACK_SIZE:
rt->SetSimtDvgWarpStkSize(alignVal);
break;
default:
RT_LOG_OUTER_MSG_WITH_FUNC_DESC(
ErrorCode::EE1003, "Setting the SIMT stack size", LimitTypeToString(type), "type",
LimitTypeToString(RT_LIMIT_TYPE_SIMT_STACK_SIZE) + " or " +
LimitTypeToString(RT_LIMIT_TYPE_SIMT_DVG_WARP_STACK_SIZE));
return RT_ERROR_DEVICE_LIMIT;
}
if ((rt->GetSimtWarpStkSize() == 0ULL) && (rt->GetSimtDvgWarpStkSize() == 0U)) {
RT_LOG_OUTER_MSG_WITH_FUNC_DESC(
ErrorCode::EE1011, "Setting the SIMT stack size", val, "val",
"simtStackSize and simtDvgWarpStackSize cannot both be 0");
return RT_ERROR_INVALID_VALUE;
}
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::DeviceSetLimit(const int32_t devId, const rtLimitType_t type, const uint32_t val)
{
RT_LOG(
RT_LOG_DEBUG, "drv devId=%u, type=%s, value=%u.", static_cast<uint32_t>(devId), LimitTypeToString(type).c_str(),
val);
rtError_t error = RT_ERROR_NONE;
Runtime* rt = Runtime::Instance();
COND_RETURN_ERROR_MSG_INNER(rt == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
if (type == RT_LIMIT_TYPE_LOW_POWER_TIMEOUT) {
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
RT_LOG(
RT_LOG_WARNING, "DeviceSetLimit, drv devId=%u, type=%s, value=%u.", static_cast<uint32_t>(devId),
LimitTypeToString(type).c_str(), val);
return error;
} else if (type == RT_LIMIT_TYPE_STACK_SIZE) {
std::unique_lock<std::mutex> lock(rt->GetSimtStackMutex());
rt->SetDeviceCustomerStackSize(val);
return RT_ERROR_NONE;
} else if (type == RT_LIMIT_TYPE_SIMD_PRINTF_FIFO_SIZE_PER_CORE) {
std::unique_lock<std::mutex> lock(rt->GetSimdFifoMutex());
return rt->SetSimdPrintFifoSize(val);
} else if (type == RT_LIMIT_TYPE_SIMT_PRINTF_FIFO_SIZE) {
std::unique_lock<std::mutex> lock(rt->GetSimtFifoMutex());
return rt->SetSimtPrintFifoSize(val);
} else {
}
error = SetLimitSizeByType(type, val);
COND_RETURN_ERROR_MSG_INNER(
error != RT_ERROR_NONE, error, "Set simt stack size failed, drv devId=%u, retCode=%#x.",
static_cast<uint32_t>(devId), static_cast<uint32_t>(error));
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::DeviceGetLimit(const rtLimitType_t type, uint32_t* val)
{
RT_LOG(RT_LOG_DEBUG, "type=%s.", LimitTypeToString(type).c_str());
Runtime* rt = Runtime::Instance();
COND_RETURN_ERROR_MSG_INNER(rt == nullptr, RT_ERROR_INSTANCE_NULL, "Runtime instance is null.");
if (type == RT_LIMIT_TYPE_LOW_POWER_TIMEOUT) {
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
RT_LOG(RT_LOG_WARNING, "DeviceGetLimit, type=%s.", LimitTypeToString(type).c_str());
*val = 0;
} else if (type == RT_LIMIT_TYPE_STACK_SIZE) {
*val = rt->GetDeviceCustomerStackSize();
} else if (type == RT_LIMIT_TYPE_SIMD_PRINTF_FIFO_SIZE_PER_CORE) {
std::unique_lock<std::mutex> lock(rt->GetSimdFifoMutex());
*val = rt->GetSimdPrintFifoSize();
} else if (type == RT_LIMIT_TYPE_SIMT_PRINTF_FIFO_SIZE) {
std::unique_lock<std::mutex> lock(rt->GetSimtFifoMutex());
*val = rt->GetSimtPrintFifoSize();
} else if (type == RT_LIMIT_TYPE_SIMT_STACK_SIZE) {
*val = static_cast<uint32_t>(rt->GetSimtWarpStkSize());
} else if (type == RT_LIMIT_TYPE_SIMT_DVG_WARP_STACK_SIZE) {
*val = rt->GetSimtDvgWarpStkSize();
} else {
RT_LOG(RT_LOG_WARNING, "Limit type is not supported, type=%s", LimitTypeToString(type).c_str());
return RT_ERROR_FEATURE_NOT_SUPPORT;
}
RT_LOG(RT_LOG_INFO, "DeviceGetLimit success, type=%s, val=%u.", LimitTypeToString(type).c_str(), *val);
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::StreamTaskAbort(Stream* const stm)
{
Stream* curStm = stm;
if (curStm == nullptr) {
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
curStm = curCtx->DefaultStream_();
NULL_STREAM_PTR_RETURN_MSG(curStm);
} else {
const bool isValid = ContextManage::CheckStreamPtrIsValid(curStm);
COND_RETURN_AND_MSG_OUTER(
!isValid, RT_ERROR_INVALID_VALUE, ErrorCode::EE1017, "Aborting a stream task", "stm",
RtFmtMsg("Stream (stream_id=%d) does not belong to any context", curStm->Id_()));
}
return curStm->StreamAbort();
}
rtError_t ApiImplDavid::StreamAbort(Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = (stm == nullptr) ? curCtx->DefaultStream_() : stm;
NULL_STREAM_PTR_RETURN_MSG(curStm);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Aborting tasks in a stream");
COND_RETURN_AND_MSG_OUTER(
((curStm->Flags() & RT_STREAM_PERSISTENT) != 0U), RT_ERROR_STREAM_INVALID, ErrorCode::EE1006, "Stream aborting",
"Aborting persistent stream",
"The stream flag contains ACL_STREAM_PERSISTENT(0x4) and the stream cannot be aborted");
return curStm->StreamAbort();
}
rtError_t ApiImplDavid::StreamStop(Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Stopping the running tasks in a stream");
if (stm->GetBindFlag()) {
const uint32_t modelId = (stm->Model_() != nullptr) ? stm->Model_()->Id_() : UINT32_MAX;
RT_LOG_OUTER_MSG_IMPL(
ErrorCode::EE1016, "Stopping tasks in a stream",
RtFmtMsg(
"Stopping tasks in the stream (stream_id=%d) that is bound to a model (model_id=%u) is not supported. "
"Unbind the stream from the model before stopping tasks",
stm->Id_(), modelId));
return RT_ERROR_STREAM_INVALID;
}
return stm->StreamStop();
}
rtError_t ApiImplDavid::StreamRecover(Stream* const stm)
{
const bool isValid = ContextManage::CheckStreamPtrIsValid(stm);
COND_RETURN_AND_MSG_OUTER(
!isValid, RT_ERROR_INVALID_VALUE, ErrorCode::EE1017, "Resuming tasks in a stream", "stm",
RtFmtMsg("Stream (stream_id=%d) does not belong to any context", stm->Id_()));
return stm->StreamRecoverAbort();
}
rtError_t ApiImplDavid::StreamTaskClean(Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Clearing tasks in a stream");
return stm->StreamTaskClean();
}
rtError_t ApiImplDavid::DeviceResourceClean(int32_t devId) { return ContextManage::DeviceResourceClean(devId); }
rtError_t ApiImplDavid::LabelGotoEx(Label* const lbl, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
stm, curCtx, RT_ERROR_STREAM_CONTEXT, "Label redirection");
COND_RETURN_AND_MSG_INVALID_CONTEXT_LABEL_WITH_FUNC_DESC(lbl, curCtx, RT_ERROR_LABEL_CONTEXT, "Label redirection");
RT_LOG_OUTER_MSG_WITH_FUNC_DESC(ErrorCode::EE1005, "label redirection");
return RT_ERROR_FEATURE_NOT_SUPPORT;
}
rtError_t ApiImplDavid::GetMemUceInfo(const uint32_t deviceId, rtMemUceInfo* memUceInfo)
{
rtError_t error;
rtErrorInfo errorInfo = {};
error = GetMemUceInfoProc(deviceId, &errorInfo);
COND_RETURN_WARN(
error == RT_ERROR_FEATURE_NOT_SUPPORT, RT_ERROR_FEATURE_NOT_SUPPORT, "Getting mem uce info is not supported.");
if (error != RT_ERROR_NONE) {
RT_LOG(RT_LOG_ERROR, "Get mem uce info failed, drv devId=%u, error=%d.", deviceId, error);
return error;
}
rtMemUceArray* memUceArray = &(errorInfo.detail.uceInfo);
memUceInfo->devid = deviceId;
memUceInfo->count = memUceArray->arraySize;
errno_t ret = memcpy_s(
memUceInfo->repairAddr, sizeof(memUceInfo->repairAddr), memUceArray->repairAddrArray,
sizeof(memUceArray->repairAddrArray));
if (ret != EOK) {
const std::string retStr = std::to_string(ret);
std::stringstream ss;
ss << std::hex << "dest=0x" << RtPtrToValue(memUceInfo->repairAddr) << ", src=0x"
<< RtPtrToValue(memUceArray->repairAddrArray) << std::dec << ", destMax=" << sizeof(memUceInfo->repairAddr)
<< ", count=" << sizeof(memUceArray->repairAddrArray) << ".";
RT_LOG_OUTER_MSG_IMPL(
ErrorCode::EE1020, "Getting memory uce info", "memcpy_s", retStr.c_str(), strerror(ret), ss.str().c_str());
return RT_ERROR_INVALID_VALUE;
}
return RT_ERROR_NONE;
}
rtError_t ApiImplDavid::DeviceTaskAbort(const int32_t devId, const uint32_t timeout)
{
return DavidDeviceTaskAbort(devId, timeout);
}
static rtError_t L2BufferErrProc(const uint32_t deviceId, rtErrorInfo* const errorInfo)
{
errorInfo->errorType = RT_ERROR_L2;
uint32_t resume_cnt = MAX_UINT32_NUM;
int32_t buf_size = sizeof(resume_cnt);
const rtError_t error = NpuDriver::GetDeviceInfoByBuff(
deviceId, MODULE_TYPE_L2BUFF, INFO_TYPE_L2BUFF_RESUME_CNT, static_cast<void*>(&resume_cnt), &buf_size);
COND_RETURN_WARN(
error == RT_ERROR_FEATURE_NOT_SUPPORT, RT_ERROR_FEATURE_NOT_SUPPORT,
"Getting fault event info is not supported.");
if ((error != RT_ERROR_NONE) || (resume_cnt == MAX_UINT32_NUM) || (buf_size != sizeof(resume_cnt))) {
RT_LOG(
RT_LOG_ERROR, "Calling drv api halGetDeviceInfoByBuff failed, resume_cnt=%u, buf_size=%d, error=%#x.",
resume_cnt, buf_size, static_cast<uint32_t>(error));
return RT_ERROR_DRV_ERR;
}
if (resume_cnt == 0U) {
errorInfo->tryRepair = 1U;
} else {
errorInfo->tryRepair = 0U;
RT_LOG(RT_LOG_INFO, "Resume cnt is not 0, recovery may have been triggered.");
}
return error;
}
static void AicoreErrorProc(const Device* const dev, rtErrorInfo* const errorInfo)
{
errorInfo->hasDetail = 1U;
errorInfo->errorType = RT_ERROR_AICORE;
const uint32_t recoverCnt = dev->GetAixErrRecoverCnt();
if (recoverCnt == 0U) {
errorInfo->tryRepair = 1U;
} else {
errorInfo->tryRepair = 0U;
RT_LOG(RT_LOG_INFO, "Aicore recover cnt is not 0 or other fault event exists, recoverCnt=%u.", recoverCnt);
}
return;
}
static void AicoreUnknownErrorProc(rtErrorInfo* const errorInfo)
{
errorInfo->errorType = RT_ERROR_AICORE;
errorInfo->hasDetail = 1U;
errorInfo->detail.aicoreErrType = RT_AICORE_ERROR_UNKNOWN;
}
static void UnknowErrorProc(const Context* const curCtx, rtErrorInfo* const errorInfo)
{
if (curCtx->GetFailureError() == RT_ERROR_NONE) {
errorInfo->errorType = RT_NO_ERROR;
} else {
errorInfo->errorType = RT_ERROR_OTHERS;
}
}
rtError_t ApiImplDavid::GetErrorVerbose(const uint32_t deviceId, rtErrorInfo* const errorInfo)
{
errorInfo->hasDetail = 0U;
errorInfo->tryRepair = 0U;
errorInfo->errorType = RT_NO_ERROR;
const uint32_t tsId = InnerThreadLocalContainer::GetTsId();
Context* const ctx = Runtime::Instance()->GetPriCtxByDeviceId(deviceId, tsId);
COND_RETURN_WARN(ctx == nullptr, RT_ERROR_NONE, "Device[%u] has no fault.", deviceId);
Device* const dev = ctx->Device_();
COND_RETURN_WARN(dev == nullptr, RT_ERROR_NONE, "Device[%u] has no fault.", deviceId);
rtError_t error = RT_ERROR_NONE;
const DeviceFaultType faultType = dev->GetDeviceFaultType();
RT_LOG(RT_LOG_DEBUG, "start GetErrorVerbose, device_id=%u, type=%u", deviceId, faultType);
switch (faultType) {
case DeviceFaultType::L2_BUFFER_ERROR:
error = L2BufferErrProc(deviceId, errorInfo);
break;
case DeviceFaultType::HBM_UCE_ERROR:
error = GetMemUceInfoProc(deviceId, errorInfo);
errorInfo->errorType = RT_ERROR_MEMORY;
break;
case DeviceFaultType::AICORE_SW_ERROR:
errorInfo->detail.aicoreErrType = RT_AICORE_ERROR_SW;
AicoreErrorProc(dev, errorInfo);
break;
case DeviceFaultType::AICORE_HW_L_ERROR:
errorInfo->detail.aicoreErrType = RT_AICORE_ERROR_HW_LOCAL;
AicoreErrorProc(dev, errorInfo);
break;
case DeviceFaultType::AICORE_UNKNOWN_ERROR:
AicoreUnknownErrorProc(errorInfo);
break;
case DeviceFaultType::LINK_ERROR:
errorInfo->errorType = RT_ERROR_LINK;
errorInfo->tryRepair = 1U;
break;
case DeviceFaultType::L3_PORT_ERROR:
errorInfo->errorType = RT_ERROR_L3_PORT;
errorInfo->tryRepair = 1U;
break;
default:
UnknowErrorProc(ctx, errorInfo);
break;
}
return error;
}
static rtError_t L2BufferErrorResume(Device* const dev, const uint32_t deviceId)
{
uint32_t buf_context = DRV_L2BUFF_CLEAN;
const rtError_t error = NpuDriver::SetDeviceInfoByBuff(
deviceId, MODULE_TYPE_L2BUFF, INFO_TYPE_L2BUFF_RESUME, static_cast<void*>(&buf_context), sizeof(buf_context));
COND_RETURN_WARN(
error == RT_ERROR_FEATURE_NOT_SUPPORT, RT_ERROR_FEATURE_NOT_SUPPORT, "L2 buffer resume is not supported.");
COND_PROC(
(error != RT_ERROR_NONE), RT_LOG(
RT_LOG_ERROR, "L2 buffer err repair failed, deviceId=%u, retCode=%#x.", deviceId,
static_cast<uint32_t>(error)));
dev->SetDeviceFaultType(DeviceFaultType::NO_ERROR);
return error;
}
static rtError_t L3PortRepairResume(Device* const dev)
{
halRepairFaultInfo repairInfo = {};
repairInfo.fault_type = HAL_REPAIR_FAULT_TYPE_UBMEM;
const rtError_t error = NpuDriver::L3PortRepair(dev->Id_(), &repairInfo);
COND_RETURN_WARN(
error == RT_ERROR_FEATURE_NOT_SUPPORT, RT_ERROR_FEATURE_NOT_SUPPORT, "L3 port resume is not supported.");
COND_PROC(
(error != RT_ERROR_NONE), RT_LOG(
RT_LOG_ERROR, "l3 port err repair failed, deviceId=%u, retCode=%#x.", dev->Id_(),
static_cast<uint32_t>(error)));
dev->SetDeviceFaultType(DeviceFaultType::NO_ERROR);
return error;
}
static void L3PortErrorStatusReset(Device* const dev)
{
dev->SetDeviceStatus(RT_ERROR_NONE);
const ReadProtect rp(&ContextDataManage::Instance().GetSetRwLock());
for (Context* const ctx : ContextDataManage::Instance().GetSetObj()) {
if (!ContextManage::IsContextOnDevice(ctx, static_cast<int32_t>(dev->Id_()))) {
continue;
}
ctx->SetStreamsStatus(RT_ERROR_NONE);
ctx->SetFailureError(RT_ERROR_NONE);
}
}
rtError_t ApiImplDavid::RepairError(const uint32_t deviceId, const rtErrorInfo* const errorInfo)
{
rtError_t error = RT_ERROR_NONE;
const uint32_t tsId = InnerThreadLocalContainer::GetTsId();
Device* const dev = Runtime::Instance()->GetDevice(deviceId, tsId);
NULL_PTR_RETURN_MSG(dev, RT_ERROR_DEVICE_NULL);
switch (errorInfo->errorType) {
case RT_ERROR_L2:
error = L2BufferErrorResume(dev, deviceId);
break;
case RT_ERROR_AICORE:
dev->SetAixErrRecoverCnt();
dev->SetDeviceFaultType(DeviceFaultType::NO_ERROR);
break;
case RT_ERROR_MEMORY:
error = MemUceErrorResume(dev, deviceId, errorInfo);
break;
case RT_ERROR_LINK:
dev->SetDeviceFaultType(DeviceFaultType::NO_ERROR);
break;
case RT_ERROR_L3_PORT:
error = L3PortRepairResume(dev);
L3PortErrorStatusReset(dev);
break;
default:
error = RT_ERROR_INVALID_VALUE;
RT_LOG(
RT_LOG_ERROR, "Does not support current error type [%s]",
ErrorTypeToString(errorInfo->errorType).c_str());
break;
}
dev->SetBaseTime();
return error;
}
rtError_t ApiImplDavid::GetStackBuffer(
const rtBinHandle binHandle, uint32_t deviceId, const uint32_t stackType, const uint32_t coreType,
const uint32_t coreId, const void** stack, uint32_t* stackSize)
{
RT_LOG(
RT_LOG_DEBUG, "Get stack buffer, bin handle %p, stackType %u, coreType %u, coreId %u", binHandle, stackType,
coreType, coreId);
return cce::runtime::GetStackBuffer(
RtPtrToPtr<const Program*>(binHandle), deviceId, stackType, coreType, coreId, stack, stackSize);
}
rtError_t ApiImplDavid::DebugReadAICore(rtDebugMemoryParam_t* const param)
{
return cce::runtime::DebugReadAICore(param);
}
rtError_t ApiImplDavid::StarsLaunchSubscribeProc(
Stream* const stm, const rtCallback_t callBackFunc, void* const fnData, const bool needSubscribe,
const uint64_t threadId)
{
rtError_t ret = RT_ERROR_NONE;
Runtime* const rtInstance = Runtime::Instance();
if (needSubscribe && !(stm->IsCapturing())) {
Notify* curNotify = nullptr;
ret = NotifyCreate(static_cast<int32_t>(stm->Device_()->Id_()), &curNotify, RT_NOTIFY_DEFAULT);
ERROR_RETURN(ret, "Call NotifyCreate failed for callback, ret=%#x.", ret);
ret = rtInstance->SubscribeCallback(threadId, stm, static_cast<void*>(curNotify));
if (ret != RT_ERROR_NONE) {
(void)NotifyDestroy(curNotify);
}
}
return CallbackLaunchForDavidWithBlock(callBackFunc, fnData, stm, threadId);
}
rtError_t ApiImplDavid::LaunchHostFunc(Stream* const stm, const rtCallback_t callBackFunc, void* const fnData)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_STREAM_PTR_RETURN_MSG(curStm);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Adding a host callback function to the stream task queue");
Runtime* const rtInstance = Runtime::Instance();
Device* const dev = curCtx->Device_();
NULL_PTR_RETURN_MSG(dev, RT_ERROR_INVALID_VALUE);
curCtx->callbackTheadMutex_.lock();
if (!curCtx->GetCallBackThreadExistFlag()) {
COND_PROC_RETURN_ERROR_MSG_INNER(
curCtx->CreateContextCallBackThread() != RT_ERROR_NONE, RT_ERROR_MEMORY_ALLOCATION,
curCtx->callbackTheadMutex_.unlock(), "Failed to create callback thread.");
curCtx->SetCallBackThreadExistFlag();
}
curCtx->callbackTheadMutex_.unlock();
const bool isNeedSubscribe = rtInstance->JudgeNeedSubscribe(curCtx->GetCallBackThreadId(), curStm, dev->Id_());
return StarsLaunchSubscribeProc(curStm, callBackFunc, fnData, isNeedSubscribe, curCtx->GetCallBackThreadId());
}
rtError_t ApiImplDavid::MemWriteValue(
const void* const devAddr, const uint64_t value, const uint32_t flag, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_STREAM_PTR_RETURN_MSG(curStm);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT, "Writing data to the specified memory");
return cce::runtime::MemWriteValue(devAddr, value, flag, curStm);
}
rtError_t ApiImplDavid::MemWaitValue(
const void* const devAddr, const uint64_t value, const uint32_t flag, Stream* const stm)
{
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
Stream* curStm = stm;
if (curStm == nullptr) {
curStm = curCtx->DefaultStream_();
NULL_STREAM_PTR_RETURN_MSG(curStm);
}
COND_RETURN_AND_MSG_INVALID_CONTEXT_STREAM_WITH_FUNC_DESC(
curStm, curCtx, RT_ERROR_STREAM_CONTEXT,
"Unblocking the data in the specified memory when the data meets certain conditions");
return cce::runtime::MemWaitValue(devAddr, value, flag, curStm);
}
rtError_t ApiImplDavid::StreamAddCondTask(rtCondTaskParams params, Stream* const stm, uint32_t flags)
{
CondHandle* realHandle = nullptr;
rtError_t error = StreamAddCondTaskParasCheck(params, stm, &realHandle);
COND_RETURN_ERROR(error != RT_ERROR_NONE, error, "condition task parameters check failed.");
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
error = GetCaptureFunc().createSubCaptureModels(curCtx, realHandle, params, stm);
ERROR_RETURN_MSG_INNER(
error, "Create sub capture model failed, condition type=%s, condition size=%u, retCode=%#x.",
CondTaskTypeToString(params.type).c_str(), params.size, static_cast<uint32_t>(error));
return cce::runtime::StreamAddCondTask(realHandle, params, stm, flags);
}
rtError_t ApiImplDavid::IpcSetMemoryAttr(const char* name, uint32_t type, uint64_t attr)
{
RT_LOG(RT_LOG_DEBUG, "Set ipc memory attribute. name=%s, type=%u, attr=%" PRIu64 ".", name, type, attr);
Context* const curCtx = CurrentContext();
CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
const rtError_t error = curCtx->Device_()->Driver_()->CheckIpcMapRoute(name, attr, curCtx->Device_()->Id_());
COND_RETURN_WITH_NOLOG(error != RT_ERROR_NONE, error);
const std::unique_lock<std::mutex> lock(Runtime::Instance()->GetIpcMemNameLock());
std::unordered_map<std::string, ipcMemInfo_t>& ipcMemNameMap = Runtime::Instance()->GetIpcMemNameMap();
std::string ipcName(name);
auto it = ipcMemNameMap.find(ipcName);
if (it == ipcMemNameMap.end()) {
ipcMemInfo_t& info = ipcMemNameMap[ipcName];
info.latestAttr = attr;
} else {
it->second.latestAttr = attr;
}
return RT_ERROR_NONE;
}
}
}
