[feat] Arch5162 aicpu support timeout-runtime-AtomGit

Mpkg_inc/aicpu_sched/runtime_thread_aicpu_plugin.h+12-1

	@@ -13,6 +13,8 @@
13		13
14	#include <cstdint>	14	#include <cstdint>
15		15
		16	+constexpr uint64_t RUNTIME_THREAD_AICPU_DEFAULT_EXECUTE_TIMEOUT_US = 28000000ULL;
		17	+
16	enum class RuntimeThreadAicpuStatus : uint32_t {	18	enum class RuntimeThreadAicpuStatus : uint32_t {
17	OK = 0U,	19	OK = 0U,
18	INVALID_PARAM,	20	INVALID_PARAM,
	@@ -29,6 +31,11 @@ enum class RuntimeThreadAicpuSqeSubtype : uint32_t {
29	AICPU = 22U,	31	AICPU = 22U,
30	};	32	};
31		33
		34	+enum class RuntimeThreadAicpuStreamErrorType : uint32_t {
		Y Y yanhaiyan18 天前当前头文件仅用于runtime，pkg_inc 为对外发布头文件，放在这里后要保证后续兼容性。建议将这个头文件放到runtime仓。 FengHaozhan 18 天前评论： 18 天前评论：
		35	+ EXECUTION_FAILED = 0U,
		36	+ EXECUTION_TIMEOUT,
		37	+};
		38	+
32	struct RuntimeThreadAicpuRuntimeHooks {	39	struct RuntimeThreadAicpuRuntimeHooks {
33	uint32_t structSize;	40	uint32_t structSize;
34	void* runtimeData;	41	void* runtimeData;
	@@ -38,10 +45,12 @@ struct RuntimeThreadAicpuRuntimeHooks {
38	void* runtimeData, void* streamHandle, void** eventHandle, uint32_t* eventId);	45	void* runtimeData, void* streamHandle, void** eventHandle, uint32_t* eventId);
39	void (destroyCompletionEvent)(void runtimeData, void* eventHandle);	46	void (destroyCompletionEvent)(void runtimeData, void* eventHandle);
40	void (*setStreamError)(	47	void (*setStreamError)(
41	- void* runtimeData, uint32_t deviceId, uint32_t tsId, uint32_t streamId, uint32_t executeResult);	48	+ void* runtimeData, uint32_t deviceId, uint32_t tsId, uint32_t streamId, void* streamHandle,
		49	+ RuntimeThreadAicpuStreamErrorType errorType, uint32_t errorDetail);
42	void (monitorThreadEnter)(void runtimeData);	50	void (monitorThreadEnter)(void runtimeData);
43	void (monitorThreadExit)(void runtimeData);	51	void (monitorThreadExit)(void runtimeData);
44	bool (isProcessExiting)(void runtimeData);	52	bool (isProcessExiting)(void runtimeData);
		53	+ uint64_t (clockGetTimeUs)(void runtimeData);
45	};	54	};
46		55
47	struct RuntimeThreadAicpuKernelRequest {	56	struct RuntimeThreadAicpuKernelRequest {
	@@ -79,6 +88,8 @@ struct RuntimeThreadAicpuPluginApi {
79	const RuntimeThreadAicpuKernelRequest* request, RuntimeThreadAicpuPreparedKernel* preparedKernel);	88	const RuntimeThreadAicpuKernelRequest* request, RuntimeThreadAicpuPreparedKernel* preparedKernel);
80	void (*releasePreparedKernel)(uint64_t taskCookie);	89	void (*releasePreparedKernel)(uint64_t taskCookie);
81	void (streamDestroyed)(void streamHandle);	90	void (streamDestroyed)(void streamHandle);
		91	+ void (*setExecuteTimeout)(uint64_t timeoutUs);
		92	+ void (*monitorExecutionTimeout)();
		Y Y yanhaiyan18 天前 `void (*monitorExecutionTimeout)();` 空参数列表未写 `(void)`。头文件含 `extern "C"` 声明属对外 C 接口，若被纯 C 翻译单元引用，`()` 为非原型（K&R）声明失去参数校验；建议统一写 `(void)`。 FengHaozhan 18 天前评论： 18 天前评论：
82	};	93	};
83		94
84	using RuntimeThreadAicpuGetPluginApiFunc = RuntimeThreadAicpuStatus (*)(	95	using RuntimeThreadAicpuGetPluginApiFunc = RuntimeThreadAicpuStatus (*)(

Msrc/aicpu_sched/runtime_thread_aicpu/runtime_thread_aicpu_plugin.cc+16-0

	@@ -50,11 +50,27 @@ void StreamDestroyed(void* const streamHandle)
50	}	50	}
51	}	51	}
52		52
		53	+void SetExecuteTimeout(const uint64_t timeoutUs)
		54	+{
		55	+ if (g_pluginService != nullptr) {
		56	+ g_pluginService->SetExecuteTimeout(timeoutUs);
		57	+ }
		58	+}
		59	+
		60	+void MonitorExecutionTimeout()
		61	+{
		62	+ if (g_pluginService != nullptr) {
		63	+ g_pluginService->MonitorExecutionTimeout();
		64	+ }
		65	+}
		66	+
53	const RuntimeThreadAicpuPluginApi PLUGIN_API = {	67	const RuntimeThreadAicpuPluginApi PLUGIN_API = {
54	.structSize = sizeof(RuntimeThreadAicpuPluginApi),	68	.structSize = sizeof(RuntimeThreadAicpuPluginApi),
55	.prepareKernel = &PrepareKernel,	69	.prepareKernel = &PrepareKernel,
56	.releasePreparedKernel = &ReleasePreparedKernel,	70	.releasePreparedKernel = &ReleasePreparedKernel,
57	.streamDestroyed = &StreamDestroyed,	71	.streamDestroyed = &StreamDestroyed,
		72	+ .setExecuteTimeout = &SetExecuteTimeout,
		73	+ .monitorExecutionTimeout = &MonitorExecutionTimeout,
58	};	74	};
59		75
60	} // namespace	76	} // namespace

Msrc/aicpu_sched/runtime_thread_aicpu/runtime_thread_aicpu_service.cc+121-9

	@@ -91,7 +91,7 @@ bool HooksAreValid(const RuntimeThreadAicpuRuntimeHooks& hooks)
91	(hooks.releaseGroupId != nullptr) && (hooks.createCompletionEvent != nullptr) &&	91	(hooks.releaseGroupId != nullptr) && (hooks.createCompletionEvent != nullptr) &&
92	(hooks.destroyCompletionEvent != nullptr) && (hooks.setStreamError != nullptr) &&	92	(hooks.destroyCompletionEvent != nullptr) && (hooks.setStreamError != nullptr) &&
93	(hooks.monitorThreadEnter != nullptr) && (hooks.monitorThreadExit != nullptr) &&	93	(hooks.monitorThreadEnter != nullptr) && (hooks.monitorThreadExit != nullptr) &&
94	- (hooks.isProcessExiting != nullptr);	94	+ (hooks.isProcessExiting != nullptr) && (hooks.clockGetTimeUs != nullptr);
95	}	95	}
96		96
97	} // namespace	97	} // namespace
	@@ -173,7 +173,8 @@ RuntimeThreadAicpuStatus RuntimeThreadAicpuService::EnsureStarted(const RuntimeT
173	aicpusd_err(	173	aicpusd_err(
174	"RuntimeThreadAicpu runtime hooks are invalid, hooks_size=%u, expected_size=%zu, "	174	"RuntimeThreadAicpu runtime hooks are invalid, hooks_size=%u, expected_size=%zu, "
175	"reserve_group_valid=%u, release_group_valid=%u, create_event_valid=%u, destroy_event_valid=%u, "	175	"reserve_group_valid=%u, release_group_valid=%u, create_event_valid=%u, destroy_event_valid=%u, "
176	- "set_stream_error_valid=%u, monitor_enter_valid=%u, monitor_exit_valid=%u, process_exit_valid=%u.",	176	+ "set_stream_error_valid=%u, monitor_enter_valid=%u, monitor_exit_valid=%u, process_exit_valid=%u, "
		177	+ "clock_valid=%u.",
177	hooks_.structSize, sizeof(RuntimeThreadAicpuRuntimeHooks),	178	hooks_.structSize, sizeof(RuntimeThreadAicpuRuntimeHooks),
178	static_cast<uint32_t>(hooks_.reserveGroupId != nullptr),	179	static_cast<uint32_t>(hooks_.reserveGroupId != nullptr),
179	static_cast<uint32_t>(hooks_.releaseGroupId != nullptr),	180	static_cast<uint32_t>(hooks_.releaseGroupId != nullptr),
	@@ -182,7 +183,8 @@ RuntimeThreadAicpuStatus RuntimeThreadAicpuService::EnsureStarted(const RuntimeT
182	static_cast<uint32_t>(hooks_.setStreamError != nullptr),	183	static_cast<uint32_t>(hooks_.setStreamError != nullptr),
183	static_cast<uint32_t>(hooks_.monitorThreadEnter != nullptr),	184	static_cast<uint32_t>(hooks_.monitorThreadEnter != nullptr),
184	static_cast<uint32_t>(hooks_.monitorThreadExit != nullptr),	185	static_cast<uint32_t>(hooks_.monitorThreadExit != nullptr),
185	- static_cast<uint32_t>(hooks_.isProcessExiting != nullptr));	186	+ static_cast<uint32_t>(hooks_.isProcessExiting != nullptr),
		187	+ static_cast<uint32_t>(hooks_.clockGetTimeUs != nullptr));
186	return RuntimeThreadAicpuStatus::INVALID_PARAM;	188	return RuntimeThreadAicpuStatus::INVALID_PARAM;
187	}	189	}
188		190
	@@ -352,6 +354,7 @@ RuntimeThreadAicpuStatus RuntimeThreadAicpuService::CreateKernelContext(
352	context->blockDim = request.blockDim;	354	context->blockDim = request.blockDim;
353	context->deviceId = request.deviceId;	355	context->deviceId = request.deviceId;
354	context->streamId = request.streamId;	356	context->streamId = request.streamId;
		357	+ context->streamHandle = request.streamHandle;
355	context->opType = (request.opType == nullptr) ? std::string() : std::string(request.opType);	358	context->opType = (request.opType == nullptr) ? std::string() : std::string(request.opType);
356	const RuntimeThreadAicpuStatus nameStatus = ResolveKernelNames(request, *context);	359	const RuntimeThreadAicpuStatus nameStatus = ResolveKernelNames(request, *context);
357	if (nameStatus != RuntimeThreadAicpuStatus::OK) {	360	if (nameStatus != RuntimeThreadAicpuStatus::OK) {
	@@ -445,17 +448,119 @@ void RuntimeThreadAicpuService::ReleasePreparedKernel(const uint64_t taskCookie)
445	void RuntimeThreadAicpuService::StreamDestroyed(void* const streamHandle)	448	void RuntimeThreadAicpuService::StreamDestroyed(void* const streamHandle)
446	{	449	{
447	EventEntry event;	450	EventEntry event;
		451	+ bool eventFound = false;
448	{	452	{
449	std::lock_guard<std::mutex> lock(resourceMutex_);	453	std::lock_guard<std::mutex> lock(resourceMutex_);
450	const auto eventIt = events_.find(streamHandle);	454	const auto eventIt = events_.find(streamHandle);
451	- if (eventIt == events_.end()) {	455	+ if (eventIt != events_.end()) {
		456	+ event = eventIt->second;
		457	+ events_.erase(eventIt);
		458	+ eventFound = true;
		459	+ }
		460	+ }
		461	+ if (eventFound) {
		462	+ hooks_.destroyCompletionEvent(hooks_.runtimeData, event.eventHandle);
		463	+ aicpusd_info("Destroy completion event, stream_id=%u, event_id=%u.", event.streamId, event.eventId);
		464	+ }
		465	+
		466	+ std::lock_guard<std::mutex> lock(executionMutex_);
		467	+ for (auto taskIt = executingTasks_.begin(); taskIt != executingTasks_.end();) {
		468	+ if (taskIt->second.streamHandle == streamHandle) {
		469	+ taskIt = executingTasks_.erase(taskIt);
		470	+ } else {
		471	+ ++taskIt;
		472	+ }
		473	+ }
		474	+}
		475	+
		476	+void RuntimeThreadAicpuService::SetExecuteTimeout(const uint64_t timeoutUs)
		477	+{
		478	+ executeTimeoutUs_.store(timeoutUs, std::memory_order_release);
		Y Y yanhaiyan18 天前 `SetExecuteTimeout` 对 timeoutUs 无取值校验：0 会使所有任务立即判定超时（`elapsed >= 0` 恒真），与 runtime 侧 never-timeout=max 的哨兵语义共同构成协议，但 0 值是否合法无校验、无注释。建议至少在注释中写明 0/max 两哨兵的语义，或对 0 返回 INVALID_PARAM（需与 runtime 侧行为对齐后决定）。 FengHaozhan 18 天前评论： 18 天前评论：
		479	+ aicpusd_info(
		480	+ "Set RuntimeThreadAicpu execute timeout, timeout_us=%llu.", static_cast<unsigned long long>(timeoutUs));
		481	+}
		482	+
		483	+RuntimeThreadAicpuStatus RuntimeThreadAicpuService::BeginExecution(
		484	+ const uint64_t taskCookie, const KernelContext& context)
		485	+{
		486	+ ExecutingTaskInfo taskInfo;
		487	+ taskInfo.taskCookie = taskCookie;
		488	+ taskInfo.startTimeUs = hooks_.clockGetTimeUs(hooks_.runtimeData);
		489	+ taskInfo.deviceId = context.deviceId;
		490	+ taskInfo.tsId = tsId_;
		491	+ taskInfo.streamId = context.streamId;
		492	+ taskInfo.taskId = context.taskId;
		493	+ taskInfo.streamHandle = context.streamHandle;
		494	+ try {
		495	+ const std::lock_guard<std::mutex> lock(executionMutex_);
		496	+ const auto result = executingTasks_.emplace(taskCookie, taskInfo);
		497	+ if (!result.second) {
		498	+ aicpusd_err(
		499	+ "Start AICPU execution monitoring failed because task is duplicated, task_cookie=%llu, "
		500	+ "stream_id=%u, task_id=%u.",
		501	+ static_cast<unsigned long long>(taskCookie), context.streamId, context.taskId);
		502	+ return RuntimeThreadAicpuStatus::INTERNAL_ERROR;
		503	+ }
		504	+ } catch (const std::bad_alloc&) {
		505	+ aicpusd_err(
		506	+ "Start AICPU execution monitoring failed because memory allocation failed, task_cookie=%llu, "
		507	+ "stream_id=%u, task_id=%u.",
		508	+ static_cast<unsigned long long>(taskCookie), context.streamId, context.taskId);
		509	+ return RuntimeThreadAicpuStatus::NO_MEMORY;
		510	+ } catch (...) {
		511	+ aicpusd_err(
		512	+ "Start AICPU execution monitoring failed, task_cookie=%llu, stream_id=%u, task_id=%u.",
		Y Y yanhaiyan18 天前是否可以打印异常类型，便于定位 FengHaozhan 18 天前评论： 18 天前评论：
		513	+ static_cast<unsigned long long>(taskCookie), context.streamId, context.taskId);
		514	+ return RuntimeThreadAicpuStatus::INTERNAL_ERROR;
		515	+ }
		516	+ return RuntimeThreadAicpuStatus::OK;
		517	+}
		518	+
		519	+void RuntimeThreadAicpuService::EndExecution(const uint64_t taskCookie)
		520	+{
		521	+ const std::lock_guard<std::mutex> lock(executionMutex_);
		522	+ (void)executingTasks_.erase(taskCookie);
		523	+}
		524	+
		525	+void RuntimeThreadAicpuService::MonitorExecutionTimeout()
		526	+{
		527	+ const uint64_t timeoutUs = executeTimeoutUs_.load(std::memory_order_acquire);
		528	+ if (timeoutUs == std::numeric_limits<uint64_t>::max()) {
		529	+ return;
		530	+ }
		531	+
		532	+ const uint64_t nowUs = hooks_.clockGetTimeUs(hooks_.runtimeData);
		533	+ std::vector<ExecutingTaskInfo> timeoutTasks;
		534	+ {
		535	+ const std::lock_guard<std::mutex> lock(executionMutex_);
		536	+ try {
		537	+ timeoutTasks.reserve(executingTasks_.size());
		Y Y yanhaiyan18 天前 executingTasks_ 为空时仍无条件调用 `clockGetTimeUs` + 加锁 + 构造空 vector。请确认是否要先判空（或 atomic 计数）快速返回。 FengHaozhan 18 天前评论： 18 天前评论：
		538	+ } catch (const std::bad_alloc&) {
		539	+ aicpusd_err("Collect RuntimeThreadAicpu timeout tasks failed because memory allocation failed.");
		Y Y yanhaiyan18 天前要不要打印task或者device的关键信息如executingTasks_.size() FengHaozhan 18 天前评论： 18 天前评论：
452	return;	540	return;
453	}	541	}
454	- event = eventIt->second;	542	+ for (auto& taskEntry : executingTasks_) {
455	- events_.erase(eventIt);	543	+ ExecutingTaskInfo& taskInfo = taskEntry.second;
		544	+ if (taskInfo.timeoutReported \|\| (nowUs < taskInfo.startTimeUs) \|\|
		545	+ ((nowUs - taskInfo.startTimeUs) < timeoutUs)) {
		546	+ continue;
		547	+ }
		548	+ timeoutTasks.emplace_back(taskInfo);
		549	+ taskInfo.timeoutReported = true;
		550	+ }
		551	+ }
		552	+
		553	+ for (const ExecutingTaskInfo& taskInfo : timeoutTasks) {
		554	+ aicpusd_err(
		555	+ "RuntimeThreadAicpu operator timed out, task_cookie=%llu, device_id=%u, ts_id=%u, stream_id=%u, "
		556	+ "task_id=%u, elapsed_us=%llu, timeout_us=%llu.",
		557	+ static_cast<unsigned long long>(taskInfo.taskCookie), taskInfo.deviceId, taskInfo.tsId, taskInfo.streamId,
		558	+ taskInfo.taskId, static_cast<unsigned long long>(nowUs - taskInfo.startTimeUs),
		559	+ static_cast<unsigned long long>(timeoutUs));
		560	+ hooks_.setStreamError(
		561	+ hooks_.runtimeData, taskInfo.deviceId, taskInfo.tsId, taskInfo.streamId, taskInfo.streamHandle,
		562	+ RuntimeThreadAicpuStreamErrorType::EXECUTION_TIMEOUT, 0U);
456	}	563	}
457	- hooks_.destroyCompletionEvent(hooks_.runtimeData, event.eventHandle);
458	- aicpusd_info("Destroy completion event, stream_id=%u, event_id=%u.", event.streamId, event.eventId);
459	}	564	}
460		565
461	bool RuntimeThreadAicpuService::SetReportedTaskId(const uint64_t taskCookie, const uint32_t taskId)	566	bool RuntimeThreadAicpuService::SetReportedTaskId(const uint64_t taskCookie, const uint32_t taskId)
	@@ -494,6 +599,12 @@ uint32_t RuntimeThreadAicpuService::ExecutePreparedKernel(const uint64_t taskCoo
494	context = std::move(contextIt->second);	599	context = std::move(contextIt->second);
495	kernelContexts_.erase(contextIt);	600	kernelContexts_.erase(contextIt);
496	}	601	}
		602	+
		603	+ const RuntimeThreadAicpuStatus status = BeginExecution(taskCookie, *context);
		604	+ if (status != RuntimeThreadAicpuStatus::OK) {
		605	+ return static_cast<uint32_t>(status);
		606	+ }
		607	+ const ExecutionStateGuard executionStateGuard(*this, taskCookie);
497	return ExecuteKernel(*context);	608	return ExecuteKernel(*context);
498	}	609	}
499		610
	@@ -580,7 +691,8 @@ RuntimeThreadAicpuStatus RuntimeThreadAicpuService::FinishReport(
580	"AICPU operator report execution failed, stream_id=%u, task_id=%u, execute_result=%#x.",	691	"AICPU operator report execution failed, stream_id=%u, task_id=%u, execute_result=%#x.",
581	static_cast<uint32_t>(report->streamId), static_cast<uint32_t>(report->taskId), executeResult);	692	static_cast<uint32_t>(report->streamId), static_cast<uint32_t>(report->taskId), executeResult);
582	hooks_.setStreamError(	693	hooks_.setStreamError(
583	- hooks_.runtimeData, deviceId_, tsId_, static_cast<uint32_t>(report->streamId), executeResult);	694	+ hooks_.runtimeData, deviceId_, tsId_, static_cast<uint32_t>(report->streamId), nullptr,
		695	+ RuntimeThreadAicpuStreamErrorType::EXECUTION_FAILED, executeResult);
584	}	696	}
585		697
586	halSqMemGetInput getInput = {};	698	halSqMemGetInput getInput = {};

Msrc/aicpu_sched/runtime_thread_aicpu/runtime_thread_aicpu_service.hpp+31-0

	@@ -33,6 +33,8 @@ public:
33	const RuntimeThreadAicpuKernelRequest& request, RuntimeThreadAicpuPreparedKernel& preparedKernel);	33	const RuntimeThreadAicpuKernelRequest& request, RuntimeThreadAicpuPreparedKernel& preparedKernel);
34	void ReleasePreparedKernel(const uint64_t taskCookie);	34	void ReleasePreparedKernel(const uint64_t taskCookie);
35	void StreamDestroyed(void* const streamHandle);	35	void StreamDestroyed(void* const streamHandle);
		36	+ void SetExecuteTimeout(const uint64_t timeoutUs);
		37	+ void MonitorExecutionTimeout();
36	static uint32_t ExecutePreparedKernelEntry(void* const cookieData);	38	static uint32_t ExecutePreparedKernelEntry(void* const cookieData);
37		39
38	private:	40	private:
	@@ -46,6 +48,7 @@ private:
46	uint32_t deviceId = 0U;	48	uint32_t deviceId = 0U;
47	uint32_t streamId = 0U;	49	uint32_t streamId = 0U;
48	uint32_t taskId = 0U;	50	uint32_t taskId = 0U;
		51	+ void* streamHandle = nullptr;
49	};	52	};
50		53
51	struct EventEntry {	54	struct EventEntry {
	@@ -54,6 +57,29 @@ private:
54	uint32_t streamId = 0U;	57	uint32_t streamId = 0U;
55	};	58	};
56		59
		60	+ struct ExecutingTaskInfo {
		61	+ uint64_t taskCookie = 0U;
		62	+ uint64_t startTimeUs = 0U;
		63	+ uint32_t deviceId = 0U;
		64	+ uint32_t tsId = 0U;
		65	+ uint32_t streamId = 0U;
		66	+ uint32_t taskId = 0U;
		67	+ void* streamHandle = nullptr;
		68	+ bool timeoutReported = false;
		69	+ };
		70	+
		71	+ class ExecutionStateGuard final {
		Y Y yanhaiyan18 天前 `ExecutionStateGuard` 未显式删除拷贝构造：引用成员 `service_` 仅使拷贝赋值被隐式删除（引用无法重绑定），拷贝构造仍隐式生成（原样复制引用，`ExecutionStateGuard copy = guard;` 可编译；类含用户声明析构，属 deprecated 隐式生成，GCC 9+ 在 -Wextra 下有 -Wdeprecated-copy 类告警）。误拷贝的后果：副本析构会对同一 taskCookie 提前/重复执行 EndExecution——若副本先析构（临时对象/内层作用域），任务被提前移出 executingTasks_，ExecuteKernel 仍在执行而超时监控已失效。建议 `ExecutionStateGuard(const ExecutionStateGuard&) = delete;`。 FengHaozhan 18 天前评论： 18 天前评论：
		72	+ public:
		73	+ ExecutionStateGuard(RuntimeThreadAicpuService& service, const uint64_t taskCookie)
		74	+ : service_(service), taskCookie_(taskCookie)
		75	+ {}
		76	+ ~ExecutionStateGuard() noexcept { service_.EndExecution(taskCookie_); }
		77	+
		78	+ private:
		79	+ RuntimeThreadAicpuService& service_;
		80	+ uint64_t taskCookie_;
		81	+ };
		82	+
57	RuntimeThreadAicpuStatus EnsureStarted(const RuntimeThreadAicpuKernelRequest& request);	83	RuntimeThreadAicpuStatus EnsureStarted(const RuntimeThreadAicpuKernelRequest& request);
58	RuntimeThreadAicpuStatus AllocateCallbackChannel();	84	RuntimeThreadAicpuStatus AllocateCallbackChannel();
59	void ReleaseCallbackChannel();	85	void ReleaseCallbackChannel();
	@@ -62,6 +88,8 @@ private:
62	RuntimeThreadAicpuStatus ResolveKernelNames(	88	RuntimeThreadAicpuStatus ResolveKernelNames(
63	const RuntimeThreadAicpuKernelRequest& request, KernelContext& context) const;	89	const RuntimeThreadAicpuKernelRequest& request, KernelContext& context) const;
64	bool SetReportedTaskId(const uint64_t taskCookie, const uint32_t taskId);	90	bool SetReportedTaskId(const uint64_t taskCookie, const uint32_t taskId);
		91	+ RuntimeThreadAicpuStatus BeginExecution(const uint64_t taskCookie, const KernelContext& context);
		92	+ void EndExecution(const uint64_t taskCookie);
65	uint32_t ExecutePreparedKernel(const uint64_t taskCookie);	93	uint32_t ExecutePreparedKernel(const uint64_t taskCookie);
66	uint32_t ExecuteKernel(KernelContext& context);	94	uint32_t ExecuteKernel(KernelContext& context);
67	void WorkerLoop();	95	void WorkerLoop();
	@@ -73,6 +101,7 @@ private:
73	std::mutex startMutex_;	101	std::mutex startMutex_;
74	std::mutex resourceMutex_;	102	std::mutex resourceMutex_;
75	std::mutex taskMutex_;	103	std::mutex taskMutex_;
		104	+ std::mutex executionMutex_;
76	std::thread worker_;	105	std::thread worker_;
77	std::atomic<bool> failed_{false};	106	std::atomic<bool> failed_{false};
78	std::atomic<uint64_t> nextTaskCookie_{1U};	107	std::atomic<uint64_t> nextTaskCookie_{1U};
	@@ -84,6 +113,8 @@ private:
84	uint32_t callbackCqId_ = 0U;	113	uint32_t callbackCqId_ = 0U;
85	std::unordered_map<void*, EventEntry> events_;	114	std::unordered_map<void*, EventEntry> events_;
86	std::unordered_map<uint64_t, std::unique_ptr<KernelContext>> kernelContexts_;	115	std::unordered_map<uint64_t, std::unique_ptr<KernelContext>> kernelContexts_;
		116	+ std::unordered_map<uint64_t, ExecutingTaskInfo> executingTasks_;
		117	+ std::atomic<uint64_t> executeTimeoutUs_{RUNTIME_THREAD_AICPU_DEFAULT_EXECUTE_TIMEOUT_US};
87	SoManager soManager_;	118	SoManager soManager_;
88	};	119	};
89		120

Msrc/runtime/api/impl/api_impl.cc+0-27

	@@ -5459,33 +5459,6 @@ rtError_t ApiImpl::GetOpTimeOutInterval(uint64_t* interval)
5459	return RT_ERROR_NONE;	5459	return RT_ERROR_NONE;
5460	}	5460	}
5461		5461
5462	-rtError_t ApiImpl::SetOpExecuteTimeOutV2(uint64_t timeout, uint64_t* actualTimeout)
Y Y yanhaiyan18 天前已过期 `#include "task_execute_time.h"` 在 SetOpExecuteTimeOutV2 迁出后已无任何符号使用，应删除。 FengHaozhan 18 天前评论： 18 天前评论：
5463	-{
5464	- Context* const curCtx = CurrentContext(); // 隐式set device
5465	- CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
5466	- Runtime* const rtInstance = Runtime::Instance();
5467	- NULL_PTR_RETURN_MSG(rtInstance, RT_ERROR_INSTANCE_NULL);
5468	- const RtTimeoutConfig& timeoutCfg = rtInstance->GetTimeoutConfig();
5469	- COND_RETURN_AND_MSG_OUTER(
5470	- (timeoutCfg.isInit == false), RT_ERROR_DEVICE_RETAIN, ErrorCode::EE1018,
5471	- "Setting the timeout interval for operator execution", "Device is not initialized, call rtSetDevice API first");
5472	-
5473	- const rtError_t error = rtInstance->SetTimeoutConfig(RT_TIMEOUT_TYPE_OP_EXECUTE, timeout, RT_TIME_UNIT_TYPE_US);
5474	- ERROR_RETURN_MSG_INNER(error, "Failed to set op execute timeout, retCode=%#x.", static_cast<uint32_t>(error));
5475	-
5476	- if ((timeoutCfg.isOpTimeoutMs) && (timeoutCfg.isCfgOpExcTaskTimeout) && (timeoutCfg.opExcTaskTimeout == 0UL)) {
5477	- *actualTimeout = MAX_UINT64_NUM; // never timeout
5478	- } else {
5479	- uint16_t credit = 0U;
5480	- TransExeTimeoutCfgToKernelCredit(timeout, credit);
5481	- actualTimeout = static_cast<uint64_t>(static_cast<double>(credit) timeoutCfg.interval);
5482	- }
5483	- RT_LOG(
5484	- RT_LOG_INFO, "set op execute timeout, timeout=%" PRIu64 "us, actualTimeout=%" PRIu64 "us.", timeout,
5485	- *actualTimeout);
5486	- return RT_ERROR_NONE;
5487	-}
5488	-
5489	rtError_t ApiImpl::StarsTaskLaunch(const void* const sqe, const uint32_t sqeLen, Stream* const stm, const uint32_t flag)	5462	rtError_t ApiImpl::StarsTaskLaunch(const void* const sqe, const uint32_t sqeLen, Stream* const stm, const uint32_t flag)
5490	{	5463	{
5491	RT_LOG(RT_LOG_DEBUG, "Stars launch, sqeLen=%u, flag=%u.", sqeLen, flag);	5464	RT_LOG(RT_LOG_DEBUG, "Stars launch, sqeLen=%u, flag=%u.", sqeLen, flag);

Msrc/runtime/api/impl/api_impl_arch5162.cc+31-0

	@@ -11,8 +11,10 @@
11	#include "context.hpp"	11	#include "context.hpp"
12	#include "error_message_manage.hpp"	12	#include "error_message_manage.hpp"
13	#include "kernel.hpp"	13	#include "kernel.hpp"
		14	+#include "runtime.hpp"
14	#include "runtime_thread_aicpu.hpp"	15	#include "runtime_thread_aicpu.hpp"
15	#include "stream.hpp"	16	#include "stream.hpp"
		17	+#include "task_execute_time.h"
16		18
17	namespace cce {	19	namespace cce {
18	namespace runtime {	20	namespace runtime {
	@@ -52,6 +54,35 @@ rtError_t ApiImpl::CpuKernelLaunchEx(
52	return LaunchRuntimeThreadAicpuKernel(this, kernel, coreDim, argsInfo, stream);	54	return LaunchRuntimeThreadAicpuKernel(this, kernel, coreDim, argsInfo, stream);
53	}	55	}
54		56
		57	+rtError_t ApiImpl::SetOpExecuteTimeOutV2(uint64_t timeout, uint64_t* actualTimeout)
		58	+{
		59	+ Context* const curCtx = CurrentContext(); // 隐式set device
		60	+ CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
		61	+ Runtime* const rtInstance = Runtime::Instance();
		62	+ NULL_PTR_RETURN_MSG(rtInstance, RT_ERROR_INSTANCE_NULL);
		63	+ const RtTimeoutConfig& timeoutCfg = rtInstance->GetTimeoutConfig();
		64	+ COND_RETURN_AND_MSG_OUTER(
		65	+ (timeoutCfg.isInit == false), RT_ERROR_DEVICE_RETAIN, ErrorCode::EE1018,
		66	+ "Setting the timeout interval for operator execution", "Device is not initialized, call rtSetDevice API first");
		67	+
		68	+ const rtError_t error = rtInstance->SetTimeoutConfig(RT_TIMEOUT_TYPE_OP_EXECUTE, timeout, RT_TIME_UNIT_TYPE_US);
		69	+ ERROR_RETURN_MSG_INNER(error, "Failed to set op execute timeout, retCode=%#x.", static_cast<uint32_t>(error));
		70	+
		71	+ if ((timeoutCfg.isOpTimeoutMs) && (timeoutCfg.isCfgOpExcTaskTimeout) && (timeoutCfg.opExcTaskTimeout == 0UL)) {
		72	+ *actualTimeout = MAX_UINT64_NUM; // never timeout
		73	+ } else {
		74	+ uint16_t credit = 0U;
		75	+ TransExeTimeoutCfgToKernelCredit(timeout, credit);
		76	+ actualTimeout = static_cast<uint64_t>(static_cast<double>(credit) timeoutCfg.interval);
		77	+ }
		78	+ ERROR_RETURN_MSG_INNER(
		Y Y yanhaiyan18 天前 error是只打用户 format，不自动附加错误码，失败时无法定位具体返回值。建议和SetTimeoutConfig一样，增加error打印 FengHaozhan 18 天前评论： 18 天前评论：
		79	+ SetRuntimeThreadAicpuExecuteTimeout(this, *actualTimeout), "Failed to set RuntimeThreadAicpu execute timeout.");
		Y Y yanhaiyan18 天前 arch5162和非arch5162的差异仅在当前2行，配置超时不是性能敏感的热路径，建议当前函数改成弱符号。所有芯片共一个函数，函数中进行地址非空判断，便于后续统一维护，也便于其他芯片扩展。 FengHaozhan 18 天前评论： 18 天前评论：
		80	+ RT_LOG(
		81	+ RT_LOG_INFO, "set op execute timeout, timeout=%" PRIu64 "us, actualTimeout=%" PRIu64 "us.", timeout,
		82	+ *actualTimeout);
		83	+ return RT_ERROR_NONE;
		84	+}
		85	+
55	rtError_t ApiImpl::FlushCache(const uint64_t base, const size_t len)	86	rtError_t ApiImpl::FlushCache(const uint64_t base, const size_t len)
56	{	87	{
57	RT_LOG(RT_LOG_INFO, "flush cache base=%" PRIu64 ", len=%zu.", base, len);	88	RT_LOG(RT_LOG_INFO, "flush cache base=%" PRIu64 ", len=%zu.", base, len);

Msrc/runtime/api/impl/api_impl_cpu_kernel.cc+28-0

	@@ -16,6 +16,7 @@
16	#include "kernel.hpp"	16	#include "kernel.hpp"
17	#include "runtime.hpp"	17	#include "runtime.hpp"
18	#include "stream.hpp"	18	#include "stream.hpp"
		19	+#include "task_execute_time.h"
19		20
20	namespace cce {	21	namespace cce {
21	namespace runtime {	22	namespace runtime {
	@@ -53,5 +54,32 @@ rtError_t ApiImpl::CpuKernelLaunchEx(
53	argsInfo->cpuParamHeadOffset);	54	argsInfo->cpuParamHeadOffset);
54	}	55	}
55		56
		57	+rtError_t ApiImpl::SetOpExecuteTimeOutV2(uint64_t timeout, uint64_t* actualTimeout)
		58	+{
		59	+ Context* const curCtx = CurrentContext(); // 隐式set device
		60	+ CHECK_CONTEXT_VALID_WITH_RETURN(curCtx, RT_ERROR_CONTEXT_NULL);
		61	+ Runtime* const rtInstance = Runtime::Instance();
		62	+ NULL_PTR_RETURN_MSG(rtInstance, RT_ERROR_INSTANCE_NULL);
		63	+ const RtTimeoutConfig& timeoutCfg = rtInstance->GetTimeoutConfig();
		64	+ COND_RETURN_AND_MSG_OUTER(
		65	+ (timeoutCfg.isInit == false), RT_ERROR_DEVICE_RETAIN, ErrorCode::EE1018,
		66	+ "Setting the timeout interval for operator execution", "Device is not initialized, call rtSetDevice API first");
		67	+
		68	+ const rtError_t error = rtInstance->SetTimeoutConfig(RT_TIMEOUT_TYPE_OP_EXECUTE, timeout, RT_TIME_UNIT_TYPE_US);
		69	+ ERROR_RETURN_MSG_INNER(error, "Failed to set op execute timeout, retCode=%#x.", static_cast<uint32_t>(error));
		70	+
		71	+ if ((timeoutCfg.isOpTimeoutMs) && (timeoutCfg.isCfgOpExcTaskTimeout) && (timeoutCfg.opExcTaskTimeout == 0UL)) {
		72	+ *actualTimeout = MAX_UINT64_NUM; // never timeout
		73	+ } else {
		74	+ uint16_t credit = 0U;
		75	+ TransExeTimeoutCfgToKernelCredit(timeout, credit);
		76	+ actualTimeout = static_cast<uint64_t>(static_cast<double>(credit) timeoutCfg.interval);
		77	+ }
		78	+ RT_LOG(
		79	+ RT_LOG_INFO, "set op execute timeout, timeout=%" PRIu64 "us, actualTimeout=%" PRIu64 "us.", timeout,
		80	+ *actualTimeout);
		81	+ return RT_ERROR_NONE;
		82	+}
		83	+
56	} // namespace runtime	84	} // namespace runtime
57	} // namespace cce	85	} // namespace cce

Msrc/runtime/core/inc/stars/stars.hpp+8-0

	@@ -18,6 +18,8 @@
18		18
19	namespace cce {	19	namespace cce {
20	namespace runtime {	20	namespace runtime {
		21	+class Device;
		22	+
21	constexpr uint8_t RT_STARS_AS31XM1X_DEFAULT_KERNEL_CREDIT = 3U; // The TS reference time is 33.5 ms.	23	constexpr uint8_t RT_STARS_AS31XM1X_DEFAULT_KERNEL_CREDIT = 3U; // The TS reference time is 33.5 ms.
22	constexpr float32_t RT_STARS_AS31XM1X_TASK_KERNEL_CREDIT_SCALE = 33.5F; // 2^24 / 500M *1000(ms)	24	constexpr float32_t RT_STARS_AS31XM1X_TASK_KERNEL_CREDIT_SCALE = 33.5F; // 2^24 / 500M *1000(ms)
23	constexpr uint8_t RT_STARS_FFTSPLUS_SUBTYPE_MIX = 0x1U;	25	constexpr uint8_t RT_STARS_FFTSPLUS_SUBTYPE_MIX = 0x1U;
	@@ -30,6 +32,12 @@ constexpr uint16_t TS_FFTS_TYPE_AIC_AIV_MIX = 5U;
30	constexpr uint8_t RT_STARS_DCACHE_LOCK_OP = 0x0AU;	32	constexpr uint8_t RT_STARS_DCACHE_LOCK_OP = 0x0AU;
31	constexpr uint8_t RT_STARS_SQE_LEN = 64U;	33	constexpr uint8_t RT_STARS_SQE_LEN = 64U;
32		34
		35	+using StarsMonitorTask = void ()(Device dev);
		36	+
		37	+// Monitor tasks run under the registry lock and must not register or unregister monitor tasks.
		38	+rtError_t RegisterStarsMonitorTask(StarsMonitorTask task);
		39	+rtError_t UnregisterStarsMonitorTask(StarsMonitorTask task);
		Y Y yanhaiyan18 天前函数定义在枚举和常量之间，是否和最后的函数放到一起更合适？ FengHaozhan 18 天前评论： 18 天前评论：
		40	+
33	enum rtStarsCqeErrorCodeType {	41	enum rtStarsCqeErrorCodeType {
34	RT_STARS_CQE_ERR_TYPE_EXCEPTION = (1ULL << 0U),	42	RT_STARS_CQE_ERR_TYPE_EXCEPTION = (1ULL << 0U),
35	RT_STARS_CQE_ERR_TYPE_TRAP = (1ULL << 1U),	43	RT_STARS_CQE_ERR_TYPE_TRAP = (1ULL << 1U),

Msrc/runtime/core/src/engine/stars/stars_engine.cc+46-0

	@@ -8,7 +8,11 @@
8	* See LICENSE in the root of the software repository for the full text of the License.	8	* See LICENSE in the root of the software repository for the full text of the License.
9	*/	9	*/
10	#include "stars_engine.hpp"	10	#include "stars_engine.hpp"
		11	+#include <algorithm>
		12	+#include <memory>
		13	+#include <mutex>
11	#include <thread>	14	#include <thread>
		15	+#include <vector>
12	#include "recycle_thread_utils.hpp"	16	#include "recycle_thread_utils.hpp"
13	#include "securec.h"	17	#include "securec.h"
14	#include "context.hpp"	18	#include "context.hpp"
	@@ -43,10 +47,39 @@ namespace {
43	constexpr uint16_t TASK_RECLAIM_MAX_NUM = 64U; // Max reclaim num per query.	47	constexpr uint16_t TASK_RECLAIM_MAX_NUM = 64U; // Max reclaim num per query.
44	constexpr uint16_t TASK_QUERY_INTERVAL_NUM = 64U; // Shared memory query interval.	48	constexpr uint16_t TASK_QUERY_INTERVAL_NUM = 64U; // Shared memory query interval.
45	constexpr uint16_t TASK_WAIT_EXECUTE_MAX_NUM = 512U; // Max number of tasks waiting to be executed on device.	49	constexpr uint16_t TASK_WAIT_EXECUTE_MAX_NUM = 512U; // Max number of tasks waiting to be executed on device.
		50	+
		51	+std::mutex g_starsMonitorTaskMutex;
		52	+std::vector<cce::runtime::StarsMonitorTask> g_starsMonitorTasks;
46	} // namespace	53	} // namespace
47		54
48	namespace cce {	55	namespace cce {
49	namespace runtime {	56	namespace runtime {
		57	+rtError_t RegisterStarsMonitorTask(const StarsMonitorTask task)
		58	+{
		59	+ COND_RETURN_ERROR(task == nullptr, RT_ERROR_INVALID_VALUE, "Stars monitor task is null.");
		60	+ const std::lock_guard<std::mutex> lock(g_starsMonitorTaskMutex);
		61	+ const auto taskIt = std::find(g_starsMonitorTasks.cbegin(), g_starsMonitorTasks.cend(), task);
		62	+ if (taskIt != g_starsMonitorTasks.cend()) {
		63	+ return RT_ERROR_NONE;
		64	+ }
		65	+
		66	+ g_starsMonitorTasks.emplace_back(task);
		67	+ return RT_ERROR_NONE;
		68	+}
		69	+
		70	+rtError_t UnregisterStarsMonitorTask(const StarsMonitorTask task)
		71	+{
		72	+ COND_RETURN_ERROR(task == nullptr, RT_ERROR_INVALID_VALUE, "Stars monitor task is null.");
		73	+ const std::lock_guard<std::mutex> lock(g_starsMonitorTaskMutex);
		74	+ const auto taskIt = std::find(g_starsMonitorTasks.cbegin(), g_starsMonitorTasks.cend(), task);
		75	+ if (taskIt == g_starsMonitorTasks.end()) {
		76	+ return RT_ERROR_NONE;
		77	+ }
		78	+
		79	+ g_starsMonitorTasks.erase(taskIt);
		80	+ return RT_ERROR_NONE;
		81	+}
		82	+
50	StarsEngine::StarsEngine(Device* const dev, Thread* const monitor, Thread* const recycle)	83	StarsEngine::StarsEngine(Device* const dev, Thread* const monitor, Thread* const recycle)
51	: Engine(dev), monitorThread_(monitor), recycleThread_(recycle)	84	: Engine(dev), monitorThread_(monitor), recycleThread_(recycle)
52	{	85	{
	@@ -1878,6 +1911,18 @@ void StarsEngine::MonitorEndGraphNotify(Device* const dev) const
1878	rawDev->PollEndGraphNotifyInfo();	1911	rawDev->PollEndGraphNotifyInfo();
1879	}	1912	}
1880		1913
		1914	+void StarsEngine::RunRegisteredMonitorTasks(Device* const dev) const
		1915	+{
		1916	+ const std::lock_guard<std::mutex> lock(g_starsMonitorTaskMutex);
		1917	+ for (const StarsMonitorTask task : g_starsMonitorTasks) {
		1918	+ try {
		1919	+ task(dev);
		1920	+ } catch (...) {
		1921	+ RT_LOG(RT_LOG_ERROR, "Stars monitor task threw an exception.");
		Y Y yanhaiyan18 天前没有任何task信息、dev数据、错误信息，无法定位哪个任务异常。 FengHaozhan 18 天前评论： 18 天前评论：
		1922	+ }
		1923	+ }
		1924	+}
		1925	+
1881	void StarsEngine::MonitoringRun()	1926	void StarsEngine::MonitoringRun()
1882	{	1927	{
1883	Device* const dev = GetDevice();	1928	Device* const dev = GetDevice();
	@@ -1906,6 +1951,7 @@ void StarsEngine::MonitoringRun()
1906	if (GetDevRunningState() != static_cast<uint32_t>(DEV_RUNNING_DOWN)) {	1951	if (GetDevRunningState() != static_cast<uint32_t>(DEV_RUNNING_DOWN)) {
1907	(void)MonitorForWatchDog(dev);	1952	(void)MonitorForWatchDog(dev);
1908	MonitorEndGraphNotify(dev);	1953	MonitorEndGraphNotify(dev);
		1954	+ RunRegisteredMonitorTasks(dev);
1909	}	1955	}
1910		1956
1911	// sleep 1s	1957	// sleep 1s

Msrc/runtime/core/src/engine/stars/stars_engine.hpp+1-0

	@@ -134,6 +134,7 @@ private:
134	static const std::vector<std::string> StarsCqeErrorDesc_;	134	static const std::vector<std::string> StarsCqeErrorDesc_;
135		135
136	void MonitoringRun();	136	void MonitoringRun();
		137	+ void RunRegisteredMonitorTasks(Device* dev) const;
137	rtError_t CreateRecycleThread(void);	138	rtError_t CreateRecycleThread(void);
138	void RecycleThreadRun(void);	139	void RecycleThreadRun(void);
139	void DestroyRecycleThread(void);	140	void DestroyRecycleThread(void);

Msrc/runtime/feature/runtime_thread_aicpu/runtime_thread_aicpu.cc+152-18

	@@ -10,9 +10,11 @@
10		10
11	#include "runtime_thread_aicpu.hpp"	11	#include "runtime_thread_aicpu.hpp"
12		12
		13	+#include <atomic>
13	#include <cstddef>	14	#include <cstddef>
14	#include <dlfcn.h>	15	#include <dlfcn.h>
15	#include <functional>	16	#include <functional>
		17	+#include <memory>
16	#include <mutex>	18	#include <mutex>
17	#include <new>	19	#include <new>
18	#include <string>	20	#include <string>
	@@ -22,6 +24,7 @@
22	#include "api.hpp"	24	#include "api.hpp"
23	#include "davinci_kernel_task.h"	25	#include "davinci_kernel_task.h"
24	#include "device.hpp"	26	#include "device.hpp"
		27	+#include "dfx_api.hpp"
25	#include "event.hpp"	28	#include "event.hpp"
26	#include "inner_thread_local.hpp"	29	#include "inner_thread_local.hpp"
27	#include "kernel.hpp"	30	#include "kernel.hpp"
	@@ -35,6 +38,7 @@
35	#include "subscribe.hpp"	38	#include "subscribe.hpp"
36	#include "task.hpp"	39	#include "task.hpp"
37	#include "task_info.hpp"	40	#include "task_info.hpp"
		41	+#include "task_scheduler_error.h"
38		42
39	namespace cce {	43	namespace cce {
40	namespace runtime {	44	namespace runtime {
	@@ -89,6 +93,8 @@ namespace {
89	constexpr char PLUGIN_SO_NAME[] = "libruntime_thread_aicpu.so";	93	constexpr char PLUGIN_SO_NAME[] = "libruntime_thread_aicpu.so";
90	constexpr char PLUGIN_QUERY_SYMBOL[] = "RuntimeThreadAicpuGetPluginApi";	94	constexpr char PLUGIN_QUERY_SYMBOL[] = "RuntimeThreadAicpuGetPluginApi";
91	constexpr char STREAM_OBSERVER_NAME[] = "Inner#RuntimeThreadAicpu";	95	constexpr char STREAM_OBSERVER_NAME[] = "Inner#RuntimeThreadAicpu";
		96	+std::mutex g_streamErrorMutex;
		97	+std::atomic<const RuntimeThreadAicpuPluginApi*> g_monitorPluginApi{nullptr};
92	static_assert(	98	static_assert(
93	static_cast<uint32_t>(RuntimeThreadAicpuSqeSubtype::AICPU) == static_cast<uint32_t>(RT_SQE_SUBTYPE_AICPU),	99	static_cast<uint32_t>(RuntimeThreadAicpuSqeSubtype::AICPU) == static_cast<uint32_t>(RT_SQE_SUBTYPE_AICPU),
94	"RuntimeThreadAicpu SQE subtype does not match the runtime protocol");	100	"RuntimeThreadAicpu SQE subtype does not match the runtime protocol");
	@@ -209,34 +215,97 @@ void DestroyCompletionEvent(void* const runtimeData, void* const eventHandle)
209		215
210	void SetStreamError(	216	void SetStreamError(
211	void* const runtimeData, const uint32_t deviceId, const uint32_t tsId, const uint32_t streamId,	217	void* const runtimeData, const uint32_t deviceId, const uint32_t tsId, const uint32_t streamId,
212	- const uint32_t executeResult)	218	+ void* const streamHandle, const RuntimeThreadAicpuStreamErrorType errorType, const uint32_t errorDetail)
213	{	219	{
214	UNUSED(runtimeData);	220	UNUSED(runtimeData);
		221	+ const bool isTimeout = errorType == RuntimeThreadAicpuStreamErrorType::EXECUTION_TIMEOUT;
		222	+ if ((errorType != RuntimeThreadAicpuStreamErrorType::EXECUTION_FAILED) && (!isTimeout)) {
		223	+ RT_LOG(
		224	+ RT_LOG_ERROR, "Set RuntimeThreadAicpu stream error failed because error type is invalid, error_type=%u.",
		225	+ static_cast<uint32_t>(errorType));
		226	+ return;
		227	+ }
		228	+ if (isTimeout && (streamHandle == nullptr)) {
		229	+ RT_LOG(RT_LOG_ERROR, "Set RuntimeThreadAicpu timeout error failed because stream handle is null.");
		230	+ return;
		231	+ }
		232	+
215	Runtime* const runtime = Runtime::Instance();	233	Runtime* const runtime = Runtime::Instance();
216	Device* const device = (runtime == nullptr) ? nullptr : runtime->GetDevice(deviceId, tsId, false);	234	Device* const device = (runtime == nullptr) ? nullptr : runtime->GetDevice(deviceId, tsId, false);
217	StreamSqCqManage* const streamManage = (device == nullptr) ? nullptr : device->GetStreamSqCqManage();	235	StreamSqCqManage* const streamManage = (device == nullptr) ? nullptr : device->GetStreamSqCqManage();
218	if (streamManage == nullptr) {	236	if (streamManage == nullptr) {
219	RT_LOG(	237	RT_LOG(
220	- RT_LOG_ERROR, "Set RuntimeThreadAicpu stream error failed, device_id=%u, ts_id=%u, stream_id=%u.", deviceId,	238	+ RT_LOG_ERROR,
221	- tsId, streamId);	239	+ "Set RuntimeThreadAicpu stream error failed, device_id=%u, ts_id=%u, stream_id=%u, error_type=%u.",
		240	+ deviceId, tsId, streamId, static_cast<uint32_t>(errorType));
222	return;	241	return;
223	}	242	}
224	- Stream* reportStream = nullptr;	243	+
225	- const rtError_t error = streamManage->GetStreamById(streamId, &reportStream);	244	+ std::shared_ptr<Stream> reportStream;
226	- Stream* const errorStream =	245	+ const rtError_t error = streamManage->GetStreamSharedPtrById(streamId, reportStream);
227	- ((error == RT_ERROR_NONE) && (reportStream != nullptr)) ? GetReportStream(reportStream) : nullptr;	246	+ if ((error != RT_ERROR_NONE) \|\| (reportStream == nullptr)) {
		247	+ RT_LOG(
		248	+ RT_LOG_ERROR,
		249	+ "Get RuntimeThreadAicpu error stream failed, device_id=%u, ts_id=%u, stream_id=%u, error_type=%u, "
		250	+ "error_detail=%#x, retCode=%#x.",
		251	+ deviceId, tsId, streamId, static_cast<uint32_t>(errorType), errorDetail, static_cast<uint32_t>(error));
		252	+ return;
		253	+ }
		254	+ if ((streamHandle != nullptr) && (reportStream->GetInnerHandle() != streamHandle)) {
		255	+ RT_LOG(
		256	+ RT_LOG_WARNING,
		257	+ "Skip RuntimeThreadAicpu stream error because stream has been reused, device_id=%u, ts_id=%u, "
		258	+ "stream_id=%u, error_type=%u.",
		259	+ deviceId, tsId, streamId, static_cast<uint32_t>(errorType));
		260	+ return;
		261	+ }
		262	+
		263	+ Stream* const errorStream = GetReportStream(reportStream.get());
228	if (errorStream == nullptr) {	264	if (errorStream == nullptr) {
229	RT_LOG(	265	RT_LOG(
230	RT_LOG_ERROR,	266	RT_LOG_ERROR,
231	- "Get RuntimeThreadAicpu error stream failed, device_id=%u, ts_id=%u, stream_id=%u, "	267	+ "Set RuntimeThreadAicpu stream error failed because report stream is null, device_id=%u, ts_id=%u, "
232	- "execute_result=%#x.",	268	+ "stream_id=%u, error_type=%u.",
233	- deviceId, tsId, streamId, executeResult);	269	+ deviceId, tsId, streamId, static_cast<uint32_t>(errorType));
234	return;	270	return;
235	}	271	}
236	- RT_LOG(	272	+
237	- RT_LOG_ERROR, "RuntimeThreadAicpu operator failed, stream_id=%u, error_stream_id=%d, execute_result=%#x.",	273	+ const std::lock_guard<std::mutex> lock(g_streamErrorMutex);
		FengHaozhan19 天前 🟡 Medium：这把局部互斥锁没有覆盖 `Stream::errCode_` 的全部访问，新增超时上报仍存在数据竞争。该函数由 Stars 监控线程调用并在这里读写 `errCode_`；但 `Stream::GetError()` 会在同步调用线程中直接读取并清零同一个普通 `uint32_t`，其他 `SetErrCode()` 调用点也不持有 `g_streamErrorMutex`。因此这把锁只能串行化两个 RuntimeThread hook，不能为 `errCode_` 建立完整的 happens-before 关系，既可能触发 C++ 未定义行为，也可能使这里的错误优先级检查被并发写入破坏。建议把“按优先级设置错误”下沉为 `Stream` 自身的同步操作，并让所有读写走同一把锁或原子 CAS；同时增加监控线程与同步线程并发的回归用例。
238	- streamId, errorStream->Id_(), executeResult);	274	+ const uint32_t currentError = errorStream->GetErrCode();
239	- errorStream->SetErrCode(static_cast<uint32_t>(RT_ERROR_HOST_FUNC_EXE_FAILED));	275	+ if (isTimeout) {
		276	+ if ((currentError != static_cast<uint32_t>(RT_ERROR_NONE)) &&
		277	+ (currentError != static_cast<uint32_t>(RT_ERROR_HOST_FUNC_EXE_FAILED))) {
		278	+ RT_LOG(
		279	+ RT_LOG_WARNING,
		280	+ "Keep existing stream error when RuntimeThreadAicpu operator times out, stream_id=%u, "
		281	+ "error_stream_id=%d, current_error=%#x.",
		282	+ streamId, errorStream->Id_(), currentError);
		283	+ return;
		284	+ }
		285	+ RT_LOG(
		286	+ RT_LOG_ERROR, "Set RuntimeThreadAicpu timeout error, stream_id=%u, error_stream_id=%d.", streamId,
		287	+ errorStream->Id_());
		288	+ errorStream->SetErrCode(static_cast<uint32_t>(TS_ERROR_AICPU_TIMEOUT));
		289	+ } else {
		290	+ RT_LOG(
		291	+ RT_LOG_ERROR, "RuntimeThreadAicpu operator failed, stream_id=%u, error_stream_id=%d, execute_result=%#x.",
		292	+ streamId, errorStream->Id_(), errorDetail);
		293	+ if (currentError == static_cast<uint32_t>(TS_ERROR_AICPU_TIMEOUT)) {
		294	+ RT_LOG(
		295	+ RT_LOG_WARNING,
		296	+ "Keep RuntimeThreadAicpu timeout error instead of overwriting it with execution failure, "
		297	+ "stream_id=%u, error_stream_id=%d.",
		298	+ streamId, errorStream->Id_());
		299	+ return;
		300	+ }
		301	+ errorStream->SetErrCode(static_cast<uint32_t>(RT_ERROR_HOST_FUNC_EXE_FAILED));
		302	+ }
		303	+}
		304	+
		305	+uint64_t RuntimeThreadAicpuClockGetTimeUs(void* const runtimeData)
		306	+{
		307	+ UNUSED(runtimeData);
		308	+ return ClockGetTimeUs();
240	}	309	}
241		310
242	void MonitorThreadEnter(void* const runtimeData)	311	void MonitorThreadEnter(void* const runtimeData)
	@@ -276,13 +345,45 @@ RuntimeThreadAicpuRuntimeHooks BuildRuntimeHooks(Api* const api)
276	.monitorThreadEnter = &MonitorThreadEnter,	345	.monitorThreadEnter = &MonitorThreadEnter,
277	.monitorThreadExit = &MonitorThreadExit,	346	.monitorThreadExit = &MonitorThreadExit,
278	.isProcessExiting = &IsProcessExiting,	347	.isProcessExiting = &IsProcessExiting,
		348	+ .clockGetTimeUs = &RuntimeThreadAicpuClockGetTimeUs,
279	};	349	};
280	}	350	}
281		351
		352	+void MonitorRuntimeThreadAicpuExecution(Device* const dev)
		353	+{
		354	+ UNUSED(dev);
		355	+ const RuntimeThreadAicpuPluginApi* const pluginApi = g_monitorPluginApi.load(std::memory_order_acquire);
		356	+ if (pluginApi != nullptr) {
		357	+ pluginApi->monitorExecutionTimeout();
		358	+ }
		359	+}
		360	+
282	class RuntimeThreadAicpuAdapter final {	361	class RuntimeThreadAicpuAdapter final {
283	public:	362	public:
284	explicit RuntimeThreadAicpuAdapter(Api* const api) : api_(api) {}	363	explicit RuntimeThreadAicpuAdapter(Api* const api) : api_(api) {}
285		364
		365	+ ~RuntimeThreadAicpuAdapter() noexcept
		366	+ {
		367	+ if (monitorTaskRegistered_ && (pluginApi_ != nullptr)) {
		Y Y yanhaiyan18 天前 `~RuntimeThreadAicpuAdapter()` 的注销逻辑不可达：`GetAdapter`（526-530）为函数级 static 泄漏式单例，`new` 后无任何 delete 路径，生产永不析构；当前仅 UT 局部对象触发。虽然插件 .so 与 g_pluginService 同为进程级永生对象，但析构代码给读者"退出期会注销"的错误预期。建议删除析构并注释单例永生设计，或提供显式 Destroy。 FengHaozhan 18 天前评论： 18 天前评论：
		368	+ (void)UnregisterStarsMonitorTask(&MonitorRuntimeThreadAicpuExecution);
		369	+ g_monitorPluginApi.store(nullptr, std::memory_order_release);
		370	+ }
		371	+ if (streamObserverRegistered_) {
		372	+ (void)StreamStateCallbackManager::Instance().RegStreamStateCallback(
		373	+ STREAM_OBSERVER_NAME, nullptr, nullptr, cce::runtime::StreamStateCallback::RTS_STREAM_STATE_CALLBACK);
		374	+ }
		375	+ }
		376	+
		377	+ rtError_t SetExecuteTimeout(const uint64_t timeoutUs)
		378	+ {
		379	+ const std::lock_guard<std::mutex> lock(mutex_);
		380	+ executeTimeoutUs_ = timeoutUs;
		381	+ if (pluginApi_ != nullptr) {
		382	+ pluginApi_->setExecuteTimeout(timeoutUs);
		383	+ }
		384	+ return RT_ERROR_NONE;
		385	+ }
		386	+
286	rtError_t PrepareKernel(	387	rtError_t PrepareKernel(
287	const RuntimeThreadAicpuKernelRequest& request, RuntimeThreadAicpuPreparedKernel& preparedKernel)	388	const RuntimeThreadAicpuKernelRequest& request, RuntimeThreadAicpuPreparedKernel& preparedKernel)
288	{	389	{
	@@ -354,22 +455,27 @@ private:
354	const bool prepareValid = (pluginApi_ != nullptr) && (pluginApi_->prepareKernel != nullptr);	455	const bool prepareValid = (pluginApi_ != nullptr) && (pluginApi_->prepareKernel != nullptr);
355	const bool releaseValid = (pluginApi_ != nullptr) && (pluginApi_->releasePreparedKernel != nullptr);	456	const bool releaseValid = (pluginApi_ != nullptr) && (pluginApi_->releasePreparedKernel != nullptr);
356	const bool streamDestroyedValid = (pluginApi_ != nullptr) && (pluginApi_->streamDestroyed != nullptr);	457	const bool streamDestroyedValid = (pluginApi_ != nullptr) && (pluginApi_->streamDestroyed != nullptr);
		458	+ const bool setTimeoutValid = (pluginApi_ != nullptr) && (pluginApi_->setExecuteTimeout != nullptr);
		FengHaozhan19 天前 🟡 Medium：在确认 `apiSize` 足够前读取了新增尾部回调。 `setTimeoutValid` 和 `monitorValid` 会先解引用 `pluginApi_->setExecuteTimeout` / `monitorExecutionTimeout`，而短表检查要到后面的 `apiSize < sizeof(RuntimeThreadAicpuPluginApi)` 才执行。若插件返回的确实是旧版或截短表，这里会越过对象边界读取，产生未定义行为；当前短表 UT 只是把完整 `g_pluginApi` 的 `structSize` 改小，因此覆盖不到该越界。建议先计算 `apiTableValid = (pluginApi_ != nullptr) && (apiSize >= sizeof(RuntimeThreadAicpuPluginApi))`，再通过短路条件读取所有回调，并补一个使用真实短存储的回归用例。
		459	+ const bool monitorValid = (pluginApi_ != nullptr) && (pluginApi_->monitorExecutionTimeout != nullptr);
357	if ((status != RuntimeThreadAicpuStatus::OK) \|\| (pluginApi_ == nullptr) \|\|	460	if ((status != RuntimeThreadAicpuStatus::OK) \|\| (pluginApi_ == nullptr) \|\|
358	(apiSize < sizeof(RuntimeThreadAicpuPluginApi)) \|\| (!prepareValid) \|\| (!releaseValid) \|\|	461	(apiSize < sizeof(RuntimeThreadAicpuPluginApi)) \|\| (!prepareValid) \|\| (!releaseValid) \|\|
359	- (!streamDestroyedValid)) {	462	+ (!streamDestroyedValid) \|\| (!setTimeoutValid) \|\| (!monitorValid)) {
360	RT_LOG(	463	RT_LOG(
361	RT_LOG_ERROR,	464	RT_LOG_ERROR,
362	"Initialize RuntimeThreadAicpu plugin interface failed, plugin_status=%u, api_valid=%u, "	465	"Initialize RuntimeThreadAicpu plugin interface failed, plugin_status=%u, api_valid=%u, "
363	- "api_size=%u, expected_size=%zu, prepare_valid=%u, release_valid=%u, stream_destroy_valid=%u.",	466	+ "api_size=%u, expected_size=%zu, prepare_valid=%u, release_valid=%u, stream_destroy_valid=%u, "
		467	+ "set_timeout_valid=%u, monitor_valid=%u.",
364	static_cast<uint32_t>(status), static_cast<uint32_t>(pluginApi_ != nullptr), apiSize,	468	static_cast<uint32_t>(status), static_cast<uint32_t>(pluginApi_ != nullptr), apiSize,
365	sizeof(RuntimeThreadAicpuPluginApi), static_cast<uint32_t>(prepareValid),	469	sizeof(RuntimeThreadAicpuPluginApi), static_cast<uint32_t>(prepareValid),
366	- static_cast<uint32_t>(releaseValid), static_cast<uint32_t>(streamDestroyedValid));	470	+ static_cast<uint32_t>(releaseValid), static_cast<uint32_t>(streamDestroyedValid),
		471	+ static_cast<uint32_t>(setTimeoutValid), static_cast<uint32_t>(monitorValid));
367	unsupported_ = true;	472	unsupported_ = true;
368	pluginApi_ = nullptr;	473	pluginApi_ = nullptr;
369	return RT_ERROR_FEATURE_NOT_SUPPORT;	474	return RT_ERROR_FEATURE_NOT_SUPPORT;
370	}	475	}
371		476
372	- const rtError_t error = StreamStateCallbackManager::Instance().RegStreamStateCallback(	477	+ pluginApi_->setExecuteTimeout(executeTimeoutUs_);
		478	+ rtError_t error = StreamStateCallbackManager::Instance().RegStreamStateCallback(
373	STREAM_OBSERVER_NAME, RtPtrToPtr<void*>(&RuntimeThreadAicpuAdapter::StreamStateCallback), this,	479	STREAM_OBSERVER_NAME, RtPtrToPtr<void*>(&RuntimeThreadAicpuAdapter::StreamStateCallback), this,
374	cce::runtime::StreamStateCallback::RTS_STREAM_STATE_CALLBACK);	480	cce::runtime::StreamStateCallback::RTS_STREAM_STATE_CALLBACK);
375	if (error != RT_ERROR_NONE) {	481	if (error != RT_ERROR_NONE) {
	@@ -379,13 +485,32 @@ private:
379	pluginApi_ = nullptr;	485	pluginApi_ = nullptr;
380	return error;	486	return error;
381	}	487	}
		488	+ streamObserverRegistered_ = true;
		489	+
		490	+ g_monitorPluginApi.store(pluginApi_, std::memory_order_release);
		491	+ error = RegisterStarsMonitorTask(&MonitorRuntimeThreadAicpuExecution);
		492	+ if (error != RT_ERROR_NONE) {
		493	+ RT_LOG(
		494	+ RT_LOG_ERROR, "Register RuntimeThreadAicpu monitor task failed, retCode=%#x.",
		495	+ static_cast<uint32_t>(error));
		496	+ (void)StreamStateCallbackManager::Instance().RegStreamStateCallback(
		497	+ STREAM_OBSERVER_NAME, nullptr, nullptr, cce::runtime::StreamStateCallback::RTS_STREAM_STATE_CALLBACK);
		498	+ streamObserverRegistered_ = false;
		499	+ g_monitorPluginApi.store(nullptr, std::memory_order_release);
		500	+ pluginApi_ = nullptr;
		501	+ return error;
		502	+ }
		503	+ monitorTaskRegistered_ = true;
382	return RT_ERROR_NONE;	504	return RT_ERROR_NONE;
383	}	505	}
384		506
385	Api* api_ = nullptr;	507	Api* api_ = nullptr;
386	std::mutex mutex_;	508	std::mutex mutex_;
		509	+ uint64_t executeTimeoutUs_ = RUNTIME_THREAD_AICPU_DEFAULT_EXECUTE_TIMEOUT_US;
387	void* libraryHandle_ = nullptr;	510	void* libraryHandle_ = nullptr;
388	const RuntimeThreadAicpuPluginApi* pluginApi_ = nullptr;	511	const RuntimeThreadAicpuPluginApi* pluginApi_ = nullptr;
		512	+ bool streamObserverRegistered_ = false;
		513	+ bool monitorTaskRegistered_ = false;
389	bool unsupported_ = false;	514	bool unsupported_ = false;
390	};	515	};
391		516
	@@ -452,6 +577,15 @@ rtError_t SubmitAicpuTask(
452		577
453	} // namespace	578	} // namespace
454		579
		580	+rtError_t SetRuntimeThreadAicpuExecuteTimeout(Api* const api, const uint64_t timeoutUs)
		581	+{
		582	+ if (api == nullptr) {
		583	+ return RT_ERROR_INVALID_VALUE;
		584	+ }
		585	+ RuntimeThreadAicpuAdapter* const adapter = GetAdapter(api);
		586	+ return (adapter == nullptr) ? RT_ERROR_MEMORY_ALLOCATION : adapter->SetExecuteTimeout(timeoutUs);
		587	+}
		588	+
455	rtError_t LaunchRuntimeThreadAicpuKernel(	589	rtError_t LaunchRuntimeThreadAicpuKernel(
456	Api* const api, const Kernel* const kernel, const uint32_t blockDim, const rtCpuKernelArgs_t* const argsInfo,	590	Api* const api, const Kernel* const kernel, const uint32_t blockDim, const rtCpuKernelArgs_t* const argsInfo,
457	Stream* const stream)	591	Stream* const stream)

Msrc/runtime/feature/runtime_thread_aicpu/runtime_thread_aicpu.hpp+2-0

	@@ -20,6 +20,8 @@ class Api;
20	class Kernel;	20	class Kernel;
21	class Stream;	21	class Stream;
22		22
		23	+rtError_t SetRuntimeThreadAicpuExecuteTimeout(Api* const api, const uint64_t timeoutUs);
		24	+
23	rtError_t LaunchRuntimeThreadAicpuKernel(	25	rtError_t LaunchRuntimeThreadAicpuKernel(
24	Api* const api, const Kernel* const kernel, const uint32_t blockDim, const rtCpuKernelArgs_t* const argsInfo,	26	Api* const api, const Kernel* const kernel, const uint32_t blockDim, const rtCpuKernelArgs_t* const argsInfo,
25	Stream* const stream);	27	Stream* const stream);

Mtests/ut/aicpu_sched/runtime_thread_aicpu/ut/runtime_thread_aicpu_ut.cc+75-3

	@@ -125,6 +125,7 @@ struct HookState {
125	uint32_t createEventCalls = 0U;	125	uint32_t createEventCalls = 0U;
126	uint32_t destroyEventCalls = 0U;	126	uint32_t destroyEventCalls = 0U;
127	uint32_t streamErrorCalls = 0U;	127	uint32_t streamErrorCalls = 0U;
		128	+ uint32_t clockCalls = 0U;
128	uint32_t monitorEnterCalls = 0U;	129	uint32_t monitorEnterCalls = 0U;
129	uint32_t monitorExitCalls = 0U;	130	uint32_t monitorExitCalls = 0U;
130	uint32_t releasedGroupId = 0U;	131	uint32_t releasedGroupId = 0U;
	@@ -132,6 +133,9 @@ struct HookState {
132	uint32_t streamErrorTsId = 0U;	133	uint32_t streamErrorTsId = 0U;
133	uint32_t streamErrorStreamId = 0U;	134	uint32_t streamErrorStreamId = 0U;
134	uint32_t streamExecuteResult = 0U;	135	uint32_t streamExecuteResult = 0U;
		136	+ void* streamErrorHandle = nullptr;
		137	+ RuntimeThreadAicpuStreamErrorType streamErrorType = RuntimeThreadAicpuStreamErrorType::EXECUTION_FAILED;
		138	+ uint64_t clockUs = 0U;
135	void* eventHandle = reinterpret_cast<void*>(0x12340000ULL);	139	void* eventHandle = reinterpret_cast<void*>(0x12340000ULL);
136	uint32_t eventId = EVENT_ID;	140	uint32_t eventId = EVENT_ID;
137	void* destroyedEventHandle = nullptr;	141	void* destroyedEventHandle = nullptr;
	@@ -194,15 +198,17 @@ void DestroyCompletionEvent(void* runtimeData, void* eventHandle)
194	}	198	}
195		199
196	void SetStreamError(	200	void SetStreamError(
197	- void* runtimeData, const uint32_t deviceId, const uint32_t tsId, const uint32_t streamId,	201	+ void* runtimeData, const uint32_t deviceId, const uint32_t tsId, const uint32_t streamId, void* const streamHandle,
198	- const uint32_t executeResult)	202	+ const RuntimeThreadAicpuStreamErrorType errorType, const uint32_t errorDetail)
199	{	203	{
200	auto* const state = static_cast<HookState*>(runtimeData);	204	auto* const state = static_cast<HookState*>(runtimeData);
201	++state->streamErrorCalls;	205	++state->streamErrorCalls;
202	state->streamErrorDeviceId = deviceId;	206	state->streamErrorDeviceId = deviceId;
203	state->streamErrorTsId = tsId;	207	state->streamErrorTsId = tsId;
204	state->streamErrorStreamId = streamId;	208	state->streamErrorStreamId = streamId;
205	- state->streamExecuteResult = executeResult;	209	+ state->streamErrorHandle = streamHandle;
		210	+ state->streamErrorType = errorType;
		211	+ state->streamExecuteResult = errorDetail;
206	}	212	}
207		213
208	void MonitorThreadEnter(void* runtimeData) { ++static_cast<HookState*>(runtimeData)->monitorEnterCalls; }	214	void MonitorThreadEnter(void* runtimeData) { ++static_cast<HookState*>(runtimeData)->monitorEnterCalls; }
	@@ -211,6 +217,13 @@ void MonitorThreadExit(void* runtimeData) { ++static_cast<HookState*>(runtimeDat
211		217
212	bool IsProcessExiting(void* runtimeData) { return static_cast<HookState*>(runtimeData)->processExiting.load(); }	218	bool IsProcessExiting(void* runtimeData) { return static_cast<HookState*>(runtimeData)->processExiting.load(); }
213		219
		220	+uint64_t ClockGetTimeUs(void* runtimeData)
		221	+{
		222	+ auto* const state = static_cast<HookState*>(runtimeData);
		223	+ ++state->clockCalls;
		224	+ return state->clockUs;
		225	+}
		226	+
214	RuntimeThreadAicpuRuntimeHooks MakeHooks(HookState& state)	227	RuntimeThreadAicpuRuntimeHooks MakeHooks(HookState& state)
215	{	228	{
216	return {	229	return {
	@@ -224,6 +237,7 @@ RuntimeThreadAicpuRuntimeHooks MakeHooks(HookState& state)
224	.monitorThreadEnter = &MonitorThreadEnter,	237	.monitorThreadEnter = &MonitorThreadEnter,
225	.monitorThreadExit = &MonitorThreadExit,	238	.monitorThreadExit = &MonitorThreadExit,
226	.isProcessExiting = &IsProcessExiting,	239	.isProcessExiting = &IsProcessExiting,
		240	+ .clockGetTimeUs = &ClockGetTimeUs,
227	};	241	};
228	}	242	}
229		243
	@@ -262,6 +276,7 @@ RuntimeThreadAicpuService::KernelContext MakeContext(
262	context.deviceId = DEVICE_ID;	276	context.deviceId = DEVICE_ID;
263	context.streamId = STREAM_ID;	277	context.streamId = STREAM_ID;
264	context.taskId = 31U;	278	context.taskId = 31U;
		279	+ context.streamHandle = reinterpret_cast<void*>(0x56780000ULL);
265	return context;	280	return context;
266	}	281	}
267		282
	@@ -664,6 +679,57 @@ TEST_F(RuntimeThreadAicpuTest, ExecutePreparedKernelConsumesContextOnce)
664	static_cast<uint32_t>(RuntimeThreadAicpuStatus::INTERNAL_ERROR));	679	static_cast<uint32_t>(RuntimeThreadAicpuStatus::INTERNAL_ERROR));
665	}	680	}
666		681
		682	+TEST_F(RuntimeThreadAicpuTest, TimeoutMonitorUsesBoundaryAndReportsEachExecutionOnce)
		683	+{
		684	+ HookState state;
		685	+ state.clockUs = 100U;
		686	+ RuntimeThreadAicpuService service(MakeHooks(state));
		687	+ service.tsId_ = TS_ID;
		688	+ RuntimeThreadAicpuService::KernelContext context = MakeContext();
		689	+ service.SetExecuteTimeout(10U);
		690	+
		691	+ ASSERT_EQ(service.BeginExecution(7U, context), RuntimeThreadAicpuStatus::OK);
		692	+ EXPECT_EQ(state.clockCalls, 1U);
		693	+ state.clockUs = 109U;
		694	+ service.MonitorExecutionTimeout();
		695	+ EXPECT_EQ(state.streamErrorCalls, 0U);
		696	+
		697	+ state.clockUs = 110U;
		698	+ service.MonitorExecutionTimeout();
		699	+ EXPECT_EQ(state.streamErrorCalls, 1U);
		700	+ EXPECT_EQ(state.streamErrorDeviceId, DEVICE_ID);
		701	+ EXPECT_EQ(state.streamErrorTsId, TS_ID);
		702	+ EXPECT_EQ(state.streamErrorStreamId, STREAM_ID);
		703	+ EXPECT_EQ(state.streamErrorHandle, context.streamHandle);
		704	+ EXPECT_EQ(state.streamErrorType, RuntimeThreadAicpuStreamErrorType::EXECUTION_TIMEOUT);
		705	+ EXPECT_EQ(state.streamExecuteResult, 0U);
		706	+
		707	+ state.clockUs = 120U;
		708	+ service.MonitorExecutionTimeout();
		709	+ EXPECT_EQ(state.streamErrorCalls, 1U);
		710	+ service.EndExecution(7U);
		711	+ EXPECT_TRUE(service.executingTasks_.empty());
		712	+}
		713	+
		714	+TEST_F(RuntimeThreadAicpuTest, TimeoutMonitorCanBeDisabledAndStreamDestroyClearsExecution)
		715	+{
		716	+ HookState state;
		717	+ state.clockUs = 100U;
		718	+ RuntimeThreadAicpuService service(MakeHooks(state));
		719	+ service.tsId_ = TS_ID;
		720	+ RuntimeThreadAicpuService::KernelContext context = MakeContext();
		721	+ ASSERT_EQ(service.BeginExecution(7U, context), RuntimeThreadAicpuStatus::OK);
		722	+
		723	+ service.SetExecuteTimeout(std::numeric_limits<uint64_t>::max());
		724	+ state.clockUs = std::numeric_limits<uint64_t>::max();
		725	+ service.MonitorExecutionTimeout();
		726	+ EXPECT_EQ(state.streamErrorCalls, 0U);
		727	+ EXPECT_EQ(state.clockCalls, 1U);
		728	+
		729	+ service.StreamDestroyed(context.streamHandle);
		730	+ EXPECT_TRUE(service.executingTasks_.empty());
		731	+}
		732	+
667	TEST_F(RuntimeThreadAicpuTest, ProcessOneReportValidatesFunctionAndCookie)	733	TEST_F(RuntimeThreadAicpuTest, ProcessOneReportValidatesFunctionAndCookie)
668	{	734	{
669	HookState state;	735	HookState state;
	@@ -702,6 +768,8 @@ TEST_F(RuntimeThreadAicpuTest, FinishReportWritesCompletionCommandAndStreamError
702	EXPECT_EQ(state.streamErrorDeviceId, DEVICE_ID);	768	EXPECT_EQ(state.streamErrorDeviceId, DEVICE_ID);
703	EXPECT_EQ(state.streamErrorTsId, TS_ID);	769	EXPECT_EQ(state.streamErrorTsId, TS_ID);
704	EXPECT_EQ(state.streamErrorStreamId, STREAM_ID);	770	EXPECT_EQ(state.streamErrorStreamId, STREAM_ID);
		771	+ EXPECT_EQ(state.streamErrorHandle, nullptr);
		772	+ EXPECT_EQ(state.streamErrorType, RuntimeThreadAicpuStreamErrorType::EXECUTION_FAILED);
705	EXPECT_EQ(state.streamExecuteResult, executeResult);	773	EXPECT_EQ(state.streamExecuteResult, executeResult);
706	EXPECT_EQ(g_driver.command.pid, 0U);	774	EXPECT_EQ(g_driver.command.pid, 0U);
707	EXPECT_EQ(g_driver.command.commandType, 15U);	775	EXPECT_EQ(g_driver.command.commandType, 15U);
	@@ -841,9 +909,13 @@ TEST_F(RuntimeThreadAicpuTest, PluginApiValidatesInputsAndReturnsCompleteTable)
841	EXPECT_NE(api->prepareKernel, nullptr);	909	EXPECT_NE(api->prepareKernel, nullptr);
842	EXPECT_NE(api->releasePreparedKernel, nullptr);	910	EXPECT_NE(api->releasePreparedKernel, nullptr);
843	EXPECT_NE(api->streamDestroyed, nullptr);	911	EXPECT_NE(api->streamDestroyed, nullptr);
		912	+ EXPECT_NE(api->setExecuteTimeout, nullptr);
		913	+ EXPECT_NE(api->monitorExecutionTimeout, nullptr);
844	EXPECT_EQ(api->prepareKernel(nullptr, nullptr), RuntimeThreadAicpuStatus::INVALID_PARAM);	914	EXPECT_EQ(api->prepareKernel(nullptr, nullptr), RuntimeThreadAicpuStatus::INVALID_PARAM);
845	api->releasePreparedKernel(0U);	915	api->releasePreparedKernel(0U);
846	api->streamDestroyed(nullptr);	916	api->streamDestroyed(nullptr);
		917	+ api->setExecuteTimeout(100U);
		918	+ api->monitorExecutionTimeout();
847	}	919	}
848		920
849	} // namespace	921	} // namespace

Mtests/ut/runtime/runtime/test/platform/arch5162/rt_utest_runtime_thread_aicpu.cc+163-1

	@@ -8,9 +8,13 @@
8	* See LICENSE in the root of the software repository for the full text of the License.	8	* See LICENSE in the root of the software repository for the full text of the License.
9	*/	9	*/
10		10
		11	+#include <atomic>
		12	+#include <condition_variable>
11	#include <cstdint>	13	#include <cstdint>
12	#include <cstring>	14	#include <cstring>
		15	+#include <mutex>
13	#include <string>	16	#include <string>
		17	+#include <thread>
14		18
15	#include <gtest/gtest.h>	19	#include <gtest/gtest.h>
16	#include <mockcpp/mockcpp.hpp>	20	#include <mockcpp/mockcpp.hpp>
	@@ -22,6 +26,7 @@
22	#include "event.hpp"	26	#include "event.hpp"
23	#include "kernel.hpp"	27	#include "kernel.hpp"
24	#include "raw_device.hpp"	28	#include "raw_device.hpp"
		29	+#include "stars_engine.hpp"
25	#include "stream.hpp"	30	#include "stream.hpp"
26	#include "stream_state_callback_manager.hpp"	31	#include "stream_state_callback_manager.hpp"
27	#undef private	32	#undef private
	@@ -42,7 +47,10 @@ struct PluginState {
42	uint32_t prepareCalls = 0U;	47	uint32_t prepareCalls = 0U;
43	uint32_t releaseCalls = 0U;	48	uint32_t releaseCalls = 0U;
44	uint32_t streamDestroyedCalls = 0U;	49	uint32_t streamDestroyedCalls = 0U;
		50	+ uint32_t setTimeoutCalls = 0U;
		51	+ uint32_t monitorCalls = 0U;
45	uint64_t releasedCookie = 0U;	52	uint64_t releasedCookie = 0U;
		53	+ uint64_t timeoutUs = 0U;
46	void* destroyedStream = nullptr;	54	void* destroyedStream = nullptr;
47	RuntimeThreadAicpuKernelRequest request = {};	55	RuntimeThreadAicpuKernelRequest request = {};
48	RuntimeThreadAicpuRuntimeHooks hooks = {};	56	RuntimeThreadAicpuRuntimeHooks hooks = {};
	@@ -80,6 +88,14 @@ void FakeStreamDestroyed(void* const streamHandle)
80	g_plugin.destroyedStream = streamHandle;	88	g_plugin.destroyedStream = streamHandle;
81	}	89	}
82		90
		91	+void FakeSetExecuteTimeout(const uint64_t timeoutUs)
		92	+{
		93	+ ++g_plugin.setTimeoutCalls;
		94	+ g_plugin.timeoutUs = timeoutUs;
		95	+}
		96	+
		97	+void FakeMonitorExecutionTimeout() { ++g_plugin.monitorCalls; }
		98	+
83	RuntimeThreadAicpuStatus FakeQueryPlugin(	99	RuntimeThreadAicpuStatus FakeQueryPlugin(
84	const RuntimeThreadAicpuRuntimeHooks* const hooks, const RuntimeThreadAicpuPluginApi** const pluginApi)	100	const RuntimeThreadAicpuRuntimeHooks* const hooks, const RuntimeThreadAicpuPluginApi** const pluginApi)
85	{	101	{
	@@ -114,6 +130,28 @@ void DummyStreamStateCallback(rtStream_t stream, rtStreamState state, void* args
114	(void)args;	130	(void)args;
115	}	131	}
116		132
		133	+std::atomic<uint32_t> g_monitorTaskCalls{0U};
		134	+std::atomic<Device*> g_monitorTaskDevice{nullptr};
		135	+std::mutex g_blockingMonitorMutex;
		136	+std::condition_variable g_blockingMonitorCondition;
		137	+bool g_blockingMonitorEntered = false;
		138	+bool g_releaseBlockingMonitor = false;
		139	+
		140	+void CountMonitorTask(Device* const dev)
		141	+{
		142	+ g_monitorTaskDevice.store(dev);
		143	+ ++g_monitorTaskCalls;
		144	+}
		145	+
		146	+void BlockingMonitorTask(Device* const dev)
		147	+{
		148	+ UNUSED(dev);
		149	+ std::unique_lock<std::mutex> lock(g_blockingMonitorMutex);
		150	+ g_blockingMonitorEntered = true;
		151	+ g_blockingMonitorCondition.notify_all();
		152	+ g_blockingMonitorCondition.wait(lock, []() { return g_releaseBlockingMonitor; });
		153	+}
		154	+
117	class RuntimeThreadAicpuTestDevice final : public RawDevice {	155	class RuntimeThreadAicpuTestDevice final : public RawDevice {
118	public:	156	public:
119	explicit RuntimeThreadAicpuTestDevice(const uint32_t deviceId) : RawDevice(deviceId) {}	157	explicit RuntimeThreadAicpuTestDevice(const uint32_t deviceId) : RawDevice(deviceId) {}
	@@ -165,6 +203,8 @@ protected:
165	.prepareKernel = &FakePrepareKernel,	203	.prepareKernel = &FakePrepareKernel,
166	.releasePreparedKernel = &FakeReleasePreparedKernel,	204	.releasePreparedKernel = &FakeReleasePreparedKernel,
167	.streamDestroyed = &FakeStreamDestroyed,	205	.streamDestroyed = &FakeStreamDestroyed,
		206	+ .setExecuteTimeout = &FakeSetExecuteTimeout,
		207	+ .monitorExecutionTimeout = &FakeMonitorExecutionTimeout,
168	};	208	};
169	g_plugin.prepared.structSize = sizeof(RuntimeThreadAicpuPreparedKernel);	209	g_plugin.prepared.structSize = sizeof(RuntimeThreadAicpuPreparedKernel);
170	g_plugin.prepared.callbackCqId = 23U;	210	g_plugin.prepared.callbackCqId = 23U;
	@@ -210,6 +250,56 @@ TEST_F(RuntimeThreadAicpuRuntimeTest, StatusConversionPreservesPublicSemantics)
210	EXPECT_EQ(ToRuntimeStatus(RuntimeThreadAicpuStatus::KERNEL_FAILED), RT_ERROR_AICPU_INTERNAL_ERROR);	250	EXPECT_EQ(ToRuntimeStatus(RuntimeThreadAicpuStatus::KERNEL_FAILED), RT_ERROR_AICPU_INTERNAL_ERROR);
211	}	251	}
212		252
		253	+TEST_F(RuntimeThreadAicpuRuntimeTest, StarsMonitorTaskRegistrationIsIdempotentAndRemovable)
		254	+{
		255	+ (void)UnregisterStarsMonitorTask(&CountMonitorTask);
		256	+ g_monitorTaskCalls.store(0U);
		257	+ g_monitorTaskDevice.store(nullptr);
		258	+ StarsEngine engine(nullptr);
		259	+ Device* const dev = reinterpret_cast<Device*>(0x12340000ULL);
		260	+ ASSERT_EQ(RegisterStarsMonitorTask(&CountMonitorTask), RT_ERROR_NONE);
		261	+ ASSERT_EQ(RegisterStarsMonitorTask(&CountMonitorTask), RT_ERROR_NONE);
		262	+ engine.RunRegisteredMonitorTasks(dev);
		263	+ EXPECT_EQ(g_monitorTaskCalls.load(), 1U);
		264	+ EXPECT_EQ(g_monitorTaskDevice.load(), dev);
		265	+ ASSERT_EQ(UnregisterStarsMonitorTask(&CountMonitorTask), RT_ERROR_NONE);
		266	+ ASSERT_EQ(UnregisterStarsMonitorTask(&CountMonitorTask), RT_ERROR_NONE);
		267	+ engine.RunRegisteredMonitorTasks(dev);
		268	+ EXPECT_EQ(g_monitorTaskCalls.load(), 1U);
		269	+}
		270	+
		271	+TEST_F(RuntimeThreadAicpuRuntimeTest, StarsMonitorTaskUnregisterWaitsForInFlightCall)
		272	+{
		273	+ (void)UnregisterStarsMonitorTask(&BlockingMonitorTask);
		274	+ {
		275	+ const std::lock_guard<std::mutex> lock(g_blockingMonitorMutex);
		276	+ g_blockingMonitorEntered = false;
		277	+ g_releaseBlockingMonitor = false;
		278	+ }
		279	+ StarsEngine engine(nullptr);
		280	+ ASSERT_EQ(RegisterStarsMonitorTask(&BlockingMonitorTask), RT_ERROR_NONE);
		281	+ std::thread monitorThread([&engine]() { engine.RunRegisteredMonitorTasks(nullptr); });
		282	+ {
		283	+ std::unique_lock<std::mutex> lock(g_blockingMonitorMutex);
		284	+ g_blockingMonitorCondition.wait(lock, []() { return g_blockingMonitorEntered; });
		285	+ }
		286	+
		287	+ std::atomic<bool> unregisterReturned{false};
		288	+ std::thread unregisterThread([&unregisterReturned]() {
		289	+ (void)UnregisterStarsMonitorTask(&BlockingMonitorTask);
		290	+ unregisterReturned.store(true);
		291	+ });
		292	+ EXPECT_FALSE(unregisterReturned.load());
		293	+ {
		294	+ const std::lock_guard<std::mutex> lock(g_blockingMonitorMutex);
		295	+ g_releaseBlockingMonitor = true;
		296	+ }
		297	+ g_blockingMonitorCondition.notify_all();
		298	+ monitorThread.join();
		299	+ unregisterThread.join();
		300	+ EXPECT_TRUE(unregisterReturned.load());
		301	+}
		302	+
213	TEST_F(RuntimeThreadAicpuRuntimeTest, InternalGroupReservationKeepsAllocationAndWaitBitmapsConsistent)	303	TEST_F(RuntimeThreadAicpuRuntimeTest, InternalGroupReservationKeepsAllocationAndWaitBitmapsConsistent)
214	{	304	{
215	CbSubscribe subscribe(2U);	305	CbSubscribe subscribe(2U);
	@@ -244,6 +334,7 @@ TEST_F(RuntimeThreadAicpuRuntimeTest, RuntimeHooksExposeExpectedFunctionsAndVali
244	EXPECT_NE(hooks.monitorThreadEnter, nullptr);	334	EXPECT_NE(hooks.monitorThreadEnter, nullptr);
245	EXPECT_NE(hooks.monitorThreadExit, nullptr);	335	EXPECT_NE(hooks.monitorThreadExit, nullptr);
246	EXPECT_NE(hooks.isProcessExiting, nullptr);	336	EXPECT_NE(hooks.isProcessExiting, nullptr);
		337	+ EXPECT_NE(hooks.clockGetTimeUs, nullptr);
247	EXPECT_EQ(CreateCompletionEvent(nullptr, nullptr, nullptr, nullptr), RuntimeThreadAicpuStatus::INVALID_PARAM);	338	EXPECT_EQ(CreateCompletionEvent(nullptr, nullptr, nullptr, nullptr), RuntimeThreadAicpuStatus::INVALID_PARAM);
248	EXPECT_EQ(ReserveGroupId(nullptr, nullptr), RuntimeThreadAicpuStatus::INVALID_PARAM);	339	EXPECT_EQ(ReserveGroupId(nullptr, nullptr), RuntimeThreadAicpuStatus::INVALID_PARAM);
249	DestroyCompletionEvent(nullptr, nullptr);	340	DestroyCompletionEvent(nullptr, nullptr);
	@@ -330,10 +421,55 @@ TEST_F(RuntimeThreadAicpuRuntimeTest, RuntimeHooksMaintainMonitorCountAndHandleM
330	EXPECT_EQ(runtime->monitorThreadNum_.Value(), monitorCount + 1U);	421	EXPECT_EQ(runtime->monitorThreadNum_.Value(), monitorCount + 1U);
331	MonitorThreadExit(nullptr);	422	MonitorThreadExit(nullptr);
332	EXPECT_EQ(runtime->monitorThreadNum_.Value(), monitorCount);	423	EXPECT_EQ(runtime->monitorThreadNum_.Value(), monitorCount);
333	- SetStreamError(nullptr, 99U, 0U, 1U, 2U);	424	+ SetStreamError(nullptr, 99U, 0U, 1U, nullptr, RuntimeThreadAicpuStreamErrorType::EXECUTION_FAILED, 2U);
		425	+ SetStreamError(
		426	+ nullptr, 99U, 0U, 1U, reinterpret_cast<void*>(0x1ULL), RuntimeThreadAicpuStreamErrorType::EXECUTION_TIMEOUT,
		427	+ 0U);
		428	+ (void)RuntimeThreadAicpuClockGetTimeUs(nullptr);
334	(void)IsProcessExiting(nullptr);	429	(void)IsProcessExiting(nullptr);
335	}	430	}
336		431
		432	+TEST_F(RuntimeThreadAicpuRuntimeTest, RuntimeErrorHookValidatesStreamIdentityAndPreservesErrorPriority)
		433	+{
		434	+ Runtime* const runtime = Runtime::Instance();
		435	+ ASSERT_NE(runtime, nullptr);
		436	+ RuntimeThreadAicpuTestDevice device(3U);
		437	+ device.streamSqCqManage_ = new StreamSqCqManage(&device);
		438	+ ASSERT_NE(device.streamSqCqManage_, nullptr);
		439	+ Stream stream(&device, 0U);
		440	+ stream.streamId_ = 17;
		441	+ stream.myself = std::shared_ptr<Stream>(&stream, [](Stream*) {});
		442	+ device.streamSqCqManage_->SetStreamIdToStream(17U, &stream);
		443	+ const uint32_t tsId = device.DevGetTsId();
		444	+ runtime->devices_[device.Id_()][tsId].SetVal(&device);
		445	+
		446	+ SetStreamError(
		447	+ nullptr, device.Id_(), tsId, 17U, reinterpret_cast<void*>(0x1ULL),
		448	+ RuntimeThreadAicpuStreamErrorType::EXECUTION_TIMEOUT, 0U);
		449	+ EXPECT_EQ(stream.GetErrCode(), static_cast<uint32_t>(RT_ERROR_NONE));
		450	+
		451	+ SetStreamError(
		452	+ nullptr, device.Id_(), tsId, 17U, stream.GetInnerHandle(), RuntimeThreadAicpuStreamErrorType::EXECUTION_TIMEOUT,
		453	+ 0U);
		454	+ EXPECT_EQ(stream.GetErrCode(), static_cast<uint32_t>(TS_ERROR_AICPU_TIMEOUT));
		455	+ SetStreamError(
		456	+ nullptr, device.Id_(), tsId, 17U, nullptr, RuntimeThreadAicpuStreamErrorType::EXECUTION_FAILED, 0x1234U);
		457	+ EXPECT_EQ(stream.GetErrCode(), static_cast<uint32_t>(TS_ERROR_AICPU_TIMEOUT));
		458	+
		459	+ stream.SetErrCode(static_cast<uint32_t>(TS_ERROR_TASK_EXCEPTION));
		460	+ SetStreamError(
		461	+ nullptr, device.Id_(), tsId, 17U, stream.GetInnerHandle(), RuntimeThreadAicpuStreamErrorType::EXECUTION_TIMEOUT,
		462	+ 0U);
		463	+ EXPECT_EQ(stream.GetErrCode(), static_cast<uint32_t>(TS_ERROR_TASK_EXCEPTION));
		464	+
		465	+ stream.SetErrCode(static_cast<uint32_t>(RT_ERROR_HOST_FUNC_EXE_FAILED));
		466	+ SetStreamError(
		467	+ nullptr, device.Id_(), tsId, 17U, stream.GetInnerHandle(), RuntimeThreadAicpuStreamErrorType::EXECUTION_TIMEOUT,
		468	+ 0U);
		469	+ EXPECT_EQ(stream.GetErrCode(), static_cast<uint32_t>(TS_ERROR_AICPU_TIMEOUT));
		470	+ runtime->devices_[device.Id_()][tsId].SetVal(nullptr);
		471	+}
		472	+
337	TEST_F(RuntimeThreadAicpuRuntimeTest, AdapterCachesUnsupportedWhenPluginCannotBeLoaded)	473	TEST_F(RuntimeThreadAicpuRuntimeTest, AdapterCachesUnsupportedWhenPluginCannotBeLoaded)
338	{	474	{
339	MOCKER(dlopen).stubs().will(returnValue(static_cast<void*>(nullptr)));	475	MOCKER(dlopen).stubs().will(returnValue(static_cast<void*>(nullptr)));
	@@ -404,6 +540,32 @@ TEST_F(RuntimeThreadAicpuRuntimeTest, AdapterPropagatesPrepareAndForwardsLifecyc
404	EXPECT_EQ(g_plugin.destroyedStream, streamHandle);	540	EXPECT_EQ(g_plugin.destroyedStream, streamHandle);
405	}	541	}
406		542
		543	+TEST_F(RuntimeThreadAicpuRuntimeTest, AdapterCachesTimeoutUntilPluginIsLoaded)
		544	+{
		545	+ MockSuccessfulDynamicLoad();
		546	+ StarsEngine engine(nullptr);
		547	+ {
		548	+ RuntimeThreadAicpuAdapter adapter(nullptr);
		549	+ ASSERT_EQ(adapter.SetExecuteTimeout(123456U), RT_ERROR_NONE);
		550	+ EXPECT_EQ(g_plugin.setTimeoutCalls, 0U);
		551	+
		552	+ RuntimeThreadAicpuKernelRequest request = {};
		553	+ RuntimeThreadAicpuPreparedKernel prepared = {};
		554	+ ASSERT_EQ(adapter.PrepareKernel(request, prepared), RT_ERROR_NONE);
		555	+ EXPECT_EQ(g_plugin.setTimeoutCalls, 1U);
		556	+ EXPECT_EQ(g_plugin.timeoutUs, 123456U);
		557	+
		558	+ engine.RunRegisteredMonitorTasks(nullptr);
		559	+ EXPECT_EQ(g_plugin.monitorCalls, 1U);
		560	+
		561	+ ASSERT_EQ(adapter.SetExecuteTimeout(654321U), RT_ERROR_NONE);
		562	+ EXPECT_EQ(g_plugin.setTimeoutCalls, 2U);
		563	+ EXPECT_EQ(g_plugin.timeoutUs, 654321U);
		564	+ }
		565	+ engine.RunRegisteredMonitorTasks(nullptr);
		566	+ EXPECT_EQ(g_plugin.monitorCalls, 1U);
		567	+}
		568	+
407	TEST_F(RuntimeThreadAicpuRuntimeTest, AdapterReportsDuplicateObserverRegistration)	569	TEST_F(RuntimeThreadAicpuRuntimeTest, AdapterReportsDuplicateObserverRegistration)
408	{	570	{
409	ASSERT_EQ(	571	ASSERT_EQ(