support irq bind-pytorch-AtomGit

已合并

support irq bind #39959

zhaoyu65创建于 7月3日从zhaoyu65/pytorch-npu： v2.7.1-IRQ 合入到 Ascend/pytorch： v2.7.1

support irq bind #39959

已合并

zhaoyu65创建于 7月3日

从zhaoyu65/pytorch-npu v2.7.1-IRQ合入到Ascend/pytorch v2.7.1

共 10 个文件变更+582-58

Mthird_party/dcmi/inc/dcmi_interface_api.h+15-0

	@@ -17,6 +17,19 @@
17		17
18	#define TOPO_INFO_MAX_LENTH 32 // topo info max length	18	#define TOPO_INFO_MAX_LENTH 32 // topo info max length
19		19
		20	+// PCIe information structure used by dcmi_get_device_pcie_info_v2
		21	+struct dcmi_pcie_info_all {
		22	+ unsigned int venderid;
		23	+ unsigned int subvenderid;
		24	+ unsigned int deviceid;
		25	+ unsigned int subdeviceid;
		26	+ int domain;
		27	+ unsigned int bdf_busid;
		28	+ unsigned int bdf_deviceid;
		29	+ unsigned int bdf_funcid;
		30	+ unsigned char reserve[32];
		31	+};
		32	+
20	DCMIDLLEXPORT int dcmi_init(void);	33	DCMIDLLEXPORT int dcmi_init(void);
21		34
22	DCMIDLLEXPORT int dcmi_get_card_num_list(int card_num, int card_list, int list_len); // card_num is the number of device.	35	DCMIDLLEXPORT int dcmi_get_card_num_list(int card_num, int card_list, int list_len); // card_num is the number of device.
	@@ -32,3 +45,5 @@ DCMIDLLEXPORT int dcmiv2_get_affinity_cpu_info_by_device_id(int dev_id, char *af
32	DCMIDLLEXPORT int dcmiv2_get_affinity_cpu_info_by_dev_id(int dev_id, char affinity_cpu, int length);	45	DCMIDLLEXPORT int dcmiv2_get_affinity_cpu_info_by_dev_id(int dev_id, char affinity_cpu, int length);
33		46
34	DCMIDLLEXPORT int dcmiv2_get_device_list(int* device_list, int* device_cnt, int list_len);	47	DCMIDLLEXPORT int dcmiv2_get_device_list(int* device_list, int* device_cnt, int list_len);
		48	+
		49	+DCMIDLLEXPORT int dcmi_get_device_pcie_info_v2(int card_id, int device_id, struct dcmi_pcie_info_all *pcie_info);

Mtorch_npu/csrc/InitNpuBindings.cpp+8-0

	@@ -25,6 +25,8 @@
25	#include "torch_npu/csrc/utils/TensorType.h"	25	#include "torch_npu/csrc/utils/TensorType.h"
26	#include "torch_npu/csrc/utils/AutocastMode.h"	26	#include "torch_npu/csrc/utils/AutocastMode.h"
27	#include "torch_npu/csrc/core/npu/NPURecovery.h"	27	#include "torch_npu/csrc/core/npu/NPURecovery.h"
		28	+#include "torch_npu/csrc/core/npu/NPUFunctions.h"
		29	+#include "torch_npu/csrc/core/npu/NPUInterruptAffinity.h"
28	#include "torch_npu/csrc/profiler/python/combined_traceback.h"	30	#include "torch_npu/csrc/profiler/python/combined_traceback.h"
29	#ifndef BUILD_LIBTORCH	31	#ifndef BUILD_LIBTORCH
30	#include "torch_npu/csrc/sanitizer/NPUTrace.h"	32	#include "torch_npu/csrc/sanitizer/NPUTrace.h"
	@@ -90,6 +92,12 @@ PyObject* THPModule_npu_shutdown(PyObject* self, PyObject* arg)
90	ASCEND_LOGE("NPUSwappedMemoryAllocator::emptyCache failed");	92	ASCEND_LOGE("NPUSwappedMemoryAllocator::emptyCache failed");
91	}	93	}
92		94
		95	+ // Restart irqbalance service on rank 0 before the process exits.
		96	+ const auto rank_id = c10_npu::option::OptionsManager::GetRankId();
		97	+ if (rank_id == 0 \|\| rank_id == -1) {
		98	+ c10_npu::restartIrqbalance();
		99	+ }
		100	+
93	ASCEND_LOGI("NPU shutdown NpuSysCtrl Finalize.");	101	ASCEND_LOGI("NPU shutdown NpuSysCtrl Finalize.");
94	c10_npu::NpuSysCtrl::SysStatus status = c10_npu::NpuSysCtrl::GetInstance().Finalize();	102	c10_npu::NpuSysCtrl::SysStatus status = c10_npu::NpuSysCtrl::GetInstance().Finalize();
95	if (status != c10_npu::NpuSysCtrl::SysStatus::FINALIZE_SUCC) {	103	if (status != c10_npu::NpuSysCtrl::SysStatus::FINALIZE_SUCC) {

Mtorch_npu/csrc/core/npu/NPUAffinityController.cpp+122-56

	@@ -1,4 +1,4 @@
1	-#include <torch_npu/csrc/core/npu/NPUAffinityController.h>	1	+#include "torch_npu/csrc/core/npu/NPUAffinityController.h"
2		2
3	#include <pthread.h>	3	#include <pthread.h>
4	#include <sys/prctl.h>	4	#include <sys/prctl.h>
	@@ -11,31 +11,36 @@
11	#include <string>	11	#include <string>
12	#include <unordered_map>	12	#include <unordered_map>
13		13
14	-#include <torch_npu/csrc/core/npu/GetAffinityCPUInfo.h>	14	+#include "torch_npu/csrc/core/npu/GetAffinityCPUInfo.h"
15	-#include <torch_npu/csrc/core/npu/NPUFunctions.h>	15	+#include "torch_npu/csrc/core/npu/NPUFunctions.h"
16	-#include <torch_npu/csrc/core/npu/NpuVariables.h>	16	+#include "torch_npu/csrc/core/npu/NpuVariables.h"
		17	+#include "torch_npu/csrc/core/npu/NPUInterruptAffinity.h"
17		18
18	namespace c10_npu {	19	namespace c10_npu {
19		20
20	namespace {	21	namespace {
21		22
22	-thread_local ThreadType local_thread = ThreadType::MAIN_THREAD;	23	+thread_local ThreadType kLocalThread = ThreadType::MAIN_THREAD;
23		24
24	-bool start_main_thread_bind = false;	25	+bool kStartMainThreadBind = false;
25	-std::mutex core_map_mutex;	26	+std::mutex kCoreMapMutex;
26	-bool lazy_bind = true;	27	+bool kLazyBind = true;
27	-bool force_bind = false;	28	+bool kForceBind = false;
		29	+bool kBindIrq = false;
		30	+// Use 2 cores for IRQ if kBindIrq is true
		31	+constexpr int kIrqCoreOffset = 2;
28		32
29	using ThreadCoreMap = std::unordered_map<ThreadType, CoreIdList>;	33	using ThreadCoreMap = std::unordered_map<ThreadType, CoreIdList>;
30		34
31	-uint32_t cpu_affinity_mode;	35	+uint32_t kCpuAffinityMode;
32	-std::vector<CoreIdList> devices_aff_cores;	36	+std::vector<CoreIdList> kDevicesAffCores;
33	-std::unordered_map<c10::DeviceIndex, ThreadCoreMap> device_thread_core_maps;	37	+std::vector<CoreIdList> kDevicesIrqCores;
		38	+std::unordered_map<c10::DeviceIndex, ThreadCoreMap> kDeviceThreadCoreMaps;
34		39
35	-const std::initializer_list<ThreadType> threadTypeList =	40	+const std::initializer_list<ThreadType> kThreadTypeList =
36	{MAIN_THREAD, ACL_THREAD, RELEASE_THREAD, WATCHDOG_THREAD, OTHER_THREAD};	41	{MAIN_THREAD, ACL_THREAD, RELEASE_THREAD, WATCHDOG_THREAD, OTHER_THREAD};
37		42
38	-const std::unordered_map<ThreadType, std::string> threadTypeToNameMap = {	43	+const std::unordered_map<ThreadType, std::string> kThreadTypeToNameMap = {
39	{MAIN_THREAD, "main_thread"},	44	{MAIN_THREAD, "main_thread"},
40	{ACL_THREAD, "acl_thread"},	45	{ACL_THREAD, "acl_thread"},
41	{RELEASE_THREAD, "release_thread"},	46	{RELEASE_THREAD, "release_thread"},
	@@ -134,7 +139,7 @@ void parseLazyBindMode(const std::string& inputStr) {
134	std::regex pattern_for_lazy_bind("lazy_bind:(\\d)");	139	std::regex pattern_for_lazy_bind("lazy_bind:(\\d)");
135	std::smatch match_for_lazy_bind;	140	std::smatch match_for_lazy_bind;
136	if (std::regex_search(inputStr, match_for_lazy_bind, pattern_for_lazy_bind)) {	141	if (std::regex_search(inputStr, match_for_lazy_bind, pattern_for_lazy_bind)) {
137	- lazy_bind = std::stoi(match_for_lazy_bind[1].str()) == 0 ? false : true;	142	+ kLazyBind = std::stoi(match_for_lazy_bind[1].str()) == 0 ? false : true;
138	}	143	}
139	}	144	}
140		145
	@@ -147,7 +152,7 @@ void parseForceMode(const std::string& inputStr) {
147	if (force_val != 0 && force_val != 1) {	152	if (force_val != 0 && force_val != 1) {
148	ASCEND_LOGE("force value must be 0 or 1, got: %d", force_val);	153	ASCEND_LOGE("force value must be 0 or 1, got: %d", force_val);
149	} else {	154	} else {
150	- force_bind = (force_val != 0);	155	+ kForceBind = (force_val != 0);
151	}	156	}
152	} else {	157	} else {
153	std::regex pattern_for_force_check("force:([^,]+)");	158	std::regex pattern_for_force_check("force:([^,]+)");
	@@ -161,6 +166,38 @@ void parseForceMode(const std::string& inputStr) {
161	}	166	}
162	}	167	}
163		168
		169	+// Parse bind_irq setting from CPU_AFFINITY_CONF
		170	+void parseBindIrqMode(
		171	+ const std::string& inputStr,
		172	+ int device_nums,
		173	+ std::vector<CoreIdList>& devices_aff_cores) {
		174	+ std::regex pattern_for_bind_irq("bind_irq:(\\d)");
		175	+ std::smatch match_for_bind_irq;
		176	+ if (std::regex_search(inputStr, match_for_bind_irq, pattern_for_bind_irq)) {
		177	+ kBindIrq = std::stoi(match_for_bind_irq[1].str()) == 0 ? false : true;
		178	+ }
		179	+ if (!kBindIrq) {
		180	+ return;
		181	+ }
		182	+ if (::geteuid() != 0) {
		183	+ ASCEND_LOGW("IRQ binding requires root privilege, skipped.");
		184	+ kBindIrq = false;
		185	+ return;
		186	+ }
		187	+ for (int i = 0; i < device_nums; ++i) {
		188	+ auto cores = devices_aff_cores[i].size();
		189	+ if (cores > kIrqCoreOffset) {
		190	+ auto it = devices_aff_cores[i].begin();
		191	+ std::advance(it, kIrqCoreOffset);
		192	+ CoreIdList subset(devices_aff_cores[i].begin(), it);
		193	+ kDevicesIrqCores[i] = subset;
		194	+ devices_aff_cores[i].erase(devices_aff_cores[i].begin(), it);
		195	+ } else {
		196	+ ASCEND_LOGW("Device-%d has %d cpu cores, irq will not be bound for this device.", i, cores);
		197	+ }
		198	+ }
		199	+}
		200	+
164	// Parse mode from CPU_AFFINITY_CONF when only digits or mode:xxx is provided	201	// Parse mode from CPU_AFFINITY_CONF when only digits or mode:xxx is provided
165	bool parseModeOnly(const std::string& inputStr, uint32_t& mode) {	202	bool parseModeOnly(const std::string& inputStr, uint32_t& mode) {
166	// Handle cases where only `mode` is provided, or `mode:` without value	203	// Handle cases where only `mode` is provided, or `mode:` without value
	@@ -254,6 +291,8 @@ void parseCPUAffinityConf(
254	int device_nums = device_count_ensure_non_zero();	291	int device_nums = device_count_ensure_non_zero();
255	devices_aff_cores.clear();	292	devices_aff_cores.clear();
256	devices_aff_cores.resize(device_nums);	293	devices_aff_cores.resize(device_nums);
		294	+ kDevicesIrqCores.clear();
		295	+ kDevicesIrqCores.resize(device_nums);
257	ASCEND_LOGD("Get device nums: %d by aclrtGetDeviceCount.", device_nums);	296	ASCEND_LOGD("Get device nums: %d by aclrtGetDeviceCount.", device_nums);
258	for (int i = 0; i < device_nums; ++i) {	297	for (int i = 0; i < device_nums; ++i) {
259	devices_aff_cores[i] = getNPUDefaultCores(i);	298	devices_aff_cores[i] = getNPUDefaultCores(i);
	@@ -279,6 +318,7 @@ void parseCPUAffinityConf(
279	parseForceMode(inputStr);	318	parseForceMode(inputStr);
280	// Parse device-specific core ranges defined by user	319	// Parse device-specific core ranges defined by user
281	parseDeviceCoreRange(inputStr, device_nums, devices_aff_cores);	320	parseDeviceCoreRange(inputStr, device_nums, devices_aff_cores);
		321	+ parseBindIrqMode(inputStr, device_nums, devices_aff_cores);
282	}	322	}
283		323
284	void printCoreRanges(	324	void printCoreRanges(
	@@ -333,37 +373,37 @@ bool checkThreadAffinityConflict(
333	}	373	}
334		374
335	bool getThreadAffinityInfo() {	375	bool getThreadAffinityInfo() {
336	- parseCPUAffinityConf(cpu_affinity_mode, devices_aff_cores);	376	+ parseCPUAffinityConf(kCpuAffinityMode, kDevicesAffCores);
337	- printCoreRanges(cpu_affinity_mode, devices_aff_cores);	377	+ printCoreRanges(kCpuAffinityMode, kDevicesAffCores);
338		378
339	- for (int i = 0; i < devices_aff_cores.size(); ++i) {	379	+ for (int i = 0; i < kDevicesAffCores.size(); ++i) {
340	- if (devices_aff_cores[i].size() > 0) {	380	+ if (kDevicesAffCores[i].size() > 0) {
341	ASCEND_LOGD(	381	ASCEND_LOGD(
342	"Device %d get cores %s.",	382	"Device %d get cores %s.",
343	i,	383	i,
344	- formatCoreRange(devices_aff_cores[i]).c_str());	384	+ formatCoreRange(kDevicesAffCores[i]).c_str());
345	}	385	}
346	}	386	}
347		387
348	- if (cpu_affinity_mode == 0) {	388	+ if (kCpuAffinityMode == 0) {
349	return false;	389	return false;
350	}	390	}
351		391
352	- if (force_bind) {	392	+ if (kForceBind) {
353	ASCEND_LOGI(	393	ASCEND_LOGI(
354	"CPU affinity force mode enabled, skipping affinity conflict detection, applying CPU_AFFINITY_CONF binding.");	394	"CPU affinity force mode enabled, skipping affinity conflict detection, applying CPU_AFFINITY_CONF binding.");
355	return true;	395	return true;
356	}	396	}
357		397
358	- return checkThreadAffinityConflict(devices_aff_cores);	398	+ return checkThreadAffinityConflict(kDevicesAffCores);
359	}	399	}
360		400
361	std::string getAffinityMapAsString(	401	std::string getAffinityMapAsString(
362	c10::DeviceIndex device_id,	402	c10::DeviceIndex device_id,
363	const ThreadCoreMap& threadCoreMap) {	403	const ThreadCoreMap& threadCoreMap) {
364	std::ostringstream oss;	404	std::ostringstream oss;
365	- for (auto thread_type : threadTypeList) {	405	+ for (auto thread_type : kThreadTypeList) {
366	- oss << threadTypeToNameMap.at(thread_type) << ": ["	406	+ oss << kThreadTypeToNameMap.at(thread_type) << ": ["
367	<< formatCoreRange(threadCoreMap.at(thread_type)) << "]";	407	<< formatCoreRange(threadCoreMap.at(thread_type)) << "]";
368	std::string end_str =	408	std::string end_str =
369	(thread_type == ThreadType::OTHER_THREAD) ? "." : "; ";	409	(thread_type == ThreadType::OTHER_THREAD) ? "." : "; ";
	@@ -377,18 +417,18 @@ ThreadCoreMap getCpuAffinityMap(
377	const std::vector<CoreIdList>& devices_aff_cores) {	417	const std::vector<CoreIdList>& devices_aff_cores) {
378	ThreadCoreMap threadCoreMap;	418	ThreadCoreMap threadCoreMap;
379	CoreIdList cores = devices_aff_cores[device_id];	419	CoreIdList cores = devices_aff_cores[device_id];
380	- if (cores.size() < threadTypeList.size()) {	420	+ if (cores.size() < kThreadTypeList.size()) {
381	ASCEND_LOGW(	421	ASCEND_LOGW(
382	"Device %d available core numbers (%zu) are insufficient for all %zu thread types and will bind available cores to all threads.",	422	"Device %d available core numbers (%zu) are insufficient for all %zu thread types and will bind available cores to all threads.",
383	device_id,	423	device_id,
384	cores.size(),	424	cores.size(),
385	- threadTypeList.size());	425	+ kThreadTypeList.size());
386	- for (auto thread_type : threadTypeList) {	426	+ for (auto thread_type : kThreadTypeList) {
387	threadCoreMap[thread_type] = cores;	427	threadCoreMap[thread_type] = cores;
388	}	428	}
389	return threadCoreMap;	429	return threadCoreMap;
390	}	430	}
391	- for (auto thread_type : threadTypeList) {	431	+ for (auto thread_type : kThreadTypeList) {
392	if (thread_type != ThreadType::OTHER_THREAD) {	432	if (thread_type != ThreadType::OTHER_THREAD) {
393	CoreId first_core = *cores.begin();	433	CoreId first_core = *cores.begin();
394	threadCoreMap[thread_type].insert(first_core);	434	threadCoreMap[thread_type].insert(first_core);
	@@ -407,16 +447,16 @@ ThreadCoreMap getCpuAffinityMap(
407		447
408	CoreIdList getCoreList(c10::DeviceIndex device_id, ThreadType type) {	448	CoreIdList getCoreList(c10::DeviceIndex device_id, ThreadType type) {
409	CoreIdList core_list;	449	CoreIdList core_list;
410	- if (cpu_affinity_mode == 0 \|\| cpu_affinity_mode == 1) {	450	+ if (kCpuAffinityMode == 0 \|\| kCpuAffinityMode == 1) {
411	- core_list = devices_aff_cores[device_id];	451	+ core_list = kDevicesAffCores[device_id];
412	} else {	452	} else {
413	- std::lock_guard<std::mutex> lock(core_map_mutex);	453	+ std::lock_guard<std::mutex> lock(kCoreMapMutex);
414	- if (device_thread_core_maps.find(device_id) ==	454	+ if (kDeviceThreadCoreMaps.find(device_id) ==
415	- device_thread_core_maps.end()) {	455	+ kDeviceThreadCoreMaps.end()) {
416	- device_thread_core_maps.emplace(	456	+ kDeviceThreadCoreMaps.emplace(
417	- device_id, getCpuAffinityMap(device_id, devices_aff_cores));	457	+ device_id, getCpuAffinityMap(device_id, kDevicesAffCores));
418	}	458	}
419	- core_list = device_thread_core_maps.at(device_id).at(type);	459	+ core_list = kDeviceThreadCoreMaps.at(device_id).at(type);
420	}	460	}
421	return core_list;	461	return core_list;
422	}	462	}
	@@ -440,36 +480,36 @@ inline bool needToSetThreadAffinity() {
440	void SetThreadType(ThreadType type) {	480	void SetThreadType(ThreadType type) {
441	// Called at the start of the thread's execution to avoid frequent triggering	481	// Called at the start of the thread's execution to avoid frequent triggering
442	// of this function.	482	// of this function.
443	- local_thread = type;	483	+ kLocalThread = type;
444	if (type == ThreadType::OTHER_THREAD \|\| type == ThreadType::MAIN_THREAD) {	484	if (type == ThreadType::OTHER_THREAD \|\| type == ThreadType::MAIN_THREAD) {
445	return;	485	return;
446	}	486	}
447	- if (prctl(PR_SET_NAME, threadTypeToNameMap.at(type).c_str()) != 0) {	487	+ if (prctl(PR_SET_NAME, kThreadTypeToNameMap.at(type).c_str()) != 0) {
448	ASCEND_LOGW(	488	ASCEND_LOGW(
449	- "Set thread name to %s failed!", threadTypeToNameMap.at(type).c_str());	489	+ "Set thread name to %s failed!", kThreadTypeToNameMap.at(type).c_str());
450	}	490	}
451	ASCEND_LOGD(	491	ASCEND_LOGD(
452	- "Set thread name to %s success.", threadTypeToNameMap.at(type).c_str());	492	+ "Set thread name to %s success.", kThreadTypeToNameMap.at(type).c_str());
453	}	493	}
454		494
455	void SetThreadAffinity(c10::DeviceIndex device_id) {	495	void SetThreadAffinity(c10::DeviceIndex device_id) {
456	- if (!needToSetThreadAffinity() \|\| local_thread == ThreadType::USER_THREAD) {	496	+ if (!needToSetThreadAffinity() \|\| kLocalThread == ThreadType::USER_THREAD) {
457	return;	497	return;
458	}	498	}
459		499
460	- CoreIdList core_list = getCoreList(device_id, local_thread);	500	+ CoreIdList core_list = getCoreList(device_id, kLocalThread);
461	std::string range_str = formatCoreRange(core_list);	501	std::string range_str = formatCoreRange(core_list);
462	if (setThreadAffinityImpl(pthread_self(), core_list)) {	502	if (setThreadAffinityImpl(pthread_self(), core_list)) {
463	ASCEND_LOGD(	503	ASCEND_LOGD(
464	"Device %d set %s affinity to %s success.",	504	"Device %d set %s affinity to %s success.",
465	device_id,	505	device_id,
466	- threadTypeToNameMap.at(local_thread).c_str(),	506	+ kThreadTypeToNameMap.at(kLocalThread).c_str(),
467	range_str.c_str());	507	range_str.c_str());
468	} else {	508	} else {
469	ASCEND_LOGE(	509	ASCEND_LOGE(
470	"Device %d set %s affinity to %s failed.",	510	"Device %d set %s affinity to %s failed.",
471	device_id,	511	device_id,
472	- threadTypeToNameMap.at(local_thread).c_str(),	512	+ kThreadTypeToNameMap.at(kLocalThread).c_str(),
473	range_str.c_str());	513	range_str.c_str());
474	}	514	}
475	}	515	}
	@@ -481,9 +521,9 @@ void SetThreadAffinity(ThreadType type) {
481	int device_index;	521	int device_index;
482	NPU_CHECK_ERROR_WITHOUT_UCE(GetDevice(&device_index));	522	NPU_CHECK_ERROR_WITHOUT_UCE(GetDevice(&device_index));
483	c10::DeviceIndex device = static_cast<c10::DeviceIndex>(device_index);	523	c10::DeviceIndex device = static_cast<c10::DeviceIndex>(device_index);
484	- local_thread = type;	524	+ kLocalThread = type;
485	- if (local_thread == ThreadType::MAIN_THREAD) {	525	+ if (kLocalThread == ThreadType::MAIN_THREAD) {
486	- start_main_thread_bind = true;	526	+ kStartMainThreadBind = true;
487	}	527	}
488	SetThreadAffinity(device);	528	SetThreadAffinity(device);
489	}	529	}
	@@ -505,7 +545,7 @@ void SetThreadAffinity(const CoreIdList core_ids) {
505	++it;	545	++it;
506	}	546	}
507	}	547	}
508	- local_thread = ThreadType::USER_THREAD;	548	+ kLocalThread = ThreadType::USER_THREAD;
509	if (setThreadAffinityImpl(pthread_self(), processed_core_ids)) {	549	if (setThreadAffinityImpl(pthread_self(), processed_core_ids)) {
510	ASCEND_LOGD(	550	ASCEND_LOGD(
511	"Set thread affinity to user-defined range %s success.",	551	"Set thread affinity to user-defined range %s success.",
	@@ -526,18 +566,18 @@ void SetThreadAffinity(int core_start, int core_end) {
526	}	566	}
527		567
528	bool NeedMainThreadBind() {	568	bool NeedMainThreadBind() {
529	- return start_main_thread_bind && (local_thread == ThreadType::MAIN_THREAD);	569	+ return kStartMainThreadBind && (kLocalThread == ThreadType::MAIN_THREAD);
530	}	570	}
531		571
532	bool SetThreadAffinityInInitialize() {	572	bool SetThreadAffinityInInitialize() {
533	- if (needToSetThreadAffinity() && !lazy_bind) {	573	+ if (needToSetThreadAffinity() && !kLazyBind) {
534	return true;	574	return true;
535	}	575	}
536	return false;	576	return false;
537	}	577	}
538		578
539	void StartMainThreadBind(c10::DeviceIndex device_id) {	579	void StartMainThreadBind(c10::DeviceIndex device_id) {
540	- if (!needToSetThreadAffinity() \|\| local_thread == ThreadType::USER_THREAD) {	580	+ if (!needToSetThreadAffinity() \|\| kLocalThread == ThreadType::USER_THREAD) {
541	return;	581	return;
542	}	582	}
543	// Make sure that the bind is only executed once per thread, otherwise, it will impact the performance of NPUGuardImpl::uncheckedSetDevice	583	// Make sure that the bind is only executed once per thread, otherwise, it will impact the performance of NPUGuardImpl::uncheckedSetDevice
	@@ -547,11 +587,37 @@ void StartMainThreadBind(c10::DeviceIndex device_id) {
547	}	587	}
548	bind_main_executed = true;	588	bind_main_executed = true;
549		589
		590	+ static std::once_flag bind_irq_flag;
		591	+ std::call_once(bind_irq_flag, []() {
		592	+ if (!kBindIrq) {
		593	+ return;
		594	+ }
		595	+ const auto rank_id = c10_npu::option::OptionsManager::GetRankId();
		596	+ if (rank_id == 0 \|\| rank_id == -1) {
		597	+ ASCEND_LOGI("Stop IRQ balance for rank %d.", rank_id);
		598	+ c10_npu::stopIrqbalance();
		599	+ }
		600	+ int device = 0;
		601	+ NPU_CHECK_ERROR(c10_npu::GetDevice(&device));
		602	+ if (!kDevicesIrqCores[device].empty()) {
		603	+ if (!c10_npu::bindIrqAffinity(device, kDevicesIrqCores[device])) {
		604	+ ASCEND_LOGW("Failed to bind IRQ affinity for device %d. "
		605	+ "This will result in 2 CPU cores remaining unused, potentially causing performance degradation. "
		606	+ "Please set 'bind_irq=0' in the CPU_AFFINITY_CONF environment variable to disable IRQ binding.",
		607	+ device);
		608	+ } else {
		609	+ ASCEND_LOGI("IRQ affinity for device %d bound successfully.", device);
		610	+ }
		611	+ } else {
		612	+ ASCEND_LOGI("IRQ cpu cores for device %d is empty, skip binding.", device);
		613	+ }
		614	+ });
		615	+
550	auto main_thread_pid = getpid();	616	auto main_thread_pid = getpid();
551	if (syscall(SYS_gettid) == main_thread_pid) {	617	if (syscall(SYS_gettid) == main_thread_pid) {
552	return;	618	return;
553	}	619	}
554	- start_main_thread_bind = true;	620	+ kStartMainThreadBind = true;
555	SetThreadAffinity(device_id);	621	SetThreadAffinity(device_id);
556	std::string thread_name = getThreadName();	622	std::string thread_name = getThreadName();
557	if (!std::regex_match(thread_name, std::regex("pt_autograd_[0-9]+"))) {	623	if (!std::regex_match(thread_name, std::regex("pt_autograd_[0-9]+"))) {
	@@ -569,14 +635,14 @@ void StartMainThreadBind(c10::DeviceIndex device_id) {
569	ASCEND_LOGD(	635	ASCEND_LOGD(
570	"Device %d set %s affinity to %s success, thread name: %s.",	636	"Device %d set %s affinity to %s success, thread name: %s.",
571	device_id,	637	device_id,
572	- threadTypeToNameMap.at(ThreadType::MAIN_THREAD).c_str(),	638	+ kThreadTypeToNameMap.at(ThreadType::MAIN_THREAD).c_str(),
573	formatCoreRange(core_list).c_str(),	639	formatCoreRange(core_list).c_str(),
574	thread_name.c_str());	640	thread_name.c_str());
575	} else {	641	} else {
576	ASCEND_LOGE(	642	ASCEND_LOGE(
577	"Device %d set %s affinity to %s failed, ret: %d, thread name: %s.",	643	"Device %d set %s affinity to %s failed, ret: %d, thread name: %s.",
578	device_id,	644	device_id,
579	- threadTypeToNameMap.at(ThreadType::MAIN_THREAD).c_str(),	645	+ kThreadTypeToNameMap.at(ThreadType::MAIN_THREAD).c_str(),
580	formatCoreRange(core_list).c_str(),	646	formatCoreRange(core_list).c_str(),
581	ret,	647	ret,
582	thread_name.c_str());	648	thread_name.c_str());

Atorch_npu/csrc/core/npu/NPUInterruptAffinity.cpp+369-0

@@ -0,0 +1,369 @@
	1	+#include "torch_npu/csrc/core/npu/NPUInterruptAffinity.h"
	2	+
	3	+#include <cstdio>
	4	+#include <cstdlib>
	5	+#include <cerrno>
	6	+#include <dirent.h>
	7	+#include <fstream>
	8	+#include <regex>
	9	+#include <set>
	10	+#include <sstream>
	11	+#include <string>
	12	+#include <vector>
	13	+
	14	+#include "torch_npu/csrc/core/npu/GetAffinityCPUInfo.h"
	15	+#include "torch_npu/csrc/core/npu/NPUFunctions.h"
	16	+#include "torch_npu/csrc/core/npu/interface/DcmiInterface.h"
	17	+#include "torch_npu/csrc/core/npu/npu_log.h"
	18	+#include "torch_npu/csrc/core/npu/NpuVariables.h"
	19	+
	20	+namespace c10_npu {
	21	+
	22	+namespace {
	23	+
	24	+// Paths and files used for IRQ affinity binding on NPU devices:
	25	+//
	26	+// /proc/irq/<irq_num>/smp_affinity
	27	+// - Each IRQ has a subdirectory under /proc/irq/.
	28	+// - Writing a hex CPU mask to the smp_affinity file pins that IRQ
	29	+// to the specified CPU core(s).
	30	+//
	31	+// /sys/bus/pci/devices/<domain:bus:device.func>/msi_irqs/
	32	+// - Each PCIe device is enumerated under /sys/bus/pci/devices/ by
	33	+// its BDF address (e.g. "0000:c1:00.0").
	34	+// - The msi_irqs/ directory lists the MSI IRQ numbers assigned to
	35	+// that device as regular files named by the IRQ number.
	36	+//
	37	+// These two mechanisms together let us discover which IRQs belong to
	38	+// a given NPU device and then redirect them to the desired CPU cores.
	39	+
	40	+constexpr const char* kIrqAffinityPath = "/proc/irq/";
	41	+constexpr const char* kSmpAffinityFile = "/smp_affinity";
	42	+constexpr const char* kPciSysfsPath = "/sys/bus/pci/devices/";
	43	+constexpr const char* kMsiIrqsDir = "/msi_irqs";
	44	+bool kNeedRestartIrqbalance = false;
	45	+
	46	+bool writeSmpAffinity(int physical_id, int irq_num, const std::string& cpu_mask) {
	47	+ std::string filepath =
	48	+ std::string(kIrqAffinityPath) + std::to_string(irq_num) + kSmpAffinityFile;
	49	+ std::ofstream file(filepath);
	50	+ if (!file.is_open()) {
	51	+ ASCEND_LOGW("Physical device-%d: failed to open %s for writing.", physical_id, filepath.c_str());
	52	+ return false;
	53	+ }
	54	+ file << cpu_mask;
	55	+ file.close();
	56	+ if (file.fail()) {
	57	+ ASCEND_LOGW("Physical device-%d: failed to write to %s.", physical_id, filepath.c_str());
	58	+ return false;
	59	+ }
	60	+ ASCEND_LOGI("Physical device-%d: smp_affinity for irq %d set to %s.", physical_id, irq_num, cpu_mask.c_str());
	61	+ return true;
	62	+}
	63	+
	64	+// Convert a CPU core number to hex affinity mask string.
	65	+// CPU 0 -> "00000001"
	66	+// CPU 31 -> "80000000"
	67	+// CPU 32 -> "00000001,00000000"
	68	+std::string cpuToMask(unsigned int cpu) {
	69	+ unsigned int group = cpu / 32;
	70	+ unsigned int bit = cpu % 32;
	71	+ unsigned int value = 1u << bit;
	72	+ char buf[512];
	73	+ int ret = std::snprintf(buf, sizeof(buf), "%08x", value);
	74	+ if (ret < 0 \|\| static_cast<size_t>(ret) >= sizeof(buf)) {
	75	+ ASCEND_LOGW("cpuToMask: snprintf failed, ret=%d", ret);
	76	+ return "";
	77	+ }
	78	+ std::string mask(buf);
	79	+ for (unsigned int i = 0; i < group; i++) {
	80	+ mask += ",00000000";
	81	+ }
	82	+ return mask;
	83	+}
	84	+
	85	+// Get the PCIe bus address for an NPU device via the DCMI interface.
	86	+// Returns empty string on failure.
	87	+std::string getPciAddress(int physical_id) {
	88	+ c10_npu::dcmi::DcmiInit();
	89	+ c10_npu::dcmi::DcmiPcieInfo pcie_info = {};
	90	+ static const auto soc = c10_npu::GetSocVersion();
	91	+ int card_id = 0, die_id = 0;
	92	+ if (soc >= c10_npu::SocVersion::Ascend910_9391 && soc < c10_npu::SocVersion::Ascend950) {
	93	+ // On A3 series, 2 dies per card, physical_id = card_id * 2 + die_id.
	94	+ card_id = physical_id / 2;
	95	+ die_id = physical_id % 2;
	96	+ } else if (soc >= c10_npu::SocVersion::Ascend910B1 && soc < c10_npu::SocVersion::Ascend310B1) {
	97	+ // On A2 series, 1 die per card.
	98	+ card_id = physical_id;
	99	+ } else {
	100	+ TORCH_CHECK(false, "Unsupported SOC version %d.", static_cast<int>(soc));
	101	+ }
	102	+ int acl_ret = c10_npu::dcmi::DcmiGetDevicePcieInfoV2(card_id, die_id, &pcie_info);
	103	+ if (acl_ret != 0) {
	104	+ ASCEND_LOGW("Physical device-%d: failed to get PCIe info, ret=%d", physical_id, acl_ret);
	105	+ return "";
	106	+ }
	107	+ ASCEND_LOGD("Physical device-%d: PCIe fields: vendor=0x%x subvendor=0x%x "
	108	+ "deviceid=0x%x subdeviceid=0x%x domain=%d bus=0x%x dev=0x%x func=0x%x",
	109	+ physical_id,
	110	+ pcie_info.venderid, pcie_info.subvenderid,
	111	+ pcie_info.deviceid, pcie_info.subdeviceid,
	112	+ pcie_info.domain,
	113	+ pcie_info.bdf_busid, pcie_info.bdf_deviceid, pcie_info.bdf_funcid);
	114	+
	115	+ // Format as "domain:bus:device.func", e.g. "0000:c1:00.0"
	116	+ // %x produces lowercase hex, matching sysfs path conventions
	117	+ char buf[32];
	118	+ int ret = std::snprintf(buf, sizeof(buf), "%04x:%02x:%02x.%01x",
	119	+ pcie_info.domain,
	120	+ pcie_info.bdf_busid,
	121	+ pcie_info.bdf_deviceid,
	122	+ pcie_info.bdf_funcid);
	123	+ if (ret < 0 \|\| static_cast<size_t>(ret) >= sizeof(buf)) {
	124	+ ASCEND_LOGW("Physical device-%d: snprintf for PCIe address failed, ret=%d", physical_id, ret);
	125	+ return "";
	126	+ }
	127	+ ASCEND_LOGI("Physical device-%d: PCIe address formatted as '%s'", physical_id, buf);
	128	+ return std::string(buf);
	129	+}
	130	+
	131	+// Collect all numeric IRQ entries from the device's msi_irqs sysfs directory.
	132	+std::set<int> collectMsiIrqs(int physical_id, const std::string& pci_addr) {
	133	+ std::set<int> irqs = {};
	134	+ std::string dir_path = std::string(kPciSysfsPath) + pci_addr + kMsiIrqsDir;
	135	+ DIR* dir = ::opendir(dir_path.c_str());
	136	+ if (!dir) {
	137	+ ASCEND_LOGW("Physical device-%d: failed to open %s for device %s.", physical_id, dir_path.c_str(), pci_addr.c_str());
	138	+ return irqs;
	139	+ }
	140	+ struct dirent* entry;
	141	+ while ((entry = ::readdir(dir)) != nullptr) {
	142	+ if (entry->d_type == DT_REG \|\| entry->d_type == DT_UNKNOWN) {
	143	+ char* end = nullptr;
	144	+ long val = std::strtol(entry->d_name, &end, 10);
	145	+ if (end != entry->d_name && *end == '\0') {
	146	+ irqs.insert(static_cast<int>(val));
	147	+ }
	148	+ }
	149	+ }
	150	+ ::closedir(dir);
	151	+ return irqs;
	152	+}
	153	+
	154	+// Scan /proc/interrupts for all sq_send_trigger_irq entries.
	155	+// Uses a function-local static cache so the scan runs only once per process.
	156	+const std::vector<int>& getSqSendTriggerIrqs(int physical_id) {
	157	+ // Use vector (not set) because this list is only traversed sequentially, never
	158	+ // looked up; vector's contiguous memory is more cache-friendly for iteration.
	159	+ static const std::vector<int> candidates = [physical_id]() -> std::vector<int> {
	160	+ std::vector<int> result;
	161	+ std::ifstream proc_interrupts("/proc/interrupts");
	162	+ if (!proc_interrupts.is_open()) {
	163	+ ASCEND_LOGW("Physical device-%d: failed to open /proc/interrupts.", physical_id);
	164	+ return result;
	165	+ }
	166	+ std::string line;
	167	+ std::regex sq_regex("sq_send_trigger_irq");
	168	+ while (std::getline(proc_interrupts, line)) {
	169	+ if (!std::regex_search(line, sq_regex)) {
	170	+ continue;
	171	+ }
	172	+ auto colon_pos = line.find(':');
	173	+ if (colon_pos == std::string::npos) {
	174	+ continue;
	175	+ }
	176	+ std::string irq_str = line.substr(0, colon_pos);
	177	+ irq_str.erase(0, irq_str.find_first_not_of(" \t"));
	178	+ char* end = nullptr;
	179	+ long val = std::strtol(irq_str.c_str(), &end, 10);
	180	+ if (end == irq_str.c_str() \|\| *end != '\0') {
	181	+ continue;
	182	+ }
	183	+ result.push_back(static_cast<int>(val));
	184	+ }
	185	+ ASCEND_LOGD("Physical device-%d: found %zu sq_send_trigger_irq in /proc/interrupts.",
	186	+ physical_id, result.size());
	187	+ return result;
	188	+ }();
	189	+ return candidates;
	190	+}
	191	+
	192	+// Find the sq_send_trigger_irq belonging to a specific device by taking the
	193	+// intersection of the device's MSI IRQ set and the global sq candidates list.
	194	+// Returns the IRQ number on success, or -1 if not found.
	195	+int findDeviceSqIrq(const std::set<int>& msi_irqs,
	196	+ const std::vector<int>& sq_candidates) {
	197	+ for (int candidate : sq_candidates) {
	198	+ if (msi_irqs.count(candidate) > 0) {
	199	+ return candidate;
	200	+ }
	201	+ }
	202	+ return -1;
	203	+}
	204	+
	205	+// Write smp_affinity for both sq_irq and cq_irq (= sq_irq + 1),
	206	+// then log the result. Returns true on success.
	207	+bool writeDeviceIrqAffinity(int physical_id,
	208	+ const std::string& pci_addr,
	209	+ int sq_irq,
	210	+ unsigned int sq_cpu,
	211	+ unsigned int cq_cpu) {
	212	+ int cq_irq = sq_irq + 1;
	213	+ std::string sq_mask = cpuToMask(sq_cpu);
	214	+ std::string cq_mask = cpuToMask(cq_cpu);
	215	+
	216	+ if (!writeSmpAffinity(physical_id, sq_irq, sq_mask)) {
	217	+ ASCEND_LOGW("Physical device-%d: Failed to write smp_affinity for sq_irq %d",
	218	+ physical_id, sq_irq);
	219	+ return false;
	220	+ }
	221	+ if (!writeSmpAffinity(physical_id, cq_irq, cq_mask)) {
	222	+ ASCEND_LOGW("Physical device-%d: Failed to write smp_affinity for cq_irq %d",
	223	+ physical_id, cq_irq);
	224	+ return false;
	225	+ }
	226	+
	227	+ ASCEND_LOGI(
	228	+ "Physical device-%d (PCI %s): sq_send_trigger_irq IRQ=%d -> CPU%u, "
	229	+ "cq_update_irq IRQ=%d -> CPU%u",
	230	+ physical_id, pci_addr.c_str(), sq_irq, sq_cpu, cq_irq, cq_cpu);
	231	+ return true;
	232	+}
	233	+
	234	+// Wraps ::system() to distinguish system-level failures (fork, OOM, etc.)
	235	+// from command execution failures. Returns the raw ::system() value:
	236	+// -1 → system() itself failed (errno logged)
	237	+// 0 → command executed successfully
	238	+// >0 → command executed but exited with non-zero status
	239	+int SystemCommand(const char* cmd) {
	240	+ if (cmd == nullptr) {
	241	+ ASCEND_LOGW("SystemCommand: cmd is nullptr.");
	242	+ return -1;
	243	+ }
	244	+ int ret = ::system(cmd);
	245	+ if (ret == -1) {
	246	+ ASCEND_LOGW("system() call failed, errno=%d, cmd=[%s]", errno, cmd);
	247	+ } else if (ret != 0) {
	248	+ ASCEND_LOGW("system execute %s command failed, ret=%d", cmd, ret);
	249	+ }
	250	+ return ret;
	251	+}
	252	+} // namespace
	253	+
	254	+bool bindIrqAffinity(int device_id, const CoreIdList& irq_cores)
	255	+{
	256	+ if (irq_cores.empty()) {
	257	+ ASCEND_LOGW("Device-%d: irq_cores is empty.", device_id);
	258	+ return false;
	259	+ }
	260	+ if (irq_cores.size() < 2) {
	261	+ ASCEND_LOGW(
	262	+ "Device-%d: irq_cores size %zu is insufficient, need at least 2 cores.",
	263	+ device_id, irq_cores.size());
	264	+ return false;
	265	+ }
	266	+
	267	+ // Take the first two cores from the ordered set
	268	+ auto it = irq_cores.begin();
	269	+ unsigned int sq_cpu = *it;
	270	+ ++it;
	271	+ unsigned int cq_cpu = *it;
	272	+
	273	+ // Step 1: Get PCIe address for this NPU device
	274	+ int32_t logic_id = 0;
	275	+ int acl_ret = c10_npu::acl::AclrtGetLogicDevIdByUserDevId(device_id, &logic_id);
	276	+ if (acl_ret != ACL_SUCCESS) {
	277	+ ASCEND_LOGW("Failed to get logic device id for user device id %d, ret = %d", device_id, acl_ret);
	278	+ return false;
	279	+ }
	280	+ ASCEND_LOGD("Device-%d: logic id = %d", device_id, logic_id);
	281	+ // In non-container scenarios, the physical device ID is the same as the logical device ID.
	282	+ int32_t physical_id = logic_id;
	283	+ std::string pci_addr = getPciAddress(physical_id);
	284	+ if (pci_addr.empty()) {
	285	+ ASCEND_LOGW("Physical device-%d: Cannot find PCIe address, skipping IRQ binding.", physical_id);
	286	+ return false;
	287	+ }
	288	+ ASCEND_LOGD("Physical device-%d: PCIe address = %s", physical_id, pci_addr.c_str());
	289	+
	290	+ // Step 2: Read the device's MSI IRQ list from sysfs
	291	+ std::set<int> msi_irqs = collectMsiIrqs(physical_id, pci_addr);
	292	+ ASCEND_LOGD("Physical device-%d: MSI IRQs count = %zu", physical_id, msi_irqs.size());
	293	+ if (msi_irqs.empty()) {
	294	+ ASCEND_LOGW(
	295	+ "Physical device-%d (PCI %s): msi_irqs folder is empty or not found, skipping IRQ binding.",
	296	+ physical_id, pci_addr.c_str());
	297	+ return false;
	298	+ }
	299	+
	300	+ // Step 3: Get all sq_send_trigger_irq entries (cached after first call)
	301	+ const std::vector<int>& sq_candidates = getSqSendTriggerIrqs(physical_id);
	302	+ if (sq_candidates.empty()) {
	303	+ ASCEND_LOGW("Physical device-%d: No sq_send_trigger_irq found in /proc/interrupts, skipping IRQ binding.", physical_id);
	304	+ return false;
	305	+ }
	306	+
	307	+ // Step 4: Find the sq_irq that belongs to THIS device
	308	+ int sq_irq = findDeviceSqIrq(msi_irqs, sq_candidates);
	309	+ if (sq_irq < 0) {
	310	+ ASCEND_LOGW(
	311	+ "Physical device-%d (PCI %s): sq_send_trigger_irq not found in MSI IRQs, skipping IRQ binding.",
	312	+ physical_id, pci_addr.c_str());
	313	+ return false;
	314	+ }
	315	+
	316	+ // Step 5: Write smp_affinity for sq_irq and cq_irq
	317	+ return writeDeviceIrqAffinity(physical_id, pci_addr, sq_irq, sq_cpu, cq_cpu);
	318	+}
	319	+
	320	+void stopIrqbalance()
	321	+{
	322	+ // Check if systemctl is available
	323	+ int ret = SystemCommand("command -v systemctl > /dev/null 2>&1");
	324	+ if (ret != 0) {
	325	+ return;
	326	+ }
	327	+ // Check if irqbalance.service is installed
	328	+ ret = SystemCommand("systemctl list-unit-files --quiet irqbalance.service > /dev/null 2>&1");
	329	+ if (ret != 0) {
	330	+ return;
	331	+ }
	332	+ // Stop if running
	333	+ ret = SystemCommand("systemctl is-active --quiet irqbalance");
	334	+ if (ret == 0) {
	335	+ ASCEND_LOGI("irqbalance service is running, attempting to stop it.");
	336	+ int stop_ret = SystemCommand("systemctl stop irqbalance");
	337	+ if (stop_ret == 0) {
	338	+ ASCEND_LOGW("To bind IRQ affinity, the irqbalance service has been stopped successfully. "
	339	+ "It will be restarted when the process exits normally. If the process exits abnormally "
	340	+ "(e.g., killed or coredump), the irqbalance service needs to be restarted manually by "
	341	+ "the user using 'sudo systemctl restart irqbalance' (requires root privileges).");
	342	+ kNeedRestartIrqbalance = true;
	343	+ }
	344	+ } else if (ret > 0) {
	345	+ ASCEND_LOGI("irqbalance service is not running, no need to stop.");
	346	+ }
	347	+ return;
	348	+}
	349	+
	350	+void restartIrqbalance()
	351	+{
	352	+ if (!kNeedRestartIrqbalance) {
	353	+ return;
	354	+ }
	355	+ // Check if systemctl is available
	356	+ // Most mainstream server operating systems released after 2015 use systemd
	357	+ // by default and support the systemctl command.
	358	+ int ret = SystemCommand("command -v systemctl > /dev/null 2>&1");
	359	+ if (ret != 0) {
	360	+ return;
	361	+ }
	362	+ ret = SystemCommand("systemctl restart irqbalance");
	363	+ if (ret != 0) {
	364	+ return;
	365	+ }
	366	+ ASCEND_LOGI("irqbalance service restarted.");
	367	+ kNeedRestartIrqbalance = false;
	368	+}
	369	+} // namespace c10_npu

Atorch_npu/csrc/core/npu/NPUInterruptAffinity.h+29-0

@@ -0,0 +1,29 @@
	1	+#pragma once
	2	+
	3	+#include <string>
	4	+#include <unordered_map>
	5	+#include <vector>
	6	+
	7	+#include "torch_npu/csrc/core/npu/NPUMacros.h"
	8	+#include "torch_npu/csrc/core/npu/GetAffinityCPUInfo.h"
	9	+
	10	+namespace c10_npu {
	11	+// The irq binding affinity feature is implemented by modifying the
	12	+// /proc/irq/{id}/smp_affinity file. Therefore, this feature requires root
	13	+// privileges on the physical machine to take effect (the irqbalance daemon
	14	+// on the physical machine cannot be stopped within a docker container).
	15	+bool bindIrqAffinity(int device_id, const CoreIdList& irq_cores);
	16	+
	17	+// When irq binding affinity is enabled, torch_npu uses the
	18	+// 'systemctl stop irqbalance' command to stop the irqbalance service,
	19	+// preventing the irqbalance service from conflicting with torch_npu's
	20	+// irq binding configuration. When the process exits normally,
	21	+// torch_npu uses the 'systemctl restart irqbalance' command to restart
	22	+// the irqbalance service. In extreme cases where the irqbalance service
	23	+// fails to restart, users need to manually restart the irqbalance
	24	+// service on the server using the 'sudo systemctl restart irqbalance' command.
	25	+void stopIrqbalance();
	26	+
	27	+C10_NPU_API void restartIrqbalance();
	28	+
	29	+} // namespace c10_npu

Mtorch_npu/csrc/core/npu/interface/AclInterface.cpp+11-0

	@@ -131,6 +131,7 @@ TORCH_NPU_LOAD_FUNC(aclrtRepairError)
131	TORCH_NPU_LOAD_FUNC(aclrtGetDeviceInfo)	131	TORCH_NPU_LOAD_FUNC(aclrtGetDeviceInfo)
132	TORCH_NPU_LOAD_FUNC(aclrtMemset)	132	TORCH_NPU_LOAD_FUNC(aclrtMemset)
133	TORCH_NPU_LOAD_FUNC(aclmdlRICaptureThreadExchangeMode)	133	TORCH_NPU_LOAD_FUNC(aclmdlRICaptureThreadExchangeMode)
		134	+TORCH_NPU_LOAD_FUNC(aclrtGetLogicDevIdByUserDevId)
134		135
135	aclprofStepInfoPtr init_stepinfo() {	136	aclprofStepInfoPtr init_stepinfo() {
136	typedef aclprofStepInfoPtr(*npdInitFunc)();	137	typedef aclprofStepInfoPtr(*npdInitFunc)();
	@@ -1850,5 +1851,15 @@ aclError AclmdlRICaptureThreadExchangeMode(aclmdlRICaptureMode* mode)
1850	return func(mode);	1851	return func(mode);
1851	}	1852	}
1852		1853
		1854	+aclError AclrtGetLogicDevIdByUserDevId(const int32_t userDevid, int32_t *const logicDevId)
		1855	+{
		1856	+ typedef aclError (AclrtGetLogicDevIdByUserDevId)(const int32_t, int32_t const);
		1857	+ static AclrtGetLogicDevIdByUserDevId func = nullptr;
		1858	+ if (func == nullptr) {
		1859	+ func = (AclrtGetLogicDevIdByUserDevId) TORCH_NPU_GET_FUNC(aclrtGetLogicDevIdByUserDevId);
		1860	+ }
		1861	+ TORCH_CHECK(func, "Failed to find function aclrtGetLogicDevIdByUserDevId", PTA_ERROR(ErrCode::NOT_FOUND));
		1862	+ return func(userDevid, logicDevId);
		1863	+}
1853	} // namespace acl	1864	} // namespace acl
1854	} // namespace c10	1865	} // namespace c10

Mtorch_npu/csrc/core/npu/interface/AclInterface.h+2-0

	@@ -386,6 +386,8 @@ bool IsExistAclrtGetDeviceInfo();
386		386
387	aclError AclrtGetDeviceInfo(uint32_t deviceId, aclrtDevAttr attr, int64_t *value);	387	aclError AclrtGetDeviceInfo(uint32_t deviceId, aclrtDevAttr attr, int64_t *value);
388		388
		389	+aclError AclrtGetLogicDevIdByUserDevId(const int32_t userDevid, int32_t *const logicDevId);
		390	+
389	aclError AclrtMemSet(void *devPtr, size_t maxCount, int32_t value, size_t count);	391	aclError AclrtMemSet(void *devPtr, size_t maxCount, int32_t value, size_t count);
390		392
391	} // namespace acl	393	} // namespace acl

Mtorch_npu/csrc/core/npu/interface/DcmiInterface.cpp+22-1

	@@ -19,6 +19,7 @@ TORCH_NPU_LOAD_FUNC(dcmi_get_affinity_cpu_info_by_device_id)
19	TORCH_NPU_LOAD_FUNC(dcmi_init)	19	TORCH_NPU_LOAD_FUNC(dcmi_init)
20	TORCH_NPU_LOAD_FUNC(dcmi_get_device_id_in_card)	20	TORCH_NPU_LOAD_FUNC(dcmi_get_device_id_in_card)
21	TORCH_NPU_LOAD_FUNC(dcmi_get_card_num_list)	21	TORCH_NPU_LOAD_FUNC(dcmi_get_card_num_list)
		22	+TORCH_NPU_LOAD_FUNC(dcmi_get_device_pcie_info_v2)
22		23
23	// dcmiv2 functions	24	// dcmiv2 functions
24	TORCH_NPU_LOAD_FUNC(dcmiv2_init)	25	TORCH_NPU_LOAD_FUNC(dcmiv2_init)
	@@ -116,6 +117,26 @@ int DcmiGetDeviceIdInCard(int card_id, int device_id_max, int mcu_id, int *cpu
116	return func(card_id, device_id_max, mcu_id, cpu_id);	117	return func(card_id, device_id_max, mcu_id, cpu_id);
117	}	118	}
118		119
119	-}	120	+int DcmiGetDevicePcieInfoV2(int card_id, int device_id, DcmiPcieInfo *pcie_info)
		121	+{
120		122
		123	+ TORCH_CHECK(pcie_info != nullptr, "pcie_info is null, please check input param.", PTA_ERROR(ErrCode::PARAM))
		124	+ using dcmiGetDevicePcieInfoV2Func = int()(int, int, DcmiPcieInfo );
		125	+ static dcmiGetDevicePcieInfoV2Func func = nullptr;
		126	+ if (func == nullptr) {
		127	+ func = (dcmiGetDevicePcieInfoV2Func)TORCH_NPU_GET_FUNC(dcmi_get_device_pcie_info_v2);
		128	+ }
		129	+ if (func == nullptr) {
		130	+ TORCH_CHECK(false, "Failed to find function dcmi_get_device_pcie_info_v2, "
		131	+ " maybe your hdk version is too low, please upgrade it.", PTA_ERROR(ErrCode::NOT_FOUND))
		132	+ }
		133	+
		134	+ DcmiPcieInfo info_value = {};
		135	+ int ret = func(card_id, device_id, &info_value);
		136	+ if (ret == 0) {
		137	+ *pcie_info = info_value;
		138	+ }
		139	+ return ret;
		140	+}
		141	+}
121	}	142	}

Mtorch_npu/csrc/core/npu/interface/DcmiInterface.h+3-0

	@@ -5,10 +5,13 @@
5	namespace c10_npu {	5	namespace c10_npu {
6	namespace dcmi {	6	namespace dcmi {
7		7
		8	+using DcmiPcieInfo = struct dcmi_pcie_info_all;
		9	+
8	int DcmiInit(void);	10	int DcmiInit(void);
9	int DcmiGetCardNumList(int card_num, int card_list, int list_len);	11	int DcmiGetCardNumList(int card_num, int card_list, int list_len);
10	int DcmiGetAffinityCpuInfoByDeviceId(int card_id, int device_id, char affinity_cpu, int length);	12	int DcmiGetAffinityCpuInfoByDeviceId(int card_id, int device_id, char affinity_cpu, int length);
11	int DcmiGetDeviceIdInCard(int card_id, int device_id_max, int mcu_id, int *cpu_id);	13	int DcmiGetDeviceIdInCard(int card_id, int device_id_max, int mcu_id, int *cpu_id);
		14	+int DcmiGetDevicePcieInfoV2(int card_id, int device_id, DcmiPcieInfo *pcie_info);
12		15
13	}	16	}
14		17

Mtorch_npu/csrc/core/npu/register/OptionsManager.h+1-1

	@@ -122,7 +122,7 @@ public:
122	static uint32_t CheckUseHcclAsyncErrorHandleEnable();	122	static uint32_t CheckUseHcclAsyncErrorHandleEnable();
123	static uint32_t CheckUseDesyncDebugEnable();	123	static uint32_t CheckUseDesyncDebugEnable();
124	C10_NPU_API static bool isACLGlobalLogOn(aclLogLevel level);	124	C10_NPU_API static bool isACLGlobalLogOn(aclLogLevel level);
125	- static int64_t GetRankId();	125	+ C10_NPU_API static int64_t GetRankId();
126	static char *GetNslbPath();	126	static char *GetNslbPath();
127	static bool CheckStatusSaveEnable();	127	static bool CheckStatusSaveEnable();
128	static std::string GetStatusSavePath() noexcept;	128	static std::string GetStatusSavePath() noexcept;