fix: gate createSubHcclComm behind ROOTINFO_SUBCOMM

已合并

fix: gate createSubHcclComm behind ROOTINFO_SUBCOMM_ENABLE env var #40483

fix: gate createSubHcclComm behind ROOTINFO_SUBCOMM_ENABLE env var #40483

已合并

limuan创建于 7月8日

共 4 个文件变更+20-5

Mtest/distributed/test_sub_comm_derivation.py+1-0

	@@ -69,6 +69,7 @@ class SubCommDerivationTest(TestCase):
69	os.environ['MASTER_PORT'] = str(port)	69	os.environ['MASTER_PORT'] = str(port)
70	os.environ['HCCL_WHITELIST_DISABLE'] = '1'	70	os.environ['HCCL_WHITELIST_DISABLE'] = '1'
71	os.environ['HCCL_NPU_SOCKET_PORT_RANGE'] = socket_port_range	71	os.environ['HCCL_NPU_SOCKET_PORT_RANGE'] = socket_port_range
		72	+ os.environ['ROOTINFO_SUBCOMM_ENABLE'] = '1'
72	torch_npu._C._logging._LogContext.GetInstance().setLogs({"torch.distributed": 20})	73	torch_npu._C._logging._LogContext.GetInstance().setLogs({"torch.distributed": 20})
73	torch_npu.npu.set_device(rank)	74	torch_npu.npu.set_device(rank)
74	dist.init_process_group(backend='hccl', world_size=world_size, rank=rank)	75	dist.init_process_group(backend='hccl', world_size=world_size, rank=rank)

Mtorch_npu/csrc/core/npu/register/OptionsManager.cpp+9-0

	@@ -61,6 +61,15 @@ bool OptionsManager::IsResumeModeEnable()
61	return isResumeModeEnable;	61	return isResumeModeEnable;
62	}	62	}
63		63
		64	+bool OptionsManager::IsSubCommRootInfoEnable()
		65	+{
		66	+ const static bool isSubCommRootInfoEnable = []() -> bool {
		67	+ int32_t enable = OptionsManager::GetBoolTypeOption("ROOTINFO_SUBCOMM_ENABLE", 0);
		68	+ return enable != 0;
		69	+ }();
		70	+ return isSubCommRootInfoEnable;
		71	+}
		72	+
64	ReuseMode OptionsManager::GetMultiStreamMemoryReuse()	73	ReuseMode OptionsManager::GetMultiStreamMemoryReuse()
65	{	74	{
66	const static ReuseMode reuseMode = []() -> ReuseMode {	75	const static ReuseMode reuseMode = []() -> ReuseMode {

Mtorch_npu/csrc/core/npu/register/OptionsManager.h+1-0

	@@ -106,6 +106,7 @@ class OptionsManager {
106	public:	106	public:
107	static bool IsHcclZeroCopyEnable();	107	static bool IsHcclZeroCopyEnable();
108	static bool IsResumeModeEnable();	108	static bool IsResumeModeEnable();
		109	+ static bool IsSubCommRootInfoEnable();
109	static ReuseMode GetMultiStreamMemoryReuse();	110	static ReuseMode GetMultiStreamMemoryReuse();
110	static bool CheckInfNanModeEnable();	111	static bool CheckInfNanModeEnable();
111	static bool CheckInfNanModeForceDisable();	112	static bool CheckInfNanModeForceDisable();

Mtorch_npu/csrc/distributed/ProcessGroupHCCL.cpp+9-5

	@@ -2679,12 +2679,16 @@ void ProcessGroupHCCL::createHCCLCommOrigin(
2679	std::vector<c10_npu::NPUStream> &streamVal,	2679	std::vector<c10_npu::NPUStream> &streamVal,
2680	int p2pRank)	2680	int p2pRank)
2681	{	2681	{
2682	- bool isSub = !options_->global_ranks_in_group.empty();	2682	+ if (c10_npu::option::OptionsManager::IsSubCommRootInfoEnable()) {
2683	- if (isSub) {	2683	+ bool isSub = !options_->global_ranks_in_group.empty();
2684	- if (createHCCLCommSub(devicesKey, devices, commType, commConfig, hcclComms, streamVal, p2pRank, true)) {	2684	+ if (isSub) {
2685	- return;	2685	+ if (createHCCLCommSub(devicesKey, devices, commType, commConfig, hcclComms, streamVal, p2pRank, true)) {
		2686	+ TORCH_NPU_HCCL_LOGW(
		2687	+ "Sub-communicator is an internal experimental feature and is incompatible with resume mode.");
		2688	+ return;
		2689	+ }
		2690	+ TORCH_NPU_HCCL_LOGI("Sub comm derivation failed in createHCCLCommOrigin, fallback to rootinfo.");
2686	}	2691	}
2687	- TORCH_NPU_HCCL_LOGI("Sub comm derivation failed in createHCCLCommOrigin, fallback to rootinfo.");
2688	}	2692	}
2689		2693
2690	HcclRootInfo hcclID;	2694	HcclRootInfo hcclID;