silentCheckV3-pytorch-AtomGit

Mtorch_npu/asd/_silent_fault_data.py+8-1

	@@ -33,4 +33,11 @@ class SilentFaultDataV2:
33	def __init__(self):	33	def __init__(self):
34	self.step_tensor = torch.zeros(1, dtype=torch.int64).npu()	34	self.step_tensor = torch.zeros(1, dtype=torch.int64).npu()
35	self.check_tensor = torch.zeros(3, dtype=torch.float).npu()	35	self.check_tensor = torch.zeros(3, dtype=torch.float).npu()
36	- self.upper_thresh, self.sigma_thresh = get_thresh()	36	+ self.upper_thresh, self.sigma_thresh = get_thresh()
		37	+
		38	+
		39	+class SilentFaultDataV3:
		40	+ def __init__(self):
		41	+ self.step_tensor = torch.zeros(1, dtype=torch.int64, device="npu")
		42	+ self.avg_tensor = None
		43	+ self.upper_thresh, self.sigma_thresh = get_thresh()

Mtorch_npu/asd/asd.py+32-3

	@@ -5,7 +5,7 @@ from torch.nn.functional import embedding as origin_embedding
5		5
6	import torch_npu	6	import torch_npu
7	from torch_npu.utils._error_code import ErrCode, pta_error	7	from torch_npu.utils._error_code import ErrCode, pta_error
8	-from ._silent_fault_data import SilentFaultData, SilentFaultDataV2	8	+from ._silent_fault_data import SilentFaultData, SilentFaultDataV2, SilentFaultDataV3
9		9
10	__all__ = []	10	__all__ = []
11		11
	@@ -95,7 +95,8 @@ def _patch_embedding(input_embedding, weight, args, *kwargs):
95		95
96	def _asd_patch():	96	def _asd_patch():
97	env_value = os.getenv("NPU_ASD_ENABLE", "0")	97	env_value = os.getenv("NPU_ASD_ENABLE", "0")
98	- if env_value.isdigit() and int(env_value) and torch_npu._C._npu_support_silentClientV2():	98	+
		99	+ if env_value.isdigit() and int(env_value) and torch_npu._C._get_silent_check_version() > 1:
99	return	100	return
100		101
101	if env_value not in ["0", "1"]:	102	if env_value not in ["0", "1"]:
	@@ -127,4 +128,32 @@ class _SilentFaultDetectorV2:
127	sfda.sigma_thresh[0], sfda.upper_thresh[1], sfda.sigma_thresh[1], asd_enable)	128	sfda.sigma_thresh[0], sfda.upper_thresh[1], sfda.sigma_thresh[1], asd_enable)
128		129
129		130
130	-_silent_fault_detector_v2 = _SilentFaultDetectorV2()	131	+_silent_fault_detector_v2 = _SilentFaultDetectorV2()
		132	+
		133	+
		134	+@_Singleton
		135	+class _SilentFaultDetectorV3:
		136	+ def __init__(self):
		137	+ self.silent_data_dict = dict()
		138	+ self.beta1 = 0.99
		139	+
		140	+ def silent_fault_check(self, idx, asd_enable, grad):
		141	+ if grad.dtype != torch.bfloat16 and grad.dtype != torch.float32:
		142	+ return
		143	+
		144	+ val = grad.pow(2).max().view(-1)
		145	+
		146	+ if idx not in self.silent_data_dict:
		147	+ self.silent_data_dict[idx] = SilentFaultDataV3()
		148	+ self.silent_data_dict[idx].avg_tensor = grad.pow(2).amax().view(-1)
		149	+ grad_max = self.silent_data_dict[idx].avg_tensor
		150	+ else:
		151	+ grad_max = val
		152	+
		153	+ sfda = self.silent_data_dict[idx]
		154	+
		155	+ torch_npu._npu_silent_check_v3(val, grad, sfda.step_tensor, grad_max, sfda.avg_tensor,
		156	+ sfda.upper_thresh[0], sfda.upper_thresh[1], self.beta1, asd_enable)
		157	+
		158	+
		159	+_silent_fault_detector_v3 = _SilentFaultDetectorV3()

Mtorch_npu/csrc/core/npu/interface/OpInterface.cpp+12-2

	@@ -13,6 +13,7 @@ namespace opapi {
13		13
14	REGISTER_LIBRARY(libopapi)	14	REGISTER_LIBRARY(libopapi)
15	LOAD_FUNCTION(aclnnSilentCheck)	15	LOAD_FUNCTION(aclnnSilentCheck)
		16	+LOAD_FUNCTION(aclnnSilentCheckV2)
16		17
17	bool IsExistAclnnSilentCheck()	18	bool IsExistAclnnSilentCheck()
18	{	19	{
	@@ -23,5 +24,14 @@ bool IsExistAclnnSilentCheck()
23	return isExist;	24	return isExist;
24	}	25	}
25		26
26	-} // namespace acl	27	+bool IsExistAclnnSilentCheckV2()
27	-} // namespace c10	28	+{
		29	+ const static bool isExistV2 = []() -> bool {
		30	+ static auto func = GET_FUNC(aclnnSilentCheckV2);
		31	+ return func != nullptr;
		32	+ }();
		33	+ return isExistV2;
		34	+}
		35	+
		36	+} // namespace opapi
		37	+} // namespace c10_npu

Mtorch_npu/csrc/core/npu/interface/OpInterface.h+5-0

	@@ -7,5 +7,10 @@ namespace opapi {
7	*/	7	*/
8	bool IsExistAclnnSilentCheck();	8	bool IsExistAclnnSilentCheck();
9		9
		10	+/**
		11	+ * This API is used to check whether aclnnSilentCheckV2 exist.
		12	+*/
		13	+bool IsExistAclnnSilentCheckV2();
		14	+
10	} // namespace opapi	15	} // namespace opapi
11	} // namespace c10_npu	16	} // namespace c10_npu

Mtorch_npu/csrc/distributed/ProcessGroupHCCL.cpp+28-10

	@@ -28,6 +28,7 @@
28	#include "torch_npu/csrc/core/npu/NPUAffinityController.h"	28	#include "torch_npu/csrc/core/npu/NPUAffinityController.h"
29	#include "torch_npu/csrc/core/npu/NPUStream.h"	29	#include "torch_npu/csrc/core/npu/NPUStream.h"
30	#include "torch_npu/csrc/core/npu/register/OptionsManager.h"	30	#include "torch_npu/csrc/core/npu/register/OptionsManager.h"
		31	+#include "torch_npu/csrc/core/npu/interface/OpInterface.h"
31	#include "torch_npu/csrc/distributed/HCCLUtils.hpp"	32	#include "torch_npu/csrc/distributed/HCCLUtils.hpp"
32	#include "torch_npu/csrc/distributed/HcclCompile.h"	33	#include "torch_npu/csrc/distributed/HcclCompile.h"
33	#include "torch_npu/csrc/distributed/ProcessGroupHCCL.hpp"	34	#include "torch_npu/csrc/distributed/ProcessGroupHCCL.hpp"
	@@ -1783,17 +1784,34 @@ void ProcessGroupHCCL::silenceCheck(at::Tensor &input, c10d::OpType opType)
1783	return;	1784	return;
1784	}	1785	}
1785	}	1786	}
1786	- if (silenceCheckCache_.find(opType) == silenceCheckCache_.end()) {	1787	+ if (c10_npu::opapi::IsExistAclnnSilentCheckV2()) {
1787	- at::Tensor stepTensor = at::zeros({1}, input.options().dtype(at::kLong));	1788	+ at::Tensor val = input.detach().pow(2).max().view(-1);
1788	- at::Tensor cacheTensor = at::zeros({3}, input.options().dtype(at::kFloat));	1789	+ at::Tensor max;
1789	- silenceCheckCache_.emplace(opType, std::make_pair(std::move(stepTensor), std::move(cacheTensor)));	1790	+ if (silenceCheckCache_.find(opType) == silenceCheckCache_.end()) {
		1791	+ at::Tensor stepTensor = at::zeros({1}, input.options().dtype(at::kLong));
		1792	+ at::Tensor avg = input.detach().pow(2).amax().view(-1);
		1793	+ max = avg;
		1794	+ silenceCheckCache_.emplace(opType, std::make_pair(std::move(stepTensor), std::move(avg)));
		1795	+ } else {
		1796	+ max = val;
		1797	+ }
		1798	+ static double beta1 = 0.99;
		1799	+ op_plugin::_npu_silent_check_v3(val, input, silenceCheckCache_[opType].first, max, silenceCheckCache_[opType].second,
		1800	+ c10_npu::option::OptionsManager::GetSilenceUpperThresh().first, c10_npu::option::OptionsManager::GetSilenceUpperThresh().second,
		1801	+ beta1, static_cast<int64_t>(c10_npu::option::OptionsManager::GetSilenceCheckFlag()));
		1802	+ } else {
		1803	+ if (silenceCheckCache_.find(opType) == silenceCheckCache_.end()) {
		1804	+ at::Tensor stepTensor = at::zeros({1}, input.options().dtype(at::kLong));
		1805	+ at::Tensor cacheTensor = at::zeros({3}, input.options().dtype(at::kFloat));
		1806	+ silenceCheckCache_.emplace(opType, std::make_pair(std::move(stepTensor), std::move(cacheTensor)));
		1807	+ }
		1808	+ at::Tensor val = at::norm(input);
		1809	+ static double min_steps = 100.0;
		1810	+ op_plugin::_npu_silent_check_v2(val, input, silenceCheckCache_[opType].second, silenceCheckCache_[opType].first, min_steps,
		1811	+ c10_npu::option::OptionsManager::GetSilenceUpperThresh().first, c10_npu::option::OptionsManager::GetSilenceSigmaThresh().first,
		1812	+ c10_npu::option::OptionsManager::GetSilenceUpperThresh().second, c10_npu::option::OptionsManager::GetSilenceSigmaThresh().second,
		1813	+ static_cast<int64_t>(c10_npu::option::OptionsManager::GetSilenceCheckFlag()));
1790	}	1814	}
1791	- at::Tensor val = at::norm(input);
1792	- static double min_steps = 100.0;
1793	- op_plugin::_npu_silent_check_v2(val, input, silenceCheckCache_[opType].second, silenceCheckCache_[opType].first, min_steps,
1794	- c10_npu::option::OptionsManager::GetSilenceUpperThresh().first, c10_npu::option::OptionsManager::GetSilenceSigmaThresh().first,
1795	- c10_npu::option::OptionsManager::GetSilenceUpperThresh().second, c10_npu::option::OptionsManager::GetSilenceSigmaThresh().second,
1796	- static_cast<int64_t>(c10_npu::option::OptionsManager::GetSilenceCheckFlag()));
1797	}	1815	}
1798		1816
1799	HcclCommConfig ProcessGroupHCCL::createHcclCommConfigWithOptions()	1817	HcclCommConfig ProcessGroupHCCL::createHcclCommConfigWithOptions()

Mtorch_npu/csrc/npu/Module.cpp+13-5

	@@ -1133,6 +1133,7 @@ PyObject* THNPModule_npu_set_call_state(PyObject* _unused, PyObject* arg)
1133	TORCH_CHECK(false, "invalid value of call_state, expected one of `forward`, `backward`", PTA_ERROR(ErrCode::PARAM));	1133	TORCH_CHECK(false, "invalid value of call_state, expected one of `forward`, `backward`", PTA_ERROR(ErrCode::PARAM));
1134	}	1134	}
1135	c10_npu::model_state().set_call_state(mode);	1135	c10_npu::model_state().set_call_state(mode);
		1136	+ ASCEND_LOGI("NPU set call state success, state is %s.", state.c_str());
1136	Py_RETURN_NONE;	1137	Py_RETURN_NONE;
1137	END_HANDLE_TH_ERRORS	1138	END_HANDLE_TH_ERRORS
1138	}	1139	}
	@@ -1152,17 +1153,24 @@ PyObject* THNPModule_npu_set_module_train_state(PyObject* _unused, PyObject* arg
1152	TORCH_CHECK(false, "invalid value of train_state, expected one of `train`, `infer`", PTA_ERROR(ErrCode::PARAM));	1153	TORCH_CHECK(false, "invalid value of train_state, expected one of `train`, `infer`", PTA_ERROR(ErrCode::PARAM));
1153	}	1154	}
1154	c10_npu::model_state().set_model_mode(mode);	1155	c10_npu::model_state().set_model_mode(mode);
		1156	+ ASCEND_LOGI("NPU set train state success, state is %s.", state.c_str());
1155	Py_RETURN_NONE;	1157	Py_RETURN_NONE;
1156	END_HANDLE_TH_ERRORS	1158	END_HANDLE_TH_ERRORS
1157	}	1159	}
1158		1160
1159	-PyObject* THNPModule_npu_support_silentClientV2(PyObject* self, PyObject* noargs)	1161	+PyObject* THNPModule_npu_get_silent_check_version(PyObject* self, PyObject* noargs)
1160	{	1162	{
1161	HANDLE_TH_ERRORS	1163	HANDLE_TH_ERRORS
1162	- if (c10_npu::opapi::IsExistAclnnSilentCheck()) {	1164	+ if (c10_npu::opapi::IsExistAclnnSilentCheckV2()) {
1163	- Py_RETURN_TRUE;	1165	+ // silent check v3
		1166	+ return PyLong_FromLong(3);
		D D dilililiwhy12025年1月6日底层特性版本和我们的版本号是不对应的吗？
1164	} else {	1167	} else {
1165	- Py_RETURN_FALSE;	1168	+ if (c10_npu::opapi::IsExistAclnnSilentCheck()) {
		1169	+ // silent check v2
		1170	+ return PyLong_FromLong(2);
		1171	+ }
		1172	+ // silent check v1
		1173	+ return PyLong_FromLong(1);
1166	}	1174	}
1167	END_HANDLE_TH_ERRORS	1175	END_HANDLE_TH_ERRORS
1168	}	1176	}
	@@ -1236,7 +1244,7 @@ static struct PyMethodDef THNPModule_methods[] = {
1236	{"_mark", (PyCFunction)THNPModule_msTxMark, METH_VARARGS, nullptr},	1244	{"_mark", (PyCFunction)THNPModule_msTxMark, METH_VARARGS, nullptr},
1237	{"_npu_set_call_state", (PyCFunction)THNPModule_npu_set_call_state, METH_O, nullptr},	1245	{"_npu_set_call_state", (PyCFunction)THNPModule_npu_set_call_state, METH_O, nullptr},
1238	{"_npu_set_module_train_state", (PyCFunction)THNPModule_npu_set_module_train_state, METH_O, nullptr},	1246	{"_npu_set_module_train_state", (PyCFunction)THNPModule_npu_set_module_train_state, METH_O, nullptr},
1239	- {"_npu_support_silentClientV2", (PyCFunction)THNPModule_npu_support_silentClientV2, METH_NOARGS, nullptr},	1247	+ {"_get_silent_check_version", (PyCFunction)THNPModule_npu_get_silent_check_version, METH_NOARGS, nullptr},
1240	{"_npu_set_threads_affinity", (PyCFunction)THNPModule_npu_set_thread_affinity, METH_NOARGS, nullptr},	1248	{"_npu_set_threads_affinity", (PyCFunction)THNPModule_npu_set_thread_affinity, METH_NOARGS, nullptr},
1241	{"_npu_reset_threads_affinity", (PyCFunction)THNPModule_npu_reset_thread_affinity, METH_NOARGS, nullptr},	1249	{"_npu_reset_threads_affinity", (PyCFunction)THNPModule_npu_reset_thread_affinity, METH_NOARGS, nullptr},
1242	{nullptr}};	1250	{nullptr}};

Mtorch_npu/utils/_asd_detector.py+2-2

	@@ -4,14 +4,14 @@ from torch_npu.asd.asd import _silent_fault_detector
4		4
5		5
6	def set_asd_loss_scale(loss_scale=1.0):	6	def set_asd_loss_scale(loss_scale=1.0):
7	- if torch_npu._C._npu_support_silentClientV2():	7	+ if torch_npu._C._get_silent_check_version() > 1:
8	return	8	return
9	_silent_fault_detector.set_asd_loss_scale(loss_scale)	9	_silent_fault_detector.set_asd_loss_scale(loss_scale)
10	return	10	return
11		11
12		12
13	def register_asd_hook(x, weight):	13	def register_asd_hook(x, weight):
14	- if torch_npu._C._npu_support_silentClientV2():	14	+ if torch_npu._C._get_silent_check_version() > 1:
15	return	15	return
16	if x is not None and x.requires_grad and x._backward_hooks is None:	16	if x is not None and x.requires_grad and x._backward_hooks is None:
17	x.register_hook(_silent_fault_detector.silent_fault_check_hook(weight))	17	x.register_hook(_silent_fault_detector.silent_fault_check_hook(weight))

Mtorch_npu/utils/_step.py+11-6

	@@ -11,7 +11,7 @@ from torch.nn import Module
11		11
12	import torch_npu	12	import torch_npu
13	from torch_npu.utils._error_code import ErrCode, pta_error	13	from torch_npu.utils._error_code import ErrCode, pta_error
14	-from torch_npu.asd.asd import _silent_fault_detector_v2	14	+from torch_npu.asd.asd import _silent_fault_detector_v2, _silent_fault_detector_v3
15		15
16		16
17	original_call = Module.__call__	17	original_call = Module.__call__
	@@ -53,7 +53,10 @@ def input_hook(idx, asd_flag):
53		53
54	if idx != "":	54	if idx != "":
55	IS_IN_BACKWARD = IS_IN_BACKWARD & 1 # 011 & 001 = 001	55	IS_IN_BACKWARD = IS_IN_BACKWARD & 1 # 011 & 001 = 001
56	- _silent_fault_detector_v2.silent_fault_check(idx, asd_flag, grad)	56	+ if torch_npu._C._get_silent_check_version() == 3:
		57	+ _silent_fault_detector_v3.silent_fault_check(idx, asd_flag, grad)
		58	+ else:
		59	+ _silent_fault_detector_v2.silent_fault_check(idx, asd_flag, grad)
57	else:	60	else:
58	IS_IN_BACKWARD = IS_IN_BACKWARD & 2 # 011 & 010 = 010	61	IS_IN_BACKWARD = IS_IN_BACKWARD & 2 # 011 & 010 = 010
59		62
	@@ -360,10 +363,12 @@ def add_perf_dump_patch():
360	"2 as `ASD opened, print error logs and raise exception`, "	363	"2 as `ASD opened, print error logs and raise exception`, "
361	"3 as `ASD opened, print debug logs and raise exception`" + pta_error(ErrCode.VALUE))	364	"3 as `ASD opened, print debug logs and raise exception`" + pta_error(ErrCode.VALUE))
362	asd_enable = int(asd_value)	365	asd_enable = int(asd_value)
363	-	366	+ if asd_enable:
364	- if asd_enable and not torch_npu._C._npu_support_silentClientV2():	367	+ if torch_npu._C._get_silent_check_version() == 1:
365	- warnings.warn(f"Warning: CANN version lower than 8.0.RC3 and currently does not support silent check 2.0 version. It will switch to 1.0 version.")	368	+ warnings.warn(f"Warning: CANN version lower than 8.0.RC3 and currently does not support silent check 2.0 version or later. It will switch to 1.0 version.")
366	- asd_enable = 0	369	+ asd_enable = 0
		370	+ elif torch_npu._C._get_silent_check_version() == 2:
		371	+ warnings.warn(f"Warning: CANN version lower than 8.0.0 and currently does not support silent check 3.0 version. It will switch to 2.0 version. The asd_detect is {asd_enable}")
367		372
368	if perf_dump_enable or asd_enable:	373	if perf_dump_enable or asd_enable:
369	Module.__call__ = _custom_call	374	Module.__call__ = _custom_call