已合并
add set_deterministic_level #30384
SCh_zx创建于 2月2日
add set_deterministic_level #30384
已合并
共 7 个文件变更+101-21
| @@ -9,8 +9,8 @@ from torch_npu.testing.common_utils import SkipIfNotGteCANNVersion | |||
| 9 | 9 | ||
| 10 | 10 | ||
| 11 | class TestCompatibleImpl(TestCase): | 11 | class TestCompatibleImpl(TestCase): |
| 12 | - def test_are_compatible_impl_enable(self): | 12 | + def test__get_deterministic_level(self): |
| 13 | - self.assertEqual(torch_npu.npu.are_compatible_impl_enable(), True) | 13 | + self.assertEqual(torch_npu.npu._get_deterministic_level(), 2) |
| 14 | 14 | ||
| 15 | 15 | ||
| 16 | def test_npu_matmul(self): | 16 | def test_npu_matmul(self): |
| @@ -27,5 +27,5 @@ class TestCompatibleImpl(TestCase): | |||
| 27 | 27 | ||
| 28 | 28 | ||
| 29 | if __name__ == "__main__": | 29 | if __name__ == "__main__": |
| 30 | - torch_npu.npu.use_compatible_impl(True) | 30 | + torch_npu.npu.set_deterministic_level(2) |
| 31 | run_tests() | 31 | run_tests() |
| @@ -992,16 +992,13 @@ | |||
| 992 | "torch_npu.npu.reset_peak_host_memory_stats": { | 992 | "torch_npu.npu.reset_peak_host_memory_stats": { |
| 993 | "signature": "()" | 993 | "signature": "()" |
| 994 | }, | 994 | }, |
| 995 | - "torch_npu.npu.use_consistent_algorithms": { | 995 | + "torch_npu.npu.set_deterministic_level": { |
| 996 | - "signature": "(is_enable)" | 996 | + "signature": "(level)" |
| 997 | - }, | ||
| 998 | - "torch_npu.npu.are_consistent_algorithms_enable": { | ||
| 999 | - "signature": "()" | ||
| 1000 | }, | 997 | }, |
| 1001 | "torch_npu.npu.use_compatible_impl": { | 998 | "torch_npu.npu.use_compatible_impl": { |
| 1002 | "signature": "(is_enable)" | 999 | "signature": "(is_enable)" |
| 1003 | }, | 1000 | }, |
| 1004 | - "torch_npu.npu.are_compatible_impl_enable": { | 1001 | + "torch_npu.npu.are_compatible_impl_enabled": { |
| 1005 | "signature": "()" | 1002 | "signature": "()" |
| 1006 | }, | 1003 | }, |
| 1007 | "torch_npu.npu.enable_deterministic_with_backward": { | 1004 | "torch_npu.npu.enable_deterministic_with_backward": { |
| @@ -18,6 +18,7 @@ static thread_local int local_device = -1; | |||
| 18 | static std::unordered_map<int8_t, aclrtContext> used_devices; | 18 | static std::unordered_map<int8_t, aclrtContext> used_devices; |
| 19 | std::recursive_mutex mtx; | 19 | std::recursive_mutex mtx; |
| 20 | thread_local int targetDeviceIndex = -1; | 20 | thread_local int targetDeviceIndex = -1; |
| 21 | +static uint32_t deterministic_level = 0; | ||
 | |||
| 21 | 22 | ||
| 22 | bool is_lazy_set_device() | 23 | bool is_lazy_set_device() |
| 23 | { | 24 | { |
| @@ -435,4 +436,13 @@ uint32_t GetResInCurrentThread(int32_t type) | |||
| 435 | return value; | 436 | return value; |
| 436 | } | 437 | } |
| 437 | 438 | ||
| 439 | +void SetDeterministicLevel(uint32_t level) | ||
| 440 | +{ | ||
| 441 | + deterministic_level = level; | ||
| 442 | +} | ||
| 443 | + | ||
| 444 | +uint32_t GetDeterministicLevel() | ||
| 445 | +{ | ||
| 446 | + return deterministic_level; | ||
| 447 | +} | ||
| 438 | } // namespace c10_npu | 448 | } // namespace c10_npu |
| @@ -2226,6 +2226,23 @@ PyObject* THNPModule_setOpTimeoutMs(PyObject* self, PyObject* arg) | |||
| 2226 | END_HANDLE_TH_ERRORS | 2226 | END_HANDLE_TH_ERRORS |
| 2227 | } | 2227 | } |
| 2228 | 2228 | ||
| 2229 | +PyObject* THNPModule_set_deterministic_level(PyObject* self, PyObject* arg) | ||
| 2230 | +{ | ||
| 2231 | + HANDLE_TH_ERRORS | ||
| 2232 | + uint32_t level = THPUtils_unpackUInt32(arg); | ||
| 2233 | + c10_npu::SetDeterministicLevel(level); | ||
| 2234 | + Py_RETURN_NONE; | ||
| 2235 | + END_HANDLE_TH_ERRORS | ||
| 2236 | +} | ||
| 2237 | + | ||
| 2238 | +PyObject* THNPModule_get_deterministic_level(PyObject* self, PyObject* noargs) | ||
| 2239 | +{ | ||
| 2240 | + HANDLE_TH_ERRORS | ||
| 2241 | + uint32_t level = c10_npu::GetDeterministicLevel(); | ||
| 2242 | + return THPUtils_packUInt32(level); | ||
| 2243 | + END_HANDLE_TH_ERRORS | ||
| 2244 | +} | ||
| 2245 | + | ||
| 2229 | static struct PyMethodDef THNPModule_methods[] = { | 2246 | static struct PyMethodDef THNPModule_methods[] = { |
| 2230 | {"_npu_init", (PyCFunction)THNPModule_initExtension, METH_NOARGS, nullptr}, | 2247 | {"_npu_init", (PyCFunction)THNPModule_initExtension, METH_NOARGS, nullptr}, |
| 2231 | {"_npu_set_run_yet_variable_to_false", (PyCFunction)THNPModule_set_run_yet_variable_to_false_wrap, METH_NOARGS, nullptr}, | 2248 | {"_npu_set_run_yet_variable_to_false", (PyCFunction)THNPModule_set_run_yet_variable_to_false_wrap, METH_NOARGS, nullptr}, |
| @@ -2308,6 +2325,8 @@ static struct PyMethodDef THNPModule_methods[] = { | |||
| 2308 | {"_aclop_stop_dump", (PyCFunction)THNPModule_aclop_stop_dump, METH_NOARGS, nullptr}, | 2325 | {"_aclop_stop_dump", (PyCFunction)THNPModule_aclop_stop_dump, METH_NOARGS, nullptr}, |
| 2309 | {"_npu_to_dlpack", (PyCFunction)THPModule_toDLPack, METH_O, nullptr}, | 2326 | {"_npu_to_dlpack", (PyCFunction)THPModule_toDLPack, METH_O, nullptr}, |
| 2310 | {"_npu_from_dlpack", (PyCFunction)THPModule_fromDLPack, METH_O, nullptr}, | 2327 | {"_npu_from_dlpack", (PyCFunction)THPModule_fromDLPack, METH_O, nullptr}, |
| 2328 | + {"_npu_set_deterministic_level", (PyCFunction)THNPModule_set_deterministic_level, METH_O, nullptr}, | ||
| 2329 | + {"_npu_get_deterministic_level", (PyCFunction)THNPModule_get_deterministic_level, METH_NOARGS, nullptr}, | ||
| 2311 | {nullptr}}; | 2330 | {nullptr}}; |
| 2312 | 2331 | ||
| 2313 | TORCH_NPU_API PyMethodDef* THNPModule_get_methods() | 2332 | TORCH_NPU_API PyMethodDef* THNPModule_get_methods() |
| @@ -132,10 +132,9 @@ __all__ = [ | |||
| 132 | "host_memory_stats_as_nested_dict", | 132 | "host_memory_stats_as_nested_dict", |
| 133 | "reset_accumulated_host_memory_stats", | 133 | "reset_accumulated_host_memory_stats", |
| 134 | "reset_peak_host_memory_stats", | 134 | "reset_peak_host_memory_stats", |
| 135 | - "use_consistent_algorithms", | 135 | + "set_deterministic_level", |
| 136 | - "are_consistent_algorithms_enable", | ||
| 137 | "use_compatible_impl", | 136 | "use_compatible_impl", |
| 138 | - "are_compatible_impl_enable" | 137 | + "are_compatible_impl_enabled" |
| 139 | ] | 138 | ] |
| 140 | 139 | ||
| 141 | from typing import Tuple, Union, List, cast, Optional | 140 | from typing import Tuple, Union, List, cast, Optional |
| @@ -571,14 +570,34 @@ def _comm_switch_nic(ranks, useBackup): | |||
| 571 | return torch_npu.distributed.distributed_c10d._comm_switch_nic(ranks, useBackup) | 570 | return torch_npu.distributed.distributed_c10d._comm_switch_nic(ranks, useBackup) |
| 572 | 571 | ||
| 573 | 572 | ||
| 574 | -def use_consistent_algorithms(is_enable): | 573 | +def set_deterministic_level(level): |
| 575 | - option = {"STRONG_CONSISTENCY": "enable" if is_enable else "disable"} | 574 | + warnings.warn("After using 'torch_npu.npu.set_deterministic_level', " |
| 576 | - torch_npu._C._npu_setOption(option) | 575 | + "please do not use 'torch.use_deterministic_algorithms' anymore, " |
| 576 | + "as it may cause unknown errors.") | ||
| 577 | + if level == 0 and torch.are_deterministic_algorithms_enabled(): | ||
| 578 | + warnings.warn("The current configuration value of 'torch_npu.npu.set_deterministic_level' " | ||
| 579 | + "conflicts with 'torch.use_deterministic_algorithms'. " | ||
| 580 | + "'torch.use_deterministic_algorithms' has been configured to 'False'") | ||
| 581 | + torch.use_deterministic_algorithms(False) | ||
| 582 | + elif level >= 1 and not torch.are_deterministic_algorithms_enabled(): | ||
| 583 | + warnings.warn("The current configuration value of 'torch_npu.npu.set_deterministic_level' " | ||
| 584 | + "conflicts with 'torch.use_deterministic_algorithms'. " | ||
| 585 | + "'torch.use_deterministic_algorithms' has been configured to 'True'") | ||
| 586 | + torch.use_deterministic_algorithms(True) | ||
| 587 | + torch_npu._C._npu_set_deterministic_level(level) | ||
| 577 | 588 | ||
| 578 | 589 | ||
| 579 | -def are_consistent_algorithms_enable(): | 590 | +def _get_deterministic_level(): |
| 580 | - consistency_value = torch_npu._C._npu_getOption("STRONG_CONSISTENCY") | 591 | + level = torch_npu._C._npu_get_deterministic_level() |
| 581 | - return consistency_value is not None and consistency_value.decode() == "enable" | 592 | + if level == 0 and torch.are_deterministic_algorithms_enabled(): |
| 593 | + level = 1 | ||
| 594 | + torch_npu.npu.set_deterministic_level(level) | ||
| 595 | + return level | ||
| 596 | + if level >= 1 and not torch.are_deterministic_algorithms_enabled(): | ||
| 597 | + level = 0 | ||
| 598 | + torch_npu.npu.set_deterministic_level(level) | ||
| 599 | + return level | ||
| 600 | + return level | ||
| 582 | 601 | ||
| 583 | 602 | ||
| 584 | def use_compatible_impl(is_enable): | 603 | def use_compatible_impl(is_enable): |
| @@ -586,7 +605,7 @@ def use_compatible_impl(is_enable): | |||
| 586 | torch_npu._C._npu_setOption(option) | 605 | torch_npu._C._npu_setOption(option) |
| 587 | 606 | ||
| 588 | 607 | ||
| 589 | -def are_compatible_impl_enable(): | 608 | +def are_compatible_impl_enabled(): |
| 590 | compatible_value = torch_npu._C._npu_getOption("COMPATIBLE_IMPL") | 609 | compatible_value = torch_npu._C._npu_getOption("COMPATIBLE_IMPL") |
| 591 | return compatible_value is not None and compatible_value.decode() == "enable" | 610 | return compatible_value is not None and compatible_value.decode() == "enable" |
| 592 | 611 | ||
并发安全: 新增的静态变量
deterministic_level被多个函数（SetDeterministicLevel和GetDeterministicLevel）访问和修改，但缺乏适当的同步机制。在多线程环境下，对deterministic_level的并发读写可能导致数据竞争（data race），进而引发不确定的行为或程序崩溃。问题类型: 并发安全 文件路径:
torch_npu/csrc/core/npu/NPUFunctions.cpp行号: 21 问题代码:static uint32_t deterministic_level = 0;修改建议:
此评论由代码审查工具自动生成