已合并
test(npu): add SupportedDevices decorator for test_fault_mode #32019
通晓宇宙创建于 3月19日
test(npu): add SupportedDevices decorator for test_fault_mode #32019
已合并
共 2 个文件变更+23-0
| @@ -5,6 +5,7 @@ import torch | |||
| 5 | import torch_npu | 5 | import torch_npu |
| 6 | from torch_npu.testing.testcase import TestCase, run_tests | 6 | from torch_npu.testing.testcase import TestCase, run_tests |
| 7 | from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU | 7 | from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU |
| 8 | +from torch_npu.testing.common_utils import SupportedDevices | ||
| 8 | 9 | ||
| 9 | 10 | ||
| 10 | class TestMode(TestCase): | 11 | class TestMode(TestCase): |
| @@ -136,6 +137,7 @@ class TestMode(TestCase): | |||
| 136 | ) | 137 | ) |
| 137 | 138 | ||
| 138 | 139 | ||
| 140 | + | ||
| 139 | def test_hccl_timeout(self): | 141 | def test_hccl_timeout(self): |
| 140 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_hccl_timeout.py') | 142 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_hccl_timeout.py') |
| 141 | process = subprocess.Popen(["torchrun", "--nproc-per-node=2", f"{path}"], shell=False, stdout=subprocess.PIPE, | 143 | process = subprocess.Popen(["torchrun", "--nproc-per-node=2", f"{path}"], shell=False, stdout=subprocess.PIPE, |
| @@ -154,6 +156,25 @@ class TestMode(TestCase): | |||
| 154 | message | 156 | message |
| 155 | ) | 157 | ) |
| 156 | 158 | ||
| 159 | + | ||
| 160 | + | ||
| 161 | + def test_hccl_timeout_950(self): | ||
| 162 | + path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_hccl_timeout.py') | ||
| 163 | + process = subprocess.Popen(["torchrun", "--nproc-per-node=2", f"{path}"], shell=False, stdout=subprocess.PIPE, | ||
| 164 | + stderr=subprocess.PIPE, text=True) | ||
| 165 | + message = process.stderr.read() | ||
| 166 | + process.stderr.close() | ||
| 167 | + process.stdout.close() | ||
| 168 | + process.terminate() | ||
| 169 | + process.wait() | ||
| 170 | + self.assertIn( | ||
| 171 | + "wait for compute device to finish failed", | ||
| 172 | + message | ||
| 173 | + ) | ||
| 174 | + self.assertIn( | ||
| 175 | + "task timeout", | ||
| 176 | + message | ||
| 177 | + ) | ||
| 157 | 178 | ||
| 158 | 179 | ||
| 159 | if __name__ == "__main__": | 180 | if __name__ == "__main__": |
| @@ -257,6 +257,7 @@ class TestMode(TestCase): | |||
| 257 | with self.assertRaisesRegex(ValueError, "loaded state dict has a different number of parameter groups"): | 257 | with self.assertRaisesRegex(ValueError, "loaded state dict has a different number of parameter groups"): |
| 258 | optimizer1.load_state_dict(optimizer2.state_dict()) | 258 | optimizer1.load_state_dict(optimizer2.state_dict()) |
| 259 | 259 | ||
| 260 | + | ||
| 260 | def test_aclopCompile(self): | 261 | def test_aclopCompile(self): |
| 261 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_aclopCompileAndExecute.py') | 262 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_aclopCompileAndExecute.py') |
| 262 | process = subprocess.Popen(["python", f"{path}"], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, | 263 | process = subprocess.Popen(["python", f"{path}"], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, |
| @@ -271,6 +272,7 @@ class TestMode(TestCase): | |||
| 271 | error | 272 | error |
| 272 | ) | 273 | ) |
| 273 | 274 | ||
| 275 | + | ||
| 274 | def test_ascyn(self): | 276 | def test_ascyn(self): |
| 275 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_aclopCompileAndExecute.py') | 277 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_aclopCompileAndExecute.py') |
| 276 | process = subprocess.Popen(["python", f"{path}"], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, | 278 | process = subprocess.Popen(["python", f"{path}"], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, |