已合并
test(npu): add SupportedDevices decorator for test_fault_mode #32020
通晓宇宙创建于 3月19日
test(npu): add SupportedDevices decorator for test_fault_mode #32020
已合并
共 2 个文件变更+23-0
| @@ -4,6 +4,7 @@ import torch | |||
| 4 | import torch_npu | 4 | import torch_npu |
| 5 | from torch_npu.testing.testcase import TestCase, run_tests | 5 | from torch_npu.testing.testcase import TestCase, run_tests |
| 6 | from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU | 6 | from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU |
| 7 | +from torch_npu.testing.common_utils import SupportedDevices | ||
| 7 | 8 | ||
| 8 | 9 | ||
| 9 | class TestMode(TestCase): | 10 | class TestMode(TestCase): |
| @@ -135,6 +136,7 @@ class TestMode(TestCase): | |||
| 135 | ) | 136 | ) |
| 136 | 137 | ||
| 137 | 138 | ||
| 139 | + | ||
| 138 | def test_hccl_timeout(self): | 140 | def test_hccl_timeout(self): |
| 139 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_hccl_timeout.py') | 141 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_hccl_timeout.py') |
| 140 | process = subprocess.Popen(["torchrun", "--nproc-per-node=2", f"{path}"], shell=False, stdout=subprocess.PIPE, | 142 | process = subprocess.Popen(["torchrun", "--nproc-per-node=2", f"{path}"], shell=False, stdout=subprocess.PIPE, |
| @@ -153,6 +155,25 @@ class TestMode(TestCase): | |||
| 153 | message | 155 | message |
| 154 | ) | 156 | ) |
| 155 | 157 | ||
| 158 | + | ||
| 159 | + | ||
| 160 | + def test_hccl_timeout_950(self): | ||
| 161 | + path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_hccl_timeout.py') | ||
| 162 | + process = subprocess.Popen(["torchrun", "--nproc-per-node=2", f"{path}"], shell=False, stdout=subprocess.PIPE, | ||
| 163 | + stderr=subprocess.PIPE, text=True) | ||
| 164 | + message = process.stderr.read() | ||
| 165 | + process.stderr.close() | ||
| 166 | + process.stdout.close() | ||
| 167 | + process.terminate() | ||
| 168 | + process.wait() | ||
| 169 | + self.assertIn( | ||
| 170 | + "wait for compute device to finish failed", | ||
| 171 | + message | ||
| 172 | + ) | ||
| 173 | + self.assertIn( | ||
| 174 | + "task timeout", | ||
| 175 | + message | ||
| 176 | + ) | ||
| 156 | 177 | ||
| 157 | 178 | ||
| 158 | if __name__ == "__main__": | 179 | if __name__ == "__main__": |
| @@ -257,6 +257,7 @@ class TestMode(TestCase): | |||
| 257 | with self.assertRaisesRegex(ValueError, "loaded state dict has a different number of parameter groups"): | 257 | with self.assertRaisesRegex(ValueError, "loaded state dict has a different number of parameter groups"): |
| 258 | optimizer1.load_state_dict(optimizer2.state_dict()) | 258 | optimizer1.load_state_dict(optimizer2.state_dict()) |
| 259 | 259 | ||
| 260 | + | ||
| 260 | def test_aclopCompile(self): | 261 | def test_aclopCompile(self): |
| 261 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_aclopCompileAndExecute.py') | 262 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_aclopCompileAndExecute.py') |
| 262 | process = subprocess.Popen(["python", f"{path}"], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, | 263 | process = subprocess.Popen(["python", f"{path}"], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, |
| @@ -271,6 +272,7 @@ class TestMode(TestCase): | |||
| 271 | error | 272 | error |
| 272 | ) | 273 | ) |
| 273 | 274 | ||
| 275 | + | ||
| 274 | def test_ascyn(self): | 276 | def test_ascyn(self): |
| 275 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_aclopCompileAndExecute.py') | 277 | path = os.path.join(os.path.dirname(__file__), '_fault_mode_cases/error_aclopCompileAndExecute.py') |
| 276 | process = subprocess.Popen(["python", f"{path}"], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, | 278 | process = subprocess.Popen(["python", f"{path}"], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, |