已合并
[fix]multi_stream_lazy_reclaim coredump error #42424
xuyun15创建于 7月22日
[fix]multi_stream_lazy_reclaim coredump error #42424
已合并
共 2 个文件变更+66-14
| @@ -1,7 +1,8 @@ | |||
| 1 | import os | 1 | import os |
| 2 | import math | 2 | import math |
| 3 | -import shutil | 3 | +import subprocess |
| 4 | -import unittest | 4 | +import sys |
| 5 | +import textwrap | ||
| 5 | import torch | 6 | import torch |
| 6 | import torch_npu | 7 | import torch_npu |
| 7 | 8 | ||
| @@ -46,5 +47,48 @@ class TestAllocator(TestCase): | |||
| 46 | self.assertEqual(torch_npu.npu.memory_allocated(), prev + math.ceil((8 * 8 * 16 * 4 + 32) / 512) * 512) | 47 | self.assertEqual(torch_npu.npu.memory_allocated(), prev + math.ceil((8 * 8 * 16 * 4 + 32) / 512) * 512) |
| 47 | del os.environ["PYTORCH_NPU_ALLOC_CONF"] | 48 | del os.environ["PYTORCH_NPU_ALLOC_CONF"] |
| 48 | 49 | ||
| 50 | + def test_multi_stream_lazy_reclaim_trigger_event(self): | ||
| 51 | + code = textwrap.dedent("""\ | ||
| 52 | + import os | ||
| 53 | + os.environ["PYTORCH_NPU_ALLOC_CONF"] = "multi_stream_lazy_reclaim:True" | ||
| 54 | + | ||
| 55 | + import time | ||
| 56 | + import torch | ||
| 57 | + import torch_npu | ||
| 58 | + | ||
| 59 | + max_event_lazy_num = 512 | ||
| 60 | + shared_stream = torch.npu.Stream() | ||
| 61 | + x_list = [] | ||
| 62 | + for i in range(max_event_lazy_num): | ||
| 63 | + x_list.append(torch.empty(16, 16, device="npu", dtype=torch.bfloat16)) | ||
| 64 | + x_list[i].record_stream(shared_stream) | ||
| 65 | + | ||
| 66 | + x = torch.empty(16, 16, device="npu", dtype=torch.bfloat16) | ||
| 67 | + x.record_stream(shared_stream) | ||
| 68 | + | ||
| 69 | + with torch.npu.stream(shared_stream): | ||
| 70 | + y = x + 0.1 | ||
| 71 | + | ||
| 72 | + del x_list | ||
| 73 | + del x | ||
| 74 | + | ||
| 75 | + time.sleep(0.1) | ||
| 76 | + dumb = torch.empty(16, 16, device="npu", dtype=torch.bfloat16) | ||
| 77 | + del dumb | ||
| 78 | + """) | ||
| 79 | + | ||
| 80 | + result = subprocess.run( | ||
| 81 | + [sys.executable, "-c", code], | ||
| 82 | + capture_output=True, | ||
| 83 | + text=True, | ||
| 84 | + ) | ||
 | |||
| 85 | + | ||
| 86 | + assert result.returncode == 0, ( | ||
| 87 | + f"Subprocess failed with return code {result.returncode}.\\n" | ||
| 88 | + f"stdout: {result.stdout}\\n" | ||
| 89 | + f"stderr: {result.stderr}" | ||
| 90 | + ) | ||
| 91 | + | ||
| 92 | + | ||
| 49 | if __name__ == '__main__': | 93 | if __name__ == '__main__': |
| 50 | run_tests() | 94 | run_tests() |
| @@ -1213,24 +1213,32 @@ public: | |||
| 1213 | TORCH_NPU_MEMORY_LOGD("Rounded size: %zu, alloc size: %zu, using %s pool on device %d", | 1213 | TORCH_NPU_MEMORY_LOGD("Rounded size: %zu, alloc size: %zu, using %s pool on device %d", |
| 1214 | size, alloc_size, pool.is_small ? "small" : "large", device); | 1214 | size, alloc_size, pool.is_small ? "small" : "large", device); |
| 1215 | 1215 | ||
| 1216 | + const bool lazy_reclaim = | ||
| 1217 | + CachingAllocatorConfig::multi_stream_lazy_reclaim() && C10_LIKELY(captures_underway.empty()); | ||
| 1218 | + bool reaped = false; | ||
| 1219 | + if (lazy_reclaim) { | ||
| 1220 | + size_t sum = 0; | ||
| 1221 | + for (auto it = npu_events.begin(); it != npu_events.end(); ++it) { | ||
| 1222 | + sum += it->second.size(); | ||
| 1223 | + } | ||
| 1224 | + if (sum > kLazyQuerySize) { | ||
| 1225 | + process_events(context); | ||
| 1226 | + reaped = true; | ||
| 1227 | + } | ||
| 1228 | + } | ||
| 1229 | + | ||
| 1216 | // First, try to get a block from the existing pool. | 1230 | // First, try to get a block from the existing pool. |
| 1217 | bool block_found = | 1231 | bool block_found = |
| 1218 | // Search pool | 1232 | // Search pool |
| 1219 | get_free_block(params) || | 1233 | get_free_block(params) || |
| 1220 | // Trigger callbacks and retry search | 1234 | // Trigger callbacks and retry search |
| 1221 | (trigger_free_memory_callbacks(params) && get_free_block(params)); | 1235 | (trigger_free_memory_callbacks(params) && get_free_block(params)); |
| 1222 | - if (CachingAllocatorConfig::multi_stream_lazy_reclaim() && C10_LIKELY(captures_underway.empty())) { | 1236 | + |
| 1223 | - // Lazy process events and free memory | 1237 | + // Out of blocks: it is safe to reap now (no un-finalized block is held yet). |
| 1224 | - size_t sum = 0; | 1238 | + // Guard with `reaped` so we never process events twice in one malloc. |
| 1225 | - for (auto it = npu_events.begin(); it != npu_events.end(); ++it) { | 1239 | + if (lazy_reclaim && !block_found && !reaped) { |
| 1226 | - sum += it->second.size(); | 1240 | + process_events(context); |
| 1227 | - } | 1241 | + block_found = get_free_block(params); |
| 1228 | - if (!block_found || sum > kLazyQuerySize) { | ||
| 1229 | - process_events(context); | ||
| 1230 | - } | ||
| 1231 | - if (!block_found) { | ||
| 1232 | - block_found = get_free_block(params); | ||
| 1233 | - } | ||
| 1234 | } | 1242 | } |
| 1235 | // Can't reuse an existing block; try to get a new one. | 1243 | // Can't reuse an existing block; try to get a new one. |
| 1236 | if (!block_found) { | 1244 | if (!block_found) { |
🟡 Medium Priority
变更行：第 80-84 行，
subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)未设置timeout参数。受影响行为：若子进程因 NPU 异常、死锁等原因挂死，
subprocess.run将无限期阻塞，导致整个测试套件/CI 流水线挂死，无任何超时保护。失败模式：NPU 驱动异常、资源耗尽或偶发死锁时，子进程永久等待 → 父进程 hang → CI 超时或被管理员强制终止。
建议：为
subprocess.run添加timeout参数，防止子进程挂死时测试无限期阻塞。