已开启
1 #1
chengpeng25创建于  1月6日
chengpeng25成员
1月6日 创建

image.png

likedislike
Cchengpeng25成员
1月6日 修改了issue 的描述
chengpeng25成员
2月13日 评论:

1

likedislike
chengpeng25成员
19 天前 评论:

repro_warm_pool.py (Linux 执行)

import os, time, threading
import psutil

LOG = "/tmp/wp_trace.log"
def wlog(msg):
with open(LOG, "a", encoding="utf-8") as f:
f.write(msg + "\n")
open(LOG, "w").close()
wlog(f"[start] pid={os.getpid()} ppid={os.getppid()}")

打点 warm_pool:torch_npu 直接调它,捕获每个进程里的调用

import torch._inductor.async_compile as ac
_orig = ac.AsyncCompile.warm_pool
def traced(cls):
wlog(f"[warm_pool] pid={os.getpid()} ppid={os.getppid()}")
return _orig(cls)
ac.AsyncCompile.warm_pool = classmethod(traced)

import torch._inductor.config as c
wlog(f"[config] compile_threads={c.compile_threads} "
f"worker_start_method={c.worker_start_method} "
f"quiesce={c.quiesce_async_compile_pool}")

import torch_npu._inductor # 触发 init:7 -> warm_pool
wlog(f"[main] import done pid={os.getpid()}")

后台线程:编译期间持续采样进程树(不是编译后抓一次)

me = psutil.Process()
stop = threading.Event()
def sampler():
n = 0
while not stop.is_set():
procs = [me] + me.children(recursive=True)
wlog(f"[tree {n}] total={len(procs)} pids={[p.pid for p in procs]}")
n += 1
stop.wait(0.5)
threading.Thread(target=sampler, daemon=True).start()

真实编译:多 kernel 拉长窗口 + 禁用缓存强制重编译

import torch, torch_npu
os.environ["TORCHINDUCTOR_FORCE_DISABLE_CACHES"] = "1"

@torch.compile
def f(x):
return (x * 3).cos().relu().sum() + (x * 2).sin().sum()

x = torch.randn(512, 512).npu()
t0 = time.time()
y = f(x)
wlog(f"[main] compile+run done in {time.time()-t0:.2f}s")
time.sleep(3)
stop.set()
time.sleep(1)

汇总

pids = sorted({ln.split("pid=")[1].split()[0] for ln in open(LOG) if "[warm_pool]" in ln}, key=int)
print("=== warm_pool 被调用的 PID ===", pids)
print("去重 PID 数:", len(pids))

likedislike
chengpeng25成员
19 天前 评论:

[start] pid=1620950 ppid=110001
[config] compile_threads=32 worker_start_method=subprocess quiesce=True
[main] import done pid=1620950
[tree 0] total=1 pids=[1620950]
[tree 1] total=1 pids=[1620950]
[tree 2] total=4 pids=[1620950, 1621744, 1621746, 1621756]
[tree 3] total=4 pids=[1620950, 1621949, 1621954, 1621972]
[tree 4] total=3 pids=[1620950, 1621949, 1621954]
[tree 5] total=4 pids=[1620950, 1622211, 1622213, 1622235]
[tree 6] total=3 pids=[1620950, 1622268, 1622270]
[tree 7] total=4 pids=[1620950, 1622268, 1622270, 1622280]
[tree 8] total=4 pids=[1620950, 1622344, 1622346, 1622356]
[tree 9] total=2 pids=[1620950, 1622471]

likedislike
chengpeng25成员
19 天前 评论:

image.png

likedislike
Cchengpeng25成员
5 天前 修改了issue 的描述