#!/usr/bin/env python3
"""Per-interface full fuzz pipeline driver (batch mode).

Usage: python3 batch_fuzz_driver.py <aclnnApiName> [--start-stage X] [--only-stage X]

Stages (in order, idempotent via .stage_done markers):
  design -> whitebox -> sample -> normal -> noncontig -> multithread -> consistent -> error -> closure

Gate rules (AGENTS.md):
  - normal precheck 1000: success_rate >= 90% required before full 10000
  - non_contig requires normal precheck passed
  - multithread requires non_contig full passed; debug 1000 gate then full 20000
  - consistent requires normal precheck passed; predebug N*2 gates (TK 100% + hash_mismatch=0) then full N*50
  - error requires normal cpp exists
  - every run: plog archive + aicore scan + xlsx update + cpp copyback
"""
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import time

BASE = '/home/30039325/Vector_fuzz_cpp_testing'
FRAME = os.path.join(BASE, os.environ.get('FUZZ_INSTANCE', 'turing_sec'), 'aclnnAPI_fuzz')
ASCEND = '/usr/local/Ascend/ascend-toolkit/latest/x86_64-linux'
OP_INDEX = '/tmp/opencode/op_index.json'
ENV = dict(os.environ)
ENV['ASCEND_HOME_PATH'] = ASCEND
ENV['LD_LIBRARY_PATH'] = ASCEND + '/lib64:' + ENV.get('LD_LIBRARY_PATH', '')
ENV['ASCEND_LOG_PATH'] = '/root/ascend/log/debug/plog'
ENV['ASCEND_OPP_PATH'] = '/usr/local/Ascend/ascend-toolkit/latest/opp'
if os.environ.get('FUZZ_CARD') is not None:
    ENV['ASCEND_RT_VISIBLE_DEVICES'] = os.environ['FUZZ_CARD']

S = '/'  # shorthand


def run(cmd, cwd=BASE, timeout=600, env=None, log_file=None, check=True):
    """Run command, return (rc, output)."""
    e = env or ENV
    if log_file:
        with open(log_file, 'a') as lf:
            p = subprocess.run(cmd, shell=True, cwd=cwd, env=e, timeout=timeout,
                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            lf.write(p.stdout.decode(errors='ignore'))
        return p.returncode, p.stdout.decode(errors='ignore')
    p = subprocess.run(cmd, shell=True, cwd=cwd, env=e, timeout=timeout,
                       stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    return p.returncode, p.stdout.decode(errors='ignore')


def count_success(logfile):
    """Count FUZZ_RESULT success/skip/faild lines."""
    n_succ = n_skip = n_fail = 0
    rets = {}
    if not os.path.exists(logfile):
        return 0, 0, 0, {}
    for line in open(logfile, encoding='utf-8', errors='ignore'):
        m = re.search(r'\[FUZZ_RESULT\] case_\d+: (\w+), ret=(-?\d+)', line)
        if not m:
            continue
        tag, ret = m.group(1), m.group(2)
        if tag == 'success':
            n_succ += 1
        elif tag == 'skip':
            n_skip += 1
        else:
            n_fail += 1
        rets[ret] = rets.get(ret, 0) + 1
    return n_succ, n_skip, n_fail, rets


def scan_aicore(plogfile):
    """Return (has_aicore, detail) per updated rule: 'the error is aivec error' OR kernel_name+ERROR line."""
    if not os.path.exists(plogfile) or os.path.getsize(plogfile) == 0:
        return False, ''
    hits = []
    for line in open(plogfile, encoding='utf-8', errors='ignore'):
        low = line.lower()
        if 'the error is aivec error' in low or 'there is an aivec error exception' in low or 'there is an aicore error exception' in low:
            hits.append(line.strip()[:200])
        elif 'kernel_name' in low and '[error]' in low:
            hits.append(line.strip()[:200])
    if hits:
        return True, ' | '.join(hits[:3])
    return False, ''


def classify_bug(success, skip, fail, rets, aicore, plogfile):
    """Simplified three-state verdict per AGENTS.md rules."""
    total = success + skip + fail
    if total == 0:
        return '需要人工确认', '无执行用例'
    if fail + skip == 0 and not aicore:
        return '不是bug', '全部用例执行通过'
    if aicore:
        return '需要人工确认', '存在aicore_error，待人工反向CPP检查'
    # classify by ret distribution
    all_env = all(r in ('561103', '561112', '361001') for r in rets if rets and r != '-1')
    # environment errors dominate
    env_cnt = sum(c for r, c in rets.items() if r in ('561103', '561112', '361001'))
    constraint_cnt = sum(c for r, c in rets.items() if r in ('161001', '161002'))
    if env_cnt > 0 and env_cnt >= (fail + skip) * 0.8:
        return '不是bug', '环境问题(kernel加载失败/设备不支持)'
    if constraint_cnt > 0 and constraint_cnt >= (fail + skip) * 0.8:
        return '不是bug', '用例违反接口约束按资料预期报错'
    # check plog for OOM
    oom_kw = False
    if os.path.exists(plogfile):
        head = open(plogfile, encoding='utf-8', errors='ignore').read(2000000)
        if re.search(r'out of memory|oom|No enough memory', head, re.I):
            oom_kw = True
    if oom_kw:
        return '不是bug', '用例shape过大OOM资源不足'
    return '需要人工确认', '失败用例需人工根因分析(ret分布:%s)' % json.dumps(rets)[:120]


def update_xlsx(api, scenario, total, success, fail, reason, bug, aicore, card=None, analyzed='是'):
    card = card if card is not None else os.environ.get('FUZZ_CARD', '0')
    cmd = (f'python3 {BASE}/scripts/update_batch_results.py --api {api} --scenario {scenario} '
           f'--total {total} --success {success} --fail {fail} --fail_reason "{reason[:180]}" '
           f'--has_bug "{bug}" --has_aicore "{aicore}" --card {card} --analyzed {analyzed}')
    rc, out = run(cmd, timeout=120)
    return rc


class Driver:
    def __init__(self, api, args):
        self.api = api
        self.args = args
        self.out = os.path.join(BASE, 'output', api)
        self.td = os.path.join(self.out, 'test-design')
        self.logs = os.path.join(self.out, 'fuzz_logs')
        os.makedirs(self.logs, exist_ok=True)
        os.makedirs(self.td, exist_ok=True)
        self.mark = os.path.join(self.td, '.stage_done')
        self.idx = json.load(open(OP_INDEX))[api] if os.path.exists(OP_INDEX) and api in json.load(open(OP_INDEX)) else {}
        self.md = self.idx.get('md') or ''
        self.header = self.idx.get('header') or ''
        self.precheck_passed = False

    def done(self, stage):
        return os.path.exists(os.path.join(self.mark, stage))

    def set_done(self, stage):
        os.makedirs(self.mark, exist_ok=True)
        open(os.path.join(self.mark, stage), 'w').write(time.strftime('%F %T'))

    def log(self, msg):
        line = '[%s] %s %s' % (time.strftime('%H:%M:%S'), self.api, msg)
        print(line, flush=True)
        with open(os.path.join(BASE, 'out_res', 'batch_driver.log'), 'a') as f:
            f.write(line + '\n')

    # ---------------- stages ----------------

    def stage_design(self):
        d = self.td
        if os.path.exists(os.path.join(d, '03_参数定义.yaml')):
            self.log('[DESIGN_REUSE] 03 已存在（含人工修正），跳过重新生成，仅刷新 05')
            run(f'python3 {BASE}/scripts/auto_gen_constraint_yaml.py --param_yaml {d}/03_参数定义.yaml --output {d}/05_约束定义.yaml', timeout=300)
            return True
        rc, o = run(f'python3 {BASE}/scripts/auto_gen_param_yaml.py {self.api}', timeout=300)
        if rc != 0:
            self.log('[DESIGN_FAIL] param yaml: ' + o[-200:])
            return False
        rc, o = run(f'python3 {BASE}/scripts/auto_gen_constraint_yaml.py --param_yaml {d}/03_参数定义.yaml --output {d}/05_约束定义.yaml', timeout=300)
        if rc != 0:
            self.log('[DESIGN_FAIL] constraint yaml: ' + o[-200:])
            return False
        run(f'python3 {BASE}/skills/ascendc-st-design/scripts/generate_test_factors.py {d}/03_参数定义.yaml {d}/04_测试因子.yaml --quiet', timeout=300)
        run(f'python3 {BASE}/skills/ascendc-st-design/scripts/generate_implicit_constraints.py {d}/04_测试因子.yaml {d}/05_约束定义.yaml', timeout=300)
        run(f'python3 {BASE}/skills/ascendc-st-design/scripts/generate_solver_config.py {d}/05_约束定义.yaml {d}/06_求解配置.yaml', timeout=300)
        self.log('[DESIGN_OK]')
        return True

    def stage_whitebox(self):
        rc, o = run(f'python3 {BASE}/skills/whitebox-attr-check/scripts/whitebox_attr_check.py --api {self.api} '
                    f'--base_root {BASE} --out {self.td}/whitebox_report.md --json {self.td}/whitebox_report.json',
                    timeout=600, log_file=os.path.join(self.logs, f'{self.api}_whitebox.log'))
        if rc == 3:
            self.log('[WHITEBOX_SKIP] 不可检(md/源码缺失)')
        elif rc == 2:
            self.log('[WHITEBOX_FINDINGS] 存在白盒发现，待复核')
        else:
            self.log('[WHITEBOX_OK] 无发现')
        return True  # non-blocking in all cases

    def stage_sample(self):
        outlog = os.path.join(self.logs, f'{self.api}_sample_output.log')
        if os.path.exists(outlog):
            self.log('[SAMPLE_SKIP] 已存在')
            return True
        lst = f'/tmp/opencode/sample_list_{self.api}.txt'
        open(lst, 'w').write(self.api + '\n')
        repos = ' '.join(os.path.join(BASE, r) for r in ('ops-math', 'ops-nn', 'ops-cv', 'ops-transformer')
                         if os.path.isdir(os.path.join(BASE, r)))
        rc, o = run(f'python3 extract_md_samples.py --list {lst} --out src/op_sample --repos {repos} --soc 950',
                    cwd=FRAME, timeout=600, log_file=os.path.join(self.logs, f'{self.api}_sample_extract.log'))
        sample_cpp = os.path.join(FRAME, 'src/op_sample', f'{self.api}.cpp')
        if not os.path.exists(sample_cpp):
            self.log('[SAMPLE_MISSING] md无可用调用示例或950不支持，跳过样例测试')
            return True
        run(f'python3 apply_sample_fixes.py src/op_sample/{self.api}.cpp', cwd=FRAME, timeout=120)
        shutil.copy(sample_cpp, os.path.join(self.out, f'{self.api}_sample.cpp'))
        # backup op_sample dir list to allow partial-failure recovery
        rc, o = run('make sample_fuzz', cwd=FRAME, timeout=1200,
                    log_file=os.path.join(self.logs, f'{self.api}_sample_make.log'))
        if rc != 0:
            # remove broken samples of other apis (up to 5 recovery rounds)
            for _round in range(5):
                broken = re.findall(r'(?:\./)?src/op_sample/(\w+)\.cpp', o)
                moved = False
                for b in set(broken):
                    if b != self.api:
                        p = os.path.join(FRAME, 'src/op_sample', b + '.cpp')
                        if os.path.exists(p):
                            shutil.move(p, p + '.broken')
                            moved = True
                if not moved:
                    break
                rc, o = run('make sample_fuzz', cwd=FRAME, timeout=1800,
                            log_file=os.path.join(self.logs, f'{self.api}_sample_make.log'))
                if rc == 0:
                    break
        if rc != 0:
            self.log('[SAMPLE_COMPILE_FAIL] 样例编译失败(是bug官方样例失败，记录待提单)')
            open(outlog, 'w').write('COMPILE_FAIL\n')
            update_xlsx(self.api, 'sample', 1, 0, 1, '官方样例编译失败', '是bug(官方样例失败)', '否')
            return True
        exe = os.path.join(FRAME, 'out', 'aclnnAPI_sample_fuzz')
        proc = subprocess.Popen([exe, self.api], cwd=os.path.join(FRAME, 'out'), env=ENV,
                                stdout=open(outlog, 'w'), stderr=subprocess.STDOUT)
        try:
            proc.wait(timeout=900)
        except subprocess.TimeoutExpired:
            proc.kill()
            proc.wait()
        pid = proc.pid
        run(f'bash {BASE}/scripts/archive_plog.sh {pid} {self.logs}/{self.api}_sample_plog.log', timeout=300)
        aic, det = scan_aicore(os.path.join(self.logs, f'{self.api}_sample_plog.log'))
        ret = proc.returncode
        if ret == 0 and not aic:
            verdict, reason = '不是bug', '样例通过'
            self.log('[SAMPLE_PASS]')
        else:
            verdict = '是bug(官方样例失败)' if ret != 0 else '是bug(样例aicore)'
            reason = f'样例执行ret={ret} aicore={aic} {det[:100]}'
            self.log(f'[SAMPLE_FAIL] ret={ret} aicore={aic}')
        update_xlsx(self.api, 'sample', 1, 0 if ret != 0 else 1, 0 if ret == 0 else 1, reason, verdict, '是' if aic else '否')
        return True

    def _gen_normal_cpp(self):
        cpp = os.path.join(self.out, f'{self.api}.cpp')
        if os.path.exists(cpp):
            # 已有 CPP（待验证版本或人工修复版本），直接使用，不再重新生成
            self.log('[NORMAL_CPP_REUSE] output 下已有 CPP，跳过重新生成')
            return True
        d = self.td
        rc, o = run(f'python3 {BASE}/skills/ascendc-fuzz-cpp-gen/scripts/st_to_fuzz_config.py '
                    f'--param_def {d}/03_参数定义.yaml --constraint_def {d}/05_约束定义.yaml '
                    f'--op_name {self.api} --output {d}/fuzz_op_config.yaml --api_doc "{self.md}"',
                    timeout=300)
        if rc != 0:
            self.log('[NORMAL_FAIL] st_to_fuzz_config: ' + o[-200:])
            return False
        rc, o = run(f'python3 {BASE}/skills/ascendc-fuzz-cpp-gen/scripts/gen_fuzz_cpp.py '
                    f'--op_name {self.api} --st_config {d}/fuzz_op_config.yaml --api_doc "{self.md}" '
                    f'--output {self.out}', timeout=300)
        if rc != 0:
            self.log('[NORMAL_FAIL] gen_fuzz_cpp: ' + o[-200:])
            return False
        cpp = os.path.join(self.out, f'{self.api}.cpp')
        # fix include if header known
        if self.header:
            snake = 'aclnn_' + re.sub(r'(?<!^)(?=[A-Z])', '_', self.api[5:]).lower()
            wrong = f'#include "aclnnop/{snake}.h"'
            right = f'#include "aclnnop/{self.header}"'
            if wrong != right and wrong in open(cpp, encoding='utf-8', errors='ignore').read():
                run(f"sed -i 's|{wrong}|{right}|' {cpp}")
        # per-API patch hook (survives regeneration)
        patch = os.path.join(BASE, 'scripts', 'patches', f'{self.api}.py')
        if os.path.exists(patch):
            rc, o = run(f'python3 {patch}', timeout=300)
            self.log('[NORMAL_CPP_PATCH] rc=%d %s' % (rc, o.strip()[-100:]))
        return True

    def _compile_op(self, cpp, subdir='op', target=''):
        """Clean op dir, copy cpp, make. Returns (rc, errmsg)."""
        opdir = os.path.join(FRAME, 'src', subdir)
        run(f'rm -rf {opdir} && mkdir -p {opdir}')
        shutil.copy(cpp, opdir)
        cmd = 'make op_fuzz' + (f' TARGET={target}' if target else '')
        rc, o = run(cmd, cwd=FRAME, timeout=1800, log_file=f'/tmp/opencode/make_{self.api}_{target or "op"}.log')
        return rc, o[-300:] if rc else ''

    def _run_fuzz_capture(self, exe, args, outlog, timeout):
        """Run fuzz binary, capture PID for plog, archive plog to logs. Returns (rc, pid)."""
        cwd = os.path.join(FRAME, 'out')
        with open(outlog, 'w') as lf:
            proc = subprocess.Popen([exe] + args, cwd=cwd, env=ENV, stdout=lf, stderr=subprocess.STDOUT)
            try:
                proc.wait(timeout=timeout)
            except subprocess.TimeoutExpired:
                proc.kill()
                proc.wait()
                lf.write('\n[DRIVER_TIMEOUT]\n')
        return proc.returncode, proc.pid

    def stage_normal(self):
        outlog = os.path.join(self.logs, f'{self.api}_fuzz_output.log')
        prelog = os.path.join(self.logs, f'{self.api}_fuzz_precheck.log')
        if os.path.exists(outlog):
            succ, skip, fail, rets = count_success(outlog)
            self.precheck_passed = True
            self.log(f'[NORMAL_SKIP] 已有全量日志 succ={succ}')
            return True
        if not self._gen_normal_cpp():
            return False
        cpp = os.path.join(self.out, f'{self.api}.cpp')
        rc, err = self._compile_op(cpp)
        if rc != 0:
            self.log('[NORMAL_FAIL] compile: ' + err)
            return False
        exe = os.path.join(FRAME, 'out', 'aclnnAPI_fuzz')
        # precheck 1000
        rc, pid = self._run_fuzz_capture(exe, [f'{self.api}_fuzz', '1000'], prelog, 1800)
        run(f'bash {BASE}/scripts/archive_plog.sh {pid} {self.logs}/{self.api}_fuzz_plog.log', timeout=300)
        succ, skip, fail, rets = count_success(prelog)
        executed = succ + skip + fail
        # DT 变异空间自然穷尽的运行（正常结束但实际条数<请求数）按实际执行条数为分母
        precheck_completed = os.path.exists(prelog) and ('program exit for run complete' in open(prelog, encoding='utf-8', errors='ignore').read()[-3000:] or executed > 0)
        denom = executed if (executed > 0 and precheck_completed and executed < 1000) else 1000
        rate = succ / denom if os.path.exists(prelog) else 0
        aic, det = scan_aicore(os.path.join(self.logs, f'{self.api}_fuzz_plog.log'))
        self.log(f'[NORMAL_PRECHECK] succ={succ} skip={skip} fail={fail} executed={executed}/{1000} rate={rate:.1%} aicore={aic}')
        bug, reason = classify_bug(succ, skip, fail, rets, aic, os.path.join(self.logs, f'{self.api}_fuzz_plog.log'))
        update_xlsx(self.api, 'normal_precheck', 1000, succ, 1000 - succ,
                    f'预检: {reason}', bug, '是' if aic else '否')
        if rate < 0.9 or not os.path.exists(prelog):
            self.log('[NORMAL_PRECHECK_FAIL] <90%，进入待人工调试清单，跳过全量')
            return False
        self.precheck_passed = True
        # full 10000
        rc, pid = self._run_fuzz_capture(exe, [f'{self.api}_fuzz', '10000'], outlog, 10800)  # 3h: 大 shape 用例防超时截断
        run(f'bash {BASE}/scripts/archive_plog.sh {pid} {self.logs}/{self.api}_fuzz_plog.log', timeout=300)
        succ, skip, fail, rets = count_success(outlog)
        executed = succ + skip + fail
        denom = executed if (executed > 0 and executed < 10000) else 10000
        aic, det = scan_aicore(os.path.join(self.logs, f'{self.api}_fuzz_plog.log'))
        rate = succ / denom
        bug, reason = classify_bug(succ, skip, fail, rets, aic, os.path.join(self.logs, f'{self.api}_fuzz_plog.log'))
        self.log(f'[NORMAL_FULL] succ={succ} rate={rate:.1%} aicore={aic} bug={bug}')
        update_xlsx(self.api, 'normal', 10000, succ, 10000 - succ, reason, bug, '是' if aic else '否')
        # report
        run(f'python3 {BASE}/skills/ascendc-fuzz-log-parser/scripts/parse_fuzz_log.py --log {outlog} '
            f'--op {self.api} --mode normal --output_json {self.logs}/{self.api}_fuzz_analysis.json '
            f'--output_report {self.logs}/{self.api}_fuzz_test_report.md', timeout=300)
        if rate >= 0.9:
            # copyback verified cpp
            shutil.copy(os.path.join(FRAME, 'src/op', f'{self.api}.cpp'),
                        os.path.join(self.out, f'{self.api}.cpp'))
            return True
        self.log('[NORMAL_FULL_FAIL] <90%，全量门禁未过，进入待人工调试清单（后续场景由 ABORT 路径处理 error）')
        return False

    def _precheck_passed(self):
        """从已有预检日志恢复判定（跨进程 resume）"""
        prelog = os.path.join(self.logs, f'{self.api}_fuzz_precheck.log')
        if os.path.exists(prelog):
            succ, skip, fail, rets = count_success(prelog)
            if (succ + skip + fail) > 0 and succ / (succ + skip + fail) >= 0.9:
                return True
        return False

    def stage_noncontig(self):
        outlog = os.path.join(self.logs, f'{self.api}_non_contig_fuzz_output.log')
        if os.path.exists(outlog):
            self.log('[NC_SKIP] 已有全量日志')
            return True
        if not self.precheck_passed and not self._precheck_passed():
            self.log('[NC_SKIP] normal 预检未通过，不满足前置条件')
            return False
        src = os.path.join(self.out, f'{self.api}.cpp')
        dst = os.path.join(self.out, f'{self.api}_non_contig.cpp')
        rc, o = run(f'python3 {BASE}/skills/ascendc-fuzz-cpp-gen/scripts/gen_non_contig_from_normal.py '
                    f'{src} {dst} {self.api} --api_doc "{self.md}"', timeout=600, log_file=f'/tmp/opencode/ncgen_{self.api}.log')
        if rc != 0 or not os.path.exists(dst):
            # fallback: gen-non-contig-from-normal skill script
            rc, o = run(f'python3 {BASE}/skills/gen-non-contig-from-normal/scripts/gen_non_contig_from_normal.py '
                        f'{src} {dst} {self.api}', timeout=600, log_file=f'/tmp/opencode/ncgen_{self.api}.log')
            if rc != 0 or not os.path.exists(dst):
                self.log('[NC_FAIL] 生成失败: ' + o[-200:])
                return False
        # fix include
        if self.header:
            txt = open(dst, encoding='utf-8', errors='ignore').read()
            snake = 'aclnn_' + re.sub(r'(?<!^)(?=[A-Z])', '_', self.api[5:]).lower()
            wrong = f'#include "aclnnop/{snake}.h"'
            right = f'#include "aclnnop/{self.header}"'
            if wrong != right and wrong in txt:
                run(f"sed -i 's|{wrong}|{right}|' {dst}")
        rc, err = self._compile_op(dst)
        if rc != 0:
            self.log('[NC_FAIL] compile: ' + err)
            return False
        exe = os.path.join(FRAME, 'out', 'aclnnAPI_fuzz')
        rc, pid = self._run_fuzz_capture(exe, [f'{self.api}_non_contig_fuzz', '10000'], outlog, 10800)
        run(f'bash {BASE}/scripts/archive_plog.sh {pid} {self.logs}/{self.api}_non_contig_fuzz_plog.log', timeout=300)
        succ, skip, fail, rets = count_success(outlog)
        aic, det = scan_aicore(os.path.join(self.logs, f'{self.api}_non_contig_fuzz_plog.log'))
        bug, reason = classify_bug(succ, skip, fail, rets, aic, os.path.join(self.logs, f'{self.api}_non_contig_fuzz_plog.log'))
        rate = succ / 10000.0
        self.log(f'[NC_FULL] succ={succ} rate={rate:.1%} aicore={aic} bug={bug}')
        update_xlsx(self.api, 'non_contig', 10000, succ, 10000 - succ, reason, bug, '是' if aic else '否')
        run(f'python3 {BASE}/skills/ascendc-fuzz-log-parser/scripts/parse_fuzz_log.py --log {outlog} '
            f'--op {self.api} --mode non_contig --output_json {self.logs}/{self.api}_non_contig_fuzz_analysis.json '
            f'--output_report {self.logs}/{self.api}_non_contig_fuzz_test_report.md', timeout=300)
        if rate >= 0.9:
            shutil.copy(os.path.join(FRAME, 'src/op', f'{self.api}_non_contig.cpp'), dst)
        return True

    def stage_multithread(self):
        outlog = os.path.join(self.logs, f'{self.api}_multithread_fuzz_output.log')
        if os.path.exists(outlog):
            self.log('[MT_SKIP] 已有全量日志')
            return True
        nc = os.path.join(self.out, f'{self.api}_non_contig.cpp')
        base_for_mt = nc
        if not os.path.exists(nc):
            # 非连续 CPP 不存在：若 normal 存在则从 normal 转换（适用于 md 全×非连续 tensorlist 等场景，scene 全 CONTINUOUS）
            normal = os.path.join(self.out, f'{self.api}.cpp')
            if not os.path.exists(normal):
                self.log('[MT_SKIP] 非连续与 normal CPP 均不存在，前置不满足')
                return False
            base_for_mt = normal
            self.log('[MT_FROM_NORMAL] nc CPP 不存在，从 normal CPP 转换（连续场景）')
        dst = os.path.join(self.out, f'{self.api}_multithread.cpp')
        # 输出tensor名（gate 1.6：输出scene禁止BROADCAST）——从 fuzz config 读取 outputs
        out_names = []
        try:
            import yaml as _yaml
            _cfg = _yaml.safe_load(open(os.path.join(self.td, 'fuzz_op_config.yaml'), encoding='utf-8'))
            out_names = list((_cfg.get(self.api, {}).get('outputs') or {}).keys())
        except Exception:
            pass
        rc, o = run(f'python3 {BASE}/scripts/convert_to_multithread.py {base_for_mt} {dst} {self.api} --outputs {",".join(out_names)}', timeout=120)
        if rc != 0:
            self.log('[MT_FAIL] convert: ' + o[-200:])
            return False
        mtdir = os.path.join(FRAME, 'src/op_multithread')
        run(f'rm -rf {mtdir} && mkdir -p {mtdir}')
        shutil.copy(dst, os.path.join(mtdir, f'{self.api}.cpp'))
        rc, o = run('make op_fuzz TARGET=multithread', cwd=FRAME, timeout=1800,
                    log_file=f'/tmp/opencode/make_{self.api}_mt.log')
        if rc != 0:
            self.log('[MT_FAIL] compile: ' + o[-300:])
            return False
        exe = os.path.join(FRAME, 'out', 'aclnnAPI_fuzz_multithread')
        # debug 1000 (5 threads)
        prelog = os.path.join(self.logs, f'{self.api}_multithread_fuzz_precheck.log')
        rc, pid = self._run_fuzz_capture(exe, ['5', '1000', '0', '5', '0'], prelog, 1800)  # ASCEND_RT_VISIBLE_DEVICES 已重编号，逻辑设备恒为 0
        run(f'bash {BASE}/scripts/archive_plog.sh {pid} {self.logs}/{self.api}_multithread_fuzz_plog.log', timeout=300)
        succ, skip, fail, rets = count_success(prelog)
        aic, det = scan_aicore(os.path.join(self.logs, f'{self.api}_multithread_fuzz_plog.log'))
        plan = 5000  # 5 threads x 1000
        rate = succ / plan
        bug, reason = classify_bug(succ, skip, fail, rets, aic, os.path.join(self.logs, f'{self.api}_multithread_fuzz_plog.log'))
        self.log(f'[MT_DEBUG] succ={succ}/{plan} rate={rate:.1%} aicore={aic}')
        update_xlsx(self.api, 'multithread_precheck', plan, succ, plan - succ, f'调试: {reason}', bug, '是' if aic else '否')
        if rate < 0.9:
            self.log('[MT_DEBUG_FAIL] <90%，跳过全量，待人工调试')
            return False
        # full 20000 per thread
        rc, pid = self._run_fuzz_capture(exe, ['5', '20000', '0', '5', '0'], outlog, 3600)  # ASCEND_RT_VISIBLE_DEVICES 已重编号，逻辑设备恒为 0
        run(f'bash {BASE}/scripts/archive_plog.sh {pid} {self.logs}/{self.api}_multithread_fuzz_plog.log', timeout=300)
        succ, skip, fail, rets = count_success(outlog)
        plan = 100000
        aic, det = scan_aicore(os.path.join(self.logs, f'{self.api}_multithread_fuzz_plog.log'))
        bug, reason = classify_bug(succ, skip, fail, rets, aic, os.path.join(self.logs, f'{self.api}_multithread_fuzz_plog.log'))
        rate = succ / plan
        self.log(f'[MT_FULL] succ={succ}/{plan} rate={rate:.1%} aicore={aic} bug={bug}')
        update_xlsx(self.api, 'multithread', plan, succ, plan - succ, reason, bug, '是' if aic else '否')
        run(f'python3 {BASE}/skills/ascendc-fuzz-log-parser/scripts/parse_fuzz_log.py --log {outlog} '
            f'--op {self.api} --mode normal --output_json {self.logs}/{self.api}_multithread_fuzz_analysis.json '
            f'--output_report {self.logs}/{self.api}_multithread_fuzz_test_report.md', timeout=300)
        if rate >= 0.9:
            shutil.copy(os.path.join(mtdir, f'{self.api}.cpp'), dst)
        return True

    def stage_consistent(self):
        outlog = os.path.join(self.logs, f'{self.api}_consistent_fuzz_output.log')
        if os.path.exists(outlog):
            self.log('[CONS_SKIP] 已有全量日志')
            return True
        if not self.precheck_passed and not self._precheck_passed():
            self.log('[CONS_SKIP] normal 预检未通过，前置不满足')
            return False
        d = self.td
        run(f'python3 {BASE}/skills/gen-consistent-from-normal/scripts/parse_md_deterministic.py '
            f'--api {self.api} --output {d}/deterministic_check.json', timeout=300)
        rc, o = run(f'python3 {BASE}/skills/ascendc-fuzz-cpp-gen/scripts/gen_tiling_key_coverage.py '
                    f'--op_name {self.api} --param_yaml {d}/03_参数定义.yaml --output {d}/08_tiling_key_coverage.yaml',
                    timeout=600, log_file=f'/tmp/opencode/tk_{self.api}.log')
        tk_n = 0
        if os.path.exists(os.path.join(d, '08_tiling_key_coverage.yaml')):
            try:
                tky = open(os.path.join(d, '08_tiling_key_coverage.yaml'), encoding='utf-8').read()
                tk_n = len(re.findall(r'^\s*-\s*tk_id:', tky, re.M))
            except Exception:
                tk_n = 0
        need_det = 0
        dj = os.path.join(d, 'deterministic_check.json')
        if os.path.exists(dj):
            try:
                need_det = int(json.load(open(dj)).get('need_deterministic', 0) or 0)
            except Exception:
                need_det = 0
        src = os.path.join(self.out, f'{self.api}.cpp')
        dst = os.path.join(self.out, f'{self.api}_consistent.cpp')
        rc, o = run(f'python3 {BASE}/skills/gen-consistent-from-normal/scripts/gen_consistent_from_normal.py '
                    f'--input {src} --output {dst} --api {self.api} '
                    f'--deterministic_json {dj if os.path.exists(dj) else ""}',
                    timeout=600, log_file=f'/tmp/opencode/consgen_{self.api}.log')
        if not os.path.exists(dst):
            self.log('[CONS_FAIL] 生成失败: ' + o[-200:])
            return False
        rc, o = run(f'python3 {BASE}/skills/gen-consistent-from-normal/scripts/verify_consistent.py '
                    f'--cpp {dst} --api {self.api} --need_deterministic {need_det}',
                    timeout=300, log_file=os.path.join(self.logs, f'{self.api}_consistent_verify.log'))
        if rc != 0:
            self.log('[CONS_VERIFY_FAIL] V1~V11 门禁未过: ' + o[-200:])
            return False
        consdir = os.path.join(FRAME, 'src/src_consistent')
        run(f'rm -rf {consdir} && mkdir -p {consdir}')
        shutil.copy(dst, consdir)
        rc, o = run('make op_fuzz TARGET=consistent', cwd=FRAME, timeout=1800,
                    log_file=f'/tmp/opencode/make_{self.api}_cons.log')
        if rc != 0:
            self.log('[CONS_FAIL] compile: ' + o[-300:])
            return False
        exe = os.path.join(FRAME, 'out', 'aclnnAPI_fuzz_consistent')
        n_pre = (tk_n * 2) if tk_n else 100
        n_full = (tk_n * 50) if tk_n else 1000
        ENV['ASCEND_OPP_PATH'] = '/usr/local/Ascend/ascend-toolkit/latest/opp'
        prelog = os.path.join(self.logs, f'{self.api}_consistent_precheck.log')
        rc, pid = self._run_fuzz_capture(exe, [f'{self.api}_consistent_fuzz', str(n_pre)], prelog, 1800)
        run(f'bash {BASE}/scripts/archive_plog.sh {pid} {self.logs}/{self.api}_consistent_fuzz_plog.log', timeout=300)
        rc_tk, o_tk = run(f'python3 {BASE}/skills/ascendc-fuzz-cpp-gen/scripts/check_tiling_precheck.py '
                          f'--log {prelog} --tiling_yaml {d}/08_tiling_key_coverage.yaml', timeout=300)
        rc_h, o_h = run(f'python3 {BASE}/skills/gen-consistent-from-normal/scripts/parse_consistent_log.py '
                        f'--log {prelog} --api {self.api}', timeout=300)
        hash_mismatch = 0
        try:
            aj = os.path.join(self.logs, f'{self.api}_consistent_fuzz_analysis.json')
            if os.path.exists(aj):
                hash_mismatch = json.load(open(aj)).get('hash_mismatch', 0)
        except Exception:
            pass
        aic, det = scan_aicore(os.path.join(self.logs, f'{self.api}_consistent_fuzz_plog.log'))
        self.log(f'[CONS_PREDEBUG] tk_gate_rc={rc_tk} hash_mismatch={hash_mismatch} aicore={aic}')
        if rc_tk != 0 or hash_mismatch > 0:
            bug, reason = ('是bug(非确定性实现hash_mismatch提单)', f'hash_mismatch={hash_mismatch}') if hash_mismatch > 0 else ('需要人工确认', 'TK覆盖未达100%: ' + o_tk[-150:])
            update_xlsx(self.api, 'consistent_precheck', n_pre, 0, n_pre, reason, bug, '是' if aic else '否')
            self.log('[CONS_PREDEBUG_FAIL] 预调试门禁未过')
            if hash_mismatch > 0:
                shutil.copy(os.path.join(consdir, f'{self.api}_consistent.cpp'), dst)
                return True  # bug found - scenario complete
            return False
        rc, pid = self._run_fuzz_capture(exe, [f'{self.api}_consistent_fuzz', str(n_full)], outlog, 3600)
        run(f'bash {BASE}/scripts/archive_plog.sh {pid} {self.logs}/{self.api}_consistent_fuzz_plog.log', timeout=300)
        rc_h, o_h = run(f'python3 {BASE}/skills/gen-consistent-from-normal/scripts/parse_consistent_log.py '
                        f'--log {outlog} --api {self.api} '
                        f'--plog {self.logs}/{self.api}_consistent_fuzz_plog.log --run_num 2', timeout=300)
        hash_mismatch = 0
        try:
            aj = os.path.join(self.logs, f'{self.api}_consistent_fuzz_analysis.json')
            if os.path.exists(aj):
                hash_mismatch = json.load(open(aj)).get('hash_mismatch', 0)
        except Exception:
            pass
        aic, det = scan_aicore(os.path.join(self.logs, f'{self.api}_consistent_fuzz_plog.log'))
        bug = '是bug(非确定性实现hash_mismatch提单)' if hash_mismatch > 0 else ('需要人工确认' if aic else '不是bug')
        reason = f'hash_mismatch={hash_mismatch}' + (f' aicore={det[:80]}' if aic else '')
        self.log(f'[CONS_FULL] hash_mismatch={hash_mismatch} aicore={aic}')
        update_xlsx(self.api, 'consistent', n_full, n_full - hash_mismatch, hash_mismatch, reason, bug, '是' if aic else '否')
        shutil.copy(os.path.join(consdir, f'{self.api}_consistent.cpp'), dst)
        return True

    def stage_error(self):
        outlog = os.path.join(self.logs, f'{self.api}_error_fuzz_output.log')
        if os.path.exists(outlog):
            self.log('[ERR_SKIP] 已有日志')
            return True
        if not os.path.exists(os.path.join(self.out, f'{self.api}.cpp')):
            self.log('[ERR_SKIP] normal CPP 不存在')
            return False
        rc, o = run(f'bash {BASE}/scripts/run_error_pipeline.sh {self.api} 60',
                    timeout=3600, log_file=f'/tmp/opencode/errpipe_{self.api}.log')
        if not os.path.exists(outlog):
            self.log('[ERR_FAIL] error pipeline: ' + o[-200:])
            return False
        succ, skip, fail, rets = count_success(outlog)
        aic, det = scan_aicore(os.path.join(self.logs, f'{self.api}_error_fuzz_plog.log'))
        rc_a, o_a = run(f'python3 {BASE}/scripts/analyze_bug_verdict.py --log {outlog} '
                        f'--plog {self.logs}/{self.api}_error_fuzz_plog.log --scenario error '
                        f'--json {self.logs}/{self.api}_error_fuzz_analysis.json', timeout=600)
        # detect crashes (FUZZ_ERROR_POINT without following FUZZ_RESULT) and unexpected success
        crash, unexpected = 0, 0
        try:
            pj = os.path.join(self.logs, f'{self.api}_error_problems.json')
            if os.path.exists(pj):
                pd = json.load(open(pj))
                crash = len(pd.get('crashes', pd.get('core_cases', [])))
                unexpected = len(pd.get('unexpected_success', []))
        except Exception:
            pass
        total_cases = succ + skip + fail
        bug = '是bug(error场景约束未生效/意外成功)' if unexpected > 0 else ('需要人工确认' if crash > 0 else '不是bug')
        reason = f'意外成功={unexpected} 进程core={crash} ret分布:{json.dumps(rets)[:100]}'
        self.log(f'[ERR_DONE] succ={succ} skip={skip} fail={fail} unexpected={unexpected} crash={crash} aicore={aic}')
        update_xlsx(self.api, 'error', total_cases, succ, total_cases - succ, reason, bug, '是' if aic else '否')
        if os.path.exists(os.path.join(FRAME, 'src/op', f'{self.api}_error.cpp')):
            shutil.copy(os.path.join(FRAME, 'src/op', f'{self.api}_error.cpp'),
                        os.path.join(self.out, f'{self.api}_error.cpp'))
        return True

    def stage_closure(self):
        # total report
        rep = os.path.join(self.logs, f'{self.api}_fuzz_total_report.md')
        lines = [f'# {self.api} 批量压测总报告\n', f'- 生成时间: {time.strftime("%F %T")}\n', '\n## 场景结果\n']
        for scen, logf in [('sample', f'{self.api}_sample_output.log'), ('normal', f'{self.api}_fuzz_output.log'),
                           ('non_contig', f'{self.api}_non_contig_fuzz_output.log'),
                           ('multithread', f'{self.api}_multithread_fuzz_output.log'),
                           ('consistent', f'{self.api}_consistent_fuzz_output.log'),
                           ('error', f'{self.api}_error_fuzz_output.log')]:
            p = os.path.join(self.logs, logf)
            if os.path.exists(p):
                succ, skip, fail, rets = count_success(p)
                lines.append(f'- **{scen}**: 执行 succ={succ} skip={skip} fail={fail}\n')
            else:
                lines.append(f'- **{scen}**: 未执行/前置不满足\n')
        open(rep, 'w', encoding='utf-8').write(''.join(lines))
        # bug filing (config-dependent, warns if absent)
        rc, o = run(f'python3 {BASE}/scripts/file_bug_issues.py --api {self.api} '
                    f'--local {BASE}/out_res/批量压测结果.xlsx --file', timeout=600)
        self.log('[CLOSURE] file_bug_issues rc=%d: %s' % (rc, o[-150:]))
        # 当日 issue 清单在线表格同步（GitCode 仓 issue清单_{日期}.xlsx）
        rc2, o2 = run(f'python3 {BASE}/scripts/sync_daily_issues_git.py', timeout=300)
        self.log('[CLOSURE] daily_issues_sync rc=%d: %s' % (rc2, o2.strip()[-100:]))
        rc, o = run(f'python3 {BASE}/scripts/file_bug_issues.py --api {self.api} '
                    f'--local {BASE}/out_res/批量压测结果.xlsx --check', timeout=300)
        self.log('[CLOSURE] check rc=%d' % rc)
        return True

    def run_stages(self):
        stages = [('design', self.stage_design), ('whitebox', self.stage_whitebox),
                  ('sample', self.stage_sample), ('normal', self.stage_normal),
                  ('noncontig', self.stage_noncontig), ('multithread', self.stage_multithread),
                  ('consistent', self.stage_consistent), ('error', self.stage_error),
                  ('closure', self.stage_closure)]
        names = [s[0] for s in stages]
        start_idx = names.index(self.args.start_stage) if self.args.start_stage else 0
        for idx, (name, fn) in enumerate(stages):
            if self.args.only_stage and name != self.args.only_stage:
                continue
            if idx < start_idx:
                continue
            if self.done(name) and not self.args.only_stage:
                self.log(f'[STAGE_SKIP] {name} done')
                continue
            try:
                ok = fn()
            except Exception as e:
                self.log(f'[STAGE_CRASH] {name}: {e}')
                ok = False
            if ok:
                self.set_done(name)
            elif name in ('normal', 'design'):
                # cannot proceed without normal
                self.log(f'[ABORT] {name} 失败，接口闭环为部分完成')
                # still run error if cpp exists? error needs normal cpp -> only if exists
                if os.path.exists(os.path.join(self.out, f'{self.api}.cpp')):
                    try:
                        if self.stage_error():
                            self.set_done('error')
                    except Exception as e:
                        self.log(f'[STAGE_CRASH] error: {e}')
                try:
                    self.stage_closure()
                    self.set_done('closure')
                except Exception:
                    pass
                return False
        return True


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('api')
    ap.add_argument('--start-stage')
    ap.add_argument('--only-stage')
    args = ap.parse_args()
    drv = Driver(args.api, args)
    ok = drv.run_stages()
    sys.exit(0 if ok else 1)


if __name__ == '__main__':
    main()