已合并
feat(ascendc-port-orchestrator): PR2/5 编排引擎+子agent (RFC #364, +OKF迁移) #609
junming222创建于 7月15日
feat(ascendc-port-orchestrator): PR2/5 编排引擎+子agent (RFC #364, +OKF迁移) #609
已合并
共 644 个文件变更+156728-0
| @@ -0,0 +1,43 @@ | |||
| 1 | +# MFU 求解器 (算子理论最大 MFU 模型) — v0 | ||
| 2 | + | ||
| 3 | +数学口径见 [`../docs/design/OPERATOR_MFU_DEFINITION.md`](../docs/design/OPERATOR_MFU_DEFINITION.md)、硬件常数见 [`../docs/design/L0_HARDWARE_CONSTANTS.md`](../docs/design/L0_HARDWARE_CONSTANTS.md)。 | ||
| 4 | + | ||
| 5 | +## 这是什么 | ||
| 6 | +给定算子（FLOPs/访存/dtype/可选通信）+ 硬件目标，算出**理论最大 MFU**（roofline 上界 ∩ 完全通算掩盖）+ 瓶颈归因（compute/mem/comm-bound）。 | ||
| 7 | + | ||
| 8 | +``` | ||
| 9 | +理论最大 MFU = T_compute / max(T_compute, T_mem, T_comm) × η_util | ||
| 10 | + T_compute = useful_FLOPs / cube_peak(dtype) | ||
| 11 | + T_mem = HBM_bytes / HBM_BW | ||
| 12 | + T_comm = comm_bytes / interconnect_BW | ||
| 13 | + η_util 默认 1.0(纯理论天花板)，待 msprof 真机标定下调 | ||
| 14 | +``` | ||
| 15 | + | ||
| 16 | +## 文件 | ||
| 17 | +- `mfu_model.py` — L0 硬件模型(910C/950PR/950DT) + L1 原语(matmul fwd/bwd) + L3 roofline 求解器 | ||
| 18 | +- `demo.py` — matmul 在三目标上的 MFU_max 示例 | ||
| 19 | + | ||
| 20 | +## 跑 | ||
| 21 | +```bash | ||
| 22 | +cd mfu && python3 demo.py | ||
| 23 | +``` | ||
| 24 | + | ||
| 25 | +## v0 结果验证 | ||
| 26 | +- 大方阵 matmul(4096³): MFU_max=1.0 compute-bound ✓ | ||
| 27 | +- GEMV(M=1): MFU_max≈0.003 memory-bound ✓ | ||
| 28 | +- **cube peak 交叉验证**: 950PR FP8 推导 973 TFLOPS ≈ 官方 1 PFLOPS ✓（"1拍 2·M·K·N FLOPs"口径成立） | ||
| 29 | + | ||
| 30 | +## 进度 | ||
| 31 | +- [x] 通算掩盖：分布式 DP 梯度 AllReduce 模型 + SuperPod384 互联(`with_data_parallel`) | ||
| 32 | +- [x] FlashAttention(cube+vector 混合) 原语 | ||
| 33 | +- [x] L2 调度演算：多级 roofline，tiling 用 **matmul I/O 下界(Hong-Kung)** = 最优 tiling 搬运量 | ||
| 34 | + - 这是"理论最优调度"对应的搬运量(理论最大 MFU), 而非某具体次优 tiling | ||
| 35 | + - 修正: 早期用 L0C-tile 计数的 2 级模型误判"910C 大矩阵 L2-bound 0.66" —— 那是次优 tiling 的人为产物; | ||
| 36 | + I/O 下界正确给出大矩阵 **compute-bound MFU=1.0**(理论最优), 实测与理论的差距归入 η_util(待 msprof) | ||
| 37 | + - 验证: 大/中矩阵 compute-bound=1.0; 瘦长(256×256×8192)/小(128³) 正确 memory-bound | ||
| 38 | +- [x] 非 matmul 原语: softmax/rmsnorm/elementwise(向量引擎, 深度 memory-bound, MFU 0.007~0.18) | ||
| 39 | + - 洞察: vec_peak 越高 MFU 越低(同搬运量) → 必须算子融合减 HBM 往返; 950DT(4TB/s) 比 950PR 优 2.5× | ||
| 40 | +- [x] 单算子 → 训练 e2e MFU 聚合(`transformer_layer_e2e`, OPERATOR_MFU_DEFINITION §4 契约) | ||
| 41 | + - Σuseful_FLOPs/(cube_peak×Σ各算子最优时间); 向量算子 FLOPs 小但耗时拖低 e2e; 给时间占比找瓶颈 | ||
| 42 | +- [ ] η_util msprof 真机标定 + 预测 vs 实测回归(需 NPU, 已申请) | ||
| 43 | +- [ ] 验证闭环：FA + a5_ops 优化 agent，验证理论能否指导优化（若无用则止损） | ||
| @@ -0,0 +1,75 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# -*- coding: utf-8 -*- | ||
| 3 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 12 | + | ||
| 13 | +"""MFU 求解器 demo: matmul 在 910C/950PR/950DT 上的理论最大 MFU(前向+后向)。""" | ||
| 14 | +from mfu_model import (HW, matmul_ops, flash_attention_ops, with_data_parallel, | ||
| 15 | + solve_mfu, solve_matmul_tiled, DTYPE_BYTES) | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +def run(M, N, K, dtype): | ||
| 19 | + print(f"\n==== matmul M={M} N={N} K={K} dtype={dtype} ====") | ||
| 20 | + fwd, bwd = matmul_ops(M, N, K, dtype) | ||
| 21 | + hdr = f"{'hw':7} {'stage':11} {'MFU_max':>7} {'bottleneck':>10} {'AI':>9} {'AI*':>9} {'peakTF':>7}" | ||
| 22 | + print(hdr) | ||
| 23 | + for hwname, hw in HW.items(): | ||
| 24 | + for op in (fwd, bwd): | ||
| 25 | + try: | ||
| 26 | + r = solve_mfu(op, hw) | ||
| 27 | + except ValueError as e: | ||
| 28 | + print(f"{hwname:7} {op.name:11} skip ({e})") | ||
| 29 | + continue | ||
| 30 | + print(f"{r['hw']:7} {r['op']:11} {r['mfu_max']:>7} {r['bottleneck']:>10} " | ||
| 31 | + f"{r['arithmetic_intensity']:>9} {r['ridge_ai']:>9} {r['peak_TFLOPS']:>7}") | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +def run_fa(B, H, S, D, dtype="bf16", causal=False): | ||
| 35 | + print(f"\n==== FlashAttention B={B} H={H} S={S} D={D} dtype={dtype} causal={causal} ====") | ||
| 36 | + fwd, bwd = flash_attention_ops(B, H, S, D, dtype, causal) | ||
| 37 | + print(f"{'hw':18} {'stage':8} {'MFU_max':>7} {'bottleneck':>10} {'AI':>8} {'AI*':>8}") | ||
| 38 | + for hw in (HW["910C"], HW["950PR"], HW["950DT"]): | ||
| 39 | + for op in (fwd, bwd): | ||
| 40 | + r = solve_mfu(op, hw) | ||
| 41 | + print(f"{r['hw']:18} {r['op']:8} {r['mfu_max']:>7} {r['bottleneck']:>10} " | ||
| 42 | + f"{r['arithmetic_intensity']:>8} {r['ridge_ai']:>8}") | ||
| 43 | + | ||
| 44 | + | ||
| 45 | +def run_dist_train(M, N, K, dtype, n_dev): | ||
| 46 | + """训练数据并行: bwd 的 dW 叠加梯度 AllReduce, 看通信是否被 compute 掩盖。""" | ||
| 47 | + print(f"\n==== 分布式训练 matmul M={M} N={N} K={K} dtype={dtype} DP={n_dev} (910C@SuperPod384) ====") | ||
| 48 | + _, bwd = matmul_ops(M, N, K, dtype) | ||
| 49 | + grad_bytes = K * N * DTYPE_BYTES[dtype] # dW 梯度大小 | ||
| 50 | + bwd_dp = with_data_parallel(bwd, grad_bytes, n_dev) | ||
| 51 | + hw = HW["910C_superpod"] | ||
| 52 | + r = solve_mfu(bwd_dp, hw) | ||
| 53 | + print(f" MFU_max={r['mfu_max']} bottleneck={r['bottleneck']} " | ||
| 54 | + f"t_compute={r['t_compute_us']}us t_comm={r['t_comm_us']}us " | ||
| 55 | + f"-> 通信{'被掩盖' if r['bottleneck']!='comm' else '未掩盖(comm-bound)'}") | ||
| 56 | + | ||
| 57 | + | ||
| 58 | +if __name__ == "__main__": | ||
| 59 | + # 大方阵(compute-bound 应 MFU_max≈1) vs 瘦长(memory-bound 应 <1) | ||
| 60 | + run(4096, 4096, 4096, "bf16") | ||
| 61 | + run(4096, 4096, 4096, "fp8") | ||
| 62 | + run(1, 4096, 4096, "bf16") # GEMV-like, memory-bound | ||
| 63 | + # FlashAttention (cube+vector 混合, FA tiling 高算术强度) | ||
| 64 | + run_fa(2, 32, 4096, 128, "bf16") | ||
| 65 | + run_fa(2, 32, 512, 128, "bf16") # 短序列 -> AI 降 | ||
| 66 | + # 分布式训练通算掩盖: 大矩阵 compute 多 -> 通信易掩盖; 小矩阵 -> 易 comm-bound | ||
| 67 | + run_dist_train(8192, 8192, 8192, "bf16", 8) | ||
| 68 | + run_dist_train(512, 8192, 8192, "bf16", 64) | ||
| 69 | + # L2 调度演算: 多级 roofline (tiling 受 L0C 限制 -> L2 re-read) | ||
| 70 | + print("\n==== tiled matmul (多级 roofline: HBM+L2, L0C限tile) ====") | ||
| 71 | + print(" (绝对值待 msprof 校准 + 加 L1 复用级; 结构示意 L0C小->L2-bound 机理)") | ||
| 72 | + for M, N, K in [(4096, 4096, 4096), (256, 256, 8192)]: | ||
| 73 | + for hwn in ("910C", "950PR", "950DT"): | ||
| 74 | + r = solve_matmul_tiled(M, N, K, HW[hwn], "bf16") | ||
| 75 | + print(f" {hwn:6} {r['op']:22} MFU={r['mfu_max']:<6} bn={r['bottleneck']}") | ||
| @@ -0,0 +1,393 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# -*- coding: utf-8 -*- | ||
| 3 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 12 | + | ||
| 13 | +"""算子理论最大 MFU 求解器 v0 (L0 硬件模型 + L1 原语 + L3 roofline 求解). | ||
| 14 | + | ||
| 15 | +数学口径见 docs/design/OPERATOR_MFU_DEFINITION.md 与 L0_HARDWARE_CONSTANTS.md。 | ||
| 16 | +- 理论最大 MFU = T_compute_ideal / T_total_optimal | ||
| 17 | +- T_total_optimal = max(T_compute, T_mem, T_comm) # 完全通算掩盖上界(理论天花板) | ||
| 18 | +- η_util(tiling/尾块/流水有效利用率) 默认 1.0, 待 msprof 真机标定后下调。 | ||
| 19 | + | ||
| 20 | +硬件常数来源: docs/archive/A5_HARDWARE_DETAILS.md + ASCEND_FLEET_910C_950DT_SUPERPOD384.md | ||
| 21 | +cube peak 用"1拍 = 2·M·K·N FLOPs"口径(910C 源实测, 950PR FP8 推导≈官方1PFLOPS 交叉验证)。 | ||
| 22 | +""" | ||
| 23 | +from dataclasses import dataclass, field | ||
| 24 | +from typing import Optional | ||
| 25 | + | ||
| 26 | +# 每个 cube dtype 的"单拍 FLOPs" = 2·M·K·N (矩阵粒度), 来自 L0 常数表 | ||
| 27 | +CUBE_FLOPS_PER_CYCLE = { | ||
| 28 | + "fp32": 2 * 16 * 1 * 16, # [16x1x16] | ||
| 29 | + "tf32": 2 * 16 * 8 * 16, # [16x8x16] | ||
| 30 | + "fp16": 2 * 16 * 16 * 16, # [16x16x16] = 8192 | ||
| 31 | + "bf16": 2 * 16 * 16 * 16, # 8192 | ||
| 32 | + "fp8": 2 * 16 * 32 * 16, # [16x32x16] = 16384 | ||
| 33 | + "int8": 2 * 16 * 32 * 16, # 16384 | ||
| 34 | + "mxfp4": 2 * 16 * 64 * 16, # [16x64x16] = 32768 | ||
| 35 | + "int4": 2 * 16 * 64 * 16, # 32768 | ||
| 36 | +} | ||
| 37 | +DTYPE_BYTES = {"fp32": 4, "tf32": 4, "fp16": 2, "bf16": 2, "fp8": 1, "int8": 1, "mxfp4": 0.5, "int4": 0.5} | ||
| 38 | + | ||
| 39 | + | ||
| 40 | +@dataclass | ||
| 41 | +class HardwareModel: | ||
| 42 | + """描述设备算力、带宽和片上存储容量的简化硬件模型。""" | ||
| 43 | + | ||
| 44 | + name: str | ||
| 45 | + n_aic: int # cube core 数 | ||
| 46 | + freq_hz: float # 主频 | ||
| 47 | + hbm_bw: float # HBM 带宽 B/s | ||
| 48 | + hbm_bytes: float # HBM 容量 B | ||
| 49 | + interconnect_bw: float = 0.0 # 卡间/die间 互联 B/s (单向有效) | ||
| 50 | + arch: str = "SIMD" # SIMD(A3) / SIMT(A5) | ||
| 51 | + l2_bw: float = 0.0 # L2 读带宽 B/s (聚合); 0=不建模L2级 | ||
| 52 | + l2_bytes: float = 0.0 # L2 容量 B | ||
| 53 | + l1_bytes: float = 0.0 # L1 容量 B (matmul 复用级) | ||
| 54 | + l0c_bytes: float = 0.0 # L0C(输出累加器)容量 B | ||
| 55 | + # 直接给定的 peak(若有官方值, 覆盖推导), key=dtype -> FLOP/s | ||
| 56 | + peak_override: dict = field(default_factory=dict) | ||
| 57 | + vec_peak: dict = field(default_factory=dict) # 向量 dtype -> FLOP/s | ||
| 58 | + | ||
| 59 | + def cube_peak(self, dtype: str) -> float: | ||
| 60 | + """返回指定数据类型的 Cube 峰值算力；显式配置优先。""" | ||
| 61 | + | ||
| 62 | + if dtype in self.peak_override: | ||
| 63 | + return self.peak_override[dtype] | ||
| 64 | + fpc = CUBE_FLOPS_PER_CYCLE.get(dtype) | ||
| 65 | + if fpc is None: | ||
| 66 | + raise ValueError(f"{self.name}: cube 不支持 dtype {dtype}") | ||
| 67 | + return self.n_aic * fpc * self.freq_hz | ||
| 68 | + | ||
| 69 | + def ridge_ai(self, dtype: str, engine="cube") -> float: | ||
| 70 | + """返回指定计算引擎的 Roofline 拐点算术强度。""" | ||
| 71 | + | ||
| 72 | + peak = self.cube_peak(dtype) if engine == "cube" else self.vec_peak[dtype] | ||
| 73 | + return peak / self.hbm_bw | ||
| 74 | + | ||
| 75 | + | ||
| 76 | +@dataclass | ||
| 77 | +class OpSpec: | ||
| 78 | + """单算子的某个阶段(fwd 或 bwd)的 FLOPs/bytes/通信 记账。""" | ||
| 79 | + name: str | ||
| 80 | + flops: float # useful FLOPs | ||
| 81 | + hbm_bytes: float # 最小 HBM 访存字节(理想 tiling, 每数据读一次) | ||
| 82 | + dtype: str = "bf16" | ||
| 83 | + engine: str = "cube" # cube / vector | ||
| 84 | + comm_bytes: float = 0.0 # 分布式通信字节(单向有效量) | ||
| 85 | + | ||
| 86 | + | ||
| 87 | +def solve_mfu(op: OpSpec, hw: HardwareModel, eta_util: float = 1.0) -> dict: | ||
| 88 | + """求理论最大 MFU + 瓶颈归因。eta_util=有效利用率(默认1=纯理论天花板)。""" | ||
| 89 | + peak = hw.cube_peak(op.dtype) if op.engine == "cube" else hw.vec_peak[op.dtype] | ||
| 90 | + t_compute = op.flops / peak | ||
| 91 | + t_mem = op.hbm_bytes / hw.hbm_bw | ||
| 92 | + t_comm = op.comm_bytes / hw.interconnect_bw if (op.comm_bytes and hw.interconnect_bw) else 0.0 | ||
| 93 | + t_total = max(t_compute, t_mem, t_comm) # 完全掩盖上界 | ||
| 94 | + mfu_max = (t_compute / t_total) * eta_util if t_total > 0 else 0.0 | ||
| 95 | + times = {"compute": t_compute, "mem": t_mem, "comm": t_comm} | ||
| 96 | + bottleneck = max(times, key=times.get) | ||
| 97 | + ai = op.flops / op.hbm_bytes if op.hbm_bytes else float("inf") | ||
| 98 | + return { | ||
| 99 | + "op": op.name, "hw": hw.name, "dtype": op.dtype, | ||
| 100 | + "mfu_max": round(mfu_max, 4), | ||
| 101 | + "bottleneck": bottleneck, | ||
| 102 | + "arithmetic_intensity": round(ai, 2), | ||
| 103 | + "ridge_ai": round(hw.ridge_ai(op.dtype, op.engine), 2), | ||
| 104 | + "peak_TFLOPS": round(peak / 1e12, 1), | ||
| 105 | + "t_compute_us": round(t_compute * 1e6, 3), | ||
| 106 | + "t_mem_us": round(t_mem * 1e6, 3), | ||
| 107 | + "t_comm_us": round(t_comm * 1e6, 3), | ||
| 108 | + "eta_util": eta_util, | ||
| 109 | + } | ||
| 110 | + | ||
| 111 | + | ||
| 112 | +# ---------- L1 原语: matmul 前向/后向 FLOPs/bytes 记账 ---------- | ||
| 113 | +def matmul_ops(M, N, K, dtype="bf16"): | ||
| 114 | + """Y=X·W, X[M,K] W[K,N]. 返回 (fwd, bwd) 两个 OpSpec。 | ||
| 115 | + bytes = 理想最小 HBM 流量 = 各 tensor 读/写一次。 | ||
| 116 | + """ | ||
| 117 | + sz = DTYPE_BYTES[dtype] | ||
| 118 | + fwd = OpSpec("matmul.fwd", flops=2 * M * N * K, | ||
| 119 | + hbm_bytes=(M * K + K * N + M * N) * sz, dtype=dtype) | ||
| 120 | + # bwd: dX = dY·Wᵀ (2MNK, 读 dY[M,N]+W[K,N], 写 dX[M,K]); dW = Xᵀ·dY (2MNK, 读 X[M,K]+dY[M,N], 写 dW[K,N]) | ||
| 121 | + bwd = OpSpec("matmul.bwd", flops=4 * M * N * K, | ||
| 122 | + hbm_bytes=(M * N + K * N + M * K + M * K + M * N + K * N) * sz, dtype=dtype) | ||
| 123 | + return fwd, bwd | ||
| 124 | + | ||
| 125 | + | ||
| 126 | +# ---------- L1 原语: FlashAttention 前向/后向 (cube+vector 混合) ---------- | ||
| 127 | +def flash_attention_ops(B, H, S, D, dtype="bf16", causal=False): | ||
| 128 | + """Q,K,V [B,H,S,D]. FA tiling 不物化 S=[B,H,S,S], 故 HBM 仅读 QKV 写 O。 | ||
| 129 | + cube FLOPs 主导(QKᵀ + P·V), softmax 走 vector(量级小, 此处计入但归一仍用 cube)。 | ||
| 130 | + bwd ≈ 2.5× fwd cube FLOPs(重算 + dQ/dK/dV)。causal 减半有效 S² 工作量。 | ||
| 131 | + """ | ||
| 132 | + sz = DTYPE_BYTES[dtype] | ||
| 133 | + causal_factor = 0.5 if causal else 1.0 | ||
| 134 | + # fwd: QKᵀ=2BHS²D, P·V=2BHS²D | ||
| 135 | + fwd_cube = 4 * B * H * S * S * D * causal_factor | ||
| 136 | + fwd = OpSpec("FA.fwd", flops=fwd_cube, | ||
| 137 | + hbm_bytes=(3 * B * H * S * D + B * H * S * D) * sz, # 读QKV + 写O | ||
| 138 | + dtype=dtype, engine="cube") | ||
| 139 | + bwd = OpSpec("FA.bwd", flops=2.5 * fwd_cube, | ||
| 140 | + hbm_bytes=(5 * B * H * S * D + 3 * B * H * S * D) * sz, # 读QKVO·dO + 写dQdKdV | ||
| 141 | + dtype=dtype, engine="cube") | ||
| 142 | + return fwd, bwd | ||
| 143 | + | ||
| 144 | + | ||
| 145 | +# ---------- L2(部分): 分布式集合通信 = 通算掩盖输入 ---------- | ||
| 146 | +def allreduce_bytes(msg_bytes, n_dev, algo="ring"): | ||
| 147 | + """带宽最优 AllReduce 每设备通信量。ring/halving-doubling 同为 2·(N-1)/N·M。""" | ||
| 148 | + if n_dev <= 1: | ||
| 149 | + return 0.0 | ||
| 150 | + return 2.0 * (n_dev - 1) / n_dev * msg_bytes | ||
| 151 | + | ||
| 152 | + | ||
| 153 | +def with_data_parallel(op: OpSpec, grad_bytes: float, n_dev: int): | ||
| 154 | + """训练数据并行: 在算子(通常 bwd 的 dW)上叠加梯度 AllReduce 通信, 走通算掩盖。 | ||
| 155 | + 返回带 comm_bytes 的新 OpSpec(浅拷贝)。 | ||
| 156 | + """ | ||
| 157 | + comm = allreduce_bytes(grad_bytes, n_dev) | ||
| 158 | + return OpSpec(op.name + f".dp{n_dev}", flops=op.flops, hbm_bytes=op.hbm_bytes, | ||
| 159 | + dtype=op.dtype, engine=op.engine, comm_bytes=comm) | ||
| 160 | + | ||
| 161 | + | ||
| 162 | +# ---------- L4: 单算子 -> 训练 e2e MFU 聚合 (OPERATOR_MFU_DEFINITION §4 契约) ---------- | ||
| 163 | +def _matmul_time(M, N, K, hw, dtype, flop_mult=1.0): | ||
| 164 | + """matmul 的 (useful_flops, T_total_optimal秒)。flop_mult: fwd=1, fwd+bwd≈3。""" | ||
| 165 | + flops, hbm, l2 = matmul_tiled_levels(M, N, K, hw, dtype) | ||
| 166 | + flops *= flop_mult | ||
| 167 | + r = solve_mfu_multilevel(flops, hw.cube_peak(dtype), | ||
| 168 | + [(hw.hbm_bw, hbm * flop_mult), (hw.l2_bw, l2 * flop_mult)]) | ||
| 169 | + return flops, max(r["t_us"].values()) / 1e6 | ||
| 170 | + | ||
| 171 | + | ||
| 172 | +def _vec_time(op, hw, flop_mult=1.0): | ||
| 173 | + r = solve_mfu(op, hw) | ||
| 174 | + t = max(r["t_compute_us"], r["t_mem_us"], r["t_comm_us"]) / 1e6 * flop_mult | ||
| 175 | + return op.flops * flop_mult, t | ||
| 176 | + | ||
| 177 | + | ||
| 178 | +def transformer_layer_e2e(B, S, d_model, d_ff, n_head, hw, dtype="bf16", training=True): | ||
| 179 | + """一层 Transformer 的训练 e2e MFU = Σuseful_FLOPs / (cube_peak × Σ各算子最优时间)。 | ||
| 180 | + 展示单算子 MFU 如何聚合到 e2e: 向量算子(norm/softmax)FLOPs 小但 memory-bound 耗时, | ||
| 181 | + 会拖低 e2e MFU —— 这正是 e2e MFU agent 要复用的契约。 | ||
| 182 | + """ | ||
| 183 | + M = B * S | ||
| 184 | + fm = 3.0 if training else 1.0 # matmul fwd+bwd≈3x flops | ||
| 185 | + fam = 3.5 if training else 1.0 # attention fwd+bwd≈3.5x | ||
| 186 | + items = [] # (name, flops, t) | ||
| 187 | + for name, N, K in [("qkv_proj", 3 * d_model, d_model), ("o_proj", d_model, d_model), | ||
| 188 | + ("ffn_up", d_ff, d_model), ("ffn_down", d_model, d_ff)]: | ||
| 189 | + items.append((name, *_matmul_time(M, N, K, hw, dtype, fm))) | ||
| 190 | + fwd, bwd = flash_attention_ops(B, n_head, S, d_model // n_head, dtype) | ||
| 191 | + items.append(("attention", *_vec_time(fwd, hw, fam))) # FA cube 主导, 用其 flops/time×mult | ||
| 192 | + for name, opf, args in [("rmsnorm_attn", rmsnorm_op, (M, d_model)), | ||
| 193 | + ("rmsnorm_ffn", rmsnorm_op, (M, d_model)), | ||
| 194 | + ("softmax_skip", elementwise_op, (M * d_ff,))]: # 占位非线性 | ||
| 195 | + items.append((name, *_vec_time(opf(*args), hw, 2.0 if training else 1.0))) | ||
| 196 | + total_flops = sum(f for _, f, _ in items) | ||
| 197 | + total_time = sum(t for _, _, t in items) | ||
| 198 | + cube_peak = hw.cube_peak(dtype) | ||
| 199 | + mfu = total_flops / (cube_peak * total_time) if total_time else 0.0 | ||
| 200 | + # 各算子耗时占比(找瓶颈) | ||
| 201 | + breakdown = sorted(((n, round(t / total_time * 100, 1)) for n, _, t in items), | ||
| 202 | + key=lambda x: -x[1]) | ||
| 203 | + return {"hw": hw.name, "e2e_MFU": round(mfu, 4), | ||
| 204 | + "total_TFLOPs": round(total_flops / 1e12, 2), | ||
| 205 | + "step_us": round(total_time * 1e6, 1), | ||
| 206 | + "time_pct": breakdown} | ||
| 207 | + | ||
| 208 | + | ||
| 209 | +# ---------- L0 硬件实例 (常数源: docs/design/L0_HARDWARE_CONSTANTS.md) ---------- | ||
| 210 | +_L2_BW = 5.28e12 # L2 读带宽聚合: 910C 48核×110GB/s≈5.28TB/s; 950PR 官方5.28TB/s (950DT 待核实,暂同) | ||
| 211 | +HW = { | ||
| 212 | + # 910C 整卡(双die ~48 AIC), HBM 3.2TB/s 128GB, 互联 784GB/s 双向→单向~392 | ||
| 213 | + "910C": HardwareModel("910C", n_aic=48, freq_hz=1.85e9, | ||
| 214 | + hbm_bw=3.2e12, hbm_bytes=128 * 2**30, | ||
| 215 | + interconnect_bw=784e9 / 2, arch="SIMD", | ||
| 216 | + l2_bw=_L2_BW, l2_bytes=192 * 2**20, l1_bytes=512 * 2**10, l0c_bytes=128 * 2**10), | ||
| 217 | + # 950PR PG(实测card2: 28 cube/56 vec, L2 112MB, soc 957b), HBM 1.6TB/s 128GB; AG变体=36 AIC | ||
| 218 | + "950PR": HardwareModel("950PR", n_aic=28, freq_hz=1.65e9, | ||
| 219 | + hbm_bw=1.6e12, hbm_bytes=128 * 2**30, | ||
| 220 | + interconnect_bw=2e12, arch="SIMT", | ||
| 221 | + l2_bw=_L2_BW, l2_bytes=112 * 2**20, l1_bytes=512 * 2**10, l0c_bytes=256 * 2**10), | ||
| 222 | + # 950DT: 官方 FP8 1P/MXFP4 2P -> 反推 cube; HBM 4TB/s 144GB, 互联 2TB/s | ||
| 223 | + "950DT": HardwareModel("950DT", n_aic=36, freq_hz=1.65e9, | ||
| 224 | + hbm_bw=4e12, hbm_bytes=144 * 2**30, | ||
| 225 | + interconnect_bw=2e12, arch="SIMT", | ||
| 226 | + l2_bw=_L2_BW, l2_bytes=128 * 2**20, l1_bytes=512 * 2**10, l0c_bytes=256 * 2**10, | ||
| 227 | + peak_override={"fp8": 1e15, "mxfp4": 2e15, | ||
| 228 | + "bf16": 0.5e15, "fp16": 0.5e15}), | ||
| 229 | +} | ||
| 230 | +# SuperPod384 = Atlas900 A3 SuperPoD: 单卡=910C, 但超节点内卡间互联 784GB/s 双向(1:1无收敛) | ||
| 231 | +HW["910C_superpod"] = HardwareModel( | ||
| 232 | + "910C@SuperPod384", n_aic=48, freq_hz=1.85e9, | ||
| 233 | + hbm_bw=3.2e12, hbm_bytes=128 * 2**30, | ||
| 234 | + interconnect_bw=784e9, arch="SIMD", | ||
| 235 | + l2_bw=_L2_BW, l2_bytes=192 * 2**20, l1_bytes=512 * 2**10, l0c_bytes=128 * 2**10) | ||
| 236 | + | ||
| 237 | +# 910C 单 die (Ascend910_9392, torch_npu 单设备粒度 = 24 AIC/64GB; HW['910C']是整卡48AIC) | ||
| 238 | +# 实测标定 2026-06-26 A3: bf16 matmul 实测峰值 ~321 TFLOPS (n=4096), 理论 363 -> η_util≈0.88, 天花板不破。 | ||
| 239 | +HW["910C_die"] = HardwareModel( | ||
| 240 | + "910C_die", n_aic=24, freq_hz=1.85e9, | ||
| 241 | + hbm_bw=1.6e12, hbm_bytes=64 * 2**30, # 单die 64GB; 单die HBM BW 待msprof(暂半卡) | ||
| 242 | + interconnect_bw=784e9 / 2, arch="SIMD", | ||
| 243 | + l2_bw=_L2_BW / 2, l2_bytes=96 * 2**20, l1_bytes=512 * 2**10, l0c_bytes=128 * 2**10) | ||
| 244 | +# 实测标定的有效利用率 η_util (2026-06-26 A3 Ascend910_9392 单die): | ||
| 245 | +# matmul 大方阵 compute-bound: 0.88 (纯 cube) | ||
| 246 | +# FlashAttention(fusion_attn 最优核): ~0.53 vs cube peak —— cube+vector 交织代价, | ||
| 247 | +# softmax(向量)阶段无法与 cube 完全掩盖 -> 混合算子天花板 < 纯 cube peak。 | ||
| 248 | +ETA_UTIL = { | ||
| 249 | + # 实测标定 2026-06-26. 关键: A5(SIMT) FA η 远高于 A3(SIMD) -> SIMT cube+vector 掩盖更好。 | ||
| 250 | + "matmul": {"910C_die": 0.88, "950PR": 0.98}, | ||
| 251 | + "flash_attention": {"910C_die": 0.53, "950PR": 0.87}, | ||
| 252 | +} | ||
| 253 | +# η-provenance (main review 2026-07-01 要求): 每个 η 的来源+日期+re-validate 触发。 | ||
| 254 | +# stale-η -> 错估 headroom, 故硬件/CANN/bisheng 版本变时必须 re-validate。 | ||
| 255 | +ETA_PROVENANCE = { | ||
| 256 | + ("matmul", "910C_die"): { | ||
| 257 | + "value": 0.88, | ||
| 258 | + "measured": "2026-06-26", | ||
| 259 | + "method": "A3 Ascend910_9392 bf16 matmul n=4096 device-time vs 理论363TF", | ||
| 260 | + "revalidate_on": "SOC/CANN/bisheng 版本变 或 主频变", | ||
| 261 | + }, | ||
| 262 | + ("matmul", "950PR"): { | ||
| 263 | + "value": 0.98, | ||
| 264 | + "measured": "2026-06-26", | ||
| 265 | + "method": "A5 950PR_957b bf16 matmul n=8192 372.7 vs 378.6TF", | ||
| 266 | + "revalidate_on": "同上", | ||
| 267 | + }, | ||
| 268 | + ("flash_attention", "910C_die"): { | ||
| 269 | + "value": 0.53, | ||
| 270 | + "measured": "2026-06-26", | ||
| 271 | + "method": "A3 npu_fusion_attention 170TF vs cube peak 363, msprof cube_mac 0.50", | ||
| 272 | + "revalidate_on": "同上 + FA kernel 版本变", | ||
| 273 | + }, | ||
| 274 | + ("flash_attention", "950PR"): { | ||
| 275 | + "value": 0.87, | ||
| 276 | + "measured": "2026-06-26", | ||
| 277 | + "method": "A5 npu_fusion_attention 330TF vs 378.6, msprof cube_mac 0.98", | ||
| 278 | + "revalidate_on": "同上", | ||
| 279 | + }, | ||
| 280 | +} | ||
| 281 | + | ||
| 282 | + | ||
| 283 | +def eta_with_provenance(op_kind, hw_name): | ||
| 284 | + """返回 (η, provenance)。无 provenance 的 η = 未标定/stale 风险, 调用方应警惕。""" | ||
| 285 | + p = ETA_PROVENANCE.get((op_kind, hw_name)) | ||
| 286 | + eta = ETA_UTIL.get(op_kind, {}).get(hw_name) | ||
| 287 | + return eta, p | ||
| 288 | + | ||
| 289 | + | ||
| 290 | +def achievable_mfu_with_overhead(flops, peak, op_kind, hw_name, t_overhead_s=0.0, default_eta=0.85): | ||
| 291 | + """v2: 理论 compute 时间 / η + 固定 launch/tail overhead -> 可达 MFU。 | ||
| 292 | + B组(2026-06-29 A3 FA)实测暴露: v1理想roofline的陡峭ridge过乐观, 小算子受 | ||
| 293 | + kernel-launch/尾块固定开销主导。加 T_overhead 项后, A3 FA MFU-vs-S 全程拟合实测±几%。 | ||
| 294 | + A3 FA fusion_attn 拟合: t_overhead≈0.115ms, η≈0.53。 | ||
| 295 | + """ | ||
| 296 | + eta = ETA_UTIL.get(op_kind, {}).get(hw_name, default_eta) | ||
| 297 | + t_compute_ideal = flops / peak | ||
| 298 | + t_total = t_compute_ideal / eta + t_overhead_s | ||
| 299 | + return {"achievable_mfu": round(t_compute_ideal / t_total, 4), | ||
| 300 | + "eta_util": eta, "t_overhead_s": t_overhead_s, | ||
| 301 | + "t_compute_ideal_s": t_compute_ideal} | ||
| 302 | + | ||
| 303 | + | ||
| 304 | +def achievable_mfu(theoretical_mfu_max, op_kind, hw_name, default_eta=0.85): | ||
| 305 | + """把实测标定的 η_util 套到理论天花板上 -> 预测"可达 MFU"。 | ||
| 306 | + 返回 (theoretical_ceiling, eta, achievable)。优化时: 实测<achievable=>还有实现空间; | ||
| 307 | + 实测≈achievable=>已接近该算子类的实际上限; 要再高需改算法(降 η 的机理)。 | ||
| 308 | + """ | ||
| 309 | + eta = ETA_UTIL.get(op_kind, {}).get(hw_name, default_eta) | ||
| 310 | + return {"theoretical_ceiling": theoretical_mfu_max, | ||
| 311 | + "eta_util": eta, | ||
| 312 | + "achievable_mfu": round(theoretical_mfu_max * eta, 4)} | ||
| 313 | + | ||
| 314 | + | ||
| 315 | +# 向量算力 (源: 950PR FP16 54TOPS FMA / 27 单算; 910C 推导~22.7T 待msprof; 950DT 暂同950PR) | ||
| 316 | +HW["950PR"].vec_peak = {"fp16": 54e12, "bf16": 54e12, "fp32": 27e12} | ||
| 317 | +HW["950DT"].vec_peak = {"fp16": 54e12, "bf16": 54e12, "fp32": 27e12} | ||
| 318 | +HW["910C"].vec_peak = {"fp16": 22.7e12, "bf16": 22.7e12, "fp32": 11.3e12} | ||
| 319 | +HW["910C_superpod"].vec_peak = HW["910C"].vec_peak | ||
| 320 | + | ||
| 321 | + | ||
| 322 | +# ---------- L1 原语: 非 matmul (向量主导, 通常 memory-bound) ---------- | ||
| 323 | +def elementwise_op(n_elem, dtype="bf16", flops_per_elem=1, n_read=1, n_write=1, name="elementwise"): | ||
| 324 | + """逐元素算子(add/gelu/激活...). 向量引擎. bytes=读n_read+写n_write个张量。""" | ||
| 325 | + sz = DTYPE_BYTES[dtype] | ||
| 326 | + return OpSpec(name, flops=n_elem * flops_per_elem, | ||
| 327 | + hbm_bytes=n_elem * (n_read + n_write) * sz, dtype=dtype, engine="vector") | ||
| 328 | + | ||
| 329 | + | ||
| 330 | +def softmax_op(rows, cols, dtype="bf16"): | ||
| 331 | + """softmax: 读1写1(融合) + 每元素~5 FLOPs(max,exp,sum,div). 强 memory-bound。""" | ||
| 332 | + n = rows * cols | ||
| 333 | + return OpSpec("softmax", flops=5 * n, hbm_bytes=n * 2 * DTYPE_BYTES[dtype], | ||
| 334 | + dtype=dtype, engine="vector") | ||
| 335 | + | ||
| 336 | + | ||
| 337 | +def rmsnorm_op(rows, cols, dtype="bf16"): | ||
| 338 | + """RMSNorm: 读x+读weight+写y, 每元素~4 FLOPs(平方/和/rsqrt/乘)。""" | ||
| 339 | + n = rows * cols | ||
| 340 | + return OpSpec("rmsnorm", flops=4 * n, | ||
| 341 | + hbm_bytes=(n + cols + n) * DTYPE_BYTES[dtype], dtype=dtype, engine="vector") | ||
| 342 | + | ||
| 343 | + | ||
| 344 | +# ---------- L2 调度演算: 多级内存 roofline + matmul tiling ---------- | ||
| 345 | +def _matmul_io_lb(M, N, K, cap_bytes, sz): | ||
| 346 | + """matmul I/O 下界 (Hong-Kung): 容量 cap 的快存下, 慢存最少搬运字节。 | ||
| 347 | + = max( 2MNK/sqrt(cap_elems) , 读A+读B+写C 各一次 )。这是**最优 tiling**的搬运量, | ||
| 348 | + 故对应"理论最优调度"(理论最大 MFU), 而非某个具体次优 tiling。""" | ||
| 349 | + import math | ||
| 350 | + cap_elems = max(cap_bytes / sz, 1.0) | ||
| 351 | + q_reuse = 2.0 * M * N * K / math.sqrt(cap_elems) | ||
| 352 | + q_min = M * K + K * N + M * N | ||
| 353 | + return max(q_reuse, q_min) * sz | ||
| 354 | + | ||
| 355 | + | ||
| 356 | +def matmul_tiled_levels(M, N, K, hw: HardwareModel, dtype="bf16"): | ||
| 357 | + """matmul 多级访存量(理论最优 tiling, I/O 下界): | ||
| 358 | + - hbm_bytes: HBM↔L2 搬运, 快存=L2 容量复用 | ||
| 359 | + - l2_bytes : L2↔L1 搬运, 快存=L1 容量复用 | ||
| 360 | + 返回 (flops, hbm_bytes, l2_bytes)。 | ||
| 361 | + """ | ||
| 362 | + sz = DTYPE_BYTES[dtype] | ||
| 363 | + hbm_bytes = _matmul_io_lb(M, N, K, hw.l2_bytes, sz) if hw.l2_bytes else (M * K + K * N + M * N) * sz | ||
| 364 | + l2_bytes = _matmul_io_lb(M, N, K, hw.l1_bytes, sz) if hw.l1_bytes else 0.0 | ||
| 365 | + return 2 * M * N * K, hbm_bytes, l2_bytes | ||
| 366 | + | ||
| 367 | + | ||
| 368 | +def solve_mfu_multilevel(flops, peak, mem_levels, comm=(0.0, 0.0), eta_util=1.0): | ||
| 369 | + """多级 roofline: mem_levels=[(bw,bytes),...] 各级并行可掩盖, comm=(bw,bytes)。 | ||
| 370 | + T_total = max(T_compute, 各级 T_mem, T_comm) (完全掩盖上界)。 | ||
| 371 | + """ | ||
| 372 | + t_compute = flops / peak | ||
| 373 | + t_levels = {f"mem_L{i}": (b / bw if bw else 0.0) for i, (bw, b) in enumerate(mem_levels)} | ||
| 374 | + t_comm = comm[1] / comm[0] if (comm[0] and comm[1]) else 0.0 | ||
| 375 | + times = {"compute": t_compute, "comm": t_comm, **t_levels} | ||
| 376 | + t_total = max(times.values()) | ||
| 377 | + mfu = (t_compute / t_total) * eta_util if t_total > 0 else 0.0 | ||
| 378 | + bottleneck = max(times, key=times.get) | ||
| 379 | + return {"mfu_max": round(mfu, 4), "bottleneck": bottleneck, | ||
| 380 | + "peak_TFLOPS": round(peak / 1e12, 1), | ||
| 381 | + "t_us": {k: round(v * 1e6, 2) for k, v in times.items()}, "eta_util": eta_util} | ||
| 382 | + | ||
| 383 | + | ||
| 384 | +def solve_matmul_tiled(M, N, K, hw: HardwareModel, dtype="bf16", n_dev=1, eta_util=1.0): | ||
| 385 | + """matmul 端到端: tiling 多级访存 + (可选)数据并行梯度通信。""" | ||
| 386 | + flops, hbm_b, l2_b = matmul_tiled_levels(M, N, K, hw, dtype) | ||
| 387 | + comm = (0.0, 0.0) | ||
| 388 | + if n_dev > 1: | ||
| 389 | + comm = (hw.interconnect_bw, allreduce_bytes(K * N * DTYPE_BYTES[dtype], n_dev)) | ||
| 390 | + peak = hw.cube_peak(dtype) | ||
| 391 | + r = solve_mfu_multilevel(flops, peak, [(hw.hbm_bw, hbm_b), (hw.l2_bw, l2_b)], comm, eta_util) | ||
| 392 | + r.update({"op": f"matmul[{M}x{N}x{K}]", "hw": hw.name, "dtype": dtype}) | ||
| 393 | + return r | ||
| @@ -0,0 +1,229 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# -*- coding: utf-8 -*- | ||
| 3 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 12 | + | ||
| 13 | +"""MFU 优化信号：给 ko/fo agent 用。输入算子实测(shape/dtype/device-time)+硬件目标, | ||
| 14 | +输出 {current_mfu, achievable_mfu, gap, bottleneck, lever, done} —— 让优化器以 | ||
| 15 | +'逼近 achievable MFU' 为迭代目标/停止判据, 替代纯 ratio≥1.12x / 5-iter 的相对指标。 | ||
| 16 | + | ||
| 17 | +用法(库): from optimizer_signal import mfu_signal | ||
| 18 | + sig = mfu_signal(flops, hbm_bytes, device_us, hw_name='910C_die', dtype='fp16', | ||
| 19 | + op_kind='matmul', n_aic_used=1) | ||
| 20 | +done 判据: sig['done'] == (current_mfu >= tau*achievable_mfu) 或 杠杆耗尽。 | ||
| 21 | +""" | ||
| 22 | +import sys | ||
| 23 | +import json | ||
| 24 | +from dataclasses import dataclass | ||
| 25 | + | ||
| 26 | +from mfu_model import HW, ETA_UTIL, achievable_mfu_with_overhead, DTYPE_BYTES | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +# op-class -> compute engine. **准度关键(main review 2026-07-01)**: peak 必须按算子类取对。 | ||
| 30 | +# matmul/FA/fused/conv -> CUBE peak; elementwise/softmax/norm/reduce/vector -> VEC peak。 | ||
| 31 | +# 用错 peak(matmul 拿 VEC) => MFU 错 => headroom 错 (roofline_eval 犯过, PR#400 修)。 | ||
| 32 | +OP_ENGINE = { | ||
| 33 | + "matmul": "cube", "flash_attention": "cube", "fused": "cube", "conv": "cube", | ||
| 34 | + "elementwise": "vector", "softmax": "vector", "rmsnorm": "vector", | ||
| 35 | + "layernorm": "vector", "reduce": "vector", "vector": "vector", | ||
| 36 | +} | ||
| 37 | + | ||
| 38 | + | ||
| 39 | +def _peak_for_op_class(hw, op_kind, dtype): | ||
| 40 | + """按算子类取正确的引擎 peak。fail-loud: vec 算子缺 dtype peak 就报错, 绝不静默回落 cube | ||
| 41 | + (静默回落=main 警告的 PR#400-类 bug: 错 peak 错 headroom)。""" | ||
| 42 | + engine = OP_ENGINE.get(op_kind) | ||
| 43 | + if engine is None: | ||
| 44 | + raise ValueError(f"unknown op_kind '{op_kind}' — 显式加进 OP_ENGINE 并确认引擎(cube/vector)") | ||
| 45 | + if engine == "cube": | ||
| 46 | + return hw.cube_peak(dtype) | ||
| 47 | + # vector | ||
| 48 | + if dtype not in hw.vec_peak: | ||
| 49 | + raise ValueError(f"{hw.name}: vector op '{op_kind}' 无 dtype '{dtype}' 的 vec_peak — " | ||
| 50 | + f"须先标定(不静默回落cube, 否则错估 headroom)") | ||
| 51 | + return hw.vec_peak[dtype] | ||
| 52 | + | ||
| 53 | + | ||
| 54 | +@dataclass(frozen=True) | ||
| 55 | +class _MfuMetrics: | ||
| 56 | + """MFU 信号计算中与输入形状无关的中间度量。""" | ||
| 57 | + | ||
| 58 | + hw: object | ||
| 59 | + peak: float | ||
| 60 | + elapsed_s: float | ||
| 61 | + current_tflops: float | ||
| 62 | + current_mfu: float | ||
| 63 | + achievable: float | ||
| 64 | + overhead_bound: bool | ||
| 65 | + | ||
| 66 | + | ||
| 67 | +def _calculate_metrics(flops, device_us, hw_name, dtype, op_kind, t_overhead_us): | ||
| 68 | + """计算当前 MFU、可达 MFU 与固定开销边界标记。""" | ||
| 69 | + hw = HW[hw_name] | ||
| 70 | + peak = _peak_for_op_class(hw, op_kind, dtype) | ||
| 71 | + elapsed_s = device_us * 1e-6 | ||
| 72 | + current_tflops = flops / elapsed_s / 1e12 | ||
| 73 | + current_mfu = flops / (peak * elapsed_s) | ||
| 74 | + t_compute_ideal = flops / peak | ||
| 75 | + overhead_s = t_overhead_us * 1e-6 if t_overhead_us is not None else 0.0 | ||
| 76 | + achievable = achievable_mfu_with_overhead( | ||
| 77 | + flops, | ||
| 78 | + peak, | ||
| 79 | + op_kind, | ||
| 80 | + hw_name, | ||
| 81 | + t_overhead_s=overhead_s, | ||
| 82 | + default_eta=0.85, | ||
| 83 | + )["achievable_mfu"] | ||
| 84 | + return _MfuMetrics( | ||
| 85 | + hw=hw, | ||
| 86 | + peak=peak, | ||
| 87 | + elapsed_s=elapsed_s, | ||
| 88 | + current_tflops=current_tflops, | ||
| 89 | + current_mfu=current_mfu, | ||
| 90 | + achievable=achievable, | ||
| 91 | + overhead_bound=t_compute_ideal / elapsed_s < 0.1, | ||
| 92 | + ) | ||
| 93 | + | ||
| 94 | + | ||
| 95 | +def _classify_bottleneck(flops, hbm_bytes, comm_bytes, metrics): | ||
| 96 | + """按 roofline 和通信时间确定当前的主要瓶颈。""" | ||
| 97 | + arithmetic_intensity = flops / hbm_bytes if hbm_bytes else float("inf") | ||
| 98 | + ridge = metrics.peak / metrics.hw.hbm_bw | ||
| 99 | + if ( | ||
| 100 | + comm_bytes | ||
| 101 | + and metrics.hw.interconnect_bw | ||
| 102 | + and comm_bytes / metrics.hw.interconnect_bw > metrics.elapsed_s * 0.5 | ||
| 103 | + ): | ||
| 104 | + return "comm", arithmetic_intensity, ridge | ||
| 105 | + if arithmetic_intensity < ridge: | ||
| 106 | + return "memory", arithmetic_intensity, ridge | ||
| 107 | + return "compute", arithmetic_intensity, ridge | ||
| 108 | + | ||
| 109 | + | ||
| 110 | +def _direction_advice( | ||
| 111 | + op_kind, | ||
| 112 | + n_aic_used, | ||
| 113 | + comm_bytes, | ||
| 114 | + bottleneck, | ||
| 115 | + arithmetic_intensity, | ||
| 116 | + ridge, | ||
| 117 | + metrics, | ||
| 118 | + tau, | ||
| 119 | +): | ||
| 120 | + """只为已验证的 Cube 算子产生具体优化方向。""" | ||
| 121 | + engine = OP_ENGINE.get(op_kind) | ||
| 122 | + validated_direction = op_kind in ("matmul", "mm_grad", "flash_attention", "fused") | ||
| 123 | + if not validated_direction: | ||
| 124 | + note = ( | ||
| 125 | + f"⚠方向弃权: op-class '{op_kind}' 未验证 —— roofline 对 vector/issue-bound " | ||
| 126 | + "会误分类(busy-ratio≠bound; 盲 issue-count+latency-hiding/double-buffer). " | ||
| 127 | + "只给 ceiling/stop; 定 bound + lever 需实测 msprof pipe-ratio." | ||
| 128 | + ) | ||
| 129 | + return [], note, False | ||
| 130 | + | ||
| 131 | + levers = [] | ||
| 132 | + if engine == "cube" and n_aic_used is not None and n_aic_used < metrics.hw.n_aic: | ||
| 133 | + multiplier = metrics.hw.n_aic / n_aic_used | ||
| 134 | + levers.append( | ||
| 135 | + f"multi-AIC: blockDim {n_aic_used}->{metrics.hw.n_aic} " | ||
| 136 | + f"(~{multiplier:.0f}x, P-P74 M/N tiling)" | ||
| 137 | + ) | ||
| 138 | + if bottleneck == "memory": | ||
| 139 | + levers.append( | ||
| 140 | + f"raise arithmetic intensity (AI={arithmetic_intensity:.0f}<ridge={ridge:.0f}): " | ||
| 141 | + "增大tile/合并/算子融合" | ||
| 142 | + ) | ||
| 143 | + if bottleneck == "compute" and metrics.current_mfu < metrics.achievable * tau: | ||
| 144 | + levers.append("compute-bound但低于achievable: 查tiling/尾块/流水(η gap)") | ||
| 145 | + if comm_bytes and bottleneck == "comm": | ||
| 146 | + levers.append("comm-bound: 梯度累积/通信压缩/通算重叠") | ||
| 147 | + return levers, None, True | ||
| 148 | + | ||
| 149 | + | ||
| 150 | +def _build_verdict(metrics, tau, applicable_direction, direction_note, levers): | ||
| 151 | + """保持 ceiling-based 停止判据和面向调用方的 verdict 文本。""" | ||
| 152 | + gap = metrics.current_mfu / metrics.achievable if metrics.achievable else 0.0 | ||
| 153 | + done = metrics.current_mfu >= tau * metrics.achievable | ||
| 154 | + if not applicable_direction: | ||
| 155 | + verdict = ( | ||
| 156 | + f"ABSTAIN(direction): at {metrics.current_mfu * 100:.1f}% MFU, " | ||
| 157 | + f"{gap * 100:.0f}% of achievable; ceiling given, lever withheld — {direction_note}" | ||
| 158 | + ) | ||
| 159 | + elif done: | ||
| 160 | + verdict = "DONE (≥%.0f%% achievable)" % (tau * 100) | ||
| 161 | + else: | ||
| 162 | + verdict = "CONTINUE: at %.1f%% MFU, %.0f%% of achievable — %s" % ( | ||
| 163 | + metrics.current_mfu * 100, | ||
| 164 | + gap * 100, | ||
| 165 | + levers[0] if levers else "?", | ||
| 166 | + ) | ||
| 167 | + return gap, done, verdict | ||
| 168 | + | ||
| 169 | + | ||
| 170 | +def mfu_signal(flops, hbm_bytes, device_us, hw_name="910C_die", dtype="fp16", | ||
| 171 | + op_kind="matmul", n_aic_used=None, comm_bytes=0.0, tau=0.8, | ||
| 172 | + t_overhead_us=None): | ||
| 173 | + """返回优化信号 dict。n_aic_used: 实际用的 cube 核数(单AIC算子=1)，用于诊断 multi-AIC 杠杆。 | ||
| 174 | + t_overhead_us: launch/tail 固定开销(若知)。**iteration-2 教训(2026-06-30)**: 小/overhead-bound | ||
| 175 | + 算子用 flat η 会 over-state headroom(mm_grad tiny shape: compute 0.74us 但跑 67us, vendor 也仅 | ||
| 176 | + 1.1% MFU -> 真 achievable≈1.1% 非 88%)。传 t_overhead_us 后用 overhead-corrected achievable, | ||
| 177 | + 避免在 overhead-bound 算子上追幻影 headroom。 | ||
| 178 | + """ | ||
| 179 | + metrics = _calculate_metrics(flops, device_us, hw_name, dtype, op_kind, t_overhead_us) | ||
| 180 | + bottleneck, arithmetic_intensity, ridge = _classify_bottleneck( | ||
| 181 | + flops, | ||
| 182 | + hbm_bytes, | ||
| 183 | + comm_bytes, | ||
| 184 | + metrics, | ||
| 185 | + ) | ||
| 186 | + levers, direction_note, applicable_direction = _direction_advice( | ||
| 187 | + op_kind, | ||
| 188 | + n_aic_used, | ||
| 189 | + comm_bytes, | ||
| 190 | + bottleneck, | ||
| 191 | + arithmetic_intensity, | ||
| 192 | + ridge, | ||
| 193 | + metrics, | ||
| 194 | + tau, | ||
| 195 | + ) | ||
| 196 | + if metrics.overhead_bound and t_overhead_us is None: | ||
| 197 | + levers = ["⚠overhead-bound (compute<10%实测): 传 t_overhead_us 才能得真achievable; " | ||
| 198 | + "flat-η会over-state headroom(iter2教训)"] + levers | ||
| 199 | + gap, done, verdict = _build_verdict( | ||
| 200 | + metrics, | ||
| 201 | + tau, | ||
| 202 | + applicable_direction, | ||
| 203 | + direction_note, | ||
| 204 | + levers, | ||
| 205 | + ) | ||
| 206 | + return { | ||
| 207 | + "current_mfu": round(metrics.current_mfu, 4), | ||
| 208 | + "current_TFLOPS": round(metrics.current_tflops, 1), | ||
| 209 | + "achievable_mfu": round(metrics.achievable, 4), | ||
| 210 | + "peak_TFLOPS": round(metrics.peak / 1e12, 1), | ||
| 211 | + "gap_to_achievable": round(gap, 3), | ||
| 212 | + "bottleneck": bottleneck, | ||
| 213 | + "overhead_bound": metrics.overhead_bound, | ||
| 214 | + "applicable_direction": applicable_direction, | ||
| 215 | + "direction_note": direction_note, | ||
| 216 | + "levers": levers, | ||
| 217 | + "done": done, | ||
| 218 | + "verdict": verdict, | ||
| 219 | + } | ||
| 220 | + | ||
| 221 | + | ||
| 222 | +if __name__ == "__main__": | ||
| 223 | + # demo: mm_grad [512,256,512] fp16 单AIC, 78us, 2 GEMM | ||
| 224 | + M, K, N = 512, 256, 512 | ||
| 225 | + flops = 2 * M * K * N * 2 | ||
| 226 | + hbm = (M * K + K * N + M * N) * DTYPE_BYTES["fp16"] * 2 | ||
| 227 | + sig = mfu_signal(flops, hbm, 78.0, "910C_die", "fp16", "matmul", n_aic_used=1) | ||
| 228 | + print("mm_grad[512,256,512] fp16 单AIC 78us:") | ||
| 229 | + print(json.dumps(sig, ensure_ascii=False, indent=2)) | ||
| @@ -0,0 +1,268 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# -*- coding: utf-8 -*- | ||
| 3 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 12 | +"""optimizer_signal / mfu_model 的测试。验证 MFU 信号对已知输入给出正确的 | ||
| 13 | +current_mfu / achievable / bottleneck / lever / done 判定。 | ||
| 14 | + | ||
| 15 | +run: python3 mfu/test_optimizer_signal.py (exit 0 = all pass) | ||
| 16 | +""" | ||
| 17 | +import logging | ||
| 18 | +import sys | ||
| 19 | +import os | ||
| 20 | +from pathlib import Path | ||
| 21 | +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | ||
| 22 | +from mfu_model import HW, DTYPE_BYTES, flash_attention_ops, solve_mfu, matmul_ops | ||
| 23 | +from optimizer_signal import mfu_signal | ||
| 24 | + | ||
| 25 | +FAILS = [] | ||
| 26 | +LOGGER = logging.getLogger(__name__) | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +def check(name, cond, detail=""): | ||
| 30 | + LOGGER.info(" [%s] %s%s", "PASS" if cond else "FAIL", name, | ||
| 31 | + f" — {detail}" if detail and not cond else "") | ||
| 32 | + if not cond: | ||
| 33 | + FAILS.append(name) | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +def test_mm_grad_single_aic_flags_continue(): | ||
| 37 | + """mm_grad 单AIC shipped op: 必须判 CONTINUE + 低 MFU + multi-AIC 杠杆(owner实证)。""" | ||
| 38 | + M, K, N = 512, 256, 512 | ||
| 39 | + flops = 2 * M * K * N * 2 | ||
| 40 | + hbm = (M * K + K * N + M * N) * DTYPE_BYTES["fp16"] * 2 | ||
| 41 | + s = mfu_signal(flops, hbm, 78.0, "910C_die", "fp16", "matmul", n_aic_used=1) | ||
| 42 | + check("mm_grad current_mfu<5%", s["current_mfu"] < 0.05, f"got {s['current_mfu']}") | ||
| 43 | + check("mm_grad NOT done", s["done"] is False) | ||
| 44 | + check("mm_grad lever 含 multi-AIC", any("multi-AIC" in l for l in s["levers"])) | ||
| 45 | + check("mm_grad gap<<1", s["gap_to_achievable"] < 0.1, f"got {s['gap_to_achievable']}") | ||
| 46 | + | ||
| 47 | + | ||
| 48 | +def test_near_ceiling_matmul_is_done(): | ||
| 49 | + """充分优化的大 matmul (用满24AIC, 实测≈理论×η): 应判 done。""" | ||
| 50 | + M = K = N = 4096 | ||
| 51 | + flops = 2 * M * K * N | ||
| 52 | + hbm = (M * K + K * N + M * N) * DTYPE_BYTES["bf16"] | ||
| 53 | + peak = HW["910C_die"].cube_peak("bf16") | ||
| 54 | + # 实测时间 = 理论compute/0.88 (即 η=0.88, 接近 achievable) -> 满24AIC | ||
| 55 | + us = flops / peak / 0.88 * 1e6 | ||
| 56 | + s = mfu_signal(flops, hbm, us, "910C_die", "bf16", "matmul", n_aic_used=24) | ||
| 57 | + check("near-ceiling done", s["done"] is True, f"mfu={s['current_mfu']} ach={s['achievable_mfu']}") | ||
| 58 | + check("near-ceiling current≈achievable", s["gap_to_achievable"] >= 0.8, f"got {s['gap_to_achievable']}") | ||
| 59 | + | ||
| 60 | + | ||
| 61 | +def test_memory_bound_op_flags_AI_lever(): | ||
| 62 | + """memory-bound 算子(瘦GEMV): 必须判 memory + AI 杠杆。""" | ||
| 63 | + M, K, N = 1, 4096, 4096 | ||
| 64 | + flops = 2 * M * K * N | ||
| 65 | + hbm = (M * K + K * N + M * N) * DTYPE_BYTES["bf16"] | ||
| 66 | + us = hbm / HW["910C_die"].hbm_bw * 1e6 # 跑在 HBM 带宽上限 | ||
| 67 | + s = mfu_signal(flops, hbm, us, "910C_die", "bf16", "matmul", n_aic_used=24) | ||
| 68 | + check("GEMV bottleneck=memory", s["bottleneck"] == "memory", f"got {s['bottleneck']}") | ||
| 69 | + check("GEMV lever 含 arithmetic intensity", any("arithmetic" in l or "AI" in l for l in s["levers"])) | ||
| 70 | + | ||
| 71 | + | ||
| 72 | +def test_ceiling_never_exceeded(): | ||
| 73 | + """天花板不被突破: 任何实测的 current_mfu 不应 > 1.0 (除非time太小=峰值口径错)。""" | ||
| 74 | + # 用实测 A3 matmul 321 TFLOPS (n=4096 bf16) vs 理论 363 | ||
| 75 | + M = K = N = 4096 | ||
| 76 | + flops = 2 * M * K * N | ||
| 77 | + us = flops / 321e12 * 1e6 | ||
| 78 | + s = mfu_signal(flops, (M * K + K * N + M * N) * 2, us, "910C_die", "bf16", "matmul", n_aic_used=24) | ||
| 79 | + check("A3实测matmul MFU≈0.88", abs(s["current_mfu"] - 0.88) < 0.05, f"got {s['current_mfu']}") | ||
| 80 | + check("天花板不破 current_mfu<=1.0", s["current_mfu"] <= 1.0) | ||
| 81 | + | ||
| 82 | + | ||
| 83 | +def test_model_matches_kb_fa_case(): | ||
| 84 | + """模型对 KB 实测 FA case (B1H8S2048D128 fp16 causal): compute-bound, AI≈512, ceiling≈17us。""" | ||
| 85 | + fwd, _ = flash_attention_ops(1, 8, 2048, 128, "fp16", causal=True) | ||
| 86 | + r = solve_mfu(fwd, HW["950PR"]) | ||
| 87 | + check("KB-FA compute-bound", r["bottleneck"] == "compute", f"got {r['bottleneck']}") | ||
| 88 | + check("KB-FA AI≈512", abs(r["arithmetic_intensity"] - 512) < 5, f"got {r['arithmetic_intensity']}") | ||
| 89 | + | ||
| 90 | + | ||
| 91 | +def test_overhead_bound_flagged_and_corrected(): | ||
| 92 | + """iter2教训: tiny mm_grad shape overhead-bound — 无overhead时须flag警告; | ||
| 93 | + 传t_overhead后achievable下修, 近vendor水平的实测应判done。 | ||
| 94 | + """ | ||
| 95 | + M, K, N = 512, 256, 512 | ||
| 96 | + flops = 2 * M * K * N * 2 | ||
| 97 | + hbm = (M * K + K * N + M * N) * DTYPE_BYTES["fp16"] * 2 | ||
| 98 | + # 无overhead: 应 flag overhead_bound + 警告lever (compute0.74us«67us) | ||
| 99 | + s = mfu_signal(flops, hbm, 67.2, "910C_die", "fp16", "matmul", n_aic_used=24) | ||
| 100 | + check("overhead_bound flagged", s.get("overhead_bound") is True, f"got {s.get('overhead_bound')}") | ||
| 101 | + check("warns flat-η over-states", any("overhead-bound" in l for l in s["levers"])) | ||
| 102 | + # 传t_overhead=65us: achievable下修到~vendor水平, v2(1.1%)≈achievable -> done | ||
| 103 | + s2 = mfu_signal(flops, hbm, 67.2, "910C_die", "fp16", "matmul", n_aic_used=24, t_overhead_us=65.0) | ||
| 104 | + check("overhead-corrected achievable low", s2["achievable_mfu"] < 0.05, f"got {s2['achievable_mfu']}") | ||
| 105 | + check("near-vendor tiny op -> done", s2["done"] is True, f"mfu={s2['current_mfu']} ach={s2['achievable_mfu']}") | ||
| 106 | + | ||
| 107 | + | ||
| 108 | +def test_peak_source_per_op_class(): | ||
| 109 | + """main accuracy-critical gate: peak 必须按算子类取对。matmul->CUBE, elementwise->VEC。 | ||
| 110 | + 用错 peak => MFU 错 => headroom 错 (PR#400-类 bug)。且 vec 缺 dtype 须 fail-loud 不静默回落 cube。 | ||
| 111 | + """ | ||
| 112 | + from optimizer_signal import _peak_for_op_class | ||
| 113 | + hw = HW["950PR"] | ||
| 114 | + cube = hw.cube_peak("fp16") # ~378 TF | ||
| 115 | + vec = hw.vec_peak["fp16"] # 54 TF | ||
| 116 | + check("matmul->CUBE peak", _peak_for_op_class(hw, "matmul", "fp16") == cube) | ||
| 117 | + check("elementwise->VEC peak", _peak_for_op_class(hw, "elementwise", "fp16") == vec) | ||
| 118 | + check("cube!=vec (否则测试无意义)", cube != vec) | ||
| 119 | + # fail-loud: vector op 缺 dtype 的 vec_peak 不静默回落 cube | ||
| 120 | + raised = False | ||
| 121 | + try: | ||
| 122 | + _peak_for_op_class(hw, "softmax", "fp8") # fp8 不在 vec_peak | ||
| 123 | + except ValueError: | ||
| 124 | + raised = True | ||
| 125 | + check("vec缺dtype fail-loud(不回落cube)", raised) | ||
| 126 | + # unknown op_kind fail-loud | ||
| 127 | + raised2 = False | ||
| 128 | + try: | ||
| 129 | + _peak_for_op_class(hw, "mystery_op", "fp16") | ||
| 130 | + except ValueError: | ||
| 131 | + raised2 = True | ||
| 132 | + check("unknown op_kind fail-loud", raised2) | ||
| 133 | + | ||
| 134 | + | ||
| 135 | +def test_eta_provenance_present(): | ||
| 136 | + """main纪律: 每个标定的 η 须有 provenance(来源+日期+revalidate触发), 防 stale-η 错估。""" | ||
| 137 | + from mfu_model import eta_with_provenance | ||
| 138 | + | ||
| 139 | + cases = [ | ||
| 140 | + ("matmul", "910C_die"), | ||
| 141 | + ("matmul", "950PR"), | ||
| 142 | + ("flash_attention", "910C_die"), | ||
| 143 | + ("flash_attention", "950PR"), | ||
| 144 | + ] | ||
| 145 | + for op, hw in cases: | ||
| 146 | + eta, prov = eta_with_provenance(op, hw) | ||
| 147 | + check( | ||
| 148 | + f"{op}/{hw} η有provenance", | ||
| 149 | + prov is not None and "measured" in prov and "revalidate_on" in prov, | ||
| 150 | + f"prov={prov}", | ||
| 151 | + ) | ||
| 152 | + | ||
| 153 | + | ||
| 154 | +def test_verification_hook_injects_mfu_ceiling(): | ||
| 155 | + """机械注入: hook 读 verification.json -> 写 mfu_ceiling 块(ko 读的字段)。 | ||
| 156 | + 验证低-MFU op 被正确标 CONTINUE + multi-AIC lever, 且 mfu_ceiling 进了 json。 | ||
| 157 | + """ | ||
| 158 | + import json | ||
| 159 | + import tempfile | ||
| 160 | + from verification_hook import inject_mfu_ceiling | ||
| 161 | + # 造一个最小 verification.json(mm_grad 单AIC 实测形态) | ||
| 162 | + vj = {"op": "mm_grad", | ||
| 163 | + "precision": {"cases": [{"MKN": [512, 256, 512], "dtype": "float16"}]}, | ||
| 164 | + "performance": {"ours_us_two_runs": {"[512, 256, 512]": [76.6, 76.6]}}} | ||
| 165 | + fd, p = tempfile.mkstemp(suffix=".json") | ||
| 166 | + os.close(fd) | ||
| 167 | + Path(p).write_text(json.dumps(vj), encoding="utf-8") | ||
| 168 | + c = inject_mfu_ceiling(p, op_kind="mm_grad", hw_name="910C_die", n_aic_used=1) | ||
| 169 | + reloaded = json.loads(Path(p).read_text(encoding="utf-8")) | ||
| 170 | + os.unlink(p) | ||
| 171 | + check("mfu_ceiling written into json", "mfu_ceiling" in reloaded) | ||
| 172 | + sig = c["per_case"].get("[512,256,512]") or list(c["per_case"].values())[0] | ||
| 173 | + check("hook flags low-MFU CONTINUE", sig["done"] is False, f"got {sig}") | ||
| 174 | + check("hook surfaces multi-AIC lever", any("multi-AIC" in l for l in sig["levers"])) | ||
| 175 | + check("hook current_mfu ~1%", sig["current_mfu"] < 0.05, f"got {sig['current_mfu']}") | ||
| 176 | + | ||
| 177 | + | ||
| 178 | +def test_pipeline_injection_via_phase_o5(): | ||
| 179 | + """真链入(main gate 要求): phase_o5 perf 写完 verification.json 后, _inject_mfu_ceiling_safe | ||
| 180 | + 机械注入 mfu_ceiling — 非死代码. op_kind 从 op-class 取(非写死). fail-safe. | ||
| 181 | + """ | ||
| 182 | + import json | ||
| 183 | + import tempfile | ||
| 184 | + import shutil | ||
| 185 | + import sys as _s | ||
| 186 | + root = Path(__file__).resolve().parents[1] | ||
| 187 | + _s.path.insert(0, str(root / "src/scripts/orchestrator")) | ||
| 188 | + try: | ||
| 189 | + import phase_o5 | ||
| 190 | + except Exception as e: | ||
| 191 | + check("phase_o5 importable", False, f"import failed: {e!r}") | ||
| 192 | + return | ||
| 193 | + ws = Path(tempfile.mkdtemp()) | ||
| 194 | + try: | ||
| 195 | + vj = {"op": "mm_grad", "target": "a3", "soc_version": "Ascend910_9392", | ||
| 196 | + "precision": {"cases": [{"MKN": [512, 256, 512], "dtype": "float16"}]}, | ||
| 197 | + "performance": {"ours_us_two_runs": {"[512, 256, 512]": [76.6, 76.6]}}} | ||
| 198 | + vp = ws / "verification.json" | ||
| 199 | + vp.write_text(json.dumps(vj)) | ||
| 200 | + (ws / "op_classification.json").write_text(json.dumps({"class": "matmul_backward"})) | ||
| 201 | + check("op_kind from op-class (not hardcoded)", getattr(phase_o5, "_mfu_op_kind")(ws, "mm_grad") == "mm_grad") | ||
| 202 | + getattr(phase_o5, "_inject_mfu_ceiling_safe")(vp, ws, "mm_grad", vj) | ||
| 203 | + out = json.loads(vp.read_text()) | ||
| 204 | + check("pipeline writes mfu_ceiling (not dead code)", "mfu_ceiling" in out) | ||
| 205 | + pc = out.get("mfu_ceiling", {}).get("per_case", {}) | ||
| 206 | + sig = list(pc.values())[0] if pc else {} | ||
| 207 | + check("pipeline per_case has done-signal", sig.get("done") is False, f"got {sig}") | ||
| 208 | + # fail-safe: garbage verification never raises | ||
| 209 | + bad = ws / "bad.json" | ||
| 210 | + bad.write_text("{not json") | ||
| 211 | + getattr(phase_o5, "_inject_mfu_ceiling_safe")(bad, ws, "mm_grad", {}) # must not raise | ||
| 212 | + check("fail-safe on bad input (no raise)", True) | ||
| 213 | + finally: | ||
| 214 | + shutil.rmtree(ws) | ||
| 215 | + | ||
| 216 | + | ||
| 217 | +def test_applicability_gate_abstains_on_vector(): | ||
| 218 | + """owner-approved 安全 gate: vector/未验证 op-class -> 方向弃权(不发误导 lever), 仍给 ceiling/stop. | ||
| 219 | + 复现 selective_scan 场景: 绝不出 raise-AI/fuse/multi-AIC(误导). | ||
| 220 | + """ | ||
| 221 | + # selective_scan-like vector op (real numbers from independent review) | ||
| 222 | + flops = int(1.72e9) | ||
| 223 | + hbm = 225 * 2**20 | ||
| 224 | + s = mfu_signal(flops, hbm, 7134.0, "950PR", "fp16", "softmax", n_aic_used=1) | ||
| 225 | + check("vector op -> applicable_direction=False", | ||
| 226 | + s["applicable_direction"] is False, f"got {s.get('applicable_direction')}") | ||
| 227 | + check("vector op -> NO misleading lever emitted", not any(("raise arithmetic" in l or "multi-AIC" in l) | ||
| 228 | + for l in s["levers"]), f"levers={s['levers']}") | ||
| 229 | + check("vector op -> direction_note explains abstain", s.get("direction_note") is not None) | ||
| 230 | + check("vector op -> verdict says ABSTAIN", "ABSTAIN" in s["verdict"]) | ||
| 231 | + check("vector op -> ceiling still given (current_mfu present)", s["current_mfu"] > 0) | ||
| 232 | + # validated op (matmul) still emits levers (gate doesn't over-suppress) | ||
| 233 | + s2 = mfu_signal(2 * 512 * 256 * 512, (512 * 256 + 256 * 512 + 512 * 512) | ||
| 234 | + * 2, 76.6, "910C_die", "fp16", "matmul", n_aic_used=1) | ||
| 235 | + check("matmul still applicable_direction=True", s2["applicable_direction"] is True) | ||
| 236 | + check("matmul still emits multi-AIC lever", any("multi-AIC" in l for l in s2["levers"])) | ||
| 237 | + | ||
| 238 | + | ||
| 239 | +def test_multi_aic_lever_engine_aware(): | ||
| 240 | + """multi-AIC lever 绝不出在 vector(AIV) 算子上(selective_scan bug: cube-28x 错用在 aic=0 的 AIV 算子).""" | ||
| 241 | + # 即便强行传 vector op + n_aic_used, 也因弃权不发 multi-AIC | ||
| 242 | + s = mfu_signal(int(1.72e9), 225 * 2**20, 7134.0, "950PR", "fp16", "elementwise", n_aic_used=1) | ||
| 243 | + check("vector op never gets cube multi-AIC lever", | ||
| 244 | + not any("multi-AIC" in l for l in s["levers"]), f"levers={s['levers']}") | ||
| 245 | + | ||
| 246 | + | ||
| 247 | +def test_matmul_flops_accounting(): | ||
| 248 | + """matmul fwd/bwd FLOPs 记账正确: bwd=2×fwd。""" | ||
| 249 | + fwd, bwd = matmul_ops(512, 512, 512, "bf16") | ||
| 250 | + check("matmul fwd flops=2MNK", fwd.flops == 2 * 512**3) | ||
| 251 | + check("matmul bwd flops=4MNK(2x fwd)", bwd.flops == 4 * 512**3) | ||
| 252 | + | ||
| 253 | + | ||
| 254 | +if __name__ == "__main__": | ||
| 255 | + for t in [test_mm_grad_single_aic_flags_continue, test_near_ceiling_matmul_is_done, | ||
| 256 | + test_memory_bound_op_flags_AI_lever, test_ceiling_never_exceeded, | ||
| 257 | + test_model_matches_kb_fa_case, test_overhead_bound_flagged_and_corrected, | ||
| 258 | + test_peak_source_per_op_class, test_eta_provenance_present, | ||
| 259 | + test_verification_hook_injects_mfu_ceiling, | ||
| 260 | + test_pipeline_injection_via_phase_o5, | ||
| 261 | + test_applicability_gate_abstains_on_vector, | ||
| 262 | + test_multi_aic_lever_engine_aware, | ||
| 263 | + test_matmul_flops_accounting]: | ||
| 264 | + LOGGER.info("%s:", t.__name__) | ||
| 265 | + t() | ||
| 266 | + LOGGER.info("%s\n%s", "=" * 50, | ||
| 267 | + "ALL PASS" if not FAILS else f"{len(FAILS)} FAILED: {FAILS}") | ||
| 268 | + sys.exit(1 if FAILS else 0) | ||
| @@ -0,0 +1,73 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# -*- coding: utf-8 -*- | ||
| 3 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 12 | + | ||
| 13 | +"""验证harness: 模型预测 MFU_max vs msprof 实测 -> η_util + 预测准确性表。 | ||
| 14 | +NPU 到位前用示例数据跑通格式; 拿到 msprof 后把 measured_us 换成实测即可。 | ||
| 15 | + | ||
| 16 | +用法: | ||
| 17 | + python3 compare.py # 跑示例(展示对照表格式) | ||
| 18 | + # 真机: 解析 msprof 输出得各算子 device time, 填入 CASES 的 measured_us | ||
| 19 | +""" | ||
| 20 | +import sys | ||
| 21 | +import os | ||
| 22 | +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | ||
| 23 | +from mfu_model import HW, flash_attention_ops, matmul_ops, solve_mfu, solve_matmul_tiled | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +def predict(case): | ||
| 27 | + """返回模型预测 (mfu_max, bottleneck, t_ideal_us)。""" | ||
| 28 | + hw = HW[case["hw"]] | ||
| 29 | + k = case["kind"] | ||
| 30 | + if k == "matmul": | ||
| 31 | + r = solve_matmul_tiled(case["M"], case["N"], case["K"], hw, case["dtype"]) | ||
| 32 | + t_ideal = min(r["t_us"].values()) # t_compute (理想) | ||
| 33 | + return r["mfu_max"], r["bottleneck"], r["t_us"].get("compute", t_ideal) | ||
| 34 | + elif k == "fa": | ||
| 35 | + fwd, _ = flash_attention_ops(case["B"], case["H"], case["S"], case["D"], | ||
| 36 | + case["dtype"], case.get("causal", False)) | ||
| 37 | + r = solve_mfu(fwd, hw) | ||
| 38 | + return r["mfu_max"], r["bottleneck"], r["t_compute_us"] | ||
| 39 | + raise ValueError(k) | ||
| 40 | + | ||
| 41 | + | ||
| 42 | +def report(cases): | ||
| 43 | + print(f"{'case':32} {'hw':8} {'pred_MFU':>8} {'bottleneck':>11} " | ||
| 44 | + f"{'t_ideal_us':>10} {'meas_us':>9} {'meas_MFU':>8} {'eta':>6}") | ||
| 45 | + for c in cases: | ||
| 46 | + mfu_max, bn, t_ideal = predict(c) | ||
| 47 | + meas = c.get("measured_us") | ||
| 48 | + if meas: | ||
| 49 | + meas_mfu = round(t_ideal / meas * mfu_max, 3) # 实测MFU = 理论compute时间/实测墙钟 × 理论MFU基 | ||
| 50 | + eta = round(meas_mfu / mfu_max, 3) if mfu_max else 0 | ||
| 51 | + ok = "" if meas >= t_ideal - 1e-6 else " ⚠天花板被突破!" | ||
| 52 | + else: | ||
| 53 | + meas_mfu = eta = "-" | ||
| 54 | + ok = "" | ||
| 55 | + print(f"{c['name']:32} {c['hw']:8} {mfu_max:>8} {bn:>11} " | ||
| 56 | + f"{round(t_ideal,2):>10} {str(meas):>9} {str(meas_mfu):>8} {str(eta):>6}{ok}") | ||
| 57 | + print("\nη_util = 实测MFU / 理论MFU_max (有效利用率); <1 正常(tiling/尾块/流水损耗)。" | ||
| 58 | + "\n实测>理论(天花板被突破)=> 模型 peak/BW 口径偏低, 需修。") | ||
| 59 | + | ||
| 60 | + | ||
| 61 | +# 待 msprof 填 measured_us 的 case 列表(measured_us=None 占位) | ||
| 62 | +CASES = [ | ||
| 63 | + {"name": "FA dense B1H8S2048D128 causal", "kind": "fa", "hw": "950PR", | ||
| 64 | + "B": 1, "H": 8, "S": 2048, "D": 128, "dtype": "fp16", "causal": True, | ||
| 65 | + "measured_us": 17.0}, # KB 实测 ceiling 0.017ms (示例: 接近理论 -> η≈1) | ||
| 66 | + {"name": "matmul 4096^3 bf16", "kind": "matmul", "hw": "950PR", | ||
| 67 | + "M": 4096, "N": 4096, "K": 4096, "dtype": "bf16", "measured_us": None}, | ||
| 68 | + {"name": "matmul 4096^3 bf16", "kind": "matmul", "hw": "910C", | ||
| 69 | + "M": 4096, "N": 4096, "K": 4096, "dtype": "bf16", "measured_us": None}, | ||
| 70 | +] | ||
| 71 | + | ||
| 72 | +if __name__ == "__main__": | ||
| 73 | + report(CASES) | ||
| @@ -0,0 +1,111 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# -*- coding: utf-8 -*- | ||
| 3 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 12 | + | ||
| 13 | +"""把 MFU 天花板信号**机械注入** verification.json —— ko/fo 读的正是这个文件。 | ||
| 14 | +这样 MFU 不是"agent 记得跑脚本才用"，而是**每次 perf 后自动进 ko 读的数据**（owner: | ||
| 15 | +建模若 ko/fo 不用就等于没用 -> 把它变成 ko 必然读到的字段）。 | ||
| 16 | + | ||
| 17 | +集成点（pipeline）: | ||
| 18 | + perf 步骤产出 verification.json 后 -> 调 inject_mfu_ceiling() -> 写入 mfu_ceiling 块 -> | ||
| 19 | + ko/fo 的 done 判据读 verification.json['mfu_ceiling']（不依赖 agent 记得跑脚本）。 | ||
| 20 | + | ||
| 21 | +FLOPs 是 op-语义: 优先读 verification 里 perf-case 的 'flops' 字段(perf 步骤知道 op 时填); | ||
| 22 | +无则用 builtin 公式(matmul/mm_grad); 都没有则该 case skip 并记原因(fail-safe, 不静默造假)。 | ||
| 23 | +""" | ||
| 24 | +import json | ||
| 25 | +import sys | ||
| 26 | +import statistics | ||
| 27 | +sys.path.insert(0, __file__.rsplit("/", 1)[0]) | ||
| 28 | +from optimizer_signal import mfu_signal | ||
| 29 | +from mfu_model import DTYPE_BYTES | ||
| 30 | + | ||
| 31 | +# builtin FLOPs 公式(op_kind -> f(M,K,N)). 扩展新 op 在此加, 或让 perf 步骤直接填 flops 字段。 | ||
| 32 | +BUILTIN_FLOPS = { | ||
| 33 | + "matmul": lambda M, K, N: 2 * M * K * N, | ||
| 34 | + "mm_grad": lambda M, K, N: 2 * M * K * N * 2, # dA + dB 两个 GEMM | ||
| 35 | +} | ||
| 36 | +BUILTIN_HBM = { # 最小 HBM 字节(理想), 用于 memory-bound 判据 | ||
| 37 | + "matmul": lambda M, K, N, sz: (M * K + K * N + M * N) * sz, | ||
| 38 | + "mm_grad": lambda M, K, N, sz: (M * N + K * N + M * K + M * K + M * N + K * N) * sz, | ||
| 39 | +} | ||
| 40 | + | ||
| 41 | + | ||
| 42 | +def _parse_mkn(shape_key): | ||
| 43 | + """'[512, 256, 512]' -> (512,256,512).""" | ||
| 44 | + return tuple(int(x) for x in shape_key.strip("[]").split(",")) | ||
| 45 | + | ||
| 46 | + | ||
| 47 | +def inject_mfu_ceiling(vjson_path, op_kind="matmul", hw_name="910C_die", | ||
| 48 | + n_aic_used=None, t_overhead_us=None, engine_op_kind=None, | ||
| 49 | + write=True): | ||
| 50 | + """读 verification.json -> 每个 perf case 算 MFU 信号 -> 写入 mfu_ceiling 块。 | ||
| 51 | + engine_op_kind: 传给 mfu_signal 的算子类(选 CUBE/VEC peak); 默认同 op_kind。 | ||
| 52 | + 返回注入的 mfu_ceiling dict。 | ||
| 53 | + """ | ||
| 54 | + with open(vjson_path) as f: | ||
| 55 | + v = json.load(f) | ||
| 56 | + perf = v.get("performance", {}) | ||
| 57 | + ours = perf.get("ours_us_two_runs") or perf.get("ours_us") or {} | ||
| 58 | + # dtype: 从 precision.cases 按 MKN 匹配, 缺省 fp16 | ||
| 59 | + dtype_by_mkn = {} | ||
| 60 | + for c in v.get("precision", {}).get("cases", []): | ||
| 61 | + mkn = tuple(c.get("MKN", [])) if "MKN" in c else None | ||
| 62 | + if mkn: | ||
| 63 | + dtype_by_mkn[mkn] = c.get("dtype", "fp16") | ||
| 64 | + | ||
| 65 | + eok = engine_op_kind or ("matmul" if op_kind in ("matmul", "mm_grad") else op_kind) | ||
| 66 | + per_case, all_done, skipped = {}, [], [] | ||
| 67 | + for shape_key, us_val in ours.items(): | ||
| 68 | + try: | ||
| 69 | + M, K, N = _parse_mkn(shape_key) | ||
| 70 | + except Exception: | ||
| 71 | + skipped.append((shape_key, "unparseable shape")) | ||
| 72 | + continue | ||
| 73 | + us = statistics.median(us_val) if isinstance(us_val, list) else float(us_val) | ||
| 74 | + dt = dtype_by_mkn.get((M, K, N), "fp16") | ||
| 75 | + dtl = {"float16": "fp16", "float32": "fp32", "bfloat16": "bf16"}.get(dt, dt) | ||
| 76 | + # flops: 优先 case 内 'flops', 否则 builtin | ||
| 77 | + ff = BUILTIN_FLOPS.get(op_kind) | ||
| 78 | + if ff is None: | ||
| 79 | + skipped.append((shape_key, f"no flops formula for op_kind '{op_kind}'")) | ||
| 80 | + continue | ||
| 81 | + flops = ff(M, K, N) | ||
| 82 | + hf = BUILTIN_HBM.get(op_kind, BUILTIN_HBM["matmul"]) | ||
| 83 | + hbm = hf(M, K, N, DTYPE_BYTES.get(dtl, 2)) | ||
| 84 | + sig = mfu_signal(flops, hbm, us, hw_name, dtl, eok, | ||
| 85 | + n_aic_used=n_aic_used, t_overhead_us=t_overhead_us) | ||
| 86 | + per_case[shape_key] = sig | ||
| 87 | + all_done.append(sig["done"]) | ||
| 88 | + | ||
| 89 | + ceiling = { | ||
| 90 | + "generated_by": "mfu/verification_hook.py (mechanical, per perf step)", | ||
| 91 | + "hw": hw_name, "op_kind": op_kind, "engine": eok, | ||
| 92 | + "per_case": per_case, | ||
| 93 | + "all_cases_done": bool(all_done) and all(all_done), | ||
| 94 | + "verdict": ("ALL cases >= 0.8*achievable -> perf DONE (ceiling-aware)" | ||
| 95 | + if (all_done and all(all_done)) | ||
| 96 | + else "CONTINUE: some case below achievable ceiling (see per_case levers)"), | ||
| 97 | + "skipped": skipped, | ||
| 98 | + "note": "ko/fo 读此块作 MFU-gated done; 非仅 ratio. peak 按 op_kind 引擎取(fail-loud).", | ||
| 99 | + } | ||
| 100 | + v["mfu_ceiling"] = ceiling | ||
| 101 | + if write: | ||
| 102 | + with open(vjson_path, "w") as f: | ||
| 103 | + json.dump(v, f, ensure_ascii=False, indent=2) | ||
| 104 | + return ceiling | ||
| 105 | + | ||
| 106 | + | ||
| 107 | +if __name__ == "__main__": | ||
| 108 | + # demo/self-test on a copy of a real verification.json | ||
| 109 | + path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/_mmgrad_vj_demo.json" | ||
| 110 | + c = inject_mfu_ceiling(path, op_kind="mm_grad", hw_name="910C_die", n_aic_used=1) | ||
| 111 | + print(json.dumps(c, ensure_ascii=False, indent=2)) | ||
| @@ -0,0 +1,124 @@ | |||
| 1 | +[ | ||
| 2 | + { | ||
| 3 | + "check": "CORE_MODE_LEAK", | ||
| 4 | + "file": "src/scripts/orchestrator/agent_dispatch.py", | ||
| 5 | + "mode": "port_a3_to_a5" | ||
| 6 | + }, | ||
| 7 | + { | ||
| 8 | + "check": "CORE_MODE_LEAK", | ||
| 9 | + "file": "src/scripts/orchestrator/finalize_checks_precision.py", | ||
| 10 | + "mode": "port_a3_to_a5" | ||
| 11 | + }, | ||
| 12 | + { | ||
| 13 | + "check": "CORE_MODE_LEAK", | ||
| 14 | + "file": "src/scripts/orchestrator/orchestrator.py", | ||
| 15 | + "mode": "port_a3_to_a5" | ||
| 16 | + }, | ||
| 17 | + { | ||
| 18 | + "check": "CORE_MODE_LEAK", | ||
| 19 | + "file": "src/scripts/orchestrator/orchestrator_cmds.py", | ||
| 20 | + "mode": "port_a3_to_a5" | ||
| 21 | + }, | ||
| 22 | + { | ||
| 23 | + "check": "CORE_MODE_LEAK", | ||
| 24 | + "file": "src/scripts/workflow/batch_dispatcher.py", | ||
| 25 | + "mode": "optimize" | ||
| 26 | + }, | ||
| 27 | + { | ||
| 28 | + "check": "GOD_FUNCTION", | ||
| 29 | + "file": "src/scripts/orchestrator/agent_dispatch.py", | ||
| 30 | + "func": "spawn_for_state" | ||
| 31 | + }, | ||
| 32 | + { | ||
| 33 | + "check": "GOD_FUNCTION", | ||
| 34 | + "file": "src/scripts/orchestrator/agent_transport.py", | ||
| 35 | + "func": "_run_streaming" | ||
| 36 | + }, | ||
| 37 | + { | ||
| 38 | + "check": "GOD_FUNCTION", | ||
| 39 | + "file": "src/scripts/orchestrator/finalize_checks_precision.py", | ||
| 40 | + "func": "_check_perf_methodology" | ||
| 41 | + }, | ||
| 42 | + { | ||
| 43 | + "check": "GOD_FUNCTION", | ||
| 44 | + "file": "src/scripts/orchestrator/finalize_dispatch.py", | ||
| 45 | + "func": "check_finalize_eligibility" | ||
| 46 | + }, | ||
| 47 | + { | ||
| 48 | + "check": "GOD_FUNCTION", | ||
| 49 | + "file": "src/scripts/orchestrator/finalize_dispatch.py", | ||
| 50 | + "func": "finalize_op" | ||
| 51 | + }, | ||
| 52 | + { | ||
| 53 | + "check": "GOD_FUNCTION", | ||
| 54 | + "file": "src/scripts/orchestrator/finalize_ge_ophost.py", | ||
| 55 | + "func": "assemble_ge_ophost" | ||
| 56 | + }, | ||
| 57 | + { | ||
| 58 | + "check": "GOD_FUNCTION", | ||
| 59 | + "file": "src/scripts/orchestrator/kb_auto_promote.py", | ||
| 60 | + "func": "run_auto_promote" | ||
| 61 | + }, | ||
| 62 | + { | ||
| 63 | + "check": "GOD_FUNCTION", | ||
| 64 | + "file": "src/scripts/orchestrator/orchestrator.py", | ||
| 65 | + "func": "run_single_op" | ||
| 66 | + }, | ||
| 67 | + { | ||
| 68 | + "check": "GOD_FUNCTION", | ||
| 69 | + "file": "src/scripts/orchestrator/orchestrator_cli.py", | ||
| 70 | + "func": "main" | ||
| 71 | + }, | ||
| 72 | + { | ||
| 73 | + "check": "GOD_FUNCTION", | ||
| 74 | + "file": "src/scripts/orchestrator/orchestrator_cmds.py", | ||
| 75 | + "func": "_cmd_port_a3" | ||
| 76 | + }, | ||
| 77 | + { | ||
| 78 | + "check": "GOD_FUNCTION", | ||
| 79 | + "file": "src/scripts/orchestrator/orchestrator_coldstart.py", | ||
| 80 | + "func": "_cold_start_reset_workspace" | ||
| 81 | + }, | ||
| 82 | + { | ||
| 83 | + "check": "GOD_FUNCTION", | ||
| 84 | + "file": "src/scripts/orchestrator/phase_o25_a3_ref.py", | ||
| 85 | + "func": "provision_a3_reference" | ||
| 86 | + }, | ||
| 87 | + { | ||
| 88 | + "check": "GOD_FUNCTION", | ||
| 89 | + "file": "src/scripts/orchestrator/phase_o25_backward.py", | ||
| 90 | + "func": "provision_backward_reference" | ||
| 91 | + }, | ||
| 92 | + { | ||
| 93 | + "check": "GOD_FUNCTION", | ||
| 94 | + "file": "src/scripts/orchestrator/phase_o5_runner.py", | ||
| 95 | + "func": "_resync_workspace_to_container" | ||
| 96 | + }, | ||
| 97 | + { | ||
| 98 | + "check": "GOD_FUNCTION", | ||
| 99 | + "file": "src/scripts/orchestrator/phase_o5_runner.py", | ||
| 100 | + "func": "ssh_runner" | ||
| 101 | + }, | ||
| 102 | + { | ||
| 103 | + "check": "GOD_FUNCTION", | ||
| 104 | + "file": "src/scripts/orchestrator/phase_o5_verify.py", | ||
| 105 | + "func": "_run_canonical_pass_a" | ||
| 106 | + }, | ||
| 107 | + { | ||
| 108 | + "check": "GOD_FUNCTION", | ||
| 109 | + "file": "src/scripts/workflow/fsm_conditions.py", | ||
| 110 | + "func": "eval_condition" | ||
| 111 | + }, | ||
| 112 | + { | ||
| 113 | + "check": "GOD_FUNCTION", | ||
| 114 | + "file": "src/scripts/workflow/workflow_critic.py", | ||
| 115 | + "func": "mode_pre_agent_spawn", | ||
| 116 | + "reason": "Existing safety-critical pre-spawn hook; preserve behavior until a dedicated refactor." | ||
| 117 | + }, | ||
| 118 | + { | ||
| 119 | + "check": "GOD_FUNCTION", | ||
| 120 | + "file": "src/scripts/workflow/workflow_critic_validators.py", | ||
| 121 | + "func": "validate_phase_o4_state", | ||
| 122 | + "reason": "Existing phase-O4 FSM validator; preserve behavior until a dedicated refactor." | ||
| 123 | + } | ||
| 124 | +] | ||
| @@ -0,0 +1,8 @@ | |||
| 1 | +#!/bin/bash | ||
| 2 | +# a5_pipeline.sh — Backwards-compatibility wrapper around npu_pipeline.sh (V3.4) | ||
| 3 | +# Pre-V3.4 callers expect this script and an A5-only execution path. | ||
| 4 | +# We now delegate to npu_pipeline.sh with TARGET forced to a5. | ||
| 5 | +# | ||
| 6 | +# New code should invoke `bash src/scripts/npu_pipeline.sh <cmd>` directly so | ||
| 7 | +# `--target=a3` / TARGET=a3 overrides work without going through this shim. | ||
| 8 | +exec env TARGET=a5 bash "$(dirname "$0")/npu_pipeline.sh" "$@" | ||
| @@ -0,0 +1,640 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 3 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 4 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 5 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 6 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 7 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 8 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 9 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 10 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 11 | +"""architecture_lint.py — mechanized architecture-conformance checks. | ||
| 12 | + | ||
| 13 | +Implements the deterministic floor of docs/design/HARNESS_DESIGN_PHILOSOPHY.md §5: where a | ||
| 14 | +design principle can be checked deterministically, it is a lint rather than left to | ||
| 15 | +review attention. This catches *plugin-architecture rot* — the "god-function / a-bunch-of- | ||
| 16 | +else" drift the owner flagged (2026-06-20): mode-specific branches leaking into core, | ||
| 17 | +and oversized core functions. | ||
| 18 | + | ||
| 19 | +Checks: | ||
| 20 | + | ||
| 21 | + 1. CORE_MODE_LEAK — a `if/elif <mode|opgen_mode|backend|target> == "<lit>"` branch | ||
| 22 | + inside a CORE module (orchestrator/ top-level + orchestrator/briefs/ + workflow/, | ||
| 23 | + not under plugins/, not a test) means mode/target-specific logic has leaked out of | ||
| 24 | + its plugin. Per HARNESS_DESIGN_PHILOSOPHY §2/§4(a): mode-specific logic belongs IN | ||
| 25 | + the plugin; the core stays mode-agnostic (mirror of OL-160, where the safety net | ||
| 26 | + keys off the canonical contract, not mode-specific names). 2026-06-20 widening | ||
| 27 | + (lint b): the original regex only matched op-gen-mode literals on `mode`; it now | ||
| 28 | + matches ANY string literal on `mode|opgen_mode|backend|target` so `target == "a3"` | ||
| 29 | + and `backend == "ascendc"` are caught too — 0-tolerance for NEW occurrences; the | ||
| 30 | + existing ones are grandfathered into the baseline. | ||
| 31 | + | ||
| 32 | + 2. PLUGIN_NAME_BRANCH — (lint a) `(_active_)?plugin.name == "<lit>"` branches in core, | ||
| 33 | + the strongest plugin-leak smell (a core module deciding behavior by comparing the | ||
| 34 | + active plugin's identity string instead of calling a plugin method). 0-NEW; the one | ||
| 35 | + existing occurrence (phase_o5_runner.py port_a3_to_a5) is grandfathered as known | ||
| 36 | + debt to convert to a plugin method. | ||
| 37 | + | ||
| 38 | + 3. GOD_FUNCTION — a function longer than --max-func-lines (default 200) in a core module | ||
| 39 | + is a god-function candidate; per §4(a) depth-over-breadth and general maintainability. | ||
| 40 | + | ||
| 41 | + 4. PLUGIN_REGISTRY_INCOMPLETE — (lint c) every registered plugin must implement the | ||
| 42 | + BasePlugin required-method set so a NEW backend has to register + implement rather | ||
| 43 | + than being core-if-branched. Derived from the protocol-conformance contract | ||
| 44 | + (plugins/tests/test_protocol_conformance.REQUIRED_METHODS). | ||
| 45 | + | ||
| 46 | + 5. FSM_HARDCODED_TRANSITION — (lint e) a hardcoded next-state/goto string literal in | ||
| 47 | + workflow/state_machine.py that bypasses the `state_transitions.jsonl`-driven | ||
| 48 | + `next_state()` table (e.g. `next_state = "await_worker"`). Transitions must come from | ||
| 49 | + the YAML/data path (`trans.get("goto")`), never a Python literal. Clean today (0 | ||
| 50 | + findings) — a forward ratchet against re-introducing hardcoded routing. | ||
| 51 | + | ||
| 52 | +Re (d) dispatch-complexity: an if/elif-chain-length sub-check was evaluated and SKIPPED — | ||
| 53 | +GOD_FUNCTION (function length) already covers the dispatch functions it would flag | ||
| 54 | +(workflow_critic._dispatch's mode-elif chain is inside a function already length-bounded), | ||
| 55 | +and the widened CORE_MODE_LEAK catches the per-branch literals directly. A separate | ||
| 56 | +chain-length metric would double-count the same lines without catching anything new. | ||
| 57 | + | ||
| 58 | +Default is REPORT-ONLY (exit 0) so it can be adopted incrementally — like | ||
| 59 | +kb_index_audit.py --report-only. Pass --strict to fail (for pre-commit / CI once the | ||
| 60 | +known baseline is burned down). A --baseline file allows grandfathering known findings | ||
| 61 | +while preventing NEW ones. | ||
| 62 | + | ||
| 63 | +Usage: | ||
| 64 | + python3 src/scripts/architecture_lint.py # report, exit 0 | ||
| 65 | + python3 src/scripts/architecture_lint.py --strict # fail on any finding | ||
| 66 | + python3 src/scripts/architecture_lint.py --json # machine-readable | ||
| 67 | + python3 src/scripts/architecture_lint.py --baseline src/scripts/.arch_lint_baseline.json | ||
| 68 | + python3 src/scripts/architecture_lint.py --regenerate-baseline src/scripts/.arch_lint_baseline.json | ||
| 69 | +""" | ||
| 70 | +from __future__ import annotations | ||
| 71 | + | ||
| 72 | +import argparse | ||
| 73 | +import ast | ||
| 74 | +import json | ||
| 75 | +import os | ||
| 76 | +import re | ||
| 77 | +import sys | ||
| 78 | + | ||
| 79 | +# Op-gen modes (the plugin names + re-entry modes). A core branch on any of these = leak. | ||
| 80 | +# `optimize` is the --optimize re-entry mode (DEBT-161/PR#11). The staleness-guard test | ||
| 81 | +# (test_architecture_lint) asserts this allowlist ⊇ every registered plugin's mode name, | ||
| 82 | +# so it cannot silently lag a new backend/mode plugin (OL-160). | ||
| 83 | +OPGEN_MODES = ("port_a3_to_a5", "backward", "optimize") | ||
| 84 | +# DEBT-161 Batch-2 precision narrowing: a `<ident> == "<lit>"` branch in core is only an | ||
| 85 | +# op-gen leak when <lit> is a REAL op-gen mode / backend / arch-target. The identifiers | ||
| 86 | +# `mode`/`target` are OVERLOADED for non-op-gen meanings that are NOT leaks: | ||
| 87 | +# mode=="foreground" → spawn execution mode (agent_transport) | ||
| 88 | +# mode=="auto"/"pre_agent_spawn" → critic phase (workflow_critic) | ||
| 89 | +# target=="__from_user_decision__" → FSM magic token (state_machine) | ||
| 90 | +# Flagging those was a false positive of the lint-(b) widening. Allowlist-by-known-value | ||
| 91 | +# fixes precision; the staleness-guard test prevents the allowlist lagging new values. | ||
| 92 | +KNOWN_BACKENDS = ("ascendc",) | ||
| 93 | +KNOWN_ARCH_TARGETS = ("a3", "a5") | ||
| 94 | +_MODE_LEAK_ALLOWLIST = { | ||
| 95 | + "mode": OPGEN_MODES, | ||
| 96 | + "opgen_mode": OPGEN_MODES, | ||
| 97 | + "backend": KNOWN_BACKENDS, | ||
| 98 | + "target": KNOWN_ARCH_TARGETS, | ||
| 99 | +} | ||
| 100 | + | ||
| @@ -0,0 +1,110 @@ | |||
| 1 | +#!/bin/bash | ||
| 2 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 3 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 4 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 5 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 6 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 7 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 8 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 9 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 10 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 11 | +# audit_aiv_count.sh — find hardcoded AIV/core constants across archive kernels | ||
| 12 | +# | ||
| 13 | +# Motivation (2026-04-27 op#5 Cumsum ko-1 Iter 1): | ||
| 14 | +# `CUMSUM_NUM_CORES = 20` was hardcoded in kernel constants. Changing it to 56 | ||
| 15 | +# (the physical AIV count on Ascend950PR) gave +169% sum-ratio lift in a single | ||
| 16 | +# iter. This script finds other archive kernels that may have the same bug. | ||
| 17 | +# | ||
| 18 | +# Usage: | ||
| 19 | +# bash src/scripts/audit_aiv_count.sh # scan default archive | ||
| 20 | +# bash src/scripts/audit_aiv_count.sh path/to/kernels # custom root | ||
| 21 | +# | ||
| 22 | +# Output: groups of (file, line, line-content) for each suspicious hardcoded | ||
| 23 | +# constant, with self-test footer confirming the script catches op#5's known | ||
| 24 | +# bug pattern. | ||
| 25 | + | ||
| 26 | +set -e | ||
| 27 | + | ||
| 28 | +ARCHIVE_ROOT="${1:-output/a3_to_a5_port/src/kernels}" | ||
| 29 | + | ||
| 30 | +if [ ! -d "$ARCHIVE_ROOT" ]; then | ||
| 31 | + echo "ERROR: archive root not found: $ARCHIVE_ROOT" >&2 | ||
| 32 | + exit 2 | ||
| 33 | +fi | ||
| 34 | + | ||
| 35 | +echo "=== Hardcoded AIV/core constants in $ARCHIVE_ROOT ===" | ||
| 36 | +echo "" | ||
| 37 | + | ||
| 38 | +# Pattern matches identifier-like names (NUM_CORES / CORE_COUNT / NUM_AIV / | ||
| 39 | +# coreNum / aivNum / blockDim / NPU_NUM / CORE_NUM and op-prefixed variants | ||
| 40 | +# like CUMSUM_NUM_CORES) followed by ':=' or '=' and an integer literal. | ||
| 41 | +# We KEEP =20 in scope because that's the exact value op#5 had. | ||
| 42 | +# We exclude 56 (correct), 1 (single-AIV intentional), 47/48 (occasional | ||
| 43 | +# single-AIV core reservation patterns documented in some kernels). | ||
| 44 | +PATTERN='\b([A-Za-z_][A-Za-z0-9_]*?)(NUM_CORES|CORE_COUNT|NUM_AIV|aivNum|blockDim|NPU_NUM|CORE_NUM|coreNum|AIVNum)\b\s*[:=]\s*[0-9]+' | ||
| 45 | + | ||
| 46 | +# Collect hits — only AscendC kernel headers and sources; skip build/, probes/, .so files | ||
| 47 | +FOUND=0 | ||
| 48 | +LAST_OP="" | ||
| 49 | + | ||
| 50 | +while IFS= read -r line; do | ||
| 51 | + if [ -z "$line" ]; then continue; fi | ||
| 52 | + # Extract <file>:<lineno>:<content> | ||
| 53 | + file="${line%%:*}" | ||
| 54 | + rest="${line#*:}" | ||
| 55 | + lineno="${rest%%:*}" | ||
| 56 | + content="${rest#*:}" | ||
| 57 | + | ||
| 58 | + # Get op directory (3 levels up from kernel/*.h typically) | ||
| 59 | + op=$(echo "$file" | sed -E 's|.*/kernels/([^/]+)/.*|\1|') | ||
| 60 | + | ||
| 61 | + if [ "$op" != "$LAST_OP" ]; then | ||
| 62 | + echo "" | ||
| 63 | + echo "── $op ──" | ||
| 64 | + LAST_OP="$op" | ||
| 65 | + fi | ||
| 66 | + | ||
| 67 | + # Strip leading ./ | ||
| 68 | + file_rel="${file#./}" | ||
| 69 | + echo " $file_rel:$lineno:$(echo "$content" | sed 's/^[[:space:]]*//')" | ||
| 70 | + FOUND=$((FOUND + 1)) | ||
| 71 | +done < <( | ||
| 72 | + # Two layout patterns coexist in the archive: | ||
| 73 | + # (a) flat: <op>/<op>_kernel.h, <op>/<op>_kernels.cpp (older) | ||
| 74 | + # (b) nested: <op>/kernel/*.h, <op>/kernel/*.cpp (newer) | ||
| 75 | + # We accept either, but reject build/probes/.opt*_bak intermediates. | ||
| 76 | + find "$ARCHIVE_ROOT" \ | ||
| 77 | + \( -name "*.h" -o -name "*.cpp" -o -name "*.hpp" \) \ | ||
| 78 | + -not -path "*/build/*" \ | ||
| 79 | + -not -path "*/probes/*" \ | ||
| 80 | + -not -name "*.opt*_bak*" \ | ||
| 81 | + -not -name "*.bak*" \ | ||
| 82 | + 2>/dev/null \ | ||
| 83 | + | xargs grep -EHn "$PATTERN" 2>/dev/null \ | ||
| 84 | + | grep -vE '=\s*56\b' \ | ||
| 85 | + | grep -vE '=\s*1\b\s*[;,)/]' \ | ||
| 86 | + | grep -vE '=\s*48\b' \ | ||
| 87 | + | grep -vE '=\s*47\b' \ | ||
| 88 | + | sort | ||
| 89 | +) | ||
| 90 | + | ||
| 91 | +echo "" | ||
| 92 | +echo "=== Total hits: $FOUND ===" | ||
| 93 | +echo "" | ||
| 94 | + | ||
| 95 | +# Self-test: verify pattern catches op#5 cumsum's known bug | ||
| 96 | +echo "=== Self-test: does the pattern match op#5 cumsum's CUMSUM_NUM_CORES = 20? ===" | ||
| 97 | +SELF_TEST_FILE="$ARCHIVE_ROOT/5_Cumsum/kernel/cumsum_kernel.h" | ||
| 98 | +SELF_TEST_FILE2="$ARCHIVE_ROOT/5_Cumsum/kernel/cumsum_tiling.h" | ||
| 99 | +for tf in "$SELF_TEST_FILE" "$SELF_TEST_FILE2"; do | ||
| 100 | + if [ -f "$tf" ]; then | ||
| 101 | + if grep -EHn "$PATTERN" "$tf" 2>/dev/null | grep -E "CUMSUM_NUM_CORES|NUM_CORES" >/dev/null; then | ||
| 102 | + echo " ✓ pattern matches CUMSUM_NUM_CORES in $(basename $tf)" | ||
| 103 | + fi | ||
| 104 | + fi | ||
| 105 | +done | ||
| 106 | + | ||
| 107 | +# Also verify the negative test: post-fix value (=56) should NOT trigger | ||
| 108 | +echo "" | ||
| 109 | +echo "=== Sanity: would the filter HIDE a corrected =56? (it should) ===" | ||
| 110 | +echo "static constexpr int TEST_NUM_CORES = 56;" | grep -EHn "$PATTERN" 2>&1 | grep -vE '=\s*56\b' || echo " ✓ correctly suppressed (=56 is not a hit)" | ||
| @@ -0,0 +1,194 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# -*- coding: utf-8 -*- | ||
| 3 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 12 | + | ||
| 13 | +"""Harness-backend conformance harness (HARNESS_BACKEND_ABSTRACTION_DESIGN.md §Conformance). | ||
| 14 | + | ||
| 15 | +Proves the CCBackend refactor did NOT weaken any canonical gate: freeze an artifact, | ||
| 16 | +run the CANONICAL gate entrypoints against it, capture {gate: verdict+reason-code}, and | ||
| 17 | +assert BIT-IDENTICAL pre-refactor vs post-refactor (later: cross-backend). | ||
| 18 | + | ||
| 19 | +**Never touches gate logic** — it REUSES `finalize_pipeline._pass_branch_gate_specs()` | ||
| 20 | +(the canonical registry) so it stays faithful + in-sync as gates evolve. This file only | ||
| 21 | +*invokes* gates and snapshots verdicts. | ||
| 22 | + | ||
| 23 | +Usage: | ||
| 24 | + capture: python3 conformance_harness.py capture <workspace> --out before.json | ||
| 25 | + diff: python3 conformance_harness.py diff before.json after.json # exit 1 if any divergence | ||
| 26 | +""" | ||
| 27 | +from __future__ import annotations | ||
| 28 | +import argparse | ||
| 29 | +import json | ||
| 30 | +import sys | ||
| 31 | +import traceback | ||
| 32 | +from pathlib import Path | ||
| 33 | + | ||
| 34 | +_ORCH = Path(__file__).resolve().parents[1] / "orchestrator" | ||
| 35 | +if str(_ORCH) not in sys.path: | ||
| 36 | + sys.path.insert(0, str(_ORCH)) | ||
| 37 | + | ||
| 38 | + | ||
| 39 | +def capture_verdicts(workspace: Path) -> dict: | ||
| 40 | + """Run every canonical gate against `workspace`, return {gate_id: verdict}. | ||
| 41 | + verdict = None (pass) | str (reason-code) | '__EXC__:...' (gate errored on this artifact) | ||
| 42 | + | '__SKIP__:...' (gate needs an input this frozen artifact lacks). | ||
| 43 | + """ | ||
| 44 | + import finalize_pipeline as fp # canonical gates live here; import = reuse, not reimplement | ||
| 45 | + | ||
| 46 | + vj = {} | ||
| 47 | + vjp = workspace / "verification.json" | ||
| 48 | + if vjp.exists(): | ||
| 49 | + try: | ||
| 50 | + vj = json.loads(vjp.read_text()) | ||
| 51 | + except Exception as e: | ||
| 52 | + vj = {"__vj_parse_error__": str(e)} | ||
| 53 | + prec = vj.get("precision", {}) if isinstance(vj, dict) else {} | ||
| 54 | + | ||
| 55 | + verdicts: dict[str, object] = {} | ||
| 56 | + | ||
| 57 | + def _run(name, fn): | ||
| 58 | + try: | ||
| 59 | + verdicts[name] = fn() | ||
| 60 | + except FileNotFoundError as e: | ||
| 61 | + verdicts[name] = f"__SKIP__: missing input: {e}" | ||
| 62 | + except Exception as e: | ||
| 63 | + verdicts[name] = f"__EXC__: {type(e).__name__}: {e}" | ||
| 64 | + | ||
| 65 | + # --- inline pre-checks (not in the pass-branch registry) --- | ||
| 66 | + _run("STALE_ORCHESTRATOR", lambda: getattr(fp, "_check_stale_orchestrator")()) | ||
| 67 | + _run("MODEL_PY_SHAPE", lambda: getattr(fp, "_check_model_py_shape")(workspace)) | ||
| 68 | + _run("PASS_A_COVERAGE", lambda: getattr(fp, "_check_pass_a_coverage")(workspace, prec)) | ||
| 69 | + | ||
| 70 | + # --- canonical pass-branch gate registry (authoritative, reused) [stage: finalize] --- | ||
| 71 | + for gate_id, pred in getattr(fp, "_pass_branch_gate_specs")(): | ||
| 72 | + name = getattr(gate_id, "value", str(gate_id)) | ||
| 73 | + if name == getattr(fp, "_PLUGIN_EXTRAS_SENTINEL", "plugin_extras"): | ||
| 74 | + # plugin-supplied extras return a dict, not a reason string — snapshot as-is | ||
| 75 | + _run("finalize:PLUGIN_EXTRAS", lambda p=pred: p(workspace, vj)) | ||
| 76 | + continue | ||
| 77 | + _run("finalize:" + name, lambda p=pred: p(workspace, vj)) | ||
| 78 | + | ||
| 79 | + # --- stage: O2.5 reference-provider contract (E) --- | ||
| 80 | + try: | ||
| 81 | + import phase_o25_a3_ref as o25 # canonical provider validators (main merged this round) | ||
| 82 | + _run("o25:model_contract", lambda: getattr(o25.validate_model_contract(workspace), "reason_code", "?")) | ||
| 83 | + except Exception as e: | ||
| 84 | + verdicts["o25:model_contract"] = f"__EXC__: import phase_o25_a3_ref: {type(e).__name__}: {e}" | ||
| 85 | + | ||
| 86 | + # --- stage: workflow_critic CHECK-LOGIC (B) — capture rejections, NOT the hook wiring --- | ||
| 87 | + try: | ||
| 88 | + _wf = str(Path(__file__).resolve().parents[1] / "workflow") | ||
| 89 | + if _wf not in sys.path: | ||
| 90 | + sys.path.insert(0, _wf) | ||
| 91 | + import workflow_critic as wc | ||
| 92 | + for nm, fn in [("wc:target_simt_compat", wc.check_target_simt_compat), | ||
| 93 | + ("wc:global_invariants", wc.check_global_invariants), | ||
| 94 | + ("wc:taxonomy_coverage", wc.check_taxonomy_coverage)]: | ||
| 95 | + def _cap(f=fn): | ||
| 96 | + rej: list = [] | ||
| 97 | + f(workspace, rej) # check-logic appends Rejection(s); empty = pass | ||
| 98 | + return None if not rej else "; ".join(sorted(getattr(r, "rule_id", str(r)) for r in rej)) | ||
| 99 | + _run(nm, _cap) | ||
| 100 | + except Exception as e: | ||
| 101 | + verdicts["wc:__import__"] = f"__EXC__: import workflow_critic: {type(e).__name__}: {e}" | ||
| 102 | + | ||
| 103 | + # --- stage: safety guard CHECK-LOGIC (C) — FIXED probes (per-tool-call, workspace-independent); | ||
| 104 | + # the key acceptance for the dispatch/hook-wiring refactor. Incl the bash-bypass airtight case. --- | ||
| 105 | + try: | ||
| 106 | + import output_read_guard as org # workflow path already on sys.path (B section) | ||
| 107 | + it_mode = next(iter(getattr(org, "_ITERATE_MODES", {"optimize"})), "optimize") | ||
| 108 | + own = "output/myproj/kernels/opA" | ||
| 109 | + _probes = { | ||
| 110 | + # fresh/cold gen: ANY output/ read denied | ||
| 111 | + "guard:fresh_output_read_deny": lambda: getattr(org, "_should_deny")( | ||
| 112 | + "output/proj/kernels/opA/verification.json", "default", "" | ||
| 113 | + ), | ||
| 114 | + # iterate on OWN archive: allowed | ||
| 115 | + "guard:iterate_own_archive_allow": lambda: getattr(org, "_should_deny")(own + "/kernel/x.so", it_mode, own), | ||
| 116 | + # iterate but CROSS archive (other op / other SoC): denied (the cheat rule) | ||
| 117 | + "guard:iterate_cross_archive_deny": lambda: getattr(org, "_should_deny")("output/other/kernels/opA/x.so", it_mode, own), | ||
| 118 | + # non-output path: not our concern | ||
| 119 | + "guard:non_output_ignored": lambda: getattr(org, "_should_deny")("/home/w/workspace/opA/kernel.h", "default", ""), | ||
| 120 | + # BASH-BYPASS airtight case (DS PoC's key finding): a bash read-cmd's extracted output frag | ||
| 121 | + # hits the SAME deny path (guard isn't tool-name-gated in a bypassable way for reads). | ||
| 122 | + "guard:bash_read_bypass_denied": lambda: ( | ||
| 123 | + bool(getattr(org, "_output_paths")("cat output/other/kernels/opA/verification.json")) | ||
| 124 | + and all( | ||
| 125 | + getattr(org, "_should_deny")(file_path, "default", "") | ||
| 126 | + for file_path in getattr(org, "_output_paths")("cat output/other/kernels/opA/verification.json") | ||
| 127 | + ) | ||
| 128 | + ), | ||
| 129 | + } | ||
| 130 | + for nm, fn in _probes.items(): | ||
| 131 | + _run(nm, fn) | ||
| 132 | + except Exception as e: | ||
| 133 | + verdicts["guard:__import__"] = f"__EXC__: {type(e).__name__}: {e}" | ||
| 134 | + | ||
| 135 | + # --- NEXT INCREMENT: --- | ||
| 136 | + # D precision: precision_eval_port_a3_two_tier (calls compare.py) — needs tensor inputs | ||
| 137 | + # (a3_capture/cpu_truth .pt); SKIP-record when a frozen archive lacks them. Low priority | ||
| 138 | + # (grades artifact; dispatch/wiring refactor can't touch precision judgement). | ||
| 139 | + # EXCLUDED from hermetic conformance (external-state-dependent, not pure check-logic): | ||
| 140 | + # - ship_claim_audit._verify_sha_on_main (hits git → non-hermetic) | ||
| 141 | + # - _maybe_port_a3_perf_remeasure (NPU device measurement, not gate-replay) | ||
| 142 | + | ||
| 143 | + return dict(sorted(verdicts.items())) | ||
| 144 | + | ||
| 145 | + | ||
| 146 | +def diff_snapshots(before: dict, after: dict) -> list[str]: | ||
| 147 | + """Return per-gate divergences (empty = bit-identical = conformant).""" | ||
| 148 | + keys = sorted(set(before) | set(after)) | ||
| 149 | + out = [] | ||
| 150 | + for k in keys: | ||
| 151 | + b, a = before.get(k, "<absent>"), after.get(k, "<absent>") | ||
| 152 | + if b != a: | ||
| 153 | + out.append(f" {k}: before={b!r} after={a!r}") | ||
| 154 | + return out | ||
| 155 | + | ||
| 156 | + | ||
| 157 | +if __name__ == "__main__": | ||
| 158 | + ap = argparse.ArgumentParser() | ||
| 159 | + sub = ap.add_subparsers(dest="cmd", required=True) | ||
| 160 | + c = sub.add_parser("capture") | ||
| 161 | + c.add_argument("workspace") | ||
| 162 | + c.add_argument("--out", default=None) | ||
| 163 | + d = sub.add_parser("diff") | ||
| 164 | + d.add_argument("before") | ||
| 165 | + d.add_argument("after") | ||
| 166 | + a = ap.parse_args() | ||
| 167 | + | ||
| 168 | + if a.cmd == "capture": | ||
| 169 | + snap = capture_verdicts(Path(a.workspace).resolve()) | ||
| 170 | + payload = {"workspace": str(Path(a.workspace).resolve()), "verdicts": snap} | ||
| 171 | + txt = json.dumps(payload, ensure_ascii=False, indent=2) | ||
| 172 | + if a.out: | ||
| 173 | + Path(a.out).write_text(txt) | ||
| 174 | + print(f"captured {len(snap)} gate verdicts -> {a.out}") | ||
| 175 | + else: | ||
| 176 | + print(txt) | ||
| 177 | + # summary | ||
| 178 | + npass = sum(1 for v in snap.values() if v is None) | ||
| 179 | + nfail = sum(1 for v in snap.values() if isinstance(v, str) and not v.startswith("__")) | ||
| 180 | + nskip = sum(1 for v in snap.values() if isinstance(v, str) and v.startswith("__SKIP__")) | ||
| 181 | + nexc = sum(1 for v in snap.values() if isinstance(v, str) and v.startswith("__EXC__")) | ||
| 182 | + print(f"# pass={npass} fail/reason={nfail} skip={nskip} exc={nexc} total={len(snap)}", file=sys.stderr) | ||
| 183 | + sys.exit(0) | ||
| 184 | + | ||
| 185 | + if a.cmd == "diff": | ||
| 186 | + before = json.loads(Path(a.before).read_text()).get("verdicts", {}) | ||
| 187 | + after = json.loads(Path(a.after).read_text()).get("verdicts", {}) | ||
| 188 | + div = diff_snapshots(before, after) | ||
| 189 | + if div: | ||
| 190 | + print("CONFORMANCE FAIL — canonical gate verdict divergence (refactor weakened/changed a gate):") | ||
| 191 | + print("\n".join(div)) | ||
| 192 | + sys.exit(1) | ||
| 193 | + print(f"CONFORMANCE PASS — all {len(set(before)|set(after))} gate verdicts bit-identical") | ||
| 194 | + sys.exit(0) | ||
| @@ -0,0 +1,191 @@ | |||
| 1 | +#!/bin/bash | ||
| 2 | +# build_archive.sh — per-iter capture of build stdout/stderr + kernel snapshot to A5 archive | ||
| 3 | +# | ||
| 4 | +# Wraps a build invocation: runs the build remotely, captures both streams + kernel snapshot | ||
| 5 | +# to /data/build_archive/<op>/<batch>/<iter>/ on A5 host. Returns the build's exit code. | ||
| 6 | +# | ||
| 7 | +# Usage (called from deploy_to_npu_lane.sh after the existing build command): | ||
| 8 | +# | ||
| 9 | +# bash src/scripts/build_archive.sh \ | ||
| 10 | +# --ssh-cmd "$SSH_CMD" \ | ||
| 11 | +# --remote-task "$REMOTE_TASK" \ | ||
| 12 | +# --build-cmd "$BUILD_CMD" \ | ||
| 13 | +# --op-slug "$ASCENDC_WORKSPACE" \ | ||
| 14 | +# --soc "$SOC_VERSION" \ | ||
| 15 | +# --lane "${LANE:-0}" \ | ||
| 16 | +# --npu "${ASCEND_VISIBLE_DEVICES:-0}" \ | ||
| 17 | +# --workspace-marker "$_WS_DIR/.last_build_archive_path" | ||
| 18 | +# | ||
| 19 | +# Optional env: BUILD_ARCHIVE_ROOT (default /data/build_archive) | ||
| 20 | +# BUILD_BATCH_ID (default orch_<UTC ts>) | ||
| 21 | +# CANDIDATE_BRIEF_HASH (default "default") | ||
| 22 | +# | ||
| 23 | +# Standalone test (without integration): | ||
| 24 | +# BUILD_ARCHIVE_ROOT=/tmp/build_archive_test bash src/scripts/build_archive.sh --dry-run | ||
| 25 | +# | ||
| 26 | +# Created 2026-05-02 per docs/research/GEPA_skill_design_2026_05_02.md §15.2 | ||
| 27 | + | ||
| 28 | +set -uo pipefail | ||
| 29 | + | ||
| 30 | +# ─── parse args ───────────────────────────────────────────────────────── | ||
| 31 | +SSH_CMD="" | ||
| 32 | +REMOTE_TASK="" | ||
| 33 | +BUILD_CMD="" | ||
| 34 | +OP_SLUG="" | ||
| 35 | +SOC="" | ||
| 36 | +LANE="0" | ||
| 37 | +NPU="0" | ||
| 38 | +WORKSPACE_MARKER="" | ||
| 39 | +DRY_RUN=0 | ||
| 40 | + | ||
| 41 | +while [ $# -gt 0 ]; do | ||
| 42 | + case "$1" in | ||
| 43 | + --ssh-cmd) SSH_CMD="$2"; shift 2;; | ||
| 44 | + --remote-task) REMOTE_TASK="$2"; shift 2;; | ||
| 45 | + --build-cmd) BUILD_CMD="$2"; shift 2;; | ||
| 46 | + --op-slug) OP_SLUG="$2"; shift 2;; | ||
| 47 | + --soc) SOC="$2"; shift 2;; | ||
| 48 | + --lane) LANE="$2"; shift 2;; | ||
| 49 | + --npu) NPU="$2"; shift 2;; | ||
| 50 | + --workspace-marker) WORKSPACE_MARKER="$2"; shift 2;; | ||
| 51 | + --dry-run) DRY_RUN=1; shift;; | ||
| 52 | + *) echo "unknown arg: $1" >&2; exit 2;; | ||
| 53 | + esac | ||
| 54 | +done | ||
| 55 | + | ||
| 56 | +# Strip trailing slash from OP_SLUG path; take basename | ||
| 57 | +OP_SLUG="${OP_SLUG##*/}" | ||
| 58 | +OP_SLUG="${OP_SLUG:-unknown_op}" | ||
| 59 | + | ||
| 60 | +ARCHIVE_ROOT="${BUILD_ARCHIVE_ROOT:-/data/build_archive}" | ||
| 61 | +BATCH_ID="${BUILD_BATCH_ID:-orch_$(date -u +%Y%m%dT%H%M%SZ)}" | ||
| 62 | +BRIEF_HASH="${CANDIDATE_BRIEF_HASH:-default}" | ||
| 63 | + | ||
| 64 | +if [ "$DRY_RUN" = "1" ]; then | ||
| 65 | + cat <<EOF | ||
| 66 | +[dry-run] Would archive build for: | ||
| 67 | + archive_root: $ARCHIVE_ROOT | ||
| 68 | + op_slug: $OP_SLUG | ||
| 69 | + batch_id: $BATCH_ID | ||
| 70 | + brief_hash: $BRIEF_HASH | ||
| 71 | + lane: $LANE | ||
| 72 | + npu: $NPU | ||
| 73 | + soc: $SOC | ||
| 74 | + build_cmd: $BUILD_CMD | ||
| 75 | +EOF | ||
| 76 | + exit 0 | ||
| 77 | +fi | ||
| 78 | + | ||
| 79 | +if [ -z "$SSH_CMD" ] || [ -z "$REMOTE_TASK" ] || [ -z "$BUILD_CMD" ]; then | ||
| 80 | + echo "[build_archive] ERROR: missing required args (--ssh-cmd, --remote-task, --build-cmd)" >&2 | ||
| 81 | + exit 2 | ||
| 82 | +fi | ||
| 83 | + | ||
| 84 | +# ─── compute next iter_id atomically on remote ────────────────────────── | ||
| 85 | +ARCHIVE_DIR_BASE="$ARCHIVE_ROOT/$OP_SLUG/$BATCH_ID" | ||
| 86 | + | ||
| 87 | +# Use shell arithmetic on remote to find the next iter number. | ||
| 88 | +# Race-safe-ish: ls + wc on existing dirs. Two concurrent builds on same op+batch | ||
| 89 | +# can collide, but that's a degenerate case (same op, same batch_id, parallel) | ||
| 90 | +# and we'd just lose one iter's logs. | ||
| 91 | +NEXT_ITER=$($SSH_CMD " | ||
| 92 | + mkdir -p '$ARCHIVE_DIR_BASE' 2>/dev/null | ||
| 93 | + n=\$(ls -d '$ARCHIVE_DIR_BASE'/iter_* 2>/dev/null | wc -l) | ||
| 94 | + printf 'iter_%03d' \$((n + 1)) | ||
| 95 | +" 2>/dev/null) | ||
| 96 | + | ||
| 97 | +if [ -z "$NEXT_ITER" ]; then | ||
| 98 | + echo "[build_archive] WARN: could not compute iter_id; falling back to ts-based" >&2 | ||
| 99 | + NEXT_ITER="iter_$(date -u +%H%M%S)" | ||
| 100 | +fi | ||
| 101 | + | ||
| 102 | +ARCHIVE_DIR="$ARCHIVE_DIR_BASE/$NEXT_ITER" | ||
| 103 | + | ||
| 104 | +# ─── prepare archive dir on remote ────────────────────────────────────── | ||
| 105 | +$SSH_CMD "mkdir -p '$ARCHIVE_DIR'" 2>/dev/null | ||
| 106 | + | ||
| 107 | +# ─── run build, capture both streams ──────────────────────────────────── | ||
| 108 | +# We run the build inside the docker container (already implicit in $BUILD_CMD) | ||
| 109 | +# and capture into the archive dir on the host filesystem (which is bind-mounted | ||
| 110 | +# to /data inside the container — both paths see the same files). | ||
| 111 | +START_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) | ||
| 112 | + | ||
| 113 | +# Pass exit code back via $?; capture into ARCHIVE_DIR/exit_code on remote | ||
| 114 | +$SSH_CMD " | ||
| 115 | + cd '$REMOTE_TASK' 2>/dev/null | ||
| 116 | + ($BUILD_CMD) > '$ARCHIVE_DIR/stdout.log' 2> '$ARCHIVE_DIR/stderr.log' | ||
| 117 | + rc=\$? | ||
| 118 | + echo \$rc > '$ARCHIVE_DIR/exit_code' | ||
| 119 | + exit \$rc | ||
| 120 | +" | ||
| 121 | +BUILD_RC=$? | ||
| 122 | + | ||
| 123 | +END_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) | ||
| 124 | + | ||
| 125 | +# ─── snapshot kernel files ────────────────────────────────────────────── | ||
| 126 | +# Tar the kernel/ directory at build-time to ARCHIVE_DIR/kernel_snapshot.tgz | ||
| 127 | +$SSH_CMD " | ||
| 128 | + cd '$REMOTE_TASK' 2>/dev/null | ||
| 129 | + if [ -d kernel ]; then | ||
| 130 | + tar czf '$ARCHIVE_DIR/kernel_snapshot.tgz' kernel/ 2>/dev/null | ||
| 131 | + fi | ||
| 132 | +" 2>/dev/null | ||
| 133 | + | ||
| 134 | +# ─── classify exit code into a one-line verdict ───────────────────────── | ||
| 135 | +case "$BUILD_RC" in | ||
| 136 | + 0) VERDICT="PASS";; | ||
| 137 | + 1) VERDICT="COMPILE_ERROR";; | ||
| 138 | + 2) VERDICT="USAGE_ERROR";; | ||
| 139 | + 10) VERDICT="INFRA_ERROR";; | ||
| 140 | + *) VERDICT="UNKNOWN_$BUILD_RC";; | ||
| 141 | +esac | ||
| 142 | + | ||
| 143 | +# ─── compress logs if large (>100 KB) ─────────────────────────────────── | ||
| 144 | +$SSH_CMD " | ||
| 145 | + for f in '$ARCHIVE_DIR/stdout.log' '$ARCHIVE_DIR/stderr.log'; do | ||
| 146 | + if [ -f \"\$f\" ] && [ \$(stat -c%s \"\$f\" 2>/dev/null || echo 0) -gt 102400 ]; then | ||
| 147 | + gzip \"\$f\" 2>/dev/null | ||
| 148 | + fi | ||
| 149 | + done | ||
| 150 | +" | ||
| 151 | + | ||
| 152 | +# ─── write meta.json ──────────────────────────────────────────────────── | ||
| 153 | +META_JSON=$(cat <<EOF | ||
| 154 | +{ | ||
| 155 | + "op": "$OP_SLUG", | ||
| 156 | + "batch_id": "$BATCH_ID", | ||
| 157 | + "iter_id": "$NEXT_ITER", | ||
| 158 | + "ts_start_utc": "$START_TS", | ||
| 159 | + "ts_end_utc": "$END_TS", | ||
| 160 | + "host": "${HOST:-unknown}", | ||
| 161 | + "container": "${CONTAINER:-unknown}", | ||
| 162 | + "soc": "$SOC", | ||
| 163 | + "lane": "$LANE", | ||
| 164 | + "npu": "$NPU", | ||
| 165 | + "candidate_brief_hash": "$BRIEF_HASH", | ||
| 166 | + "build_exit_code": $BUILD_RC, | ||
| 167 | + "verdict": "$VERDICT" | ||
| 168 | +} | ||
| 169 | +EOF | ||
| 170 | +) | ||
| 171 | +$SSH_CMD "cat > '$ARCHIVE_DIR/meta.json' <<'METAEOF' | ||
| 172 | +$META_JSON | ||
| 173 | +METAEOF" 2>/dev/null | ||
| 174 | + | ||
| 175 | +# ─── workspace-side breadcrumb (so trace_serializer can find the dir) ─── | ||
| 176 | +if [ -n "$WORKSPACE_MARKER" ]; then | ||
| 177 | + echo "$ARCHIVE_DIR" > "$WORKSPACE_MARKER" 2>/dev/null || true | ||
| 178 | +fi | ||
| 179 | + | ||
| 180 | +# Convenience symlinks (preserve existing /tmp/ascendc_last_build.* contract) | ||
| 181 | +$SSH_CMD " | ||
| 182 | + ln -sf '$ARCHIVE_DIR/stderr.log' /tmp/ascendc_last_build.stderr 2>/dev/null || \ | ||
| 183 | + ln -sf '$ARCHIVE_DIR/stderr.log.gz' /tmp/ascendc_last_build.stderr 2>/dev/null | ||
| 184 | + ln -sf '$ARCHIVE_DIR/stdout.log' /tmp/ascendc_last_build.stdout 2>/dev/null || \ | ||
| 185 | + ln -sf '$ARCHIVE_DIR/stdout.log.gz' /tmp/ascendc_last_build.stdout 2>/dev/null | ||
| 186 | +" 2>/dev/null | ||
| 187 | + | ||
| 188 | +# ─── stdout summary line ──────────────────────────────────────────────── | ||
| 189 | +echo "[build_archive] $OP_SLUG/$BATCH_ID/$NEXT_ITER -> $VERDICT (rc=$BUILD_RC)" | ||
| 190 | + | ||
| 191 | +exit $BUILD_RC | ||
| @@ -0,0 +1,25 @@ | |||
| 1 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 10 | + | ||
| 11 | +"""CANN-source-learn carve-out (P0x v2 — DEBT-077). | ||
| 12 | + | ||
| 13 | +Deterministic scanners + utilities for the aog-cann-learner sub-agent and | ||
| 14 | +aog-knowledge-maintain Mode 5. Each scanner produces a numeric/boolean score | ||
| 15 | +that the skill caller validates BEFORE merging any candidate KB entry. | ||
| 16 | + | ||
| 17 | +Modules: | ||
| 18 | + identifier_scanner — C34a: deterministic CANN-vs-public-API denylist scan | ||
| 19 | + compile_gate — C34b: bishengir-clang -fsyntax-only against public headers | ||
| 20 | + copy_shape — C34c: token n-gram contiguous-overlap detection | ||
| 21 | + kb_overlap — C35: reason-code-based overlap with existing KB | ||
| 22 | + | ||
| 23 | +See docs/design/KB_DESIGN_NOTES.md#cann-learn-agent-design-v2 for architecture + threat model. | ||
| 24 | +""" | ||
| 25 | +__version__ = "0.1.0-dev" | ||
| @@ -0,0 +1,426 @@ | |||
| 1 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software; you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ---------------------------------------------------------------------------------------------------------- | ||
| 10 | +"""Production spawn adapter — wires aog-cann-learner agent into Mode 5. | ||
| 11 | + | ||
| 12 | +Provides `spawn_cann_learner_agent` matching the contract Mode 5 expects: | ||
| 13 | + | ||
| 14 | + callable(op, workspace, module_path, sealed_dir, run_id, kb_root, | ||
| 15 | + api_catalog_path) → dict with keys: | ||
| 16 | + - sealed_files: list[Path] | ||
| 17 | + - summary_path: Path | ||
| 18 | + - candidate_paths: list[Path] | ||
| 19 | + - cann_files_read: list[Path] | ||
| 20 | + | ||
| 21 | +Builds the brief, invokes the configured harness backend for | ||
| 22 | +aog-cann-learner, parses returned envelope | ||
| 23 | ++ workspace artifacts, returns the dict Mode 5 will independently re-validate. | ||
| 24 | + | ||
| 25 | +Non-trivial structural choices: | ||
| 26 | + | ||
| 27 | +1. **Brief is mostly cited path lists** (per agent's SKILL.md scope) — | ||
| 28 | + not free-form prompt. The agent reads `module_path`, writes sealed | ||
| 29 | + notes + public summary.json + appends to `patterns/unverified/candidates.md`. | ||
| 30 | + | ||
| 31 | +2. **The agent reports candidate filenames in handoff line** so we can | ||
| 32 | + collect them; we also do a directory diff (kb_root/patterns/unverified/ | ||
| 33 | + listing before/after) as defense in depth. | ||
| 34 | + | ||
| 35 | +3. **cann_files_read is reconstructed** from the agent's `cann_learn_summary.json` | ||
| 36 | + `cann_files_read` field (per schema). If absent, falls back to | ||
| 37 | + listing module_path's files (over-conservative for re-scan purposes — | ||
| 38 | + prefer false-positive C34a/C34c hits over false-negatives). | ||
| 39 | +""" | ||
| 40 | +from __future__ import annotations | ||
| 41 | + | ||
| 42 | +import json | ||
| 43 | +import sys | ||
| 44 | +import time | ||
| 45 | +from pathlib import Path | ||
| 46 | + | ||
| 47 | +_HERE = Path(__file__).resolve() | ||
| 48 | +# Add src/scripts/orchestrator to path so agent_transport imports work. | ||
| 49 | +sys.path.insert(0, str(_HERE.parents[1] / "orchestrator")) | ||
| 50 | + | ||
| 51 | +import agent_transport # noqa: E402 | ||
| 52 | +# Harness-decoupling: spawn via the Backend (CC plugin), not agent_transport directly. | ||
| 53 | +from backends import get_backend # noqa: E402 | ||
| 54 | + | ||
| 55 | +_backend = get_backend() | ||
| 56 | + | ||
| 57 | + | ||
| 58 | +# Default per-spawn wall-clock cap. The agent does Read-only scanning | ||
| 59 | +# + 4 self-checks; should comfortably finish under 30 min on any reasonable | ||
| 60 | +# module. Bump if observed runs need more. | ||
| 61 | +DEFAULT_TIMEOUT_SEC = 2400 # 40 min | ||
| 62 | + | ||
| 63 | + | ||
| 64 | +def build_cann_learner_brief( | ||
| 65 | + *, | ||
| 66 | + op: str, | ||
| 67 | + workspace: Path, | ||
| 68 | + module_path: Path, | ||
| 69 | + sealed_dir: Path, | ||
| 70 | + run_id: str, | ||
| 71 | + kb_root: Path, | ||
| 72 | + api_catalog_path: Path, | ||
| 73 | + extraction_mode: str = "kernel_structural", | ||
| 74 | +) -> str: | ||
| 75 | + """Construct the brief that aog-cann-learner agent receives. | ||
| 76 | + | ||
| 77 | + Per agent's SKILL.md the brief should be terse — agent already knows | ||
| 78 | + its own protocol from the system prompt. The brief just provides | ||
| 79 | + op-specific paths + the run-id token. | ||
| 80 | + """ | ||
| 81 | + slug = f"{op.lower().replace('_', '')}-cl-1" | ||
| 82 | + | ||
| 83 | + _sealed_rel = sealed_dir.relative_to(workspace.parent.parent) if sealed_dir.is_absolute() else sealed_dir | ||
| 84 | + workspace_abs = workspace.resolve() | ||
| 85 | + module_abs = module_path.resolve() | ||
| 86 | + kb_root_abs = kb_root.resolve() | ||
| 87 | + api_catalog_abs = api_catalog_path.resolve() | ||
| 88 | + | ||
| 89 | + cann_strategy_path = workspace_abs / "cann_strategy_inference.md" | ||
| 90 | + summary_path = workspace_abs / "cann_learn_summary.json" | ||
| 91 | + candidates_path = kb_root_abs / "patterns" / "unverified" / "candidates.md" | ||
| 92 | + | ||
| 93 | + return f"""{slug} — cann_learn spawn | ||
| 94 | + | ||
| 95 | +OP: {op} | ||
| 96 | +RUN_ID: {run_id} | ||
| 97 | +WORKSPACE (absolute): {workspace_abs} | ||
| 98 | +MODULE_PATH (CANN source root, read-only — your scope): {module_abs} | ||
| 99 | +KB_ROOT (existing KB for cross-reference, READ): {kb_root_abs} | ||
| 100 | +API_CATALOG (public AscendC API allowlist, READ): {api_catalog_abs} | ||
| 101 | + | ||
| 102 | +# Required reading (mandatory, before Phase A) | ||
| 103 | +- src/skills/references/shared/ANTI_PRESSURE_PROTOCOLS.md # P1-P8 catalog | ||
| 104 | +- {cann_strategy_path} # researcher's strategy_inference for this op | ||
| 105 | + (this is the "what KB already inferred" baseline — use to seed Phase A | ||
| 106 | + pre-scan + C35 reason-code matcher) | ||
| 107 | + | ||
| 108 | +# Output paths (write here, ONLY here) | ||
| 109 | +- Sealed (your private working dir, agent caller will archive on exit): | ||
| 110 | + {sealed_dir.resolve() / 'source_notes.md'} | ||
| 111 | + {sealed_dir.resolve() / 'extraction_drafts.md'} | ||
| 112 | +- Public sanitized summary (JSON-only schema, validated by skill caller): | ||
| 113 | + {summary_path} | ||
| 114 | +- Candidate KB additions (append-only): | ||
| 115 | + {candidates_path} | ||
| 116 | + | ||
| 117 | +# Phases | ||
| 118 | +A. KB pre-scan against {kb_root_abs} for op-class keywords from | ||
| 119 | + cann_strategy_inference.md. Build "what KB already knows" list. | ||
| 120 | + Output to sealed/source_notes.md §pre_scan. | ||
| 121 | +B. Read 2-5 files within {module_abs} (header + impl + tiling). | ||
| 122 | + Capture algorithm structure + public-API equivalents + internal-only | ||
| 123 | + primitives (flag explicitly). | ||
| 124 | +C. Pattern extraction: rewrite each insight in PUBLIC-API surface terms. | ||
| 125 | + Layered shape: title (principle) + body (abstract) + concrete-anchor | ||
| 126 | + (3-5 line snippet, public AscendC API only) + evidence + other-instances. | ||
| 127 | + Append candidates to {candidates_path} with `derived-from: cann-source` | ||
| 128 | + metadata. Drafts in sealed/extraction_drafts.md. | ||
| 129 | +D. Self-review: run C34a / C34b / C34c / C35 on your own output. Drop | ||
| 130 | + leaking candidates BEFORE writing summary.json. Verdict + scores in | ||
| 131 | + summary.json `self_review_verdict` + `checks` block. | ||
| 132 | +E. Exit handoff (last line of stdout): | ||
| 133 | + `→ orchestrator: cann_learn_done — kept N candidates, M metadata-fix proposals, leak_score=X` | ||
| 134 | + OR | ||
| 135 | + `→ orchestrator: cann_learn_blocked — <reason>` | ||
| 136 | + | ||
| 137 | +{_mode_specific_section(op, extraction_mode, kb_root_abs, candidates_path)} | ||
| 138 | + | ||
| 139 | +**Anti-patterns (REJECTED)** | ||
| 140 | +- Verbatim CANN code (even renamed identifiers — C34c catches token n-gram) | ||
| 141 | +- CANN-internal namespace / class / template / macro names in candidate body (C34a) | ||
| 142 | +- Hand-prose inside summary.json outside schema (validator rejects) | ||
| 143 | +- Writing to kernel/ or canonical KB files (G11 hook rejects) | ||
| 144 | +- Patterns with NO public-API equivalent (drop, don't ship infeasible) | ||
| 145 | +- Self-reporting verdict=PASS while leaving leaks (skill caller re-scans) | ||
| 146 | + | ||
| 147 | +# Required summary.json field — extraction_mode | ||
| 148 | + | ||
| 149 | +You MUST include `"extraction_mode": "{extraction_mode}"` as a top-level field | ||
| 150 | +in `cann_learn_summary.json`. This tells the skill caller which scanner | ||
| 151 | +profile + KB destination to apply. Mode 5 caller validates the value. | ||
| 152 | + | ||
| 153 | +ITER BUDGET: single shot. Self-review failure → candidate dropped, no respawn. | ||
| 154 | +""" | ||
| 155 | + | ||
| 156 | + | ||
| 157 | +def _mode_specific_section(op: str, extraction_mode: str, kb_root_abs: Path, candidates_path: Path) -> str: | ||
| 158 | + """Build the mode-specific Phase B/C scope + extraction-question section. | ||
| 159 | + | ||
| 160 | + Mode 5 (kernel_structural) — DEFAULT, the historical scope. Read 2-5 | ||
| 161 | + kernel files (header + impl + tiling). Extract algorithm structure + | ||
| 162 | + public-API equivalents. Candidates → patterns/unverified/candidates.md | ||
| 163 | + with prefix CAND-* / promotion path to P-P canonical. | ||
| 164 | + | ||
| 165 | + Mode 6 (build_system) NEW (2026-05-21) — read CMakeLists.txt + register | ||
| 166 | + + op_proto + apt.cpp files. Extract build-system-level recipes | ||
| 167 | + (per-source-file compile flag isolation; multi-target binary registration; | ||
| 168 | + launch macro routing). Candidates → target/ascendc/build_system/candidates.md | ||
| 169 | + with prefix CAND-BSP-* / promotion path to BSP-N canonical. | ||
| 170 | + | ||
| 171 | + Origin: FA Pattern A iter 1-5 (~$53 spend) empirically falsified all | ||
| 172 | + worker-scope hypotheses for V220 MIX_AIC_1_2 cube-internal sync. The | ||
| 173 | + remaining root-cause candidates live in build-system glue (per-source | ||
| 174 | + flag isolation / FFTSCNT mailbox / register attribute metadata), NOT in | ||
| 175 | + kernel.h. Mode 5 brief explicitly restricted scope to kernel files; the | ||
| 176 | + info needed wasn't reachable. Mode 6 extends scope to address this. | ||
| 177 | + """ | ||
| 178 | + if extraction_mode == "build_system": | ||
| 179 | + bs_candidates = kb_root_abs / "target" / "ascendc" / "build_system" / "candidates.md" | ||
| 180 | + return f"""# Specific extraction questions for this op (Mode 6: build_system) | ||
| 181 | + | ||
| 182 | +{op} extraction in Mode 6 is BUILD-SYSTEM-FOCUSED — the kernel structural | ||
| 183 | +patterns are out of scope (Mode 5 covers those). Focus Phase B/C on the | ||
| 184 | +HOST-SIDE GLUE that defines how the kernel binary is built + launched: | ||
| 185 | + | ||
| 186 | +- **Per-source-file compile flag isolation**: which `target_compile_definitions` | ||
| 187 | + / `add_compile_definitions` apply to which source files. Especially | ||
| 188 | + `-DASCENDC_MATMUL_AICORE` — does it apply uniformly or only to cube cpp? | ||
| 189 | +- **Multi-target binary registration**: how the same op registers different | ||
| 190 | + arch variants (V220 vs V351) via `register_*.cpp` / `op_proto*.cpp`. | ||
| 191 | +- **Launch macro routing**: what `KERNEL_TASK_TYPE_DEFAULT` arguments appear | ||
| 192 | + in which `*_apt.cpp` files, gated by what `__NPU_ARCH__` macros. | ||
| 193 | +- **CMake build dependency chains**: `add_dependencies` revealing build-time | ||
| 194 | + ordering constraints (e.g. tiling lib must build before kernel cpp). | ||
| 195 | +- **Binary attribute metadata**: how `aclrtLaunchKernel` is called with | ||
| 196 | + arch-specific attributes — does the op's host glue inject FFTSCNT mailbox | ||
| 197 | + init metadata that a simpler standalone launch path doesn't replicate? | ||
| 198 | +- **CMake-time API surface gates**: `if(ASCENDC_ENABLE_*)` blocks revealing | ||
| 199 | + what subsystems are conditionally compiled. | ||
| 200 | + | ||
| 201 | +## Scope reminder for Mode 6 file reads (Phase B) | ||
| 202 | + | ||
| 203 | +Within `module_path`, READ: | ||
| 204 | +- `CMakeLists.txt` and any `*.cmake` files (top-level + per-subdir) | ||
| 205 | +- `register_*.cpp` (host-side op registration) | ||
| 206 | +- `op_proto*.{{cpp,h}}` (host-side op definition) | ||
| 207 | +- `op_kernel/*_apt.cpp` (kernel adapter / launch entry — host-facing) | ||
| 208 | +- `BUILD` / `BUILD.bazel` if present | ||
| 209 | +- Top-level kernel.h (FOR CONTEXT ONLY — the launch macro it references; | ||
| 210 | + do NOT extract kernel-structural patterns in Mode 6, those are Mode 5's | ||
| 211 | + scope) | ||
| 212 | + | ||
| 213 | +DO NOT read internal `common/*` headers, shared utility headers, or recurse. | ||
| 214 | + | ||
| 215 | +## Output path (Mode 6 specific) | ||
| 216 | + | ||
| 217 | +Candidates → `{bs_candidates}` (NOT the patterns/unverified/candidates.md | ||
| 218 | +that Mode 5 uses — that's reserved for kernel-structural). | ||
| 219 | + | ||
| 220 | +Use prefix `CAND-BSP-*` for candidates (CANN Build-System Pattern), promotable | ||
| 221 | +to canonical `BSP-N` entries in `target/ascendc/build_system/PRINCIPLES.md`. | ||
| 222 | + | ||
| 223 | +## Generality requirement (Mode 6) | ||
| 224 | + | ||
| 225 | +Each BSP candidate must be applicable to AT LEAST 2 distinct op-classes | ||
| 226 | +(e.g. fused-attention + fused-quant-matmul) — the build-system patterns | ||
| 227 | +are higher-level than algorithm patterns and should generalize broadly. | ||
| 228 | + | ||
| 229 | +If a pattern is op-class-specific (e.g. only flash_attention_score uses it), | ||
| 230 | +flag it as `narrow_scope: true` in the candidate metadata and drop unless | ||
| 231 | +the body explains WHY no generalization is possible.""" | ||
| 232 | + | ||
| 233 | + # default: Mode 5 (kernel_structural) — preserves historical behavior | ||
| 234 | + return f"""# Specific extraction questions for this op (Mode 5: kernel_structural) | ||
| 235 | + | ||
| 236 | +{op} is a fused operator. The skill caller wants generalizable patterns | ||
| 237 | +applicable to multiple fused ops (FA / MoE / fused-norm / KV-cache / RoPE). | ||
| 238 | +Focus your Phase B/C on: | ||
| 239 | +- How CANN structures multi-stage tile orchestration (Q·K^T → softmax → ·V chains) | ||
| 240 | +- Public-API expression of online softmax (max-reduction + exp + accumulate) | ||
| 241 | +- Multi-output API contract conventions (main return + auxiliary tensors | ||
| 242 | + like softmax_max / softmax_sum, tile-coord-tagged outputs) | ||
| 243 | +- UB budget partitioning for fused ops (Q/K/V cache + scores + attn + out) | ||
| 244 | +- dtype templating decisions (when to fp32-promote vs stay fp16) | ||
| 245 | + | ||
| 246 | +These questions seed Phase A pre-scan keywords AND Phase C `other-instances-predicted` | ||
| 247 | +generality (so candidates aren't FA-specific). | ||
| 248 | + | ||
| 249 | +## Scope reminder for Mode 5 file reads (Phase B) | ||
| 250 | + | ||
| 251 | +Read 2-5 files (header + impl + tiling) within `module_path`. DO NOT recurse | ||
| 252 | +beyond passed scope. Keep notes on: | ||
| 253 | +- Algorithm structure (loops, dispatch, reduction tree, branch shape) | ||
| 254 | +- Public-API equivalents (which AscendC API would express the same idea) | ||
| 255 | +- Internal-only primitives — flag explicitly as non-portable | ||
| 256 | + | ||
| 257 | +## Output path (Mode 5 default) | ||
| 258 | + | ||
| 259 | +Candidates → `{candidates_path}` (patterns/unverified/candidates.md)""" | ||
| 260 | + | ||
| 261 | + | ||
| 262 | +def _list_unverified_candidates(kb_root: Path) -> set[Path]: | ||
| 263 | + """Snapshot of candidate files in patterns/unverified/ (excluding markers).""" | ||
| 264 | + return _list_unverified_candidates_at(kb_root / "patterns" / "unverified") | ||
| 265 | + | ||
| 266 | + | ||
| 267 | +def _list_unverified_candidates_at(dir_path: Path) -> set[Path]: | ||
| 268 | + """Snapshot of candidate files in `dir_path` (excluding markers). | ||
| 269 | + | ||
| 270 | + Mode 6 (build_system) uses `target/ascendc/build_system/` as the | ||
| 271 | + candidates dir; Mode 5 uses `patterns/unverified/`. Shared listing | ||
| 272 | + logic factored here. | ||
| 273 | + """ | ||
| 274 | + if not dir_path.exists(): | ||
| 275 | + return set() | ||
| 276 | + candidates: set[Path] = set() | ||
| 277 | + for candidate in dir_path.iterdir(): | ||
| 278 | + if ( | ||
| 279 | + candidate.is_file() | ||
| 280 | + and not candidate.name.startswith(".") | ||
| 281 | + and candidate.suffix == ".md" | ||
| 282 | + and candidate.name != "candidates.md" | ||
| 283 | + ): | ||
| 284 | + candidates.add(candidate) | ||
| 285 | + return candidates | ||
| 286 | + | ||
| 287 | + | ||
| 288 | +def _extract_cann_files_read(summary_path: Path, module_path: Path) -> list[Path]: | ||
| 289 | + """Read summary.json's cann_files_read field; fallback to module_path enumeration.""" | ||
| 290 | + if summary_path.exists(): | ||
| 291 | + try: | ||
| 292 | + d = json.loads(summary_path.read_text()) | ||
| 293 | + files = d.get("cann_files_read", []) | ||
| 294 | + if isinstance(files, list) and files: | ||
| 295 | + return [Path(f) for f in files] | ||
| 296 | + except (json.JSONDecodeError, OSError): | ||
| 297 | + pass | ||
| 298 | + | ||
| 299 | + # Fallback: enumerate module_path files (over-conservative — re-scan | ||
| 300 | + # may flag patterns that didn't actually read those files, but better | ||
| 301 | + # than missing real leaks). | ||
| 302 | + if module_path.is_dir(): | ||
| 303 | + return list(module_path.rglob("*.h")) + list(module_path.rglob("*.cpp")) | ||
| 304 | + elif module_path.is_file(): | ||
| 305 | + return [module_path] | ||
| 306 | + return [] | ||
| 307 | + | ||
| 308 | + | ||
| 309 | +def spawn_cann_learner_agent( | ||
| 310 | + *, | ||
| 311 | + op: str, | ||
| 312 | + workspace: Path, | ||
| 313 | + module_path: Path, | ||
| 314 | + sealed_dir: Path, | ||
| 315 | + run_id: str, | ||
| 316 | + kb_root: Path, | ||
| 317 | + api_catalog_path: Path, | ||
| 318 | + timeout_sec: int = DEFAULT_TIMEOUT_SEC, | ||
| 319 | + extraction_mode: str = "kernel_structural", | ||
| 320 | +) -> dict: | ||
| 321 | + """Spawn aog-cann-learner agent and collect output artifacts. | ||
| 322 | + | ||
| 323 | + Returns dict matching Mode 5's spawn_agent_func contract. | ||
| 324 | + | ||
| 325 | + Raises agent_transport.AgentTransportError if claude CLI fails. | ||
| 326 | + | ||
| 327 | + extraction_mode: "kernel_structural" (default, historical Mode 5 behavior) | ||
| 328 | + or "build_system" (Mode 6, 2026-05-21, extends scope to CMakeLists.txt + | ||
| 329 | + register + op_proto + apt.cpp for build-system pattern extraction). | ||
| 330 | + """ | ||
| 331 | + brief = build_cann_learner_brief( | ||
| 332 | + op=op, workspace=workspace, module_path=module_path, | ||
| 333 | + sealed_dir=sealed_dir, run_id=run_id, kb_root=kb_root, | ||
| 334 | + api_catalog_path=api_catalog_path, | ||
| 335 | + extraction_mode=extraction_mode, | ||
| 336 | + ) | ||
| 337 | + | ||
| 338 | + # Snapshot candidates dir before spawn (for diff after). | ||
| 339 | + # Mode 6 routes candidates to target/ascendc/build_system/candidates.md | ||
| 340 | + # instead of patterns/unverified/candidates.md. | ||
| 341 | + if extraction_mode == "build_system": | ||
| 342 | + candidates_dir = kb_root / "target" / "ascendc" / "build_system" | ||
| 343 | + candidates_md = candidates_dir / "candidates.md" | ||
| 344 | + candidates_dir.mkdir(parents=True, exist_ok=True) # may not exist yet | ||
| 345 | + cands_before = _list_unverified_candidates_at(candidates_dir) | ||
| 346 | + else: | ||
| 347 | + candidates_dir = kb_root / "patterns" / "unverified" | ||
| 348 | + candidates_md = candidates_dir / "candidates.md" | ||
| 349 | + cands_before = _list_unverified_candidates(kb_root) | ||
| 350 | + cands_md_size_before = candidates_md.stat().st_size if candidates_md.exists() else 0 | ||
| 351 | + | ||
| 352 | + spawn_started_at = time.time() | ||
| 353 | + | ||
| 354 | + # Spawn — STREAMING mode for diagnostic visibility (P0aau-c35.d, 2026-05-09). | ||
| 355 | + # First M2 attempt with --output-format json hit 40min timeout with zero | ||
| 356 | + # output and no diagnostic trace. Switching to stream-json + verbose so | ||
| 357 | + # tool_use events tee to disk and we can see what the agent did even | ||
| 358 | + # if it ultimately stalls. Performance penalty is acceptable (~10% per | ||
| 359 | + # tool call); diagnostic value is critical for first real run. | ||
| 360 | + extra_dirs: list[str] = [] | ||
| 361 | + cwd = Path.cwd().resolve() | ||
| 362 | + for must_read in (workspace.resolve(), module_path.resolve(), kb_root.resolve()): | ||
| 363 | + if not str(must_read).startswith(str(cwd)): | ||
| 364 | + extra_dirs.extend(["--add-dir", str(must_read)]) | ||
| 365 | + | ||
| 366 | + # Tee stream events into sealed dir for forensics. | ||
| 367 | + tee_path = sealed_dir / "spawn_stream.jsonl" | ||
| 368 | + | ||
| 369 | + result = _backend.dispatch( | ||
| 370 | + "aog-cann-learner", brief, kind="agent", mode="streaming", | ||
| 371 | + tee_path=tee_path, timeout=timeout_sec, cwd=cwd, | ||
| 372 | + extra_args=extra_dirs or None, | ||
| 373 | + ) | ||
| 374 | + # (permission_mode="bypassPermissions" was explicit; it's spawn_agent_streaming's default → faithful.) | ||
| 375 | + | ||
| 376 | + spawn_duration = time.time() - spawn_started_at | ||
| 377 | + | ||
| 378 | + # Collect output artifacts. | ||
| 379 | + summary_path = workspace / "cann_learn_summary.json" | ||
| 380 | + | ||
| 381 | + # Sealed dir contents. | ||
| 382 | + sealed_files: list[Path] = [] | ||
| 383 | + if sealed_dir.exists(): | ||
| 384 | + sealed_files = [f for f in sealed_dir.rglob("*") if f.is_file()] | ||
| 385 | + | ||
| 386 | + # Candidate files: anything new in candidates_dir + grow in candidates.md. | ||
| 387 | + if extraction_mode == "build_system": | ||
| 388 | + cands_after = _list_unverified_candidates_at(candidates_dir) | ||
| 389 | + else: | ||
| 390 | + cands_after = _list_unverified_candidates(kb_root) | ||
| 391 | + new_cand_files = sorted(cands_after - cands_before) | ||
| 392 | + | ||
| 393 | + # If candidates.md grew, treat its entire current content as a "candidate path" | ||
| 394 | + # for re-scan purposes — Mode 5's revalidate runs C34a / C34c against EACH | ||
| 395 | + # candidate path's contents. | ||
| 396 | + candidate_paths: list[Path] = list(new_cand_files) | ||
| 397 | + if candidates_md.exists() and candidates_md.stat().st_size > cands_md_size_before: | ||
| 398 | + candidate_paths.append(candidates_md) | ||
| 399 | + | ||
| 400 | + cann_files_read = _extract_cann_files_read(summary_path, module_path) | ||
| 401 | + | ||
| 402 | + # Write a small spawn audit record into sealed_dir for forensics. | ||
| 403 | + if sealed_dir.exists(): | ||
| 404 | + (sealed_dir / "spawn_audit.json").write_text(json.dumps({ | ||
| 405 | + "run_id": run_id, | ||
| 406 | + "op": op, | ||
| 407 | + "spawn_duration_sec": round(spawn_duration, 2), | ||
| 408 | + "agent_output_text": result.output_text, | ||
| 409 | + "agent_is_error": result.is_error, | ||
| 410 | + "agent_cost_usd": result.cost_usd, | ||
| 411 | + "n_sealed_files": len(sealed_files), | ||
| 412 | + "n_new_cand_files": len(new_cand_files), | ||
| 413 | + "candidates_md_size_delta": ( | ||
| 414 | + candidates_md.stat().st_size - cands_md_size_before | ||
| 415 | + if candidates_md.exists() else 0 | ||
| 416 | + ), | ||
| 417 | + "n_cann_files_read": len(cann_files_read), | ||
| 418 | + }, indent=2)) | ||
| 419 | + | ||
| 420 | + return { | ||
| 421 | + "sealed_files": sealed_files, | ||
| 422 | + "summary_path": summary_path if summary_path.exists() else None, | ||
| 423 | + "candidate_paths": candidate_paths, | ||
| 424 | + "cann_files_read": cann_files_read, | ||
| 425 | + "_agent_result": result, # extra; Mode 5 ignores; useful for debugging | ||
| 426 | + } | ||