"""每日 issue 在线表格同步至 GitCode 仓（Alice_zhou/issue，当日日期命名）
功能定位：每天提的 issue 自动更新到 GitCode 仓的"当日在线表格"里——仓内路径默认根目录
`issue日报_{日期}.xlsx`（config issue_tracker.json 的 daily_issue_repo_path 可配，支持 {date}
占位符），表格包含 **aclnn接口名 / 测试场景 / 问题 / issue链接**（附 日期/提单时间 辅助列），
供在线查看当日全部已提 issue。
数据来源（按优先级合并，行键=接口名+测试场景+issue链接，幂等去重）：
1. out_res/issues/{日期}/daily_issues.json（主数据源——file_bug_issues.py 每次提单/复用
成功后实时写入，含 issue 标题与 bug 判定）
2. 历史数据回退：out_res/issues/{日期}/filed.json（key=api_场景/api_error_caseN 等，
value=issue URL）+ summary.md（补充 bug 判定列）
3. 结果表格（--xlsx 可选）：是否存在bug 以"是bug"开头且 问题单 列非空的行（多链接拆行）
多机安全合并（同 sync_batch_results_git.py 机制，不丢任何机器的数据）：
- 行键 = aclnn接口名 + 测试场景 + issue链接
- 同步时先 GET 远端 xlsx → 以远端为基底，将本机行 upsert 合并进去（键冲突时本机行优先）
- 远端存在而本机没有的行（其他机器提的 issue）原样保留，禁止整表覆盖
- PUT 携带远端文件 sha（乐观锁）；多机并发推送冲突（409/sha 失配）时自动重拉远端重新
合并重试（最多 --retries 次）；远端文件不存在时 POST 创建
触发链路（自动，无需手动）：
- file_bug_issues.py 每轮提单成功后自动调用 sync_daily_issues()（即时提单 A / 23:59 兜底 B）
- scheduled_update.py 每日 4 轮整点兜底补推当日 issue 表（实时同步失败/漏推的兜底）
API（已实测验证，同 scripts/sync_batch_results_git.py）：
- GET {api_base}/repos/{owner}/{repo}/contents/{path}?ref={branch}
- POST {api_base}/repos/{owner}/{repo}/contents/{path} JSON {content(base64), message, branch}
- PUT {api_base}/repos/{owner}/{repo}/contents/{path} JSON {content, message, branch, sha}
- 认证：PRIVATE-TOKEN 头（config issue_tracker.json access_token，支持 ${GITCODE_TOKEN}）
用法:
python scripts/sync_daily_issues_git.py # 同步今天（生成+推送）
python scripts/sync_daily_issues_git.py --date 2026-11-13 # 指定日期
python scripts/sync_daily_issues_git.py --dry-run # 仅生成本地 issue_daily.xlsx 不推送
python scripts/sync_daily_issues_git.py --xlsx out_res/批量压测结果.xlsx # 附加结果表格数据源
python scripts/sync_daily_issues_git.py --repo_path issues/daily_{date}.xlsx # 自定义仓内路径
python scripts/sync_daily_issues_git.py --json # JSON 输出
开关：环境变量 DAILY_ISSUE_GIT_SYNC=0/1 > config daily_issue_git_sync（默认 true）
"""
import argparse
import base64
import datetime
import io
import json
import os
import re
import socket
import sys
import time
try:
import openpyxl
except ImportError:
openpyxl = None
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from sync_batch_results_git import GitCodeContentsClient
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
DEFAULT_CONFIG = "config/issue_tracker.json"
DEFAULT_REPO_PATH_TPL = "issue日报_{date}.xlsx"
MAX_FILE_BYTES = 50 * 1024 * 1024
ISSUES_ROOT = os.path.join("out_res", "issues")
HEADERS = ["日期", "aclnn接口名", "测试场景", "问题", "issue链接", "提单时间"]
_SCENARIO_RE = re.compile(
r"^(aclnn[A-Za-z0-9]+)_(normal|non_contig|multithread|consistent|error|sample)(.*)$")
def log(tag, msg):
print("[{}] {}".format(tag, msg))
def _today():
return datetime.date.today().strftime("%Y-%m-%d")
def _cell_str(value):
return str(value).strip() if value is not None else ""
def _parse_filed_key(key):
"""filed.json 键 → (api, scenario)（如 aclnnAbs_error_case20 → (aclnnAbs, error)）。"""
m = _SCENARIO_RE.match(str(key or ""))
if m:
return m.group(1), m.group(2)
return str(key or ""), "-"
def _parse_summary_md(path):
"""summary.md 行 `- [api/scenario](url) — bug判定` → {url: (api, scenario, bug)}。"""
out = {}
if not os.path.isfile(path):
return out
try:
for line in open(path, encoding="utf-8"):
m = re.match(r"^-\s+\[([^\]/]+)/([^\]]+)\]\((https?://[^\)]+)\)\s*[—-]\s*(.*)$", line.strip())
if m:
out[m.group(3)] = (m.group(1), m.group(2), m.group(4))
except Exception:
pass
return out
def _rows_from_result_xlsx(xlsx_path, date_str):
"""结果表格兜底数据源：是bug 且 问题单列非空的行（多链接拆行）。"""
rows = []
if not xlsx_path or not os.path.isfile(xlsx_path) or openpyxl is None:
return rows
try:
wb = openpyxl.load_workbook(xlsx_path, read_only=True)
ws = wb.active
rows_raw = list(ws.iter_rows(values_only=True))
wb.close()
if not rows_raw:
return rows
header = [str(c).strip() if c is not None else "" for c in rows_raw[0]]
idx = {h: i for i, h in enumerate(header)}
if "问题单" not in idx:
return rows
for row in rows_raw[1:]:
if not row or not any(str(c).strip() for c in row if c is not None):
continue
def col(name, default=""):
i = idx.get(name)
return ("" if i is None or i >= len(row) or row[i] is None else str(row[i]).strip()) or default
bug = col("是否存在bug")
if not bug.startswith("是bug"):
continue
issue_cell = col("问题单")
if not issue_cell:
continue
reason = col("执行失败原因", "无")
problem = bug if reason == "无" else "{}（{}）".format(bug, reason[:60])
for url in re.split(r"[\n;；]", issue_cell):
url = url.strip()
if url.startswith("http"):
rows.append({"api": col("接口名"), "scenario": col("测试场景", "-"),
"problem": problem, "issue": url,
"time": col("更新时间", "")})
except Exception as exc:
log("WARN", "结果表格 {} 解析失败（跳过该数据源）: {}".format(xlsx_path, exc))
return rows
def collect_daily_rows(date_str, xlsx=None):
"""收集当日已提 issue 行（daily_issues.json 主源 → filed.json+summary.md 回退 → 结果表格可选）。
返回行列表，每行 {api, scenario, problem, issue, time}；行键=(api, scenario, issue) 幂等去重，
先到者优先（daily_issues.json 含准确 issue 标题，故优先级最高）。"""
day_dir = os.path.join(ISSUES_ROOT, date_str)
collected = []
seen = set()
def add(api, scenario, problem, issue, t=None):
api = _cell_str(api)
scenario = _cell_str(scenario) or "-"
issue = _cell_str(issue)
problem = _cell_str(problem) or "-"
t = _cell_str(t)
if not api or not issue.startswith("http"):
return
key = (api, scenario, issue)
if key in seen:
return
seen.add(key)
collected.append({"api": api, "scenario": scenario, "problem": problem,
"issue": issue, "time": t})
daily_path = os.path.join(day_dir, "daily_issues.json")
if os.path.isfile(daily_path):
try:
data = json.load(open(daily_path, encoding="utf-8-sig"))
except ValueError:
data = None
if isinstance(data, list):
for r in data:
if isinstance(r, dict):
add(r.get("api"), r.get("scenario"), r.get("problem"), r.get("issue"), r.get("time"))
filed_path = os.path.join(day_dir, "filed.json")
if os.path.isfile(filed_path) and not collected:
summary_map = _parse_summary_md(os.path.join(day_dir, "summary.md"))
try:
filed = json.load(open(filed_path, encoding="utf-8-sig"))
except ValueError:
filed = {}
if isinstance(filed, dict):
for key, url in filed.items():
url = _cell_str(url)
if not url.startswith("http"):
continue
if url in summary_map:
api, scenario, bug = summary_map[url]
add(api, scenario, bug, url)
else:
api, scenario = _parse_filed_key(key)
add(api, scenario, "是bug（当日已提单，详见 filed.json）", url)
if xlsx:
for r in _rows_from_result_xlsx(xlsx, date_str):
add(r["api"], r["scenario"], r["problem"], r["issue"], r["time"])
return collected
def build_local_workbook(rows, date_str, out_path):
"""生成/刷新本地当日 issue 表（out_res/issues/{日期}/issue_daily.xlsx，行按 接口名/场景/时间 排序）。"""
if openpyxl is None:
raise RuntimeError("openpyxl 未安装（pip install openpyxl）")
os.makedirs(os.path.dirname(out_path), exist_ok=True)
wb = openpyxl.Workbook()
ws = wb.active
ws.title = "issue日报"
ws.append(HEADERS)
for r in sorted(rows, key=lambda x: (x["api"], x["scenario"], x["time"] or "")):
ws.append([r.get("date") or date_str, r["api"], r["scenario"], r["problem"], r["issue"], r["time"]])
for col_idx, width in ((1, 12), (2, 26), (3, 14), (4, 80), (5, 60), (6, 20)):
ws.column_dimensions[openpyxl.utils.get_column_letter(col_idx)].width = width
for row_no in range(2, ws.max_row + 1):
url = _cell_str(ws.cell(row=row_no, column=5).value)
if url.startswith("http"):
ws.cell(row=row_no, column=5).hyperlink = url
wb.save(out_path)
return out_path
def _row_key(ws, row):
return (_cell_str(ws.cell(row=row, column=2).value),
_cell_str(ws.cell(row=row, column=3).value),
_cell_str(ws.cell(row=row, column=5).value))
def merge_local_into_remote(remote_bytes, local_xlsx):
"""以远端 xlsx 为基底合并本机行（行键=接口名+测试场景+issue链接，本机行优先），
返回 (合并后 xlsx 字节, merged, appended)。远端解析失败时抛异常（禁止整表覆盖远端）。"""
if openpyxl is None:
raise RuntimeError("openpyxl 未安装，无法合并（pip install openpyxl）")
remote_wb = openpyxl.load_workbook(io.BytesIO(remote_bytes))
remote_ws = remote_wb.active
local_wb = openpyxl.load_workbook(local_xlsx)
local_ws = local_wb.active
if remote_ws.max_row < 1 or _cell_str(remote_ws.cell(row=1, column=2).value) != HEADERS[1]:
if remote_ws.max_row >= 1 and _cell_str(remote_ws.cell(row=1, column=2).value) == "":
header_row = 1
else:
header_row = remote_ws.max_row + 1
for col_idx, name in enumerate(HEADERS, start=1):
remote_ws.cell(row=header_row, column=col_idx, value=name)
remote_map = {}
for row in range(2, remote_ws.max_row + 1):
key = _row_key(remote_ws, row)
if key[0] and key[2]:
remote_map[key] = row
merged, appended = 0, 0
for row in range(2, local_ws.max_row + 1):
key = _row_key(local_ws, row)
if not (key[0] and key[2]):
continue
if key in remote_map:
target = remote_map[key]
merged += 1
else:
target = remote_ws.max_row + 1
appended += 1
for col_idx in range(1, len(HEADERS) + 1):
value = local_ws.cell(row=row, column=col_idx).value
cell = remote_ws.cell(row=target, column=col_idx, value=value)
if col_idx == 5 and _cell_str(value).startswith("http"):
cell.hyperlink = _cell_str(value)
buf = io.BytesIO()
remote_wb.save(buf)
return buf.getvalue(), merged, appended
def daily_sync_enabled(config_path=DEFAULT_CONFIG):
"""同步开关：环境变量 DAILY_ISSUE_GIT_SYNC > config daily_issue_git_sync（默认 true）。"""
env = os.environ.get("DAILY_ISSUE_GIT_SYNC", "").strip()
if env:
return env not in ("0", "false", "off", "no"), "env DAILY_ISSUE_GIT_SYNC={}".format(env)
if os.path.isfile(config_path):
try:
cfg = json.load(open(config_path, encoding="utf-8-sig"))
enabled = cfg.get("daily_issue_git_sync", True)
return bool(enabled), "config daily_issue_git_sync={}".format(enabled)
except Exception:
return True, "config 解析失败，按默认启用"
return True, "默认启用（无配置文件）"
def sync_daily_issues(date_str=None, config_path=DEFAULT_CONFIG, repo_path=None,
retries=3, dry_run=False, xlsx=None):
"""将当日已提 issue 汇总为在线表格并推送 GitCode 仓（Alice_zhou/issue）。
返回 (ok, url, detail)：ok=True 表示已推送 / 无需推送（当日无记录）/ dry-run 已生成；
ok=False 表示失败（配置缺失/网络失败等，可重试）。"""
date_str = date_str or _today()
day_dir = os.path.join(ISSUES_ROOT, date_str)
local_xlsx = os.path.join(day_dir, "issue_daily.xlsx")
enabled, why = daily_sync_enabled(config_path)
if not enabled:
return True, "", "同步开关关闭（{}），跳过推送".format(why)
rows = collect_daily_rows(date_str, xlsx=xlsx)
if not rows:
return True, "", "当日（{}）无已提 issue 记录（daily_issues.json/filed.json 均无），跳过推送".format(date_str)
build_local_workbook(rows, date_str, local_xlsx)
if dry_run:
return True, local_xlsx, "dry-run：已生成本地表格（{} 行）: {}".format(len(rows), local_xlsx)
if os.path.getsize(local_xlsx) > MAX_FILE_BYTES:
return False, "", "本地表格超过 {}MB 上限".format(MAX_FILE_BYTES // (1024 * 1024))
try:
client = GitCodeContentsClient(config_path)
except Exception as exc:
return False, "", "配置加载失败: {}".format(exc)
try:
cfg = json.load(open(config_path, encoding="utf-8-sig"))
except Exception:
cfg = {}
tpl = repo_path or cfg.get("daily_issue_repo_path") or DEFAULT_REPO_PATH_TPL
path = tpl.replace("{date}", date_str)
host = socket.gethostname()
commit_msg = "issues-daily: {} {} ({} rows) from {}".format(
date_str, os.path.basename(path), len(rows), host)
last_err = ""
for attempt in range(1, retries + 1):
code, info = client.get_file(path)
if code == 200:
sha = info.get("sha", "")
content_b64 = info.get("content", "") or ""
remote_bytes = base64.b64decode(content_b64) if content_b64 else b""
try:
merged_bytes, merged, appended = merge_local_into_remote(remote_bytes, local_xlsx)
except Exception as exc:
return False, "", "远端文件合并失败（禁止覆盖远端，已中止）: {}".format(exc)
payload_b64 = base64.b64encode(merged_bytes).decode("ascii")
code2, resp = client.update_file(path, payload_b64, commit_msg, sha)
if code2 in (200, 201):
url = info.get("html_url", "") or info.get("download_url", "")
return True, url, "merge(remote+local) merged={} appended={} rows={} host={}".format(
merged, appended, len(rows), host)
last_err = "PUT 失败 [{}]: {}".format(code2, json.dumps(resp, ensure_ascii=False)[:200])
if code2 in (409, 400, 422):
time.sleep(min(1.0 * attempt, 3.0))
continue
elif code == 404:
payload_b64 = base64.b64encode(open(local_xlsx, "rb").read()).decode("ascii")
code2, resp = client.create_file(path, payload_b64, commit_msg)
if code2 in (200, 201):
code3, info2 = client.get_file(path)
url = info2.get("html_url", "") or info2.get("download_url", "") if code3 == 200 else ""
return True, url, "create(new) rows={} host={}".format(len(rows), host)
last_err = "POST 失败 [{}]: {}".format(code2, json.dumps(resp, ensure_ascii=False)[:200])
time.sleep(min(1.0 * attempt, 3.0))
else:
last_err = "GET 失败 [{}]: {}".format(code, json.dumps(info, ensure_ascii=False)[:200])
time.sleep(min(1.0 * attempt, 3.0))
return False, "", "重试 {} 次仍失败: {}".format(retries, last_err)
def main():
parser = argparse.ArgumentParser(
description="每日 issue 在线表格同步至 GitCode 仓（Alice_zhou/issue 的 issue日报_{date}.xlsx，多机合并推送）")
parser.add_argument("--date", default=None, help="日期 YYYY-MM-DD（默认今天）")
parser.add_argument("--xlsx", default=None,
help="可选：附加结果表格数据源（如 out_res/批量压测结果.xlsx，是bug且问题单非空行拆行）")
parser.add_argument("--repo_path", default=None,
help="仓内路径模板，默认取 config 的 daily_issue_repo_path（缺省 issue日报_{date}.xlsx，支持 {date} 占位符）")
parser.add_argument("--config", default=DEFAULT_CONFIG, help="配置文件路径")
parser.add_argument("--retries", type=int, default=3, help="并发冲突重试次数，默认 3")
parser.add_argument("--dry-run", action="store_true", help="仅生成本地 issue_daily.xlsx 不推送")
parser.add_argument("--json", action="store_true", help="以 JSON 输出结果")
args = parser.parse_args()
date_str = args.date or _today()
ok, url, detail = sync_daily_issues(date_str, args.config, args.repo_path,
args.retries, args.dry_run, args.xlsx)
if args.json:
print(json.dumps({"ok": ok, "date": date_str, "url": url, "detail": detail},
ensure_ascii=False))
else:
if ok and url:
log("DAILY_SYNC", "成功 → {} ({})".format(url, detail))
elif ok:
log("DAILY_SYNC", detail)
else:
log("DAILY_SYNC", "失败: {}".format(detail))
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())