"""Apply the same recipe to every file in a folder, with a content-hash cache
so re-runs only process what changed. A recipe is a list of script steps;
{in} and {out} are substituted, and the output of one step feeds the next.
Recipe (batch.json):
{
"glob": "*.mp4",
"output_dir": "out",
"suffix": "_final",
"steps": [
["silence.py", "{in}", "--threshold", "-38", "-o", "{out}"],
["loudness.py", "{in}", "-o", "{out}"],
["export.py", "{in}", "--preset", "youtube", "-o", "{out}"]
]
}
or use a render project for every file: {"project": "project.json", "clip_key": 0}
Examples:
python3 batch.py ~/Footage --recipe batch.json
python3 batch.py ~/Footage --recipe batch.json --dry-run
python3 batch.py ~/Footage --recipe batch.json --force # ignore the cache
python3 batch.py ~/Footage --recipe batch.json --watch 30 # poll the folder every 30 s
"""
import argparse
import difflib
import hashlib
import json
import os
import sys
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
from _common import STATE, add_common, apply_common, child_args, die, emit, info, run_tool, MEDIA_EXT as _MEDIA_EXT
from _common.runner import (with_flags, HW_FORCED_ENV, OWNED_ENV, _OutputLock, add_hw_orchestrator_args, copy_to_sibling, env_hw, read_json,
owned_identity, stop_children)
from _common.emit import absorb_stage_hw, audit, note, record_skip, shown, steps_encoder_report, warn
from _common.decision import NO_DEFAULT_ENCODE_EXT, brand_asset_files, rebase_project_paths
from _common.runner import parse_json
HERE = Path(__file__).resolve().parent
MEDIA_EXT = {e for e in _MEDIA_EXT if e not in (".png", ".jpg", ".jpeg", ".webp")}
ALLOWED_STEP_SCRIPTS = {p.name for p in HERE.glob("*.py") if not p.name.startswith("_")}
def file_key(path: Path) -> str:
"""A cheap change detector: the name, the size, the modification time in nanoseconds and the
first and last MiB. A rewrite of the middle of a file keeps its size and its ends, so the
modification time is what tells it apart: in whole seconds (up to 3.8.4) two writes within
one second shared a key, and the old output was served as cached. A file whose mtime is reset
to the earlier value (``touch -r``, a copy that preserves it) or a filesystem that keeps
whole seconds (FAT keeps two) still shares one; hashing every byte would cost a read of the
whole file per item per run (docs/design-decisions.md, "After 3.8.4")."""
st = path.stat()
h = hashlib.sha1(usedforsecurity=False)
h.update(f"{path.name}|{st.st_size}|{st.st_mtime_ns}".encode())
with open(path, "rb") as fh:
h.update(fh.read(1 << 20))
if st.st_size > 2 << 20:
fh.seek(-(1 << 20), os.SEEK_END)
h.update(fh.read(1 << 20))
return h.hexdigest()
def side_input_file(value: str) -> Path:
"""The file a side-input argument reads: the argument itself, or, when that is no file and it
ends in `:SUFFIX`, the part before the last colon when that is one (caption.py's
`--srt FILE:LANG`, which split_srt_lang() reads the same way). Keying the whole token hashed a
path that does not exist, so an edited subtitle file kept its key (#510 review)."""
path = Path(value)
try:
if path.is_file() or ":" not in value:
return path
head = Path(value.rpartition(":")[0])
return head if value.rpartition(":")[0] and head.is_file() else path
except OSError:
return path
def is_brand_value(step: List[Any], j: int) -> bool:
"""Whether step[j] is the value of --brand (`--brand X` or `--brand=X`)."""
arg = str(step[j])
return arg.startswith("--brand=") or (j > 0 and str(step[j - 1]) == "--brand")
def key_brand_assets(h: Any, path: Path) -> None:
"""Hash the logo and font file a brand file names into `h`: the brand file's own bytes do not
change when one of them is replaced under its name (#510 review; render.py's stage key and a
project recipe's key already cover them)."""
for key, asset in brand_asset_files(str(path)):
a = Path(asset)
try:
ident = file_key(a) if a.is_file() else "-"
except OSError:
ident = "-"
h.update(f"brand:{key}\0{asset}\0{ident}\0".encode())
def side_inputs_key(recipe: Dict[str, Any]) -> str:
"""The files a recipe's steps name besides {in}/{out} (check.py --compare REF, a LUT, a logo),
keyed by their own content: editing or removing one is a change the cache must see, or a QC
pass is served its old verdict against a reference that changed or is gone (#428 review).
Read on every pass, so a --watch loop sees the edit too."""
h = hashlib.sha1(usedforsecurity=False)
for step in recipe.get("steps") or []:
for j, arg in enumerate(step) if isinstance(step, list) else []:
if j == 0 or not isinstance(arg, str) or "{" in arg:
continue
value = arg.split("=", 1)[1] if arg.startswith("-") and "=" in arg else arg
if value.startswith("-") or not value:
continue
path = side_input_file(value)
try:
if path.is_file():
h.update(f"{value}\0{file_key(path)}\0".encode())
if is_brand_value(step, j):
key_brand_assets(h, path)
continue
except OSError:
pass
h.update(f"{value}\0-\0".encode())
return h.hexdigest()[:8]
def item_side_inputs_key(recipe: Dict[str, Any], src: Path) -> "tuple[str, Optional[Dict[str, Any]]]":
"""The side inputs a step names through a placeholder, for one item (#510): `--image {in}.png`
in the first step is a file beside this item's source, and editing it is a change the cache
must see as side_inputs_key() sees a literal path's. Returns (key part, None), "" when the
recipe names none, or ("", why) when one cannot be resolved before the item runs: a later
step's {in} is an intermediate that does not exist yet, so its file cannot be keyed, and the
item is not served from the cache at all."""
h = hashlib.sha1(usedforsecurity=False)
named = False
for i, step in enumerate(recipe.get("steps") or []):
for j, arg in enumerate(step) if isinstance(step, list) else []:
if j == 0 or not isinstance(arg, str) or "{in}" not in arg or "{out}" in arg:
continue
value = arg.split("=", 1)[1] if arg.startswith("-") and "=" in arg else arg
if value == "{in}" or "{in}" not in value:
continue
if i > 0:
return "", {"reason": "side_input_unresolved", "step": i, "argument": arg,
"message": f"step {i} names {arg!r}, a file derived from an intermediate that does not "
"exist before the item runs, so the cache cannot key it; the item always runs"}
path = side_input_file(value.replace("{in}", str(src)))
named = True
try:
if path.is_file():
h.update(f"{path}\0{file_key(path)}\0".encode())
if is_brand_value(step, j):
key_brand_assets(h, path)
continue
except OSError:
pass
h.update(f"{path}\0-\0".encode())
return (h.hexdigest()[:8] if named else ""), None
def caseless_glob(pattern: str) -> str:
"""`pattern` with every letter outside a [...] class matched in either case: "*.mp4" finds
GOPR0001.MP4, as it already did on macOS's and Windows' default filesystems, so a recipe means
the same files on every platform (A82: on Linux a camera folder was an empty batch). pathlib's
case_sensitive= needs Python 3.12; 3.10 is the floor."""
out: List[str] = []
in_class = False
first = -1
for i, ch in enumerate(pattern):
if in_class:
out.append(ch)
if ch == "]" and i > first:
in_class = False
elif ch == "[":
first = i + 2 if pattern[i + 1:i + 2] == "!" else i + 1
in_class = "]" in pattern[first + 1:]
out.append(ch if in_class else "[[]")
elif ch.isalpha() and ch.lower() != ch.upper():
out.append(f"[{ch.lower()}{ch.upper()}]")
else:
out.append(ch)
return "".join(out)
def recipe_key(recipe: Dict[str, Any]) -> str:
project_content = ""
if recipe.get("project"):
try:
project_content = Path(recipe["project"]).read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as exc:
entry = {"code": "recipe_project_unreadable", "path": str(recipe["project"])}
if not any(all(w.get(k) == v for k, v in entry.items()) for w in STATE.warnings if isinstance(w, dict)):
warn(f"recipe.project {recipe['project']} could not be read ({exc}); the cache key covers the recipe only",
"recipe_project_unreadable", path=str(recipe["project"]), error=str(exc))
else:
project_content += "\0" + project_files_key(recipe, project_content)
return hashlib.sha1((json.dumps(recipe, sort_keys=True) + "\0" + project_content).encode(), usedforsecurity=False).hexdigest()[:12]
def project_files_key(recipe: Dict[str, Any], text: str) -> str:
"""The files a project recipe's project names (music, cues, the brand and its logo and font,
overlay images and videos, the chapters file, every clip but the one each item replaces),
each keyed by its own content as file_key() keys a source (A80): an edited cue file or music
bed changes the output, so it has to change the key. The list is render.py's own
project_input_files(), read against the project file's directory as render reads it. A
project that does not parse adds nothing: each item then fails reading it."""
try:
proj = parse_json(text)
except ValueError:
return ""
if not isinstance(proj, dict):
return ""
from render import project_input_files
base = Path(str(recipe["project"])).resolve().parent
clips = proj.get("clips")
idx = int(recipe.get("clip_key") or 0)
if isinstance(clips, list) and idx < len(clips) and isinstance(clips[idx], dict):
proj = dict(proj, clips=[dict(c, src=None) if i == idx else c for i, c in enumerate(clips)])
def rel(p: Any) -> str:
return str(p) if os.path.isabs(str(p)) else str(base / str(p))
h = hashlib.sha1(usedforsecurity=False)
for name in sorted(set(project_input_files(proj, rel))):
path = Path(name)
try:
ident = file_key(path) if path.is_file() else "-"
except OSError:
ident = "-"
h.update(f"{name}\0{ident}\0".encode())
return h.hexdigest()
def gpu_key() -> str:
"""The GPU setting the steps resolve, as a suffix for the item cache key: a VideoToolbox file is
not the CPU's bytes, so a --hw output is never served to a --no-hw run or the other way round
(render.py's cache_key folds the same two variables in). Read after apply_common(), which has
exported --hw/--no-hw to the steps. '' when every step encodes on the CPU -- no flag and no
$FFMPEG_SKILL_HW, or --no-hw -- so the keys of a cache written before --hw existed still hit."""
if not env_hw():
return ""
return ":hw" if os.environ.get(HW_FORCED_ENV) == "1" else ":hw-env"
JOBS_CAP = 8
_LOG = threading.local()
_OWN_OUTPUTS: Dict[str, Dict[str, Dict[str, int]]] = {}
_OWN_LOCK = threading.Lock()
_WATCHED: Dict[str, Any] = {}
_STOP = threading.Event()
def log(message: str) -> None:
"""info(), unless this thread is a --jobs worker -- then the line is buffered and flushed in
file order when the item finishes, so a parallel run's log reads exactly like a serial one."""
buf = getattr(_LOG, "buffer", None)
if buf is None:
info(message)
else:
buf.append(message)
class ItemError(Exception):
"""An item that cannot be processed, with the structured `error` its results row carries.
process() raises it instead of die(): die() inside an item printed a second JSON document
and left the row with error "1" (str(SystemExit))."""
def __init__(self, error: Dict[str, Any]) -> None:
super().__init__(error.get("message"))
self.error = error
def step_error(index: Optional[int], tool: Optional[str], doc: Any) -> Dict[str, Any]:
"""A failed step's own failure, for its item's row: {step, tool, kind, message, code, reason,
details, hint, exit_code} from the step's --json document (keys it did not report are left
out; step and tool are always there)."""
err = doc.get("error") if isinstance(doc, dict) and isinstance(doc.get("error"), dict) else {}
out: Dict[str, Any] = {"step": index, "tool": tool}
for k in ("kind", "message", "code", "reason", "details", "hint"):
if err.get(k) is not None:
out[k] = err[k]
if isinstance(doc, dict) and doc.get("exit_code") is not None:
out["exit_code"] = doc["exit_code"]
return out
def run_step(argv: List[str], per_call: "Optional[float]" = None,
owned: "Optional[Path]" = None) -> "tuple[bool, Optional[dict]]":
"""Returns (ok, doc): doc is the step's own --json result document; a failed step that
printed none that parses gets one in the failure shape (error.reason step_output_unparsed,
the last stderr line as the message), so its row still says why. {in}/{out} are already resolved by the caller before argv reaches here, so forcing
--json on every step (harmless: no step reads another step's stdout, only its output path)
costs nothing and lets a caller like process() read back tool-specific fields, e.g. cut.py's
`reencoded`, without adding a bespoke flag per field it might someday want. `owned`: the
owned_config() file the step reads under runner.OWNED_ENV."""
script = argv[0]
if script not in ALLOWED_STEP_SCRIPTS:
raise ItemError({"step": None, "tool": script, "kind": "input", "reason": "step_script_not_allowed",
"message": f"recipe step names a script that isn't one of this skill's own tools: {script!r} "
"(must be a bare filename like 'silence.py', found in scripts/)"})
if _STOP.is_set():
return False, {"status": "failed", "exit_code": 130,
"error": {"kind": "interrupted", "message": f"interrupted before {script} started"}}
cmd = [str(HERE / script)] + with_flags(argv[1:], child_args() + (["--json"] if "--json" not in argv[1:] else []))
log(" → " + " ".join(os.path.basename(c) if i < 1 else c for i, c in enumerate(cmd)))
env = dict(os.environ, **{OWNED_ENV: str(owned)}) if owned else None
proc = run_tool(cmd, per_call=per_call, env=env)
try:
doc = json.loads(proc.stdout)
except ValueError:
doc = None
if proc.returncode != 0:
log(" " + "\n ".join(proc.stderr.strip().splitlines()[-4:]))
if not isinstance(doc, dict):
doc = {"status": "failed", "exit_code": proc.returncode,
"error": {"reason": "step_output_unparsed",
"message": (proc.stderr.strip().splitlines() or [f"{script} exited {proc.returncode} and printed nothing"])[-1][:300]}}
return False, doc
for line in proc.stderr.splitlines():
if line.startswith("warning:"):
log(" " + line)
return True, doc
def _run_buffered(fn, *a):
"""Run a worker and hand back (its result, the log lines it produced)."""
r = fn(*a)
return r, list(getattr(_LOG, "lines", None) or [])
def final_path(src: Path, recipe: Dict[str, Any], outdir: Path) -> Path:
suffix = recipe.get("suffix", "_out")
final_ext = recipe.get("ext") or src.suffix.lstrip(".") or "mp4"
if not recipe.get("ext") and src.suffix.lower() in NO_DEFAULT_ENCODE_EXT:
final_ext = "mp4"
steps = recipe.get("steps") or []
if not recipe.get("ext") and not recipe.get("project") and steps and not step_writes(steps[-1]) and not writes_nothing(recipe):
final_ext = intermediate_ext(src)
return outdir / f"{src.stem}{suffix}.{final_ext}"
def intermediate_ext(src: Path) -> str:
"""The extension of an item's intermediate step files: the source's own for an audio file,
else mp4 (the tools' encoders write x264/AAC, which .webm and friends do not take)."""
return src.suffix.lstrip(".") if src.suffix.lower() in (".wav", ".mp3", ".m4a", ".flac") else "mp4"
RECIPE_STRINGS = ("glob", "project", "output_dir", "suffix", "ext")
RECIPE_KEYS = RECIPE_STRINGS + ("clip_key", "steps")
def check_recipe_types(recipe: Dict[str, Any]) -> None:
"""Refuse a recipe value of the wrong JSON type before anything runs (A18): a number where a
path or glob goes, an absolute glob (pathlib refuses it), a step that is not a list. Each was
a traceback, or a failure of every item naming the step's first character."""
for key in recipe:
if key not in RECIPE_KEYS:
near = difflib.get_close_matches(str(key), RECIPE_KEYS, n=1, cutoff=0.6)
die(f"recipe: unknown key {key!r}" + (f" (did you mean {near[0]!r}?)" if near
else f" (valid keys: {', '.join(sorted(RECIPE_KEYS))})"), kind="input")
for key in RECIPE_STRINGS:
value = recipe.get(key)
if value is not None and not isinstance(value, str):
die(f'recipe "{key}" must be a string, got {type(value).__name__} {value!r:.60}', kind="input")
glob = recipe.get("glob")
if glob and (os.path.isabs(glob) or glob.startswith(("/", "\\"))):
die(f'recipe "glob" {glob!r} is an absolute path: it matches names inside the batch folder, e.g. "*.mp4" '
'or "sub/*.mov"', kind="input")
key = recipe.get("clip_key")
try:
index = int(key) if key is not None and not isinstance(key, (bool, float)) else (0 if key is None else -1)
except (TypeError, ValueError):
index = -1
if index < 0:
die(f'recipe "clip_key" must be a clip index (0, 1, ...), got {key!r:.60}', kind="input")
steps = recipe.get("steps")
if steps is not None and not isinstance(steps, list):
die(f'recipe "steps" must be a list of steps, each ["script.py", args...], got {type(steps).__name__}',
kind="input")
for i, st in enumerate(steps or []):
if not isinstance(st, list) or not st:
die(f'recipe step {i} must be a list ["script.py", args...], got {type(st).__name__} {st!r:.60}',
kind="input")
STEP_WRITES_WITHOUT_OUT = "step_writes_without_out"
def names_output(step: Any) -> bool:
"""Whether a step names where it writes itself (-o/--output with a path that is not {out}): a
side product the caller placed on purpose (look.py {in} -o {in}.png), not an implicit output."""
for tok in (step[1:] if isinstance(step, list) else []):
tok = str(tok)
if tok in ("-o", "--output") or tok.startswith("--output=") or (tok.startswith("-o") and not tok.startswith("--") and len(tok) > 2):
return True
return False
def work_files(work: Path) -> "set":
"""The names of the files directly in an item's work directory, batch's own excepted (the
owned*.json snapshots, the owned-report directory, hidden temporaries)."""
try:
return {p.name for p in work.iterdir()
if p.is_file() and not p.name.startswith(".") and not (p.name.startswith("owned") and p.suffix == ".json")}
except OSError:
return set()
def wrote_without_out(i: int, tool: str, path: str, src: Path) -> Dict[str, Any]:
"""The error of a step that reports an output file while naming no {out}: the batch stops."""
written = not STATE.dry_run and os.path.isfile(path)
where = (f"this run wrote {path} for {src.name} before stopping and left it there; no later item ran"
if written else "nothing was written" + (" (dry run)" if STATE.dry_run else ""))
return {"step": i, "tool": tool, "kind": "input", "reason": STEP_WRITES_WITHOUT_OUT,
"message": (f"step {i} ({tool}) names no {{out}} but writes {path}, its own default output: the next "
f"step does not read it and the batch does not deliver it; {where}"),
"hint": f"give step {i} \"-o\", \"{{out}}\" so its file is the one the next step reads and the batch delivers",
"details": {"step": i, "tool": tool, "file": path, "written": written}}
def step_writes(step: Any) -> bool:
"""Whether a recipe step writes a file: it names {out} (check.py and probe.py do not)."""
return isinstance(step, list) and any("{out}" in str(a) for a in step[1:])
def writes_nothing(recipe: Dict[str, Any]) -> bool:
"""A steps recipe in which no step names {out}: it measures each file and writes none."""
return not recipe.get("project") and not any(step_writes(st) for st in recipe.get("steps") or [])
def row_output(src: Path, recipe: Dict[str, Any], outdir: Path) -> Dict[str, Any]:
"""An item's output fields for its row: `output` is always the final path, a string as it has
been since 1.x (a --json key never changes type within a major), and a recipe that writes
nothing adds `written: false` (#415, #428 review)."""
fields: Dict[str, Any] = {"output": str(final_path(src, recipe, outdir))}
if writes_nothing(recipe):
fields["written"] = False
return fields
def own_expected(src: Path, path: "Path | str") -> "Optional[Dict[str, int]]":
"""The identity this process left `path` with, if it delivered a file there for `src`."""
with _OWN_LOCK:
return _OWN_OUTPUTS.get(str(src), {}).get(os.path.realpath(str(path)))
def is_own_output(src: Path, final: Path) -> bool:
"""Whether `final` is still the file this process delivered there for `src` (_OWN_OUTPUTS)."""
expected = own_expected(src, final)
return expected is not None and owned_identity(str(final)) == expected
def record_own(src: Path, committed: "Dict[str, Dict[str, int]]", work: Path) -> None:
"""Remember the files this item delivered, each with the identity it had when it was written
(a step's report_committed(), deliver() for the final path), never as it is now: a sidecar
edited while a later step ran no longer matches, so the item's next pass refuses it. Paths in
the item's work directory go with it and are skipped. Replaces what an earlier pass of the item
recorded."""
base = os.path.realpath(str(work))
mine: Dict[str, Dict[str, int]] = {}
for p, ident in committed.items():
real = os.path.realpath(p)
if real == base or real.startswith(base + os.sep) or not isinstance(ident, dict):
continue
mine[real] = ident
with _OWN_LOCK:
_OWN_OUTPUTS[str(src)] = mine
def owned_config(src: Path, work: Path, i: int) -> Path:
"""The file a step reads under runner.OWNED_ENV: a snapshot of what this process delivered for
`src`, taken right before the step starts (one file per step, since a step reads it once), and
the directory where each of its processes reports the paths it passed the output check for."""
report = work / "owned-report"
report.mkdir(exist_ok=True)
with _OWN_LOCK:
replaceable = dict(_OWN_OUTPUTS.get(str(src), {}))
cfg = work / f"owned{i}.json"
cfg.write_text(json.dumps({"replaceable": replaceable, "report": str(report)}), encoding="utf-8")
return cfg
def owned_reports(work: Path) -> "Dict[str, Dict[str, int]]":
"""{path: identity when written} for every file the item's steps reported committing
(owned_config's report directory). A path a step was allowed to write but never reported
writing is not in it."""
found: Dict[str, Dict[str, int]] = {}
for f in sorted((work / "owned-report").glob("*.json")) if (work / "owned-report").is_dir() else []:
try:
doc = json.loads(f.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
committed = doc.get("committed") if isinstance(doc, dict) else None
if isinstance(committed, dict):
found.update({p: i for p, i in committed.items() if isinstance(p, str) and isinstance(i, dict)})
return found
def deliver(src: str, final: Path, expected: "Optional[Dict[str, int]]" = None) -> "Optional[Dict[str, int]]":
"""Move an item's last written intermediate to its final path, under the output guard every
tool applies: an existing file is replaced only with --overwrite, or when it is still exactly
`expected` -- the file this process delivered there on an earlier pass -- checked under the
output lock, so a file another process put there since is refused. A dry run writes nothing."""
if STATE.dry_run:
if final.exists() and not STATE.overwrite:
raise ItemError({"step": None, "tool": None, "kind": "input", "reason": "output_exists",
"message": f"refusing to overwrite existing output {str(final)!r}: pass --overwrite to replace it"})
log(f" [dry-run] would deliver the last written file to {final}")
return
def locked(msg: str, hint: str, details: Dict[str, Any]) -> None:
raise ItemError({"step": None, "tool": None, "kind": "input", "reason": "output_locked",
"message": msg, "hint": hint, "details": details})
with _OutputLock(str(final), on_locked=locked):
if final.exists() and not STATE.overwrite and not (expected is not None and owned_identity(str(final)) == expected):
raise ItemError({"step": None, "tool": None, "kind": "input", "reason": "output_exists",
"message": f"refusing to overwrite existing output {str(final)!r}: pass --overwrite to replace it"})
try:
os.replace(src, final)
except OSError:
tmp = None
try:
tmp = copy_to_sibling(str(src), str(final))
os.replace(tmp, final)
except OSError as exc:
if tmp:
try:
os.remove(tmp)
except OSError:
pass
raise ItemError({"step": None, "tool": None, "kind": "output",
"message": f"could not deliver {src} to {final}: {exc}"})
ident = owned_identity(str(final))
log(f" delivered the last written file to {final}")
return ident
def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path,
deadline: "Optional[float]" = None) -> Dict[str, Any]:
final = final_path(src, recipe, outdir)
t0 = time.time()
track = not STATE.dry_run and not STATE.overwrite
delivered: Dict[str, Dict[str, int]] = {}
def owned(i: int) -> "Optional[Path]":
return owned_config(src, work, i) if track else None
def budget() -> "Optional[float]":
"""What is left of the BATCH's time limit -- not a fresh one per item. A --timeout is a
promise about the whole run, so a queue of 40 files cannot quietly take 40 timeouts."""
if deadline is None:
return None
return max(1.0, deadline - time.monotonic())
if recipe.get("project"):
try:
proj = read_json(str(recipe["project"]))
except (OSError, ValueError) as e:
raise ItemError({"step": None, "tool": "render.py", "kind": "input", "reason": "recipe_project_unreadable",
"message": f"recipe.project: {recipe['project']} could not be read as JSON: {e}"})
if not isinstance(proj, dict):
raise ItemError({"step": None, "tool": "render.py", "kind": "input", "reason": "recipe_project_unreadable",
"message": f"recipe.project: {recipe['project']} is not a JSON object"})
idx = int(recipe.get("clip_key") or 0)
proj.setdefault("clips", [{}])
while len(proj["clips"]) <= idx:
proj["clips"].append({})
proj = rebase_project_paths(proj, Path(str(recipe["project"])).resolve().parent)
proj["clips"][idx]["src"] = str(src.resolve())
proj["output"] = str(final.resolve())
pj = work / "project.json"
pj.write_text(json.dumps(proj, indent=2), encoding="utf-8")
owned_cfg = owned(0)
if not STATE.dry_run:
with _OWN_LOCK:
_WATCHED[str(src)] = (str(final), owned_identity(str(final)))
ok, _doc = run_step(["render.py", str(pj)], budget(), owned=owned_cfg)
if ok:
with _OWN_LOCK:
_WATCHED.pop(str(src), None)
step_docs = [("render", _doc)]
failure = None if ok else step_error(0, "render.py", _doc)
cut_reencoded: List[bool] = []
cut_reasons: List[str] = []
cut_possible: List[List[str]] = []
else:
steps = recipe.get("steps") or []
if not steps:
raise ItemError({"step": None, "tool": None, "kind": "input", "reason": "recipe_has_no_steps",
"message": "recipe needs steps or project"})
cur = str(src)
ok = True
failure = None
cut_reencoded = []
step_docs = []
cut_reasons = []
cut_possible = []
for i, step in enumerate(steps):
last = i == len(steps) - 1
out = str(final) if last else str(work / f"step{i}.{intermediate_ext(src)}")
writes = step_writes(step)
argv = [str(a).replace("{in}", cur).replace("{out}", out) for a in step]
if last and writes and track and is_own_output(src, final):
log(f" {final.name} is this run's own earlier output, unchanged since: it may be replaced")
owned_cfg = owned(i)
before = set() if last else work_files(work)
if last and writes and not STATE.dry_run:
with _OWN_LOCK:
_WATCHED[str(src)] = (str(final), owned_identity(str(final)))
step_ok, doc = run_step(argv, budget(), owned=owned_cfg)
if step_ok:
with _OWN_LOCK:
_WATCHED.pop(str(src), None)
step_docs.append((Path(str(argv[0])).stem if argv else "step", doc))
if step_ok and argv and argv[0] == "cut.py" and isinstance(doc, dict) and "reencoded" in doc:
cut_reencoded.append(bool(doc["reencoded"]))
cut_reasons.extend(r for r in (doc.get("reencode_reason") or []) if isinstance(r, str))
cut_possible.append([r for r in (doc.get("possible_reencode_reason") or []) if isinstance(r, str)])
if not step_ok:
ok = False
failure = step_error(i, str(argv[0]) if argv else None, doc)
break
reported = doc.get("output") if isinstance(doc, dict) else None
if not writes and isinstance(reported, str) and reported and not names_output(step):
raise ItemError(wrote_without_out(i, str(argv[0]), os.path.abspath(reported), src))
if writes and not last:
lost = sorted(set(work_files(work)) - before - {os.path.basename(out)})
if lost:
record_skip(f"{src.name}: step {i} ({argv[0]}) also wrote {', '.join(lost)} beside its intermediate; "
"it is removed with the item's work directory, not delivered (only the last step's "
"side files land beside the output)",
"batch_side_file_not_delivered", file=str(src), step=i, tool=str(argv[0]), files=lost)
if writes:
cur = out
if ok and cur != str(final) and not writes_nothing(recipe):
try:
ident = deliver(cur, final, expected=own_expected(src, final))
if ident:
delivered[str(final)] = ident
except ItemError as exc:
ok, failure = False, exc.error
if ok and track:
record_own(src, {**owned_reports(work), **delivered}, work)
result: Dict[str, Any] = {"file": str(src), **row_output(src, recipe, outdir), "ok": ok, "seconds": round(time.time() - t0, 1)}
if failure:
result["error"] = failure
unverified = []
for idx, (name, doc) in enumerate(step_docs):
if ok and isinstance(doc, dict) and doc.get("verified") is not True:
unverified.append({"step": idx, "tool": name,
"failed": [str(v.get("step")) + (f" ({v['stage']})" if v.get("stage") else "")
for v in doc.get("verification") or [] if isinstance(v, dict) and not v.get("ok")]})
result["verified"] = bool(ok and not STATE.dry_run and step_docs and not unverified)
if unverified and not STATE.dry_run:
result["unverified"] = unverified
diag: Dict[str, List[Any]] = {}
for name, doc in step_docs:
for bucket, entries in ((doc or {}).get("diagnostics") or {}).items() if isinstance(doc, dict) else []:
diag.setdefault(bucket, []).extend(dict(e, step=name) for e in entries if isinstance(e, dict))
if diag:
result["diagnostics"] = diag
if cut_reencoded:
result["cut_reencoded"] = cut_reencoded
result["cut_reencode_reasons"] = cut_reasons
if STATE.dry_run:
result["cut_possible_reencode_reason"] = cut_possible
result.update(steps_encoder_report(step_docs))
return result
def output_identity(path: str) -> "Optional[Dict[str, int]]":
"""What a cache entry remembers of the output it served: size and mtime_ns (a stat, no read)."""
try:
st = os.stat(path)
except OSError:
return None
return {"size": st.st_size, "mtime_ns": st.st_mtime_ns}
def cached_output_changed(hit: Dict[str, Any], recipe: "Optional[Dict[str, Any]]" = None) -> "Optional[Dict[str, Any]]":
"""None when the entry's output is still the file this batch wrote; else why it is not served:
output_missing, output_changed (size or mtime differ: something replaced or edited it),
entry_unverifiable (an entry written by 2.10.11 or earlier recorded no identity, so nothing can say
the file is the one it describes), or entry_predates_output_check: the recipe has a step that
names no {out}, and the entry was stored before such a step's own output was checked (#510), so
a step that writes its default output (metadata.py {in}) was cached as one that writes nothing."""
steps = (recipe or {}).get("steps") or []
if not (recipe or {}).get("project") and any(not step_writes(st) for st in steps) and not hit.get("output_checked"):
return {"reason": "entry_predates_output_check", "output": hit.get("output"),
"message": "the entry was stored before batch checked what a step naming no {out} writes"}
if hit.get("written") is False or ("output" in hit and hit["output"] is None):
return None
out = str(hit.get("output") or "")
now = output_identity(out) if out else None
if now is None:
return {"reason": "output_missing", "output": out, "message": "the output is gone"}
expected = hit.get("output_identity")
if not isinstance(expected, dict):
return {"reason": "entry_unverifiable", "output": out,
"message": "the cache entry predates output identities, so the file cannot be checked"}
if expected != now:
return {"reason": "output_changed", "output": out, "expected": expected, "actual": now,
"message": "the output changed since the batch wrote it"}
return None
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("folder")
ap.add_argument("--recipe", required=True, help="batch.json")
ap.add_argument("--force", action="store_true", help="ignore the cache and redo everything")
ap.add_argument("--watch", type=float, help="keep polling the folder every N seconds")
ap.add_argument("--jobs", default="1", metavar="N",
help="process N files at once, or 'auto' for min(cpu_count, 4). Capped at "
"min(N, cpu_count, 8): every item is already an ffmpeg that threads "
"across cores, so more than a few contend rather than go faster. "
"Default 1, which is 1.16's behaviour exactly.")
ap.add_argument("--work", help="work directory for intermediates (default: <output_dir>/.work)")
add_common(ap)
add_hw_orchestrator_args(ap)
args = ap.parse_args()
apply_common(args)
started = time.time()
with _OWN_LOCK:
_OWN_OUTPUTS.clear()
_WATCHED.clear()
folder = Path(args.folder).resolve()
if not folder.is_dir():
die(f"not a folder: {folder}")
try:
recipe = read_json(args.recipe)
except (OSError, ValueError) as exc:
die(f"cannot read recipe: {exc}", kind="input")
if not isinstance(recipe, dict):
die("cannot read recipe: not a JSON object", kind="input")
check_recipe_types(recipe)
if recipe.get("project") and not os.path.isabs(recipe["project"]):
recipe["project"] = str((Path(args.recipe).resolve().parent / recipe["project"]))
if not recipe.get("project"):
if not recipe.get("steps"):
die("recipe needs steps or project")
for st in recipe["steps"]:
name = st[0] if isinstance(st, list) and st else st
if not isinstance(name, str) or name not in ALLOWED_STEP_SCRIPTS:
die(f"recipe step names a script that isn't one of this skill's own tools: {name!r} "
f"(must be a bare filename like 'silence.py', found in scripts/)")
if writes_nothing(recipe):
note("no recipe step names {out}: the steps only read each file, so no output is written "
"(each row says written: false; its output path is not created)")
outdir = Path(recipe.get("output_dir") or (folder / "out"))
if not outdir.is_absolute():
outdir = folder / outdir
outdir = outdir.resolve()
if outdir == folder:
die(f"output_dir {recipe.get('output_dir')!r} is the input folder {folder}: its outputs would mix with "
"or replace the batch inputs -- set an \"output_dir\" outside it (the default is out/)", kind="input")
work = Path(args.work) if args.work else outdir / ".work"
created = [d for d in (outdir, work) if not d.exists()]
outdir.mkdir(parents=True, exist_ok=True)
work.mkdir(parents=True, exist_ok=True)
item_started = {"item": False}
if created:
import atexit
def _remove_empty_dirs() -> None:
if not STATE.dry_run and item_started["item"]:
return
for d in sorted(created, key=lambda p: len(str(p)), reverse=True):
try:
if not any(d.iterdir()):
d.rmdir()
except OSError:
pass
atexit.register(_remove_empty_dirs)
cache_path = outdir / ".ffskill_cache.json"
cache: Dict[str, Any] = {}
if cache_path.exists() and not args.force:
try:
cache = json.loads(cache_path.read_text(encoding="utf-8"))
if not isinstance(cache, dict):
raise ValueError("not a JSON object")
except (OSError, ValueError) as exc:
size = cache_path.stat().st_size if cache_path.exists() else None
warn(f"the batch cache {cache_path} could not be read ({exc}); starting with an empty cache, so every item runs again",
"batch_cache_reset", path=str(cache_path), size=size, error=str(exc))
audit("batch_cache_reset", {"path": str(cache_path), "size": size, "error": str(exc)},
default_path=str(outdir / ".ffskill-audit.jsonl"))
cache = {}
glob = recipe.get("glob") or "*"
cpus = os.cpu_count() or 1
requested = min(cpus, 4) if str(args.jobs).lower() == "auto" else None
if requested is None:
try:
requested = int(args.jobs)
except ValueError:
die(f"--jobs {args.jobs!r}: a whole number, or 'auto'", kind="input")
if requested < 1:
die("--jobs must be at least 1", kind="input")
jobs = max(1, min(requested, cpus, JOBS_CAP))
if jobs != requested:
info(f"--jobs {requested} capped to {jobs} (min of the request, {cpus} CPU(s) and the "
f"{JOBS_CAP}-job ceiling): each item is already a multi-threaded encode")
shared_budget = bool(args.timeout) or jobs > 1
deadline = time.monotonic() + STATE.timeout if (shared_budget and STATE.timeout) else None
cache_lock = threading.Lock()
timed_out = {"hit": False}
interrupted: Dict[str, Any] = {"hit": False, "signal": None}
refused: Dict[str, Any] = {"hit": False}
_STOP.clear()
import signal as _signal
def _interrupt(signum: int, frame: Any) -> None:
if interrupted["signal"] is not None:
return
interrupted["signal"] = signum
_STOP.set()
stop_children()
raise KeyboardInterrupt
if threading.current_thread() is threading.main_thread():
for sig in (getattr(_signal, "SIGINT", None), getattr(_signal, "SIGTERM", None)):
if sig is not None:
try:
_signal.signal(sig, _interrupt)
except (ValueError, OSError):
pass
last_seen: Dict[str, "tuple[int, int]"] = {}
shown_empty = {"done": False}
def settle_poll(files: List[Path]) -> "set":
if not args.watch:
return set()
unsettled = set()
now = time.time()
current: Dict[str, "tuple[int, int]"] = {}
for src in files:
try:
st = src.stat()
except OSError:
continue
sig = (st.st_size, st.st_mtime_ns)
current[str(src)] = sig
before = last_seen.get(str(src))
if (before != sig) if before is not None else (now - st.st_mtime < args.watch):
unsettled.add(str(src))
last_seen.clear()
last_seen.update(current)
return unsettled
def one_pass() -> List[Dict[str, Any]]:
files = sorted(p for p in folder.glob(caseless_glob(glob)) if p.is_file() and p.suffix.lower() in MEDIA_EXT and outdir not in p.parents)
if not files and not shown_empty["done"]:
shown_empty["done"] = True
note(f"no media file in {folder} matches the recipe's glob {glob!r} (matched without regard to case); "
"nothing was processed")
by_final: Dict[str, List[Path]] = {}
for src in files:
by_final.setdefault(str(final_path(src, recipe, outdir)).casefold(), []).append(src)
collisions = {key: srcs for key, srcs in by_final.items() if len(srcs) > 1 and not writes_nothing(recipe)}
if collisions:
detail = "; ".join(" / ".join(sorted({final_path(s, recipe, outdir).name for s in srcs}))
+ " <- " + ", ".join(s.name for s in srcs) for srcs in collisions.values())
die(f"{len(collisions)} output filename collision(s) in this batch (names that differ only in case are one "
f"file on macOS and Windows) -- rename the sources, "
f"or add a distinguishing \"suffix\"/\"ext\" per run, or split into separate globs: {detail}")
if not writes_nothing(recipe):
inputs = {str(p.resolve()).casefold() for p in files}
hits = [s for s in files if str(final_path(s, recipe, outdir).resolve()).casefold() in inputs]
if hits:
die(f"{len(hits)} final output path(s) would replace a batch input ({', '.join(s.name for s in hits)}); "
"the output_dir is the input folder -- set a different \"output_dir\" or a non-empty \"suffix\"",
kind="input")
def item_work(i: int, src: Path) -> Path:
"""Where this item's intermediates go: a new directory of its own under the work
directory, removed with everything in it when the item ends (run_item()). The step
file names are stem-derived, so two items with one stem (clip.mp4, clip.mov) sharing a
directory refused each other's files, and a later run refused the files an earlier one
left there as someone else's (#415); parallel items had their own directories since
1.17, serial ones now do too."""
import tempfile
return Path(tempfile.mkdtemp(prefix=f"{i}-", dir=str(work)))
def run_item(i: int, src: Path) -> Dict[str, Any]:
"""process() in the item's own work directory, which goes when it ends, however it ends."""
import shutil
item_started["item"] = True
iw = item_work(i, src)
try:
return process(src, recipe, outdir, iw, deadline)
finally:
shutil.rmtree(iw, ignore_errors=True)
def store(key: "Optional[str]", r: Dict[str, Any]) -> None:
if not key or not (r["ok"] and not STATE.dry_run):
return
try:
now = item_key(Path(r["file"]))
except OSError:
now = None
if now != key:
record_skip(f"{Path(r['file']).name}: its source or a side input changed while it was processed, so its "
"output is not cached (the next run makes it again)", "batch_cache_inputs_changed", file=r["file"])
return
with cache_lock:
for stale in [k for k, v in cache.items() if r["output"] and r.get("written", True) and k != key and isinstance(v, dict) and v.get("output") == r["output"]]:
del cache[stale]
cache[key] = dict(r, output_identity=output_identity(r["output"]) if r["output"] and r.get("written", True) else None,
output_checked=True)
tmp = cache_path.parent / f"{cache_path.name}.tmp{os.getpid()}.{threading.get_ident()}"
tmp.write_text(json.dumps(cache, indent=2), encoding="utf-8")
os.replace(tmp, cache_path)
slots: "List[Optional[Dict[str, Any]]]" = [None] * len(files)
pending: List[tuple] = []
cache_miss: Dict[int, Dict[str, Any]] = {}
pass_key = recipe_key(recipe) + gpu_key() + side_inputs_key(recipe)
def item_key(src: Path) -> "Optional[str]":
"""The cache key of one item as its files are now: file_key(src), the recipe, the GPU
setting and the side inputs (literal and per item); None when one cannot be keyed.
Recomputed by store(), so the side inputs are read again then (pass_key's included)."""
per, why = item_side_inputs_key(recipe, src)
if why:
return None
base = recipe_key(recipe) + gpu_key() + side_inputs_key(recipe)
return f"{file_key(src)}:{base}" + (":" + per if per else "")
polled = settle_poll(files)
for i, src in enumerate(files):
if str(src) in polled:
info(f"skip (still being written) {src.name}: it changed since the last poll")
slots[i] = {"file": str(src), **row_output(src, recipe, outdir), "ok": False, "verified": False,
"seconds": 0.0, "skipped": "settling"}
continue
key = f"{file_key(src)}:{pass_key}"
per_item, unkeyed = item_side_inputs_key(recipe, src)
if unkeyed:
cache_miss[i] = unkeyed
info(f"{src.name}: {unkeyed['message']}")
pending.append((i, src, None))
continue
if per_item:
key += ":" + per_item
hit = cache.get(key)
if isinstance(hit, dict) and not args.force:
why = cached_output_changed(hit, recipe)
if why is None:
info(f"skip (cached) {src.name}")
slots[i] = {**{k: v for k, v in hit.items() if k not in ("output_identity", "output_checked")}, "cached": True}
continue
cache_miss[i] = why
facts = {k: v for k, v in why.items() if k not in ("reason", "message")}
msg = f"{src.name}: the cached output {hit.get('output')} is not served ({why['message']}); the item runs again"
if why["reason"] == "output_changed":
warn(msg, "batch_cache_output_changed", file=str(src), **facts)
elif why["reason"] == "entry_unverifiable":
warn(msg, "batch_cache_entry_unverifiable", file=str(src), **facts)
pending.append((i, src, key))
def timed_out_row(src: Path) -> "Dict[str, Any]":
timed_out["hit"] = True
return {"file": str(src), **row_output(src, recipe, outdir),
"ok": False, "seconds": 0.0, "skipped": "timeout"}
def failed_row(src: Path, exc: BaseException) -> "Dict[str, Any]":
"""A worker that raised is a failed item, not a dead run. A die() inside a thread
raises SystemExit through fut.result() and used to take the whole process down
before the summary and the per-item table were printed -- so the one thing the user
needed, which item failed and which succeeded, was the thing they did not get.
`error` is structured: an ItemError's own, else what the exception says (its class
in `exception`), never str(SystemExit) -- which was the exit code, "1"."""
if isinstance(exc, ItemError):
error = dict(exc.error)
elif isinstance(exc, SystemExit):
error = {"step": None, "tool": None, "exception": "SystemExit", "exit_code": exc.code,
"message": f"the item stopped with exit code {exc.code}; its message is on stderr"}
else:
error = {"step": None, "tool": None, "exception": exc.__class__.__name__,
"message": (str(exc) or exc.__class__.__name__)[:300]}
log(f" failed: {error['message']}")
return {"file": str(src), **row_output(src, recipe, outdir),
"ok": False, "verified": False, "seconds": 0.0, "error": error}
def interrupted_row(src: Path, started: bool, r: "Optional[Dict[str, Any]]" = None) -> "Dict[str, Any]":
"""An item the interrupt reached: `interrupted: true` when it was running (its steps
were stopped), `skipped: "interrupted"` when it had not started."""
row = dict(r) if r else {"file": str(src), **row_output(src, recipe, outdir), "seconds": 0.0}
row.update(ok=False, verified=False)
if started:
row["interrupted"] = True
with _OWN_LOCK:
watched = _WATCHED.pop(str(src), None)
if watched and owned_identity(watched[0]) not in (None, watched[1]):
row["kept"] = watched[0]
row.setdefault("error", {"step": None, "tool": None, "kind": "interrupted",
"message": "interrupted while this item was running; its running step was stopped"})
else:
row["skipped"] = "interrupted"
return row
def refused_row(src: Path) -> "Dict[str, Any]":
"""An item not started because an earlier one's step wrote a file without naming {out}."""
return {"file": str(src), **row_output(src, recipe, outdir), "ok": False, "verified": False,
"seconds": 0.0, "skipped": "recipe_refused"}
def is_refusal(r: "Optional[Dict[str, Any]]") -> bool:
return bool(r) and ((r or {}).get("error") or {}).get("reason") == STEP_WRITES_WITHOUT_OUT
def with_miss(i: int, r: Dict[str, Any]) -> Dict[str, Any]:
if i in cache_miss:
r["cache_miss"] = {k: v for k, v in cache_miss[i].items() if k != "message"}
return r
if jobs == 1 or len(pending) < 2:
for n, (i, src, key) in enumerate(pending):
if deadline is not None and time.monotonic() >= deadline:
slots[i] = timed_out_row(src)
continue
info(f"=== {src.name}")
try:
r = run_item(i, src)
except KeyboardInterrupt:
interrupted["hit"] = True
slots[i] = interrupted_row(src, True)
for j, later, _k in pending[n + 1:]:
slots[j] = interrupted_row(later, False)
break
except BaseException as exc:
if isinstance(exc, SystemExit) and not exc.code:
raise
slots[i] = with_miss(i, failed_row(src, exc))
if is_refusal(slots[i]):
refused["hit"] = True
for j, later, _k in pending[n + 1:]:
slots[j] = refused_row(later)
break
continue
slots[i] = with_miss(i, r)
store(key, r)
return [r for r in slots if r is not None]
import concurrent.futures
def work_one(i: int, src: Path, key: str) -> Dict[str, Any]:
_LOG.buffer = [f"=== {src.name}"]
try:
r = run_item(i, src)
if not interrupted["hit"]:
store(key, r)
return with_miss(i, r)
finally:
r_lines, _LOG.buffer = _LOG.buffer, None
setattr(_LOG, "lines", r_lines)
lines: "Dict[int, List[str]]" = {}
with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool:
futures: "Dict[Any, tuple]" = {}
queue = list(pending)
in_flight: "set" = set()
try:
while queue or in_flight:
while queue and len(in_flight) < jobs and not refused["hit"]:
if deadline is not None and time.monotonic() >= deadline:
break
i, src, key = queue.pop(0)
fut = pool.submit(_run_buffered, work_one, i, src, key)
futures[fut] = (i, src)
in_flight.add(fut)
if not in_flight:
break
done_now, in_flight = concurrent.futures.wait(
in_flight, return_when=concurrent.futures.FIRST_COMPLETED)
in_flight = set(in_flight)
for fut in done_now:
i, src = futures[fut]
try:
slots[i], lines[i] = fut.result()
except BaseException as exc:
slots[i] = failed_row(src, exc)
lines[i] = [f"=== {src.name}", f" failed: {exc}"]
if is_refusal(slots[i]):
refused["hit"] = True
for i, src, key in queue:
slots[i] = refused_row(src) if refused["hit"] else timed_out_row(src)
except KeyboardInterrupt:
interrupted["hit"] = True
for fut in futures:
fut.cancel()
info("interrupted: reporting what had already finished")
if interrupted["hit"]:
for fut, (i, src) in futures.items():
if slots[i] is not None:
continue
if fut.cancelled():
slots[i] = interrupted_row(src, False)
continue
try:
r, lines[i] = fut.result()
except BaseException:
r = None
slots[i] = r if (r and r.get("ok")) else interrupted_row(src, True, r)
for i, src, _key in queue:
if slots[i] is None:
slots[i] = interrupted_row(src, False)
for i, src in enumerate(files):
for line in lines.get(i, []):
info(line)
return [r for r in slots if r is not None]
results = one_pass()
if args.watch and not refused["hit"]:
info(f"watching {folder} every {args.watch:g}s (Ctrl-C to stop)")
import signal
try:
while True:
previous = signal.signal(signal.SIGINT, signal.default_int_handler)
try:
time.sleep(args.watch)
finally:
signal.signal(signal.SIGINT, previous)
results = one_pass()
if interrupted["hit"] or refused["hit"]:
break
except KeyboardInterrupt:
info("watch stopped")
for r in results:
absorb_stage_hw(Path(r["file"]).name, r)
done = sum(1 for r in results if r["ok"])
info(f"{done}/{len(results)} processed, {sum(1 for r in results if r.get('cached'))} from cache")
if not args.json:
for r in results:
print(f"{'OK ' if r['ok'] else 'FAIL'} {r['file']} -> {r['output'] or '(no output: the steps only read it)'}" + (" (cached)" if r.get("cached") else ""))
if interrupted["hit"]:
stopped = sum(1 for r in results if r.get("interrupted"))
not_started = sum(1 for r in results if r.get("skipped") == "interrupted")
kept = [r["kept"] for r in results if r.get("kept")]
die(f"interrupted after {done} of {len(results)} item(s): {stopped} stopped while running and "
f"{not_started} not started; the finished outputs are kept"
+ ((f", and {', '.join(kept)} " + ("was written by a step that was stopped and is" if len(kept) == 1 else
"were written by steps that were stopped and are") + " kept, unverified")
if kept else ""),
code=128 + int(interrupted["signal"] or 2), kind="interrupted", output=None, dry_run=STATE.dry_run, results=results,
processed=done, total=len(results), jobs=jobs, jobs_requested=requested,
timed_out=timed_out["hit"])
if refused["hit"]:
rows = [r for r in results if ((r.get("error") or {}).get("reason")) == STEP_WRITES_WITHOUT_OUT]
first = rows[0]["error"]
files = [r["error"]["details"]["file"] for r in rows]
written = [f for r, f in zip(rows, files) if r["error"]["details"].get("written")]
not_run = sum(1 for r in results if r.get("skipped") == "recipe_refused")
die(first["message"] + (f" ({len(rows)} items in flight wrote one each: {', '.join(written)})" if len(written) > 1 else "")
+ (f"; {not_run} item(s) not started" if not_run else ""),
kind="input", hint=first.get("hint"), output=None, dry_run=STATE.dry_run, results=results,
processed=done, total=len(results),
error_extra={"reason": STEP_WRITES_WITHOUT_OUT,
"details": dict(first["details"], files=files, written_files=written, not_started=not_run)})
if timed_out["hit"]:
skipped = [r["file"] for r in results if r.get("skipped") == "timeout"]
die(f"the batch's {STATE.timeout:g} s budget ran out with {len(skipped)} item(s) not "
f"started: {', '.join(os.path.basename(f) for f in skipped[:5])}"
+ (" ..." if len(skipped) > 5 else "")
+ ". --timeout is the whole run's limit, not each item's; raise it or split the folder.",
code=124, kind="timeout", output=None, dry_run=STATE.dry_run, results=results,
processed=done, total=len(results), jobs=jobs, jobs_requested=requested,
timed_out=True)
if done != len(results):
failed_files = [r["file"] for r in results if not r["ok"]]
settling = sum(1 for r in results if r.get("skipped") == "settling")
die(f"{len(results) - done} of {len(results)} items failed: {', '.join(failed_files[:5])}" + (" ..." if len(failed_files) > 5 else "")
+ (f" ({settling} of them still being written when the watch stopped, skipped: settling, not processed)" if settling else ""),
kind="verification", output=None, dry_run=STATE.dry_run, results=results, processed=done, total=len(results))
all_cuts = [r for res in results for r in (res.get("cut_reencoded") or [])]
cut_summary = None
reasons: Optional[Dict[str, int]] = None
if all_cuts:
copied = sum(1 for r in all_cuts if not r)
cut_summary = {"calls": len(all_cuts), "stream_copy": copied, "reencoded": len(all_cuts) - copied,
"stream_copy_rate": round(copied / len(all_cuts), 3)}
reasons = {}
for res in results:
if "cut_reencode_reasons" not in res:
unknown = sum(1 for r in res.get("cut_reencoded") or [] if r)
if unknown:
reasons["unknown"] = reasons.get("unknown", 0) + unknown
for r in res.get("cut_reencode_reasons") or []:
reasons[r] = reasons.get(r, 0) + 1
why = (" (reencode_reason: " + ", ".join(f"{k} x{v}" for k, v in reasons.items()) + ")") if reasons else ""
if STATE.dry_run:
planned = [r for res in results if not res.get("cached") for r in (res.get("cut_reencoded") or [])]
stored = [r for res in results if res.get("cached") for r in (res.get("cut_reencoded") or [])]
parts = []
if planned:
p_copied = sum(1 for r in planned if not r)
parts.append(f"{p_copied}/{len(planned)} call(s) planned as a lossless stream copy "
f"({p_copied / len(planned):.0%}), which a real run may still re-encode (each planned row's "
f"cut_possible_reencode_reason); {len(planned) - p_copied} planned to re-encode a range or its --segments join")
if stored:
s_copied = sum(1 for r in stored if not r)
parts.append(f"{len(stored)} call(s) in rows a real run serves from the cache (cached: true), as the run that "
f"stored them measured: {s_copied} stayed lossless stream-copy, {len(stored) - s_copied} re-encoded")
info("cut.py: " + "; ".join(parts) + why)
else:
info(f"cut.py: {copied}/{len(all_cuts)} call(s) stayed fully lossless stream-copy "
f"({cut_summary['stream_copy_rate']:.0%}), {len(all_cuts) - copied} re-encoded a range or its --segments join{why}")
unverified = [shown(os.path.basename(r["file"])) for r in results if r.get("ok") and not r.get("verified")]
if unverified and not STATE.dry_run:
note(f"{len(unverified)} item(s) completed but did not verify everything they wrote (see each row's unverified): "
+ ", ".join(unverified[:5]) + (" ..." if len(unverified) > 5 else ""))
emit(None, verification=[{"step": "item", "file": r["file"], "ok": bool(r.get("verified"))} for r in results]
if not STATE.dry_run else [],
verification_paths=[] if args.watch else [r["file"] if r.get("written") is False else r["output"] for r in results],
results=results, processed=done, total=len(results),
jobs=jobs, jobs_requested=requested, wall_seconds=round(time.time() - started, 1),
item_seconds_total=round(sum(float(r.get("seconds") or 0) for r in results), 1),
timed_out=timed_out["hit"], cut_stream_copy=cut_summary, cut_reencode_reasons=reasons)
return 0
if __name__ == "__main__":
sys.exit(main())