"""Detect the time offset between two recordings by audio cross-correlation
and (optionally) write a synced output, with optional clock-drift correction.
Pure standard library: both tracks are decoded by ffmpeg to mono 8 kHz PCM,
reduced to a loudness envelope, cross-correlated with an FFT implemented in
Python (coarse, 20 ms), then refined by direct correlation at 1 ms.
Offset semantics: a positive offset means the SECOND input starts LATER
than the reference (it began recording after it), so its sound is delayed by
that amount (its head padded) to line up; a negative offset means it started
earlier, and its first -offset seconds are dropped.
This aligns two AUDIO tracks to each other; it does not check or guarantee
lip sync (mouth movement matching the audio). It assumes each recording's
own audio is already correctly timed against its own picture, which holds
for ordinary cameras and phones (same device, same clock) but not for a
capture device with its own internal audio/video offset. There is no
face or mouth detection anywhere in this codebase to verify that; the only
way to confirm the final result actually looks in sync is to watch it.
Examples:
python3 sync.py camera.mp4 lavmic.wav # print offset only
python3 sync.py camera.mp4 lavmic.wav --replace-audio -o synced.mp4
python3 sync.py camA.mp4 camB.mp4 --trim-second -o camB_synced.mp4
python3 sync.py cam.mp4 mic.wav --max-offset 60 --json
python3 sync.py cam.mp4 recorder.wav --fix-drift --replace-audio # long takes: fix clock drift too
"""
import argparse
import math
import os
import sys
from typing import Dict, List
from _common.runner import fail_if_classified, refuse_output_is_input
from _common import STATE, video_args, add_common, apply_common, emit, aac_args, audio_codec_for, default_output, die, ffmpeg_base, info, probe, run, decode_pcm_mono, rms_envelope
from _common.emit import record_skip, warn
from _common.decision import SEEK_MARGIN, frame_rate, grid_at_or_after, length_output_t, silence_filter, stream_starts
from _common.timeline import output_timecode_args
from _common.runner import default_selection_maps
from _common.emit import input_length
SR = 8000
def sync_meaning(offset: float) -> str:
"""The sentence a caller reads next to offset_seconds.
A zero offset used to take the else branch of `later if offset > 0` and say the second
file starts earlier. Values that print as 0.000 s (under half a millisecond) say the two
recordings start together; anything larger keeps later/earlier."""
if abs(offset) < 5e-4:
return "second starts together with the reference"
return "second starts %.3fs %s than reference" % (abs(offset), "later" if offset > 0 else "earlier")
MIN_OVERLAP_FRACTION = 0.35
OVERLAP_WEIGHT_EXP = 0.5
def decode_mono(path: str, seconds: float, start: float = 0.0) -> List[float]:
return decode_pcm_mono(path, SR, seconds, start)
def envelope(samples: List[float], step: int) -> List[float]:
"""RMS energy per full block, mean-removed so silence does not correlate."""
return rms_envelope(samples, step, full_blocks_only=True, remove_mean=True)
def fft(a: List[complex]) -> List[complex]:
"""Iterative radix-2 Cooley-Tukey FFT. len(a) must be a power of two."""
n = len(a)
a = list(a)
j = 0
for i in range(1, n):
bit = n >> 1
while j & bit:
j ^= bit
bit >>= 1
j ^= bit
if i < j:
a[i], a[j] = a[j], a[i]
length = 2
while length <= n:
ang = -2 * math.pi / length
wlen = complex(math.cos(ang), math.sin(ang))
half = length // 2
for i in range(0, n, length):
w = 1 + 0j
for k in range(half):
u = a[i + k]
v = a[i + k + half] * w
a[i + k] = u + v
a[i + k + half] = u - v
w *= wlen
length <<= 1
return a
def ifft(a: List[complex]) -> List[complex]:
n = len(a)
conj = [x.conjugate() for x in a]
out = fft(conj)
return [x.conjugate() / n for x in out]
def cross_correlate(ref: List[float], other: List[float], max_lag: int):
"""Normalised cross-correlation over the overlapping region only.
The raw FFT correlation sum grows with the overlap length, so with a 60 s window a
correct 28 s offset (32 s overlap) loses to a wrong 2 s offset (58 s overlap) on
music-like material. Dividing each lag by the energy of the overlapping parts
(prefix sums, O(1) per lag) makes lags comparable and turns the peak value into a
real similarity score in 0..1 that doubles as the confidence.
"""
n = 1
while n < len(ref) + len(other):
n <<= 1
fa = fft([complex(x) for x in ref] + [0j] * (n - len(ref)))
fb = fft([complex(x) for x in other] + [0j] * (n - len(other)))
prod = [x * y.conjugate() for x, y in zip(fa, fb)]
corr = ifft(prod)
def prefix(v: List[float]) -> List[float]:
out = [0.0]
acc = 0.0
for x in v:
acc += x * x
out.append(acc)
return out
pr, po = prefix(ref), prefix(other)
lr, lo = len(ref), len(other)
max_lag = min(max_lag, n // 2 - 1)
best_lag, best_val, second = 0, -float("inf"), -float("inf")
min_overlap = max(10, int(MIN_OVERLAP_FRACTION * min(lr, lo)))
scores = []
for lag in range(-max_lag, max_lag + 1):
r0, r1 = max(0, lag), min(lr, lo + lag)
if r1 - r0 < min_overlap:
continue
e_ref = pr[r1] - pr[r0]
e_oth = po[r1 - lag] - po[r0 - lag]
denom = math.sqrt(e_ref * e_oth)
if denom <= 0:
continue
val = corr[lag % n].real / denom
val *= ((r1 - r0) / min(lr, lo)) ** OVERLAP_WEIGHT_EXP
scores.append((val, lag))
if val > best_val:
best_val, best_lag = val, lag
by_lag = dict((lag, val) for val, lag in scores)
for val, lag in scores:
if abs(lag - best_lag) > 5 and val > second and \
val >= by_lag.get(lag - 1, -float("inf")) and val >= by_lag.get(lag + 1, -float("inf")):
second = val
conf = max(0.0, min(1.0, best_val))
if second > -float("inf") and best_val > 0:
margin = (best_val - second) / best_val
conf *= min(1.0, 0.5 + margin)
return best_lag, conf
def refine(ref_s: List[float], oth_s: List[float], coarse_offset: float, fine_step: int, window_s: float) -> float:
"""Direct correlation at fine resolution around a coarse estimate (+/- window_s)."""
ref_e = envelope(ref_s, fine_step)
oth_e = envelope(oth_s, fine_step)
centre = int(round(coarse_offset * SR / fine_step))
span = int(window_s * SR / fine_step)
best_lag, best_val = centre, -float("inf")
n = min(len(ref_e), len(oth_e))
for lag in range(centre - span, centre + span + 1):
lo, hi = max(0, lag), min(n, n + lag)
if hi - lo < 10:
continue
val = 0.0
for i in range(lo, hi):
val += ref_e[i] * oth_e[i - lag]
val /= (hi - lo)
if val > best_val:
best_val, best_lag = val, lag
return best_lag * fine_step / SR
def measure_offset(ref_path: str, oth_path: str, start: float, seconds: float, step_ms: float, max_offset: float, fine_ms: float):
"""Return (offset_seconds, confidence) for a window starting at `start` in both files."""
ref_s = decode_mono(ref_path, seconds, start)
oth_s = decode_mono(oth_path, seconds, start)
step = max(1, int(SR * step_ms / 1000))
ref = envelope(ref_s, step)
oth = envelope(oth_s, step)
if len(ref) < 10 or len(oth) < 10:
short = [p for p, env in ((ref_path, ref), (oth_path, oth)) if len(env) < 10]
die(f"not enough audio to analyse in {' and '.join(short)} "
f"(needs at least {10 * step_ms / 1000:g} s of audio from {start:g} s)", kind="input")
max_lag = int(max_offset * SR / step)
lag, score = cross_correlate(ref, oth, max_lag)
offset = lag * step / SR
if fine_ms and fine_ms < step_ms:
offset = refine(ref_s, oth_s, offset, max(1, int(SR * fine_ms / 1000)), step_ms / 1000 * 2)
return offset, max(0.0, min(1.0, score))
def measure_one(reference: str, path: str, args) -> Dict:
"""Everything sync.py measures for ONE other source against the reference: offset,
confidence and (with --fix-drift) the same end-of-file drift measurement main() has always
made for a single pair. Used for every source when N>1, and for the historical single-pair
CLI shape when N==1 -- one measurement, so the two shapes cannot disagree."""
offset, score = measure_offset(reference, path, 0.0, args.analyze_seconds, args.step_ms, args.max_offset, args.fine_ms)
drift_ratio = 1.0
drift_info = None
skipped_reason = None
if args.fix_drift:
ref_dur = input_length(probe(reference))
sec_dur = input_length(probe(path))
overlap_end = min(ref_dur, sec_dur + offset)
head_len = min(args.analyze_seconds, overlap_end)
tail_start = overlap_end - args.drift_window
if tail_start <= head_len / 2 + 5:
skipped_reason = "too_short"
record_skip(f"{path} too short to measure drift reliably; skipping drift correction", "drift_correction_skipped",
path=path, reason=skipped_reason, overlap_seconds=round(overlap_end, 3),
drift_window=args.drift_window)
else:
ref_start = tail_start
sec_start = tail_start - offset
if sec_start < 0:
ref_start -= sec_start
sec_start = 0.0
ref_s = decode_mono(reference, args.drift_window, ref_start)
oth_s = decode_mono(path, args.drift_window, sec_start)
step = max(1, int(SR * args.step_ms / 1000))
lag, end_score = cross_correlate(envelope(ref_s, step), envelope(oth_s, step), int(2.0 * SR / step))
residual = lag * step / SR
if args.fine_ms:
residual = refine(ref_s, oth_s, residual, max(1, int(SR * args.fine_ms / 1000)), args.step_ms / 1000 * 2)
head_mid = head_len / 2
tail_mid = ref_start + args.drift_window / 2
elapsed = tail_mid - head_mid
if elapsed > 0 and end_score > 0.1:
drift_ratio = 1.0 - residual / elapsed
offset = offset + (drift_ratio - 1.0) * head_mid
drift_info = {"residual_at_end_seconds": round(residual, 4), "measured_over_seconds": round(elapsed, 2),
"drift_ppm": round((drift_ratio - 1) * 1e6, 1),
"meaning": "second file runs %.1f ppm %s (%.3fs over %.0fs); %s" % (
abs(drift_ratio - 1) * 1e6, "long/slow" if drift_ratio > 1 else "short/fast", abs(residual), elapsed,
("it would be resampled to match in the file this run plans (--dry-run writes nothing)" if STATE.dry_run
else "it is resampled to match in the file this run writes")
if args.replace_audio or args.trim_second else
"nothing is resampled in this measure-only run (--replace-audio or --trim-second writes the corrected file)"),
"confidence": round(end_score, 3)}
else:
skipped_reason = "low_confidence"
record_skip(f"could not measure drift with confidence for {path}; skipping drift correction",
"drift_correction_skipped", path=path, reason=skipped_reason,
confidence=round(max(0.0, end_score), 3), measured_over_seconds=round(elapsed, 2))
return {"path": path, "offset_seconds": round(offset, 4), "offset_s": round(offset, 4),
"confidence": round(max(0.0, min(1.0, score)), 3), "drift_ratio": drift_ratio, "drift": drift_info,
"drift_ppm": (drift_info["drift_ppm"] if drift_info else None), "drift_skipped_reason": skipped_reason}
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("reference", help="reference recording (usually the camera video)")
ap.add_argument("second", help="recording to align (external audio or second camera)")
ap.add_argument("more_sources", nargs="*", metavar="SOURCE3...", default=[],
help="(1.18) additional recordings beyond `second` to align to the same "
"reference -- a third camera, a second external recorder. With none of "
"these, sync.py keeps its original 2-source shape (offset_seconds/"
"confidence/drift at the top level, --replace-audio/--trim-second "
"available). With 1+, all sources (second plus these) are measured and "
"reported as one offsets JSON {reference, sources: [{path, offset_s, "
"confidence, drift_ppm}]}, and --replace-audio/--trim-second refuse: "
"each writes ONE synced output and there is more than one source")
ap.add_argument("-o", "--output", help="output file when writing a synced result (one-source runs only)")
ap.add_argument("--max-offset", type=float, default=30.0, help="largest offset to search in seconds (default 30)")
ap.add_argument("--analyze-seconds", type=float, default=120.0, help="how much audio to analyse from each file (default 120)")
ap.add_argument("--step-ms", type=float, default=20.0, help="coarse envelope resolution in ms for the FFT search (default 20)")
ap.add_argument("--fine-ms", type=float, default=1.0, help="fine resolution in ms for the refinement pass, 0 to skip (default 1)")
ap.add_argument("--fix-drift", action="store_true", help="also measure the offset near the END and correct clock drift by resampling the second file")
ap.add_argument("--drift-window", type=float, default=60.0, help="seconds of audio analysed at each end for drift (default 60)")
mode = ap.add_mutually_exclusive_group()
mode.add_argument("--replace-audio", action="store_true", help="write reference video with the second file's audio, aligned")
mode.add_argument("--trim-second", action="store_true", help="write the second file shifted so it lines up with the reference")
ap.set_defaults(crf=18)
add_common(ap)
args = ap.parse_args()
apply_common(args)
if args.max_offset <= 0:
die(f"--max-offset must be more than 0 seconds (the largest offset searched), got {args.max_offset:g}")
if args.analyze_seconds <= 0:
die(f"--analyze-seconds must be more than 0 seconds, got {args.analyze_seconds:g}")
if args.analyze_seconds > 900:
die(f"--analyze-seconds {args.analyze_seconds:g}: the window is decoded into memory; 900 s is the ceiling")
args.sources = [args.second] + list(args.more_sources)
for p in (args.reference,) + tuple(args.sources):
if not probe(p).get("audio"):
die(f"{p} has no audio stream to correlate")
n_sources = len(args.sources)
if (args.replace_audio or args.trim_second) and n_sources > 1:
die("--replace-audio/--trim-second write ONE synced output and need exactly one SOURCE; "
f"{n_sources} were given. Run sync.py once per source, or drop the flag and read the "
"offsets JSON.", kind="input")
measured = [measure_one(args.reference, p, args) for p in args.sources]
for p, m in zip(args.sources, measured):
if m["confidence"] < 0.1:
warn(f"{p}: low correlation confidence; check that it shares an audio event with the reference",
"sync_low_confidence", path=p, confidence=m["confidence"])
sources_block = [{"path": p, "offset_s": m["offset_s"], "confidence": m["confidence"],
**({"drift_ppm": m["drift_ppm"], "drift_skipped_reason": m["drift_skipped_reason"]} if args.fix_drift else {})}
for p, m in zip(args.sources, measured)]
if n_sources == 1:
args.second = args.sources[0]
m = measured[0]
offset, score, drift_ratio, drift_info = m["offset_seconds"], m["confidence"], m["drift_ratio"], m["drift"]
result = {
"reference": args.reference,
"second": args.second,
"offset_seconds": offset,
"confidence": score,
"meaning": sync_meaning(offset),
"sources": sources_block,
}
if drift_info:
result["drift"] = drift_info
if args.fix_drift:
result["drift_skipped_reason"] = m["drift_skipped_reason"]
else:
offset = drift_ratio = drift_info = None
result = {"reference": args.reference, "sources": sources_block}
info(f"measured {n_sources} sources against {args.reference}: " +
", ".join(f"{s['path']}: {s['offset_s']:+.3f}s (confidence {s['confidence']:.2f})" for s in sources_block))
if args.replace_audio or args.trim_second:
output = args.output or default_output(args.reference if args.replace_audio else args.second, "synced", "mp4")
refuse_output_is_input(output, args.reference, args.second)
delay_ms = int(round(offset * 1000))
head_trim = -offset if offset < 0 else 0.0
second_meta = probe(args.second)
has_video = bool(second_meta.get("video"))
sec_sr = (second_meta.get("audio") or {}).get("sample_rate") or 48000
drift_af: List[str] = []
if abs(drift_ratio - 1.0) > 1e-7:
drift_af = [f"asetrate={sec_sr * drift_ratio:.6f}", f"aresample={sec_sr}"]
on_clock = ["aresample=async=1:first_pts=0"] if stream_starts(second_meta)[1] > 0.0005 else []
if args.replace_audio:
cmd = ffmpeg_base() + ["-i", args.reference]
if head_trim > 0:
cmd += ["-ss", f"{head_trim:.4f}"]
cmd += ["-i", args.second, "-map", "0:v:0", "-map", "1:a:0", "-c:v", "copy"]
if os.path.splitext(output)[1].lower() in (".mp4", ".mov", ".m4v"):
cmd += ["-movflags", "+faststart"]
ref_dur = probe(args.reference).get("duration")
lead = [silence_filter(sec_sr, lead=offset)] if offset > 0 and round(offset * sec_sr) else []
af_parts = on_clock + drift_af + lead + ["apad"]
cmd += ["-af", ",".join(af_parts)]
ref_t = length_output_t(ref_dur, (second_meta.get("audio") or {}).get("sample_rate"), runs_on=True) if ref_dur else None
cmd += aac_args() + (["-t", ref_t] if ref_dur else ["-shortest"]) + [output]
proc = run(cmd, check=False)
fail_if_classified(proc)
if proc.returncode != 0:
cmd = [c for c in cmd if c != "copy"]
idx = cmd.index("-c:v"); del cmd[idx]
cmd = cmd[:-1] + video_args(probe(args.reference) if args.replace_audio else probe(args.second), args.crf) + [output]
run(cmd)
else:
if head_trim > 0 and not drift_af and not has_video:
cmd = ffmpeg_base() + ["-ss", f"{head_trim:.4f}", "-i", args.second] + default_selection_maps(second_meta, output) + ["-c", "copy"]
if os.path.splitext(output)[1].lower() in (".mp4", ".mov", ".m4v"):
cmd += ["-movflags", "+faststart"]
cmd += ["-avoid_negative_ts", "make_zero", output]
proc = run(cmd, check=False)
fail_if_classified(proc)
if proc.returncode != 0:
cmd = ffmpeg_base() + ["-ss", f"{head_trim:.4f}", "-i", args.second] + default_selection_maps(second_meta, output) + (video_args(probe(args.reference) if args.replace_audio else probe(args.second), args.crf) if has_video else []) + audio_codec_for(output) + [output]
run(cmd)
else:
cmd = ffmpeg_base()
out_seek: List[str] = []
if head_trim > 0:
m = min(SEEK_MARGIN, head_trim)
cmd += ["-ss", f"{head_trim - m:.6f}"]
if m > 0:
out_seek = ["-ss", f"{m / (drift_ratio if drift_af else 1.0):.6f}"]
cmd += ["-i", args.second] + out_seek + default_selection_maps(second_meta, output)
af_parts = on_clock + drift_af
if offset > 0 and round(offset * sec_sr):
af_parts.append(silence_filter(sec_sr, lead=offset))
if has_video:
vf = []
if delay_ms > 0:
vf.append(f"tpad=start_duration={offset:.4f}")
if drift_af:
vf.append(f"setpts=PTS/{drift_ratio:.9f}")
if vf:
cmd += ["-vf", ",".join(vf)]
cmd += video_args(probe(args.reference) if args.replace_audio else probe(args.second), args.crf)
v0, rate = stream_starts(second_meta)[0], frame_rate(second_meta)
first = (grid_at_or_after(max(round(head_trim, 4), v0), v0, rate) - v0 if head_trim > 0
else -offset if delay_ms > 0 else 0.0)
cmd += output_timecode_args(second_meta, first)
if af_parts:
cmd += ["-af", ",".join(af_parts)]
cmd += audio_codec_for(output) + [output]
run(cmd)
result["output"] = output
info(f"wrote {output}")
if args.json:
emit(result.get("output"), **{k: v for k, v in result.items() if k != "output"})
elif n_sources == 1:
print(f"offset: {result['offset_seconds']:+.3f}s ({result['meaning']}), confidence {result['confidence']:.2f}")
if drift_info:
print(f"drift: {drift_info['drift_ppm']:+.1f} ppm ({drift_info['residual_at_end_seconds']:+.3f}s over {drift_info['measured_over_seconds']:.0f}s), confidence {drift_info['confidence']:.2f}")
if "output" in result:
print(result["output"])
else:
for s in sources_block:
line = f"{s['path']}: {s['offset_s']:+.3f}s, confidence {s['confidence']:.2f}"
if args.fix_drift:
line += f", drift {s['drift_ppm']:+.1f} ppm" if s["drift_ppm"] is not None else ""
print(line)
return 0
if __name__ == "__main__":
sys.exit(main())