diff --git a/SKILL.md b/SKILL.md index 2020338..be3bfc8 100644 --- a/SKILL.md +++ b/SKILL.md @@ -300,6 +300,13 @@ table and every CPU/threading knob: **`references/hevc-conversion-policy.md`**. ## Support files +- `scripts/library_run.py` — the whole-library driver that enforces the policy above: bpp tiering + (copy / nvenc / x265), `-maxrate` at 1.0x the source **video** bitrate + a size assertion, a + per-frame SSIM gate with a one-step quality retry, audio matched to the source, local staging, + SHA-256 read-back before an atomic replace, a JSONL ledger, a run lock, E-core pinning for the + CPU tier, `--tier` to run tiers separately, `--dry-run` for the classification plan, + `--keep-local` to pilot without touching anything, and `--origin-root` to redo files that an + earlier bad setting bloated, sourcing them from a snapshot of the originals (never deleting it). - `scripts/transcode.py` — the tool. `--dry-run`, `--codec`, `--cq/--crf`, `--audio`, `--channels`, `--cpu`, `--keep`, `--overwrite`, `--no-decode-check`, `--lock`. - `references/compatibility-matrix.md` — per-platform codec/container facts with citations, diff --git a/references/hevc-conversion-policy.md b/references/hevc-conversion-policy.md index 493accd..7877df7 100644 --- a/references/hevc-conversion-policy.md +++ b/references/hevc-conversion-policy.md @@ -256,6 +256,31 @@ Defaults that follow from this, and the gates that enforce them, are in `scripts video bitrate may not exceed the source's), a one-step quality retry when the SSIM gate fails, and `-maxrate` = 1.0x the source's **video** bitrate. +## 12. Re-doing files the old settings bloated (use the ORIGINAL, not the converted file) + +When a library was already converted with the wrong settings, the output is **bigger than the +original** and re-encoding it again would be a second-generation loss. The fix is to redo it from +the **pre-conversion original**, which usually still exists in a dated snapshot of the library +(measured: a 2026-09-25 snapshot held all 582 files / 324 GiB of the course library). + +Measured audit of such a set (54 files converted at `-cq 27` + a fixed 256k audio): + +| by the ORIGINAL's bpp | files | live/original | what they needed | +|---|---|---|---| +| copy (<0.025) | **25** | up to **124.8%** | no video re-encode at all - the video should have been copied | +| nvenc (0.025-0.040) | 26 | 106-122% | cq 31 instead of 27 | +| x265 (>0.040) | 3 | - | crf 26 | + +`scripts/library_run.py --origin-root ` implements this safely: + +* a file whose live copy is already HEVC is re-done **only if it is bigger than its original** + (`> 1.01x`); otherwise it is skipped - so the run is idempotent without any extra bookkeeping; +* the encode reads the **original**, the tier is classified from the **original's** bpp, and the + gates compare against the **original** - the converted file is never used as a reference; +* `dst` stays the live path, so the output replaces the (bloated) converted file; +* **the original is never modified or deleted** - `os.remove(src)` is suppressed whenever the + source came from the origin tree. Getting this backwards would destroy the only clean copy. + ## 9. CPU and GPU tiers in parallel Different tiers can encode **at the same time** — NVENC is a fixed-function ASIC, x265 is CPU. diff --git a/scripts/library_run.py b/scripts/library_run.py index c044fea..7380f46 100644 --- a/scripts/library_run.py +++ b/scripts/library_run.py @@ -338,16 +338,31 @@ def already_hevc(src, info): return True -def process(src, args): +def process(src, args, dest=None, origin=None): info = T.probe(src) if not info: return "FAIL unreadable", None if info["dur"] < 1.0: return "FAIL stub (%.1fs) - needs a re-download, not a transcode" % info["dur"], None - dst = dst_path_for(src) + live = dest or dst_path_for(src) + if info["vcodec"] == "hevc" and info["tag"] == "hvc1" and info["pix"] in ("yuv420p",) \ and not T.unusable_timeline(info, src)[0]: - return "SKIP already HEVC hvc1", None + # Already converted. Only worth touching if the ORIGINAL is available and the current + # file is BIGGER than it: that is precisely the old settings' damage (a fixed cq27 above + # the source's own quality plus a fixed 256k audio). A file that is already smaller than + # its original is left alone, which makes the whole run naturally idempotent. + if not (origin and os.path.exists(origin)): + return "SKIP already HEVC hvc1", None + o_size = os.path.getsize(origin) + if info["size"] <= o_size * 0.99: + return "SKIP already HEVC hvc1 (%.0f MB <= original %.0f MB)" % ( + info["size"] / 1e6, o_size / 1e6), None + log(" re-doing from the original: live %.0f MB > original %.0f MB" % ( + info["size"] / 1e6, o_size / 1e6)) + src, info = origin, T.probe(origin) + if not info: + return "FAIL original unreadable", None # ---- policy: classify the source, then work only on the requested tier (if any) ---------- vbps = src_video_bps(src, info) @@ -363,10 +378,11 @@ def process(src, args): tag = hashlib.sha1(src.encode("utf-8")).hexdigest()[:12] loc_src = os.path.join(STAGE, tag + "_src" + os.path.splitext(src)[1].lower()) loc_out = os.path.join(STAGE, tag + "_out.mp4") - remote_tmp = dst + ".new.mp4" + remote_tmp = live + ".new.mp4" for p in (loc_src, loc_out): if os.path.exists(p): os.remove(p) + origin_used = bool(origin and os.path.abspath(src) == os.path.abspath(origin)) try: if os.path.exists(remote_tmp): os.remove(remote_tmp) @@ -433,16 +449,18 @@ def process(src, args): pass return "FAIL read-back mismatch local=%s/%d remote=%s/%d" % (h_loc[:12], n_loc, h_rem[:12], n_rem), None # 6. atomic replace, then confirm on the share - os.replace(remote_tmp, dst) - if os.path.abspath(dst) != os.path.abspath(src): + os.replace(remote_tmp, live) + # NEVER delete the source when it came from the snapshot origin: that is the archive of + # the originals, and removing it would destroy the only clean copy of the library. + if not origin_used and os.path.abspath(live) != os.path.abspath(src): os.remove(src) - back = T.probe(dst) + back = T.probe(live) tag_needed = (tier != "copy") if not back or back["vcodec"] != expect_vc \ or (tag_needed and back["tag"] != "hvc1") \ or abs(back["dur"] - info["dur"]) > max(1.0, info["dur"] * 0.01): return "FAIL read-back probe: %s" % (back and (back["vcodec"], back["tag"], back["dur"]),), None - ok, msg = probe_ok(dst, info, expect_vc) + ok, msg = probe_ok(live, info, expect_vc) if not ok: return "FAIL post-replace gate on share: %s" % msg, None return "ok", {"sha256": h_loc, "bytes": n_loc, "src_bytes": n_in, @@ -479,6 +497,10 @@ def main(): ap.add_argument("--pools", type=int, default=X265_POOLS, help="x265 threads (match the E-core mask)") ap.add_argument("--tier", choices=["copy", "nvenc", "x265"], help="only touch files classified into this tier (run tiers separately)") + ap.add_argument("--origin-root", default=None, + help="tree holding the PRE-CONVERSION originals (e.g. a snapshot); a file that " + "is already HEVC and BIGGER than its original is re-done from that " + "original, and the original is never modified or deleted") ap.add_argument("--audio", default=None, help="audio bitrate; default = the SOURCE's own bitrate (never a fixed 256k)") ap.add_argument("--nvenc-preset", default=NVENC_PRESET) @@ -535,8 +557,12 @@ def main(): break log("--- %d/%d %s" % (i, len(files), os.path.relpath(f, ROOT))) t0 = time.time() + origin = None + if args.origin_root: + cand = os.path.join(args.origin_root, os.path.relpath(f, ROOT)) + origin = cand if os.path.exists(cand) else None try: - status, extra = process(f, args) + status, extra = process(f, args, dest=dst_path_for(f), origin=origin) except Exception as e: status, extra = "FAIL %s: %s" % (type(e).__name__, e), None el = time.time() - t0 @@ -564,4 +590,5 @@ def main(): sys.exit(1) -main() +if __name__ == "__main__": + main()