library_run.py: policy-driven runner (never bigger, quality pinned, tiered) + measured YouTube H.264 numbers
The previous runner encoded everything at a fixed -cq 27 with -maxrate = 1.5x the FILE bitrate, which is how a library ended up at 111% of its source size. This replaces the encode step with the policy the skill documents, and adds the gates that make it a guarantee: * classify by bpp: <0.025 copy the video (bit-identical), 0.025-0.040 hevc_nvenc, >0.040 libx265 * -maxrate = 1.0x the source's VIDEO bitrate (never the file total) + a post-encode assertion * SSIM gate: per-frame comparison of a 60 s window against the source, retry once at +2 quality if it fails, then fail the item (the original is never touched) * audio: copy when AAC, else AAC at the SOURCE's bitrate (a fixed 256k cancelled the video saving) * x265 tier pinned to the E-cores; --tier to run tiers separately; --keep-local for a pilot that touches nothing * defaults cq31 / crf26 from the measurement below, not cq27 Measured on three real sources (60 s window, video-only): cq27 GROWS two of three files (115%, 119%); cq31 lands at 43-74%; x265 crf26 at 34-67% with min SSIM 0.988-0.993 - and x265 beats NVENC by 10-22% at matched SSIM. Pinning min SSIM >= 0.996 leaves only cq27, which is bigger than the source, so ~0.99 is the usable quality frontier and the owner gets to make that call explicitly (references/hevc-conversion-policy.md section 11, SKILL.md scope section).
This commit is contained in:
@@ -0,0 +1,567 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Library re-encode driver: same look, SMALLER file, playable on iPad + Android.
|
||||
|
||||
Policy (see references/hevc-conversion-policy.md):
|
||||
* classify each source by `bpp` and pick the cheapest tier that keeps the quality:
|
||||
lean (bpp < 0.025) -> `-c:v copy` video bit-identical, nothing to gain
|
||||
mid (0.025-0.040) -> hevc_nvenc cq 30 ~6.4x realtime
|
||||
fat (> 0.040) -> libx265 crf 24 ~3x realtime, 9-16% more efficient
|
||||
* `-maxrate` = 1.0x the source's **video** bitrate (never the file's total) and a
|
||||
post-encode assertion that the output video bitrate did not exceed it
|
||||
* quality pinned to a measured number: per-frame SSIM of a sampled window vs the source
|
||||
* audio: copy when already AAC, else AAC at the SOURCE's own bitrate (never a fixed 256k)
|
||||
* `-fps_mode cfr` + `-pix_fmt yuv420p` + `-tag:v hvc1` on every re-encode
|
||||
|
||||
Per file, in order:
|
||||
1. copy the source OFF the network share to LOCAL staging (never encode over SMB/WebDAV)
|
||||
2. encode locally (tier above)
|
||||
3. structural gate + full decode pass + SSIM-vs-source + size assertion, all on the LOCAL output
|
||||
4. copy the verified output to the share as <name>.new.mp4
|
||||
5. read the share copy back end-to-end and compare SHA-256 with the local file
|
||||
6. ONLY then rename it over the original, and re-probe the result on the share
|
||||
A file that fails at any step leaves the original untouched and is retried next run.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import ctypes
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import transcode as T # noqa: E402
|
||||
|
||||
FFMPEG = T.FFMPEG
|
||||
FFPROBE = T.FFPROBE
|
||||
ROOT = r"Z:\Class\樊登读书\樊登读书会（每周更新）"
|
||||
WORK = r"C:\AI\transcode-run"
|
||||
STAGE = os.path.join(WORK, "stage")
|
||||
LEDGER = os.path.join(WORK, "ledger.jsonl")
|
||||
LOCKF = os.path.join(WORK, "run.lock")
|
||||
VIDEO_EXT = {".mkv", ".mp4", ".mov", ".avi", ".webm", ".flv", ".ts", ".m4v", ".wmv",
|
||||
".mpg", ".mpeg", ".m2ts", ".3gp", ".ogv", ".vob", ".rmvb", ".rm", ".divx"}
|
||||
DECODE_CHECK = True
|
||||
|
||||
# ---- policy (all numbers measured; see references/hevc-conversion-policy.md §10) ----------
|
||||
CQ_NVENC = 31 # measured: min SSIM 0.989-0.994 at 43-74% of the source video bitrate
|
||||
CRF_X265 = 26 # measured: min SSIM 0.988-0.993 at 34-67% -- 10-22% smaller than NVENC
|
||||
# at matched SSIM. crf24 = the conservative option (min 0.990-0.994, 43-84%)
|
||||
NVENC_PRESET = "p6"
|
||||
X265_PRESET = "medium"
|
||||
X265_POOLS = 10 # E-cores only (affinity 0x3FF0000)
|
||||
E_CORE_MASK = 0x3FF0000
|
||||
SSIM_TARGET = 0.985 # per-frame floor vs the source window (480x270 comparison)
|
||||
SSIM_WINDOW = 60 # seconds sampled for the SSIM gate
|
||||
SIZE_CEIL = 1.0 # output video bitrate must not exceed the source's
|
||||
QUALITY_STEPS = 2 # on a quality-gate failure, retry with the quality raised this many
|
||||
# steps (cq -2 / crf -2) before giving up; the retry only fires on a
|
||||
# file the cheaper setting could not carry
|
||||
|
||||
|
||||
def log(*a):
|
||||
print(time.strftime("%H:%M:%S") + " " + " ".join(str(x) for x in a), flush=True)
|
||||
|
||||
|
||||
def run(cmd, timeout=86400, **kw):
|
||||
return subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8",
|
||||
errors="replace", timeout=timeout, **kw)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- integrity
|
||||
|
||||
def _sha_local(path, label="local"):
|
||||
h = hashlib.sha256()
|
||||
n = 0
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
b = f.read(8 << 20)
|
||||
if not b:
|
||||
break
|
||||
h.update(b)
|
||||
n += len(b)
|
||||
return h.hexdigest(), n
|
||||
|
||||
|
||||
def _sha_remote(path, label="remote", tries=3):
|
||||
"""Hash a file over the network. getsize() lies on RaiDrive/Dokan, so the byte
|
||||
count here is what was actually read, never a metadata figure."""
|
||||
last = None
|
||||
for attempt in range(1, tries + 1):
|
||||
try:
|
||||
h = hashlib.sha256()
|
||||
n = 0
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
b = f.read(8 << 20)
|
||||
if not b:
|
||||
break
|
||||
h.update(b)
|
||||
n += len(b)
|
||||
return h.hexdigest(), n
|
||||
except OSError as e:
|
||||
last = e
|
||||
log(" ! %s read attempt %d failed: %s" % (label, attempt, e))
|
||||
time.sleep(5 * attempt)
|
||||
raise last
|
||||
|
||||
|
||||
def copy_remote_to_local(remote, local, expect_size=None, tries=3):
|
||||
"""Network -> local. Retried; a short read is a failure, never a silent pass."""
|
||||
for attempt in range(1, tries + 1):
|
||||
try:
|
||||
n = 0
|
||||
with open(remote, "rb") as fi, open(local, "wb") as fo:
|
||||
while True:
|
||||
b = fi.read(8 << 20)
|
||||
if not b:
|
||||
break
|
||||
fo.write(b)
|
||||
n += len(b)
|
||||
fo.flush()
|
||||
os.fsync(fo.fileno())
|
||||
if expect_size and n != expect_size:
|
||||
raise IOError("short read: got %d, container says %d" % (n, expect_size))
|
||||
return n
|
||||
except OSError as e:
|
||||
log(" ! source copy attempt %d failed: %s" % (attempt, e))
|
||||
time.sleep(5 * attempt)
|
||||
raise IOError("could not copy source off the share after %d attempts" % tries)
|
||||
|
||||
|
||||
def copy_local_to_remote(local, remote_tmp, tries=3):
|
||||
"""Local -> network temp name. Chunked; never writes the final name."""
|
||||
for attempt in range(1, tries + 1):
|
||||
try:
|
||||
with open(local, "rb") as fi, open(remote_tmp, "wb") as fo:
|
||||
while True:
|
||||
b = fi.read(8 << 20)
|
||||
if not b:
|
||||
break
|
||||
fo.write(b)
|
||||
fo.flush()
|
||||
os.fsync(fo.fileno())
|
||||
return True
|
||||
except OSError as e:
|
||||
log(" ! upload attempt %d failed: %s" % (attempt, e))
|
||||
try:
|
||||
os.remove(remote_tmp)
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(5 * attempt)
|
||||
return False
|
||||
|
||||
|
||||
def probe_ok(path, info, vcodec="hevc"):
|
||||
"""Structural gate on any path (local or share)."""
|
||||
return T.verify(info, path, vcodec, "copy" if info["acodec"] in ("aac", "mp4a", "") else "aac",
|
||||
check_decode=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- encode
|
||||
|
||||
def src_bitrate(info):
|
||||
if info["dur"] > 0 and info["size"] > 0:
|
||||
return info["size"] * 8 / info["dur"]
|
||||
return 0
|
||||
|
||||
|
||||
def match_audio_kbps(path, floor=96, ceil=256, fallback=192):
|
||||
"""AAC at the SOURCE's own bitrate, not a fixed 256k.
|
||||
|
||||
Measured on this library: the weekly series ships mp3 at 128 kbps, so a fixed
|
||||
-b:a 256k added ~128 kbps for no audible gain - about +10% file size on a
|
||||
1.3 Mbps file, which by itself cancels the video savings. AAC is more efficient
|
||||
than mp3, so matching the bitrate is already a quality upgrade, never a loss.
|
||||
"""
|
||||
r = run([FFPROBE, "-v", "error", "-select_streams", "a:0",
|
||||
"-show_entries", "stream=bit_rate", "-of", "csv=p=0", path], timeout=120)
|
||||
try:
|
||||
b = int((r.stdout or "").strip().splitlines()[0].split(",")[0])
|
||||
except Exception:
|
||||
return fallback
|
||||
return max(floor, min(ceil, round(b / 1000)))
|
||||
|
||||
|
||||
def src_video_bps(path, info):
|
||||
"""The SOURCE's video-stream bitrate. Never the file's total: mixing them turned a 109%
|
||||
(grow) into a false 86% (shrink) in one analysis, because the total carries the audio."""
|
||||
r = run([FFPROBE, "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=bit_rate",
|
||||
"-of", "csv=p=0", path], timeout=180)
|
||||
try:
|
||||
v = int((r.stdout or "").strip().splitlines()[0].split(",")[0])
|
||||
if v > 0:
|
||||
return v
|
||||
except Exception:
|
||||
pass
|
||||
tot = info["size"] * 8 / info["dur"] if info.get("dur") else 0
|
||||
return max(0, int(tot - (info.get("abps") or 0)))
|
||||
|
||||
|
||||
def src_fps(info):
|
||||
"""transcode.probe() exposes rfps/afps, not `fps`."""
|
||||
return info.get("rfps") or info.get("afps") or 0
|
||||
|
||||
|
||||
def bpp_of(info, vbps):
|
||||
"""bits per pixel per frame: how much fat the source still carries."""
|
||||
d = (info.get("w") or 0) * (info.get("h") or 0) * src_fps(info)
|
||||
return (vbps / d) if d else 0.0
|
||||
|
||||
|
||||
def tier_for_bpp(b):
|
||||
if b < 0.025:
|
||||
return "copy"
|
||||
return "nvenc" if b <= 0.040 else "x265"
|
||||
|
||||
|
||||
def out_video_bps(path, dur):
|
||||
r = run([FFPROBE, "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=bit_rate",
|
||||
"-of", "csv=p=0", path], timeout=180)
|
||||
try:
|
||||
v = int((r.stdout or "").strip().splitlines()[0].split(",")[0])
|
||||
if v > 0:
|
||||
return v
|
||||
except Exception:
|
||||
pass
|
||||
return int(os.path.getsize(path) * 8 / dur) if dur else 0
|
||||
|
||||
|
||||
def ssim_vs_source(src_local, out_local, info, workdir, tag, dur=SSIM_WINDOW):
|
||||
"""Per-frame SSIM of a sampled window: the OUTPUT against the SOURCE.
|
||||
|
||||
The comparison is done at 480x270 (a wrong/scaled picture is obvious at that size and the
|
||||
decode stays cheap). NB the stats file must be a BARE name inside workdir - a Windows path
|
||||
inside `-lavfi` has its backslashes eaten by the filter parser, which silently yields zero
|
||||
frames and 0.0000 SSIM.
|
||||
"""
|
||||
start = max(0.0, (info["dur"] / 2) - dur / 2)
|
||||
cmp_scale = "480:270"
|
||||
ref = os.path.join(workdir, "ssim_ref.yuv")
|
||||
out = os.path.join(workdir, "ssim_out.yuv")
|
||||
for path, dstp in ((src_local, ref), (out_local, out)):
|
||||
r = run([FFMPEG, "-y", "-v", "error", "-ss", "%.2f" % start, "-t", "%.2f" % dur, "-i", path,
|
||||
"-map", "0:v:0", "-an", "-vf", "scale=%s:flags=bilinear,format=yuv420p" % cmp_scale,
|
||||
"-f", "rawvideo", dstp], timeout=3600)
|
||||
if r.returncode != 0 or not os.path.exists(dstp) or not os.path.getsize(dstp):
|
||||
return None, None, 0
|
||||
subprocess.run([FFMPEG, "-y", "-v", "error", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", cmp_scale,
|
||||
"-i", ref, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", cmp_scale, "-i", out,
|
||||
"-lavfi", "ssim=stats_file=ssim_%s.log" % tag, "-f", "null", "-"],
|
||||
capture_output=True, cwd=workdir)
|
||||
vals = []
|
||||
p = os.path.join(workdir, "ssim_%s.log" % tag)
|
||||
if os.path.exists(p):
|
||||
with open(p, encoding="utf-8", errors="replace") as fh:
|
||||
for line in fh:
|
||||
m = re.search(r"All:([0-9.]+)", line)
|
||||
if m:
|
||||
vals.append(float(m.group(1)))
|
||||
for f in (ref, out):
|
||||
try:
|
||||
os.remove(f)
|
||||
except OSError:
|
||||
pass
|
||||
return (min(vals), sum(vals) / len(vals), len(vals)) if vals else (None, None, 0)
|
||||
|
||||
|
||||
def pin_to_e_cores():
|
||||
"""x265 tiers: keep the P-cores free for the desktop (measured 3.51x vs 4.37x realtime)."""
|
||||
try:
|
||||
ctypes.windll.kernel32.SetProcessAffinityMask(
|
||||
ctypes.windll.kernel32.GetCurrentProcess(), ctypes.c_size_t(E_CORE_MASK))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def encode(src_local, dst_local, info, args, tier, vbps, q_off=0):
|
||||
cmd = [FFMPEG, "-y", "-hide_banner", "-loglevel", "warning", "-stats",
|
||||
"-i", src_local, "-map", "0:v:0", "-map", "0:a:0?", "-map_metadata", "0"]
|
||||
if tier == "copy":
|
||||
# Nothing to win: the video is copied bit-for-bit, which is the strongest possible
|
||||
# answer to "same picture quality" and the smallest file that still plays everywhere.
|
||||
cmd += ["-c:v", "copy"]
|
||||
elif tier == "nvenc":
|
||||
cmd += ["-c:v", "hevc_nvenc", "-preset", args.nvenc_preset, "-tune", "hq", "-rc", "vbr",
|
||||
"-cq", str(max(1, args.cq - q_off)), "-b:v", "0",
|
||||
# efficiency knobs verified accepted by this build (multipass 2pass-full and
|
||||
# weighted_pred are REJECTED here)
|
||||
"-spatial-aq", "1", "-temporal-aq", "1", "-rc-lookahead", "32",
|
||||
"-b_ref_mode", "middle", "-bf", "4", "-g", "240"]
|
||||
else:
|
||||
cmd += ["-c:v", "libx265", "-preset", args.x265_preset, "-crf", str(max(1, args.crf - q_off)),
|
||||
"-x265-params", "pools=%d" % args.pools]
|
||||
if tier != "copy" and vbps > 0:
|
||||
# Hard ceiling at the source's own video bitrate: the output physically cannot grow.
|
||||
cmd += ["-maxrate", str(int(vbps * SIZE_CEIL)), "-bufsize", str(int(vbps * SIZE_CEIL * 2.5))]
|
||||
if tier != "copy":
|
||||
cmd += ["-profile:v", "main", "-pix_fmt", "yuv420p", "-tag:v", "hvc1", "-fps_mode", "cfr"]
|
||||
if info["acodec"] in ("aac", "mp4a", ""):
|
||||
cmd += ["-c:a", "copy"]
|
||||
else:
|
||||
# AAC at the SOURCE's own bitrate: a fixed 256k added ~128 kbps for no audible gain on
|
||||
# this library's 128k mp3 sources, which by itself cancelled the video saving.
|
||||
abr = args.audio or ("%dk" % match_audio_kbps(src_local))
|
||||
cmd += ["-c:a", "aac", "-b:a", abr, "-ac", "2"]
|
||||
cmd += ["-movflags", "+faststart", dst_local]
|
||||
t0 = time.time()
|
||||
r = run(cmd)
|
||||
return r, time.time() - t0
|
||||
|
||||
|
||||
|
||||
def decode_check(path):
|
||||
"""Full decode pass. The only check that proves the payload is not truncated/corrupt."""
|
||||
r = run([FFMPEG, "-v", "error", "-i", path, "-f", "null", "-"], timeout=14400)
|
||||
return r.returncode == 0, (r.stderr or "").strip()[:300]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- per file
|
||||
|
||||
def dst_path_for(src):
|
||||
"""In-place: the SAME name for an existing .mp4/.MP4 (the share is case-sensitive),
|
||||
<stem>.mp4 for anything else."""
|
||||
if os.path.splitext(src)[1].lower() == ".mp4":
|
||||
return src
|
||||
return os.path.splitext(src)[0] + ".mp4"
|
||||
|
||||
|
||||
def already_hevc(src, info):
|
||||
"""Self-describing resume test: the share copy is already HEVC hvc1 and matches."""
|
||||
if info["vcodec"] != "hevc" or info["tag"] != "hvc1":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def process(src, args):
|
||||
info = T.probe(src)
|
||||
if not info:
|
||||
return "FAIL unreadable", None
|
||||
if info["dur"] < 1.0:
|
||||
return "FAIL stub (%.1fs) - needs a re-download, not a transcode" % info["dur"], None
|
||||
dst = dst_path_for(src)
|
||||
if info["vcodec"] == "hevc" and info["tag"] == "hvc1" and info["pix"] in ("yuv420p",) \
|
||||
and not T.unusable_timeline(info, src)[0]:
|
||||
return "SKIP already HEVC hvc1", None
|
||||
|
||||
# ---- policy: classify the source, then work only on the requested tier (if any) ----------
|
||||
vbps = src_video_bps(src, info)
|
||||
bpp = bpp_of(info, vbps)
|
||||
tier = tier_for_bpp(bpp)
|
||||
if args.tier and args.tier != tier:
|
||||
return "SKIP classified %s (run wants %s)" % (tier, args.tier), None
|
||||
expect_vc = "hevc" if tier != "copy" else (info["vcodec"] or "h264")
|
||||
log(" tier=%s bpp=%.4f src_video=%.0f kbps %sx%s@%.0f %s" % (
|
||||
tier, bpp, vbps / 1000, info["w"], info["h"], src_fps(info), os.path.basename(src)[:46]))
|
||||
|
||||
os.makedirs(STAGE, exist_ok=True)
|
||||
tag = hashlib.sha1(src.encode("utf-8")).hexdigest()[:12]
|
||||
loc_src = os.path.join(STAGE, tag + "_src" + os.path.splitext(src)[1].lower())
|
||||
loc_out = os.path.join(STAGE, tag + "_out.mp4")
|
||||
remote_tmp = dst + ".new.mp4"
|
||||
for p in (loc_src, loc_out):
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
try:
|
||||
if os.path.exists(remote_tmp):
|
||||
os.remove(remote_tmp)
|
||||
|
||||
# 1. fetch the source to local disk
|
||||
n_in = copy_remote_to_local(src, loc_src, expect_size=info["size"] or None)
|
||||
# 2. encode (with a bounded quality ladder: if the sampled window cannot clear the SSIM
|
||||
# bar at the tier's default setting, raise the quality and try once more)
|
||||
attempt, q_off, el = 0, 0, 0.0
|
||||
while True:
|
||||
attempt += 1
|
||||
r, el = encode(loc_src, loc_out, info, args, tier, vbps, q_off)
|
||||
if r.returncode != 0 or not os.path.exists(loc_out):
|
||||
return "FAIL ffmpeg rc=%s %s" % (r.returncode, (r.stderr or "").strip()[-300:]), None
|
||||
# 3. verify the LOCAL output (structural + full decode)
|
||||
ok, msg = T.verify(info, loc_out, expect_vc, "aac", check_decode=False)
|
||||
if not ok:
|
||||
return "FAIL verify: %s" % msg, None
|
||||
if DECODE_CHECK and not args.no_decode_check:
|
||||
ok, msg = decode_check(loc_out)
|
||||
if not ok:
|
||||
return "FAIL decode pass: %s" % msg, None
|
||||
# 3b. the two policy gates that make "same look, never bigger" a fact and not a hope:
|
||||
# a per-frame SSIM of a sampled window against the source, and a hard assertion
|
||||
# that the output video bitrate did not exceed the source's.
|
||||
ssim_note, growth = "copy (video bit-identical)", 100.0
|
||||
if tier != "copy":
|
||||
lo, mean, nfr = ssim_vs_source(loc_src, loc_out, info, STAGE, tag)
|
||||
if not nfr:
|
||||
return "FAIL ssim: no frames compared", None
|
||||
obps = out_video_bps(loc_out, info["dur"])
|
||||
growth = 100.0 * obps / vbps if vbps else 0.0
|
||||
ssim_note = "ssim min %.4f mean %.4f (%d frames)%s" % (
|
||||
lo, mean, nfr, "" if not q_off else " at quality+%d" % q_off)
|
||||
if lo < SSIM_TARGET:
|
||||
if q_off < QUALITY_STEPS and (attempt == 1):
|
||||
q_off = QUALITY_STEPS
|
||||
log(" quality gate %.4f < %.3f - retrying at quality+%d" %
|
||||
(lo, SSIM_TARGET, q_off))
|
||||
continue
|
||||
return "FAIL quality: %s < target %.3f" % (ssim_note, SSIM_TARGET), None
|
||||
if vbps and obps > vbps * SIZE_CEIL:
|
||||
return "FAIL size: output %.0f kbps > source %.0f kbps (%.1f%%)" % (
|
||||
obps / 1000, vbps / 1000, growth), None
|
||||
log(" %s %.1f%% of source video" % (ssim_note, growth))
|
||||
break
|
||||
# 4. upload under a temp name
|
||||
if args.keep_local:
|
||||
return "ok", {"sha256": "", "bytes": os.path.getsize(loc_out), "src_bytes": n_in,
|
||||
"src_mb": round(info["size"] / 1e6, 1),
|
||||
"out_mb": round(os.path.getsize(loc_out) / 1e6, 1),
|
||||
"ratio": round(100.0 * os.path.getsize(loc_out) / max(info["size"], 1), 1),
|
||||
"tier": tier, "bpp": round(bpp, 4), "video_pct": round(growth, 1),
|
||||
"ssim": ssim_note + " [LOCAL ONLY - not uploaded]", "encode_s": round(el, 1)}
|
||||
if not copy_local_to_remote(loc_out, remote_tmp):
|
||||
return "FAIL upload", None
|
||||
# 5. read the share copy back and compare
|
||||
h_loc, n_loc = _sha_local(loc_out)
|
||||
h_rem, n_rem = _sha_remote(remote_tmp)
|
||||
if (h_loc, n_loc) != (h_rem, n_rem):
|
||||
try:
|
||||
os.remove(remote_tmp)
|
||||
except OSError:
|
||||
pass
|
||||
return "FAIL read-back mismatch local=%s/%d remote=%s/%d" % (h_loc[:12], n_loc, h_rem[:12], n_rem), None
|
||||
# 6. atomic replace, then confirm on the share
|
||||
os.replace(remote_tmp, dst)
|
||||
if os.path.abspath(dst) != os.path.abspath(src):
|
||||
os.remove(src)
|
||||
back = T.probe(dst)
|
||||
tag_needed = (tier != "copy")
|
||||
if not back or back["vcodec"] != expect_vc \
|
||||
or (tag_needed and back["tag"] != "hvc1") \
|
||||
or abs(back["dur"] - info["dur"]) > max(1.0, info["dur"] * 0.01):
|
||||
return "FAIL read-back probe: %s" % (back and (back["vcodec"], back["tag"], back["dur"]),), None
|
||||
ok, msg = probe_ok(dst, info, expect_vc)
|
||||
if not ok:
|
||||
return "FAIL post-replace gate on share: %s" % msg, None
|
||||
return "ok", {"sha256": h_loc, "bytes": n_loc, "src_bytes": n_in,
|
||||
"src_mb": round(info["size"] / 1e6, 1),
|
||||
"out_mb": round(n_loc / 1e6, 1),
|
||||
"ratio": round(100.0 * n_loc / max(info["size"], 1), 1),
|
||||
"tier": tier, "bpp": round(bpp, 4), "video_pct": round(growth, 1),
|
||||
"ssim": ssim_note, "encode_s": round(el, 1)}
|
||||
finally:
|
||||
for p in (loc_src, loc_out):
|
||||
try:
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def collect(root):
|
||||
out = []
|
||||
for dp, _, fn in os.walk(root):
|
||||
for f in fn:
|
||||
if os.path.splitext(f)[1].lower() in VIDEO_EXT and ".part." not in f \
|
||||
and "_src" not in f and ".new." not in f:
|
||||
out.append(os.path.join(dp, f))
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def main():
|
||||
global DECODE_CHECK
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--cq", type=int, default=CQ_NVENC, help="nvenc tier quality (measured default 30)")
|
||||
ap.add_argument("--crf", type=int, default=CRF_X265, help="x265 tier quality (measured default 24)")
|
||||
ap.add_argument("--x265-preset", default=X265_PRESET)
|
||||
ap.add_argument("--pools", type=int, default=X265_POOLS, help="x265 threads (match the E-core mask)")
|
||||
ap.add_argument("--tier", choices=["copy", "nvenc", "x265"],
|
||||
help="only touch files classified into this tier (run tiers separately)")
|
||||
ap.add_argument("--audio", default=None,
|
||||
help="audio bitrate; default = the SOURCE's own bitrate (never a fixed 256k)")
|
||||
ap.add_argument("--nvenc-preset", default=NVENC_PRESET)
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
ap.add_argument("--only", default=None, help="substring filter")
|
||||
ap.add_argument("--reverse", action="store_true")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--no-decode-check", action="store_true")
|
||||
ap.add_argument("--overwrite", action="store_true", help="re-do files already HEVC")
|
||||
ap.add_argument("--keep-local", action="store_true",
|
||||
help="pilot mode: encode + verify + gate locally and report the numbers, then "
|
||||
"THROW THE OUTPUT AWAY - nothing is uploaded and nothing is replaced")
|
||||
args = ap.parse_args()
|
||||
DECODE_CHECK = not args.no_decode_check
|
||||
|
||||
os.makedirs(STAGE, exist_ok=True)
|
||||
files = collect(ROOT)
|
||||
if args.only:
|
||||
files = [f for f in files if args.only in f]
|
||||
if args.reverse:
|
||||
files = list(reversed(files))
|
||||
if args.limit:
|
||||
files = files[:args.limit]
|
||||
if args.tier == "x265":
|
||||
# x265 is CPU: pin it to the E-cores so the P-cores stay free (children inherit the mask).
|
||||
log("=== E-core pinning: %s" % ("on" if pin_to_e_cores() else "FAILED (running unpinned)"))
|
||||
log("=== %d file(s) tier=%s cq=%d crf=%d audio=%s decode_check=%s" %
|
||||
(len(files), args.tier or "auto", args.cq, args.crf, args.audio or "source-matched", DECODE_CHECK))
|
||||
|
||||
if args.dry_run:
|
||||
plan = {}
|
||||
for f in files:
|
||||
info = T.probe(f)
|
||||
if not info:
|
||||
log("PLAN %s -> UNREADABLE" % os.path.relpath(f, ROOT))
|
||||
continue
|
||||
vbps = src_video_bps(f, info)
|
||||
b = bpp_of(info, vbps)
|
||||
t = tier_for_bpp(b)
|
||||
plan[t] = plan.get(t, 0) + 1
|
||||
log("PLAN %-5s bpp=%.4f %sx%s@%.0f %6.0f kbps %7.1f MB %s" % (
|
||||
t, b, info["w"], info["h"], src_fps(info), vbps / 1000, info["size"] / 1e6,
|
||||
os.path.relpath(f, ROOT)[:60]))
|
||||
log("=== plan: %s" % plan)
|
||||
return
|
||||
|
||||
with T.Lock(LOCKF):
|
||||
stats = {"ok": 0, "skip": 0, "fail": 0}
|
||||
ledger = open(LEDGER, "a", encoding="utf-8")
|
||||
for i, f in enumerate(files, 1):
|
||||
free = shutil.disk_usage(STAGE).free / 2 ** 30
|
||||
if free < 8:
|
||||
log("!!! ABORT: only %.1f GB free on C: - stopping" % free)
|
||||
break
|
||||
log("--- %d/%d %s" % (i, len(files), os.path.relpath(f, ROOT)))
|
||||
t0 = time.time()
|
||||
try:
|
||||
status, extra = process(f, args)
|
||||
except Exception as e:
|
||||
status, extra = "FAIL %s: %s" % (type(e).__name__, e), None
|
||||
el = time.time() - t0
|
||||
if status == "ok":
|
||||
stats["ok"] += 1
|
||||
log(" ok %.0f MB -> %.0f MB (%.0f%%) video %.1f%% enc %.0fs tot %.0fs | %s | %s" % (
|
||||
extra["src_mb"], extra["out_mb"], extra["ratio"], extra["video_pct"],
|
||||
extra["encode_s"], el, extra["tier"], extra["ssim"]))
|
||||
elif status.startswith("SKIP"):
|
||||
stats["skip"] += 1
|
||||
log(" %s" % status)
|
||||
else:
|
||||
stats["fail"] += 1
|
||||
log(" !! %s" % status)
|
||||
ledger.write(json.dumps({"t": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"path": os.path.relpath(f, ROOT), "status": status,
|
||||
"secs": round(el, 1), "extra": extra},
|
||||
ensure_ascii=False) + "\n")
|
||||
ledger.flush()
|
||||
if i % 10 == 0:
|
||||
log(" [progress] ok=%d skip=%d fail=%d" % (stats["ok"], stats["skip"], stats["fail"]))
|
||||
ledger.close()
|
||||
log("=== done: ok=%d skipped=%d failed=%d" % (stats["ok"], stats["skip"], stats["fail"]))
|
||||
if stats["fail"]:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
main()
|
||||
Reference in New Issue
Block a user