From c45cdda9833331a7bad1b49ca5ce90963865793f Mon Sep 17 00:00:00 2001 From: hoelee Date: Mon, 12 Oct 2026 17:17:50 +0800 Subject: [PATCH] =?UTF-8?q?engine:=20=E6=96=B0=E5=A2=9E=20--lock-tag?= =?UTF-8?q?=EF=BC=8C=E5=85=81=E8=AE=B8=E5=90=8C=E4=B8=80=20tier=20?= =?UTF-8?q?=E7=9A=84=E7=AC=AC=E4=BA=8C=E4=B8=AA=20runner=20=E6=8B=BF?= =?UTF-8?q?=E8=87=AA=E5=B7=B1=E7=9A=84=E9=94=81=E5=B9=B6=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 锁本来是按 tier 的（run-x265.lock），这是防两个编码器抢同一单元的正确默认值； 但当单进程吃不满机器时（720p x265 = wpp(12 rows)，pools=22 实测 165 fps、整机 73%）， 控制面要拆核并行两个 x265：第二个 runner 必须有自己的锁名，否则永远退避在 engine lock busy。 --lock-tag x265-w2 → run-x265-w2.lock，留空 = 旧行为。references/pitfalls.md 补第 13 条。 --- references/pitfalls.md | 12 ++++++++++++ scripts/library_run.py | 10 +++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/references/pitfalls.md b/references/pitfalls.md index e197fcd..c1f4be5 100644 --- a/references/pitfalls.md +++ b/references/pitfalls.md @@ -39,3 +39,15 @@ Rule: treat `MemoryError` as a sibling of `OSError` in **every** share helper, and have the per-item handler delete `.new.mp4` **unconditionally** - never gate that delete on `os.path.exists()`, whose answer on this share can be stale. + +## 13. 同一 tier 想并发：必须给第二个 runner 自己的锁（2026-10-12） + +`--tier x265` 的锁是**每档一把**（`run-x265.lock`），这是防止两个编码器抢同一个单元/输出的正确 +默认值。但当一个 x265 进程吃不饱机器时（720p 的 x265 banner 是 `frame threads 3 / wpp(12 rows)`， +实测 `pools=22` 只有 165 fps、整机 73% 忙），控制面会把核拆成两组、让两个 x265 **并行**跑不同文件 —— +这时第二个 runner 拿同一把锁就会一直退避在 "engine lock busy"。 + +`--lock-tag ` 就是给它的出口：锁文件名变成 `run-.lock`（留空 = 沿用 tier 名， +行为与以前完全一致）。控制面给 cpu2/cpu3 分别传 `--lock-tag x265-w2` / `x265-w3`。 +两个 runner 处理的是**不同文件**，每个 item 的暂存产物都带 PID+tag，所以不会互相覆盖； +`--affinity-mask` 用来让两份负载落在不重叠的核上（实测 2×10 线程 = 244 fps vs 单进程 22 线程 = 165 fps，+48%）。 diff --git a/scripts/library_run.py b/scripts/library_run.py index c4f918f..0f1a83c 100644 --- a/scripts/library_run.py +++ b/scripts/library_run.py @@ -1245,6 +1245,9 @@ def main(): ap.add_argument("--crf", type=int, default=CRF_X265, help="x265 tier quality (measured default 24)") ap.add_argument("--x265-preset", default=X265_PRESET) ap.add_argument("--pools", type=int, default=X265_POOLS, help="x265 threads (match the affinity mask)") + ap.add_argument("--lock-tag", default="", + help="lock name suffix (run-.lock) so a second worker of the SAME tier can " + "encode concurrently on its own core set; empty = use the tier name") ap.add_argument("--affinity-mask", default="0x%X" % E_CORE_MASK, help="hex CPU mask for the x265 tier (default = the E-core mask); the mask is " "inherited by the ffmpeg children and must agree with --pools") @@ -1317,7 +1320,12 @@ def main(): # One lock PER TIER, not one global lock: the copy tier uses no encoder, the x265 tier is CPU # and the nvenc tier is the GPU ASIC, so they are safe to run side by side whereas two runs of # the same tier would race on dst and starve a shared unit. - lockf = os.path.join(WORK, "run-%s.lock" % (args.tier or ("single" if args.path else "all"))) + # `--lock-tag` lets a SECOND worker of the same tier take its own lock: the lock exists to stop two + # encoders racing on the same unit/output, and the control plane splits the cores explicitly + # (measured 2026-10-12: two 720p x265 processes with 10 threads on disjoint core sets = +48% + # aggregate throughput over one process with 22 threads, because x265 caps at wpp(12 rows)). + lockf = os.path.join(WORK, "run-%s.lock" % (args.lock_tag or args.tier + or ("single" if args.path else "all"))) log("=== %d file(s) tier=%s cq=%d crf=%d audio=%s decode_check=%s lock=%s" % (len(files), args.tier or "auto", args.cq, args.crf, args.audio or "source-matched", DECODE_CHECK, os.path.basename(lockf)))