diff --git a/public/banners/passbolt-hang-three-failure-modes.png b/public/banners/passbolt-hang-three-failure-modes.png new file mode 100644 index 0000000..dcabc98 Binary files /dev/null and b/public/banners/passbolt-hang-three-failure-modes.png differ diff --git a/public/og/passbolt-hang-three-failure-modes.png b/public/og/passbolt-hang-three-failure-modes.png new file mode 100644 index 0000000..ac981c6 Binary files /dev/null and b/public/og/passbolt-hang-three-failure-modes.png differ diff --git a/scripts/banner-gen/generate.mjs b/scripts/banner-gen/generate.mjs index 2ed0c66..e7084c4 100644 --- a/scripts/banner-gen/generate.mjs +++ b/scripts/banner-gen/generate.mjs @@ -323,35 +323,53 @@ const BANNERS = { }, 'how-i-made-my-own-songs-with-suno-ai': { - titlebar: 'hoelee@studio — suno v6 · 7 songs', + titlebar: 'hoelee@studio — suno v6 · 7 songs', + lines: [ + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'write lyrics · craft style prompt (200 chars)' }, + { t: 'prompt', text: 'INFO' }, { t: 'cmd', text: '[Verse][Chorus][Bridge] · [Whispered][Belted]' }, + { t: 'prompt', text: 'INFO' }, { t: 'cmd', text: 'generate 3–5 takes → keep best' }, + { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'AI vocalist mispronounces 忘川 — respell test clip' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'v6: section edit · single-line swap · voices' }, + { t: 'prompt', text: '' }, { t: 'ok', text: '→ 7 songs hosted · embedded above ✓' }, + ], + flow: [ + { n: '1', label: 'lyrics first' }, + { n: '2', label: 'style prompt' }, + { n: '3', label: 'metatags' }, + { n: '4', label: 'iterate takes' }, + { n: '5', label: 'ship ✓' }, + ], + }, + + 'passbolt-hang-three-failure-modes': { + titlebar: 'root@dsm — passbolt incident', + lines: [ + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'GET / → 504 Gateway Timeout (every minute, worse under load)' }, + { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'cron: job is still running since ... (1m elapsed)' }, + { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'SMTP Setting errors: fingerprint null' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'fix = bind-mount passbolt.php · full fingerprint · remove ssl.force' }, + { t: 'prompt', text: '' }, { t: 'ok', text: '→ cron 0.4s · UI 302 · healthcheck green ✓' }, + ], + flow: [ + { n: '1', label: '504 hang' }, + { n: '2', label: 'fingerprint null', err: true }, + { n: '3', label: 'mount config' }, + { n: '4', label: 'redirect loop', err: true }, + { n: '5', label: 'fixed ✓' }, + ], + }, + }; + + const DEFAULT_BANNER = { + titlebar: 'root@host — shell', lines: [ - { t: 'prompt', text: '$' }, { t: 'cmd', text: 'write lyrics · craft style prompt (200 chars)' }, - { t: 'prompt', text: 'INFO' }, { t: 'cmd', text: '[Verse][Chorus][Bridge] · [Whispered][Belted]' }, - { t: 'prompt', text: 'INFO' }, { t: 'cmd', text: 'generate 3–5 takes → keep best' }, - { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'AI vocalist mispronounces 忘川 — respell test clip' }, - { t: 'prompt', text: '$' }, { t: 'cmd', text: 'v6: section edit · single-line swap · voices' }, - { t: 'prompt', text: '' }, { t: 'ok', text: '→ 7 songs hosted · embedded above ✓' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'engineering · devops · self-hosting' }, + { t: 'prompt', text: '' }, { t: 'ok', text: 'read the full post →' }, ], flow: [ - { n: '1', label: 'lyrics first' }, - { n: '2', label: 'style prompt' }, - { n: '3', label: 'metatags' }, - { n: '4', label: 'iterate takes' }, - { n: '5', label: 'ship ✓' }, + { n: '1', label: 'start' }, { n: '2', label: 'work' }, { n: '3', label: 'ship' }, ], - }, -}; - -const DEFAULT_BANNER = { - titlebar: 'root@host — shell', - lines: [ - { t: 'prompt', text: '$' }, { t: 'cmd', text: 'engineering · devops · self-hosting' }, - { t: 'prompt', text: '' }, { t: 'ok', text: 'read the full post →' }, - ], - flow: [ - { n: '1', label: 'start' }, { n: '2', label: 'work' }, { n: '3', label: 'ship' }, - ], -}; + }; // ---------- read frontmatter ---------- const postPath = join(ROOT, 'src', 'content', 'posts', `${slug}.md`); diff --git a/scripts/og-gen/generate.mjs b/scripts/og-gen/generate.mjs index 7140521..c7fc581 100644 --- a/scripts/og-gen/generate.mjs +++ b/scripts/og-gen/generate.mjs @@ -110,9 +110,14 @@ const TERMINALS = {
$enhance_prompt:false · same-origin paths→ 1 room out ✓
`, 'how-i-made-my-own-songs-with-suno-ai': ` -
$suno "indie folk · playful whistles · sprite-girl vocals"
-
 lyrics + metatags → 7 songs shipped ✓
-
$v6: section edits · single-line swaps · voices
`, +
$suno "indie folk · playful whistles · sprite-girl vocals"
+
 lyrics + metatags → 7 songs shipped ✓
+
$v6: section edits · single-line swaps · voices
`, + + 'passbolt-hang-three-failure-modes': ` +
$curl -I https://pass.hoelee.com
+
 504 · ERR_TOO_MANY_REDIRECTS · fingerprint null
+
$mount passbolt.php · full fingerprint→ fixed ✓
`, }; const DEFAULT_TERMINAL = ` diff --git a/src/content/posts/passbolt-hang-three-failure-modes.md b/src/content/posts/passbolt-hang-three-failure-modes.md new file mode 100644 index 0000000..19b2c5b --- /dev/null +++ b/src/content/posts/passbolt-hang-three-failure-modes.md @@ -0,0 +1,191 @@ +--- +title: "Passbolt UI Kept Hanging — Three Failure Modes From One Missing Config File" +description: "My self-hosted Passbolt returned 504s for weeks: a null GPG fingerprint stalled email delivery, a container recreate wiped the fix, and ssl.force behind a proxy caused a redirect loop." +pubDate: 2026-09-13 +category: devops +tags: [passbolt, docker, gpg, smtp, reverse-proxy, portainer] +ogImage: /og/passbolt-hang-three-failure-modes.png +banner: /banners/passbolt-hang-three-failure-modes.png +--- + +I run Passbolt — an open-source password manager for teams — on a Synology +NAS in Docker, behind an nginx reverse proxy. For a stretch of weeks it +developed a nasty habit: the web UI would hang, return `504 Gateway Timeout`, +and refuse to hand over a password right when I needed it. This is the story +of chasing that hang to its root — and the twist where my own "fix" caused a +brand-new failure mode. + +## Why it matters + +A password manager is the one app you cannot afford to be flaky. When I'm +mid-conversation with a client and need a server credential, a loading spinner +is not an option. The symptoms looked random — sometimes up, sometimes +504 — but the cause turned out to be a single missing file, plus two mistakes +stacked on top of it. If you self-host Passbolt in Docker, one of these three +is probably in your future. + +## The symptom: cron hangs, and the UI follows + +Passbolt runs a cron job every minute that processes its email queue — account +recovery links, share notifications, test messages. Normally it completes in +under a second. Mine was taking 30–60 seconds every single run. + +Why does that matter? Behind the scenes Passbolt serves the web UI through a +pool of PHP-FPM workers. When the cron job stalls on email sending, it ties up +a worker. Enough stalled crons, and the pool is exhausted — so ordinary page +requests queue up, nginx times out, and you get the 504. + +The logs told the story plainly: + +```text +not starting: job is still running since ... (1m elapsed) +``` + +## Root cause 1: a null GPG fingerprint + +Passbolt stores its SMTP settings encrypted with the server's GPG key. To use +them, it needs to know which key to decrypt with — a value read from +`passbolt.gpg.serverFingerprint`. That value comes from a config file, +`/etc/passbolt/passbolt.php`. + +That file didn't exist. + +Without it, the fingerprint resolved to `null`, GPG decryption of the SMTP +settings failed, and every cron iteration that tried to send mail hung on the +failure. The healthcheck surfaced it as: + +```text +SMTP Setting errors: ... setDecryptKeyFromFingerprint(): +Argument #1 ($fingerprint) must be of type string, null given +``` + +There was also one genuinely dead email in the queue — an old `SMTP timeout` +record that had exhausted its retries months ago and would never send. It got +re-scanned every minute, adding to the stall. + +```sql +DELETE FROM email_queue WHERE sent = 0 AND send_tries >= 4; +``` + +## Root cause 2: the fix that didn't survive (the relapse) + +Here's the part that stung. I had fixed this exact bug once before — by +creating `/etc/passbolt/passbolt.php` *inside the container*. Two weeks later +the hang came back, and the file was gone. + +The reason: I'd recreated the container during maintenance, and a Docker +container's writable layer is **ephemeral by design**. Anything written to the +filesystem inside the container (rather than into a mounted volume) vanishes on +recreate. My fix had the shelf life of the container, not of the deployment. + +The durable version is three pieces that all live *outside* the container: + +1. A config file on the host, bind-mounted into the image at + `/etc/passbolt/passbolt.php`: + +```php + [ + 'fullBaseUrl' => env('APP_FULL_BASE_URL', 'https://pass.example.com'), + ], + 'passbolt' => [ + 'gpg' => [ + 'serverFingerprint' => 'A3DD9B762D48722C10CF88DDB5372E46A54E1419', + ], + ], +]; +``` + +2. The bind mount in compose — the part that makes it survive recreate: + +```yaml +volumes: + - /volume1/docker/passbolt/session-config/passbolt.php:/etc/passbolt/passbolt.php:ro +``` + +3. The correct environment variable name. I'd been using + `PASSBOLT_GPG_SERVER_FINGERPRINT`, which Passbolt silently ignores. The + real variable is `PASSBOLT_GPG_SERVER_KEY_FINGERPRINT` — and it wants the + **full 40-character** fingerprint, not a truncated one: + +```yaml +PASSBOLT_GPG_SERVER_KEY_FINGERPRINT: "A3DD9B762D48722C10CF88DDB5372E46A54E1419" +``` + +## Root cause 3: the redirect loop I caused myself + +With the hang fixed, I upgraded Passbolt from 5.14.3 to 5.15.0. Then, browsing +to `https://pass.example.com/`, the browser threw: + +```text +ERR_TOO_MANY_REDIRECTS +``` + +I assumed the upgrade had broken something. It hadn't. The culprit was +`ssl.force`, which I'd left in my config file: + +```php +'passbolt' => [ + 'ssl' => [ + 'force' => true, + ], +], +``` + +Passbolt's SSL-force middleware inspects the request's scheme, and behind a +TLS-terminating reverse proxy that scheme is **always `http`** — because the +proxy (nginx) terminates the TLS connection, then forwards plain HTTP to the +container. So the middleware sees `http`, dutifully redirects to +`https://same-url`, nginx forwards it back as `http`, and the cycle repeats +forever. + +The fix is to understand who owns SSL. If your reverse proxy already enforces +HTTPS at the edge, the app must **not** try to force it again. Removing +`ssl.force` from the config (leaving TLS to nginx) broke the loop immediately. + +The general rule, which applies well beyond Passbolt to any app behind a +reverse proxy: **either** the proxy terminates TLS, **or** the app does — never +both, and never configure `ssl.force` in the app while the proxy already +handles it. + +## What I'd do differently + +1. **Config lives in volumes, not in containers.** The moment I wrote a file + into `docker exec`, I'd committed to redoing it on the next recreate. The + reflex should be: does this need to survive a `docker compose up`? Then it + goes in a bind mount, not the writable layer. +2. **Read the actual variable names.** The fingerprint env var cost me a clean + diagnosis because I trusted a wrong name that Passbolt ignored without + complaint. When a config value "doesn't work," check the upstream reference + for the exact key before assuming anything else is wrong. +3. **Test the public URL, not just the healthcheck.** The healthcheck passed + every time and missed the redirect loop entirely because it never exercised + the real public path through the proxy. A two-second `curl -I` against the + live URL would have caught it instantly. + +## The result + +The hang is gone: cron completes in under a second (down from 30–60s), the +email queue is clear, and the web UI responds in milliseconds instead of +504-ing. Passbolt is now running the current 5.15.0, with every GPG and SMTP +healthcheck green — and the fix is written in a way that survives the next +container recreate. + +The lesson worth carrying: a self-hosted service that "sometimes hangs" is +rarely a mystery. It's usually one missing config value, surfaced three +different ways. Fix the config, not the symptom. + +--- + +## Want stable self-hosted infrastructure for your business? + +If you run services like Passbolt, a password manager, email, or a dashboard +and they're flaky — hanging UIs, 504s, mysteries that vanish on restart — I +diagnose and fix exactly these problems. I work with Docker, reverse proxies, +and self-hosted stacks, and I hand everything back documented so the next +person (or future you) isn't left guessing. + +Reach me at [me@hoelee.com](mailto:me@hoelee.com) or WhatsApp +[+60 12-797 2969](https://wa.me/60127972969), or see what I do at +[hoelee.com](https://hoelee.com). \ No newline at end of file diff --git a/src/content/posts/zh/passbolt-hang-three-failure-modes.md b/src/content/posts/zh/passbolt-hang-three-failure-modes.md new file mode 100644 index 0000000..458bcff --- /dev/null +++ b/src/content/posts/zh/passbolt-hang-three-failure-modes.md @@ -0,0 +1,167 @@ +--- +title: "Passbolt 界面反复卡死——一个缺失的配置文件引发的三种故障模式" +description: "自托管的 Passbolt 连续几周返回 504：GPG 指纹为空卡住邮件投递，重建容器抹掉了修复，代理后面的 ssl.force 又造成无限重定向。" +pubDate: 2026-09-13 +category: devops +tags: [passbolt, docker, gpg, smtp, reverse-proxy, portainer] +ogImage: /og/passbolt-hang-three-failure-modes.png +banner: /banners/passbolt-hang-three-failure-modes.png +--- + +我在 Synology NAS 上用 Docker 跑 Passbolt——一个开源的团队密码管理器，前面 +挂着一层 nginx 反向代理。连着几周的某个坏习惯让我头疼：Web 界面动不动就 +卡死，返回 `504 Gateway Timeout`，偏偏在正需要密码的时候拒绝交出任何凭据。 +这是追查那个卡死毛病到根因的记录——以及我自己那个"修复"又引出一个全新故障 +模式的转折。 + +## 为什么这件事值得写 + +密码管理器是唯一一个不能掉链子的应用。当我正和客户谈到一半、需要服务器凭据 +的时候，转圈的加载动画是绝对不能接受的。症状看起来毫无规律——有时正常、有时 +504——但真正的根源只是一个缺失的文件，外加叠在上面的两个错误。如果你也在 +Docker 里自托管 Passbolt，这三个坑里大概有一个正在前面等你。 + +## 症状：cron 卡住，界面跟着遭殃 + +Passbolt 每分钟跑一个 cron 任务处理它的邮件队列——找回链接、分享通知、测试 +邮件。正常情况下一秒以内跑完。我这边每次要花 30–60 秒。 + +这为什么要紧？Passbolt 靠一组 PHP-FPM worker 提供 Web 界面。当 cron 任务在 +发邮件时卡住，就会占住一个 worker。卡住的 cron 攒多了，worker 池就被耗尽—— +普通页面请求开始排队，nginx 超时，于是你看到 504。 + +日志把话说得很明白： + +```text +not starting: job is still running since ... (1m elapsed) +``` + +## 根源一：GPG 指纹为 null + +Passbolt 用服务器 GPG 密钥加密存储 SMTP 设置。要用这些设置，它必须知道用哪把 +密钥解密——这个值来自 `passbolt.gpg.serverFingerprint`，由配置文件 +`/etc/passbolt/passbolt.php` 提供。 + +这个文件不存在。 + +于是指纹解析成 `null`，SMTP 设置的 GPG 解密失败，每个想发邮件的 cron 迭代都 +卡在这个失败上。健康检查把它摊开来给你看： + +```text +SMTP Setting errors: ... setDecryptKeyFromFingerprint(): +Argument #1 ($fingerprint) must be of type string, null given +``` + +队列里还有一封名副其实的死邮件——一条几个月前就耗尽重试次数、永远发不出去的 +`SMTP timeout` 记录。它每一分钟都被重新扫描一遍，雪上加霜。 + +```sql +DELETE FROM email_queue WHERE sent = 0 AND send_tries >= 4; +``` + +## 根源二：没活下来的修复（复发） + +这里才是真正扎心的地方。这个 bug 我此前其实修过一次——通过在*容器内部*创建 +`/etc/passbolt/passbolt.php`。两个星期后卡死回来了，文件也消失了。 + +原因：维护时我重建过容器，而 Docker 容器的可写层**天生就是临时的**。写进容器 +文件系统（而不是挂载卷）的任何东西，都会在重建时消失。我的修复有效期和容器 +一样长，而不是和部署一样长。 + +真正持久的三件套，全部活在*容器之外*： + +1. 一个位于宿主机上的配置文件，通过 bind mount 挂进镜像的 + `/etc/passbolt/passbolt.php`： + +```php + [ + 'fullBaseUrl' => env('APP_FULL_BASE_URL', 'https://pass.example.com'), + ], + 'passbolt' => [ + 'gpg' => [ + 'serverFingerprint' => 'A3DD9B762D48722C10CF88DDB5372E46A54E1419', + ], + ], +]; +``` + +2. compose 里的 bind mount——让它在重建后存活的关键： + +```yaml +volumes: + - /volume1/docker/passbolt/session-config/passbolt.php:/etc/passbolt/passbolt.php:ro +``` + +3. 正确的环境变量名。我之前一直用 `PASSBOLT_GPG_SERVER_FINGERPRINT`，而 + Passbolt 会静默忽略它。真正的变量是 `PASSBOLT_GPG_SERVER_KEY_FINGERPRINT` + ——而且它要的是**完整的 40 位指纹**，不是截短的： + +```yaml +PASSBOLT_GPG_SERVER_KEY_FINGERPRINT: "A3DD9B762D48722C10CF88DDB5372E46A54E1419" +``` + +## 根源三：我自己造成的重定向循环 + +卡死修好之后，我把 Passbolt 从 5.14.3 升到 5.15.0。然后，打开 +`https://pass.example.com/`，浏览器直接报： + +```text +ERR_TOO_MANY_REDIRECTS +``` + +我以为是升级弄坏了什么。其实没有。罪魁祸首是我留在配置文件里的 `ssl.force`： + +```php +'passbolt' => [ + 'ssl' => [ + 'force' => true, + ], +], +``` + +Passbolt 的 SSL 强制中间件会检查请求的 scheme，而在 TLS 终止型反向代理后面， +这个 scheme **永远是 `http`**——因为代理（nginx）终结 TLS 连接后，把纯 HTTP +转发给容器。中间件看到 `http`，尽职尽责地重定向到 `https://同一条URL`，nginx +再把它当 `http` 转回来，循环往复，永无止境。 + +解法在于想清楚到底谁负责 SSL。如果反向代理已经在边缘强制 HTTPS，应用就**不该** +再强制一遍。把 `ssl.force` 从配置里删掉（把 TLS 交给 nginx），循环立刻消失。 + +这条通用规则不止适用于 Passbolt，所有挂在反向代理后面的应用都一样：**要么代理 +终结 TLS，要么应用自己终结——两者永不兼得**。代理已经在处理的时候，千万别在 +应用里再开 `ssl.force`。 + +## 换了我现在会怎么做 + +1. **配置放进卷，不放容器。** 我一旦用 `docker exec` 写文件，就等于答应了下一次 + 重建时重做一遍。正确反应应当是：这东西需要扛过 `docker compose up` 吗？需要 + 就进 bind mount，而不是可写层。 +2. **先查真实的变量名。** 那个指纹环境变量让我多走了弯路，因为我信了一个 Passbolt + 根本不认的名字，而它不认的时候也不吭声。当某个配置值"不起作用"时，先去上游 + 文档核对准确的 key，再怀疑别的。 +3. **测公网 URL，别只信健康检查。** 健康检查每次通过，却完全没发现重定向循环—— + 因为它从不走真实的公网代理路径。一条两秒的 `curl -I` 就能立刻暴露问题。 + +## 结果 + +卡死消失了：cron 从 30–60 秒降到 1 秒以内跑完，邮件队列清空，Web 界面毫秒级 +响应而不是 504。Passbolt 现在跑着最新的 5.15.0，所有 GPG 和 SMTP 健康检查全绿 +——而且这个修复是写成能扛过下一次容器重建的。 + +值得带走的经验：一个"偶尔卡死"的自托管服务，很少是真正的谜团。通常就是一个 +缺失的配置值，只是以三种不同的方式浮出水面。修配置，不要修症状。 + +--- + +## 想要稳定的自托管基础设施？ + +如果你跑着 Passbolt、密码管理器、邮箱或仪表盘之类的服务，而且它们时不时抽风——界面 +卡死、504、重启就消失的灵异问题——这正是我诊断和修复的范围。我熟悉 Docker、 +反向代理和各种自托管技术栈，交付时会把一切整理成文档，让下一个人（或未来的你） +不用靠猜。 + +联系我：[me@hoelee.com](mailto:me@hoelee.com) 或 WhatsApp +[+60 12-797 2969](https://wa.me/60127972969)，也可以看看我在 +[hoelee.com](https://hoelee.com) 做的事情。 \ No newline at end of file