diff --git a/public/banners/adding-a-domain-to-someone-elses-mail-server.png b/public/banners/adding-a-domain-to-someone-elses-mail-server.png new file mode 100644 index 0000000..dcda91d Binary files /dev/null and b/public/banners/adding-a-domain-to-someone-elses-mail-server.png differ diff --git a/public/banners/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png b/public/banners/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png new file mode 100644 index 0000000..4043073 Binary files /dev/null and b/public/banners/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png differ diff --git a/public/banners/my-backup-was-using-someone-elses-api-quota.png b/public/banners/my-backup-was-using-someone-elses-api-quota.png new file mode 100644 index 0000000..cf531b0 Binary files /dev/null and b/public/banners/my-backup-was-using-someone-elses-api-quota.png differ diff --git a/public/banners/space-reclamation-wasnt-stuck-it-was-waiting.png b/public/banners/space-reclamation-wasnt-stuck-it-was-waiting.png new file mode 100644 index 0000000..9bebf5b Binary files /dev/null and b/public/banners/space-reclamation-wasnt-stuck-it-was-waiting.png differ diff --git a/public/og/adding-a-domain-to-someone-elses-mail-server.png b/public/og/adding-a-domain-to-someone-elses-mail-server.png new file mode 100644 index 0000000..10a6d21 Binary files /dev/null and b/public/og/adding-a-domain-to-someone-elses-mail-server.png differ diff --git a/public/og/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png b/public/og/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png new file mode 100644 index 0000000..55642f0 Binary files /dev/null and b/public/og/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png differ diff --git a/public/og/my-backup-was-using-someone-elses-api-quota.png b/public/og/my-backup-was-using-someone-elses-api-quota.png new file mode 100644 index 0000000..610d911 Binary files /dev/null and b/public/og/my-backup-was-using-someone-elses-api-quota.png differ diff --git a/public/og/space-reclamation-wasnt-stuck-it-was-waiting.png b/public/og/space-reclamation-wasnt-stuck-it-was-waiting.png new file mode 100644 index 0000000..1323006 Binary files /dev/null and b/public/og/space-reclamation-wasnt-stuck-it-was-waiting.png differ diff --git a/scripts/banner-gen/generate.mjs b/scripts/banner-gen/generate.mjs index 0b453fd..3008f93 100644 --- a/scripts/banner-gen/generate.mjs +++ b/scripts/banner-gen/generate.mjs @@ -1288,6 +1288,74 @@ BANNERS['two-ways-a-javascript-file-stops-working-and-looks-fine'] = { ], }; +BANNERS['my-backup-was-using-someone-elses-api-quota'] = { + titlebar: "root@dsm - 1 TB seed, 0.27%", + lines: [ + { t: 'prompt', text: "$" }, { t: 'cmd', text: "rclone about gdrive-restic" }, + { t: 'dim', text: "2.88 GB of ~1075 GB uploaded - 0 snapshots" }, { t: 'err', text: "403 Quota exceeded - consumer projects/202264815644" }, + { t: 'prompt', text: "$" }, { t: 'cmd', text: "own OAuth client - Desktop app - Publish app" }, + { t: 'ok', text: "the seed spends my quota now" }, { t: 'hl', text: "the callback accepts ONE request: no reload, no second tab" }, + { t: 'dim', text: "old side: 1.012 TiB -> 3.800 GiB reclaimed" }, { t: 'dim', text: "guid = sha256(id): repointing a plan cannot fork history" }, + ], + flow: [ + { n: '1', label: "find what writes it" }, + { n: '2', label: "repoint the repo" }, + { n: '3', label: "own the client" }, + { n: '4', label: "seed" }, + ], +}; + +BANNERS['space-reclamation-wasnt-stuck-it-was-waiting'] = { + titlebar: "root@nas - volume1, 09:29", + lines: [ + { t: 'prompt', text: "$" }, { t: 'cmd', text: "df -h /volume1 (every 5 min)" }, + { t: 'dim', text: "08:59-09:24 flat at 1134.1 GiB" }, { t: 'ok', text: "09:29 +69.0 GiB - Class snapshot rotated" }, + { t: 'ok', text: "09:34 +46.8 GiB - Video snapshot rotated" }, { t: 'err', text: "15:34 +5.0 GiB - the recycle bin held almost nothing" }, + { t: 'hl', text: "retention 1: a delete frees at tomorrow's rotation" }, { t: 'dim', text: "the afternoon wrote -47.2 GiB: reclaim was a rounding error" }, + { t: 'dim', text: "volume1 98% used - volume2 100%" }, + ], + flow: [ + { n: '1', label: "sample the number" }, + { n: '2', label: "read the schedule" }, + { n: '3', label: "know retention" }, + { n: '4', label: "alert on the trend" }, + ], +}; + +BANNERS['adding-a-domain-to-someone-elses-mail-server'] = { + titlebar: "mail.hoelee.com - one more domain", + lines: [ + { t: 'prompt', text: "$" }, { t: 'cmd', text: "add domain + MX/SPF/DKIM/DMARC" }, + { t: 'ok', text: "inbound works first try - RCPT_MAILCOW_DOMAIN" }, { t: 'err', text: "outbound: 554 antispam policy (4 RBLs clean, no rDNS)" }, + { t: 'err', text: "via relay: 550 From header sender domain not verified" }, { t: 'hl', text: "PTR is per IP - every domain shares one reverse record" }, + { t: 'dim', text: "forwarding needs Sieve, and still cannot leave the box" }, { t: 'dim', text: "a relay needs a verified sender per domain, forever" }, + { t: 'prompt', text: "$" }, { t: 'cmd', text: "POP pull on the receiving side" }, + ], + flow: [ + { n: '1', label: "four records" }, + { n: '2', label: "test from outside" }, + { n: '3', label: "check the reverse" }, + { n: '4', label: "pull, don't forward" }, + ], +}; + +BANNERS['embedding-fonts-so-a-pdf-looks-the-same-everywhere'] = { + titlebar: "report.pdf - what is really inside", + lines: [ + { t: 'prompt', text: "$" }, { t: 'cmd', text: "pdffonts report.pdf" }, + { t: 'err', text: "before: one CJK face + Roboto, bold was synthetic" }, { t: 'dim', text: "variable font: parser reads glyf only, ignores fvar -> Thin 100" }, + { t: 'ok', text: "after: 8 static faces, 4 families, embedded" }, { t: 'hl', text: "font-weight 600 -> rewritten to bold for the PDF engine" }, + { t: 'dim', text: "GB2312 subsetting cut real name characters: 犇 骉 頔 祎 堃" }, { t: 'dim', text: "glyph lint fails the build instead of printing a box" }, + { t: 'prompt', text: "$" }, { t: 'cmd', text: "page counts: zh 23/23, en 28/29, bm 30/30" }, + ], + flow: [ + { n: '1', label: "files in the repo" }, + { n: '2', label: "static cuts only" }, + { n: '3', label: "map two weights" }, + { n: '4', label: "lint every glyph" }, + ], +}; + // ---------- read frontmatter ---------- const postPath = join(ROOT, 'src', 'content', 'posts', `${slug}.md`); let category = 'devops'; diff --git a/scripts/og-gen/generate.mjs b/scripts/og-gen/generate.mjs index 00a4c22..7ce9698 100644 --- a/scripts/og-gen/generate.mjs +++ b/scripts/og-gen/generate.mjs @@ -392,6 +392,26 @@ TERMINALS['two-ways-a-javascript-file-stops-working-and-looks-fine'] = `
 TypeError: t is not a function (a parameter named t)
$typeof t === 'function' ?lang=pseudo clean-> script reaches the end
`; +TERMINALS['my-backup-was-using-someone-elses-api-quota'] = ` +
$restic backup --repo gdrive:restic
+
 403 RATE_LIMIT_EXCEEDED - consumer: projects/202264815644
+
$own OAuth client + Publish app-> the quota is mine
`; + +TERMINALS['space-reclamation-wasnt-stuck-it-was-waiting'] = ` +
$synostgreclaim --get-all-status
+
 state running, schedule 168x1, run_all_time: always true
+
$sample free space every 5 min-> +121 GiB, then -47 GiB written
`; + +TERMINALS['adding-a-domain-to-someone-elses-mail-server'] = ` +
$dig +short MX newdomain.com -> mail.hoelee.com
+
 554 5.7.1 Mail rejected due to antispam policy
+
$reverse lookup 41.216.187.102-> nothing: no PTR
`; + +TERMINALS['embedding-fonts-so-a-pdf-looks-the-same-everywhere'] = ` +
$pdffonts report.pdf
+
 before: one CJK face, bold was synthetic
+
$8 static faces embedded-> same look everywhere
`; + // ---------- read frontmatter ---------- const postPath = join(ROOT, 'src', 'content', 'posts', `${slug}.md`); if (!existsSync(postPath)) { diff --git a/src/content/posts/adding-a-domain-to-someone-elses-mail-server.md b/src/content/posts/adding-a-domain-to-someone-elses-mail-server.md new file mode 100644 index 0000000..e294601 --- /dev/null +++ b/src/content/posts/adding-a-domain-to-someone-elses-mail-server.md @@ -0,0 +1,154 @@ +--- +title: "Adding a Domain to Someone Else's Mail Server" +description: "Receiving mail took four DNS records and worked first try. Sending from it, and forwarding it, hit two walls that no DNS record can fix." +pubDate: 2025-05-06 +updatedDate: 2026-10-16 +category: notes +tags: ["email", "dns", "mailcow", "smtp", "deliverability"] +ogImage: /og/adding-a-domain-to-someone-elses-mail-server.png +banner: /banners/adding-a-domain-to-someone-elses-mail-server.png +draft: false +--- + +## The task looked like a DNS job + +A domain needed mail. The mail server is my own VPS running mailcow, so this +is "add the domain, add the records, create the mailbox" — the kind of task +that takes twenty minutes and that I have done several times. + +It took longer, because two of the things I assumed about mail servers are +not true, and because of one property of the internet that no amount of +server configuration can change. + +## Inbound: four records and a real test + +Inbound mail needs exactly four things, and all four live in the domain's +own DNS (Cloudflare here): + +```text +MX ninelore.com → mail.hoelee.com (priority 10) +TXT @ → v=spf1 mx -all +TXT dkim._domainkey → (the selector the MTA publishes) +TXT _dmarc → (policy + reporting address) +``` + +Configured and then tested the only way that counts: mail sent **from +outside** the estate (through a third-party relay) into the new address, and +then read the server's own log line — `rspamd` recording +`RCPT_MAILCOW_DOMAIN(ninelore.com)` as the message was accepted into the +mailbox. Receiving worked on the first try. Inbound is genuinely the easy +half, and it is easy precisely because it depends only on records you +control. + +## Outbound is where the ground moves + +Two separate walls, each with its own error, and neither is a config mistake +you can fix by editing a zone file. + +**Wall one: the anti-spam policy that is really a missing PTR.** A direct +delivery attempt to the destination provider came back: + +```text +554 5.7.1 Mail rejected due to antispam policy +``` + +Before blaming the remote, I checked the usual causes: four major +blocklists, all clean, and the domain's own SPF/DKIM/DMARC all correct. The +remaining explanation was the one with no DNS record in my control — the +sending IP had no reverse DNS. A receiving server that cannot map the +connecting IP back to a name treats it as a source it cannot vouch for, and +no amount of correct SPF outweighs that. + +**Wall two: relaying your way around it does not work either.** The obvious +next move is to send everything through the outbound relay that already +works. That attempt produced: + +```text +550-From header sender domain not verified (gmail.com) +``` + +Read that error carefully, because it kills a whole family of designs. The +relay enforces its verified-sender policy against the **From header domain** +— and a *forwarded* message keeps the original sender's From. So the moment +you forward mail from an outside sender, you are asking your relay to send +as `gmail.com`, which it will never allow. mailcow makes this worse in a way +that is worth knowing before you try: it selects the relay by **sender** +domain, so a message with an outside From does not even reach the relay you +configured. + +## The forwarding feature that does not exist + +The plan was to create the mailbox and forward a copy to my main address. +That is two surprises in a row: + +- mailcow has no "mailbox forwarding" setting. Writing the mailbox address + into an alias' `goto` list is rejected (`is_alias_or_mailbox`) because + every mailbox already carries a self-alias record and the code skips the + address when computing destinations. +- Keeping the mailbox *and* forwarding requires a **Sieve filter**, which is + a different mechanism with a different failure mode — and it lands on wall + one and wall two anyway, because the message still has to leave the box. + +So I enabled **nothing**. A forward whose outbound path cannot deliver does +not fail quietly: it sends the rejection back to the innocent original +sender — backscatter — and every one of those bounces spends the reputation +of an IP that has none to spare. The stable answer for "this new mailbox +should reach my inbox" turned out to be a **POP pull on the receiving +side**: my mail client fetches from `mail.hoelee.com:995` and nothing has to +be sent from the VPS at all. + +## PTR is one per IP, not one per domain + +This is the part I keep having to explain, including to myself: + +- The reverse record is a property of the **IP address**. There is no + per-domain PTR, and there is nothing to add the next time a domain is + added. +- Its correct value is the name the mail server itself announces — in this + case `mail.hoelee.com`, which is what postfix sends as its + HELO/myhostname. The matching forward record + (`mail.hoelee.com → 41.216.187.102`) already exists, and the pair is what + receivers verify: **forward-confirmed rDNS**. One half without the other + is worth nothing. +- Every domain on that IP is judged by that same reverse record. A hundred + domains, one PTR. +- Who sets it: whoever runs the reverse zone. Here the SOA for both + `216.41.in-addr.arpa` and `187.216.41.in-addr.arpa` is the VPS host's own + nameserver, so the record is a provider-side change — look for an rDNS/PTR + field in the VM or network panel first, and otherwise open a ticket with + the exact value above. + +I checked the current state rather than assuming: the forward lookup +resolves (`mail.hoelee.com → 41.216.187.102`) and the reverse lookup for +that IP still returns nothing. So direct delivery continues to be rejected, +and the only outbound path that works is the relay — with its +verified-sender requirement, which new domains have to satisfy there too. + +## Three things to expect, so you are not surprised later + +1. **PTR is not a reputation.** It removes a hard rejection; it does not + make you trusted. A new IP typically needs weeks of ordinary mail before + the big providers stop folding it into spam, and starting from a fresh IP + means starting that clock from zero. +2. **PTR fixes your own outbound and does nothing for forwarding.** A + forwarded message carries someone else's From, and if that domain + publishes `p=reject` — banks and large providers do — it is blocked at + the far end no matter how clean your setup is. +3. **Relaying has a per-domain precondition.** If a domain sends through a + relay, that domain must be registered and verified as a sender with the + relay. It is an extra step per domain forever, and it is independent of + everything in your own DNS. + +## What I would do differently + +1. **Decide mailbox-or-forward before creating the domain.** Forwarding is + the option with the longest list of things that must be true; a POP pull + on the receiving side has almost none. +2. **Check the sending IP's reverse record before promising anyone mail.** + One reverse lookup would have told me which of the two walls I was about + to hit. +3. **Treat a receiving-side pull as a first-class design, not a + workaround.** "The message never has to leave my server" removes three + whole categories of failure at once. +4. **Keep the four inbound records as a checklist.** That half is boring, + verifiable, and was the part that worked first try. diff --git a/src/content/posts/embedding-fonts-so-a-pdf-looks-the-same-everywhere.md b/src/content/posts/embedding-fonts-so-a-pdf-looks-the-same-everywhere.md new file mode 100644 index 0000000..28a48bb --- /dev/null +++ b/src/content/posts/embedding-fonts-so-a-pdf-looks-the-same-everywhere.md @@ -0,0 +1,158 @@ +--- +title: "Embedding Fonts So a PDF Looks the Same on Every Machine" +description: "Four families, eight faces, two renderers and one CSS quirk: what it took to stop a generated report depending on whatever fonts the server happened to have." +pubDate: 2025-01-22 +updatedDate: 2026-10-16 +category: engineering +tags: ["pdf", "fonts", "mpdf", "typography", "php"] +ogImage: /og/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png +banner: /banners/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png +draft: false +--- + +## The requirement was one sentence + +A report generator has to produce a document that looks the same everywhere: +Chinese headings in a serif, Chinese body text in a sans, Latin headings and +cover numerals in a display serif, Latin body text and tables in Inter. +Weights matter — medium for emphasis, semibold for headings — across three +languages and two PDF renderers. + +And the fonts must travel with the document. Not "the server has them +installed", not "the reader probably has them" — **embedded**, so a print +shop in another country sees the same pages. + +The old pipeline failed on the last clause in a way that was invisible: it +depended on fonts installed on the server by a package manager, and it only +ever embedded a single CJK face. The second weight did not exist, so "bold" +was the renderer's synthetic approximation, and the only way to notice was +to open the PDF and look at what was actually inside it. + +## Rule one: read the fonts back out of the finished PDF + +"Embedded" is a claim about the output, so I measured the output. The same +document, rendered before and after: + +| | before | after | +|---|---|---| +| zh page count | 23 | **23** | +| en page count | 28 | **29** | +| bm page count | 30 | **30** | +| fonts actually embedded | one CJK face + Roboto | Source Han Serif Medium + SemiBold · Source Han Sans Regular + Medium · Cormorant Garamond Medium + SemiBold · Inter Regular + SemiBold + Medium · Source Han Sans for the CJK runs inside Latin reports | +| render time / peak memory | baseline | same order (±10%, noise) | +| file size | baseline | roughly +3% | + +The glyph counts per face are the receipt: in the Chinese report, 15,988 +glyphs land in the body sans, 2,247 in its medium weight, 634 in the heading +serif medium, 222 in its semibold. That is the difference between "the font +is on the server" and "the font is in the file" — a reader in another +country sees the same four faces, and a page count that moved by exactly one +page in one language (the Latin metrics are wider than what they replaced) +is the honest cost. + +## Rule two: static instances only, and the reason is in the parser + +The obvious modern choice — ship one variable font per family — is wrong +here, for two measured reasons: + +1. **The PDF engine's font parser reads `glyf`/`loca` only.** Feed it an + OpenType file with PostScript/CFF outlines and it refuses: + `Fonts with postscript outlines are not supported`. A variable font is + not the problem; CFF outlines are. +2. **It does not read the `fvar` table at all**, so a variable font silently + resolves to its *default instance* — and the default instances of the CJK + families are the thinnest ones (Thin 100 and ExtraLight 200, visible in + the `name` table). Every page would render in a hairline weight, with no + error anywhere in the pipeline. + +So the build step takes the upstream variable fonts and **instantiates +static cuts**: it runs the instancer, then rewrites `name`, +`OS/2.usWeightClass` and `head.macStyle` so the file identifies itself as +the weight it now is (the source's own name still says "Thin"). A manifest +records family, weight class, sha256, glyph count and code-point count for +every file, and the licence files ship next to them. + +## Rule three: the engine speaks two weights, so map onto two slots + +The renderer's CSS handling honours exactly `normal` and `bold` — its source +says so in as many words ("Does not support: bolder, lighter, 100..900"). +Numeric weights are therefore meaningless to it, while the headless-Chromium +renderer in the same pipeline honours them properly. + +The mapping that works: each family declares an **R slot** and a **B slot** +(regular-ish and heavier), `font-weight: 500` lands on the R slot of the +intended family, and `600` is **rewritten to `bold`** on the PDF side only, +which lands on the B slot. Chromium keeps `600` as written. Two tests pin +it: the B slot must be heavier than the R slot (read from `usWeightClass`, +not assumed), and `font-weight: 600` must never appear in the stylesheet +handed to the PDF engine. + +One subtlety worth the paragraph: a "medium" weight is not an emphasis +mechanism in this pipeline, it is a *family member*. Chinese headings use +the serif's medium as their regular and its semibold for emphasis; Latin +headings do the same with the display serif. Getting that wrong is invisible +on a screen and obvious in print. + +## Rule four: do not subset against a character set you do not control + +The first attempt subset the CJK fonts to the union of two national +character sets and produced a ~29 MB bundle instead of ~50 MB. Then I ran +the name probe and found these were cut: + +```text +犇 骉 頔 祎 廸 玧 祾 赟 堃 +``` + +Every one of them is a real given name character. The report prints a +customer's name on the cover, and the name is free input — so a subset-based +build will eventually put a tofu box on the most prominent page of the +product. Full coverage is ~31k code points and roughly 50 MB of embedded +fonts; that is a fine price for never showing a customer an empty rectangle. + +## Rule five: make missing glyphs a build failure, not a surprise + +The gate runs three checks and exits non-zero on any of them: + +- every font file declared in the manifest is present, and its sha256 + matches the file on disk; +- every character in the content library (all three languages) plus the + brand strings, plus probe classes — simplified, traditional, CJK + punctuation, full-width forms, mixed-script, digits, Latin extended, + symbols — plus the name characters, has a glyph **in the face that will + actually print it**; +- the lookup uses the PDF engine's own font parser rather than a third-party + library, so the check cannot disagree with the renderer. + +The known edges are written down instead of hidden. Dingbats and emoji (✅ ✔) +exist in none of these text fonts and the engine cannot draw colour emoji, +so they are kept out of the report body. The display serif has only 974 +glyphs and does not draw symbols like ① ★ ℃, which is why the +content-library check covers *all* slots rather than just the body. And CJK +characters inside a Latin report must go through the dedicated CJK slot — +two tests keep the renderers from disagreeing about that. + +## What I would do differently + +1. **Choose faces and weights before styling anything.** Retrofitting a + second weight after the layout exists means re-checking every page count + by hand. +2. **Read the renderer's source before choosing a font format.** The + variable-font decision looked like a taste question and was actually a + parser question — two files settled it in ten minutes. +3. **Treat everything you ship inside a document as a licence artifact.** + Font files enter the repo with their licences, or they enter someone + else's legal review. +4. **Measure what got embedded.** Page count, glyph-per-face counts, file + size, render time: all four moved, and three of them moved in ways I + would have guessed wrong. + +## The result + +Before: one CJK face and a UI font, with synthetic bold, depending on what +the server had installed. After: four families and eight static faces +embedded in the file, weights that are real members of their families, page +counts within one page of the previous layout, and a lint gate that fails +the build when a character has no glyph — so the failure mode moved from "a +customer sees a box" to "CI refuses the release". Which is the same trade as +everywhere else in this pipeline: make the mistake impossible to ship rather +than easy to notice. diff --git a/src/content/posts/my-backup-was-using-someone-elses-api-quota.md b/src/content/posts/my-backup-was-using-someone-elses-api-quota.md new file mode 100644 index 0000000..883f62b --- /dev/null +++ b/src/content/posts/my-backup-was-using-someone-elses-api-quota.md @@ -0,0 +1,169 @@ +--- +title: "My Backup Was Crawling Because I Was Using Someone Else's API Quota" +description: "A 1 TB restic seed stalled at zero bytes. The log named a Google project I did not own: rclone's shared OAuth client, quota pooled with every rclone user alive." +pubDate: 2026-10-15 +updatedDate: 2026-10-16 +category: notes +tags: ["rclone", "backup", "google-drive", "oauth", "restic"] +ogImage: /og/my-backup-was-using-someone-elses-api-quota.png +banner: /banners/my-backup-was-using-someone-elses-api-quota.png +draft: false +--- + +## The subscription ended, and I had to move a terabyte + +The call came in the least convenient way: the OneDrive account holding my +nightly backups had no subscription any more, and I had a 1.5-year Google +subscription sitting unused. Move the backup, keep the history, don't lose a +night. + +The first thing I hit was not a technical problem: I could not remember +*how* the backup was wired. rclone? Duplicati? Something Synology shipped? +On a three-host estate with 78 compose stacks — the one in +[Running Production Infrastructure Solo](/posts/running-production-infrastructure-solo/) +— "which tool writes my backups" is a question I should have been able to +answer from memory and could not. So step one was reading the machine. + +## What was actually there + +Two rclone configs, which is the kind of detail that bites: the GUI config +that Backrest (the restic front end) uses, plus a **separate stale CLI +copy** that a cron job or a human shell had been using at some point. Change +one and you have changed half the system. + +The migration itself is boring when done in the right order, and I did it +read-first: + +```text +rclone.conf append [gdrive-*] + [gdrive-*-crypt] +rclone/rclone.conf (CLI copy) same two remotes +Portainer stack 177 env → the new crypt remote; config :ro → :rw +restic init new repo on Google Drive +Backrest SetConfig dsm-repo URI → the new location +``` + +Before touching the plan URIs I checked one property in the source, because +everything depended on it: a restic repository's identity is +`guid = sha256(id)` — a stable value derived from the repo id, **not** from +its URL. So repointing a plan at a new backend cannot silently fork the +history. All six plans came out byte-identical to the originals, and the +repo `id`/`guid` were untouched. + +## Then the seed stopped moving, and the log named a stranger + +The new repo seeded to 2.88 GB of roughly 1075 GB — 0.27% — and then slowed +to a crawl. The live log had the answer: + +```text +'Read requests per minute' ... reason: RATE_LIMIT_EXCEEDED +consumer: projects/202264815644 <- not my project +quota_limit_value: 840000 +``` + +`202264815644` is **rclone's own shared OAuth project**. My terabyte seed +was drawing on a per-minute read quota pooled with every rclone user on +earth, which is why it degraded from ~750 KB/s to nothing while my NAS, my +disk and my link were all idle. By the time I measured, the seed was fully +stalled — **0 bytes in 90 seconds**, 159 errors, still `rateLimitExceeded` — +and the six nightly plans had all failed with the same wall: + +```text +Fatal: unable to save snapshot: unexpected HTTP response (500) +googleapi: Error 403: Quota exceeded for quota metric 'Queries' + and limit 'Requests per minute' ... +``` + +"Slow" and "blocked" look similar in a progress bar and are completely +different problems. Ninety seconds of sampled zeros is what made the call. + +## You need your own OAuth client — and the console is unavoidable + +There is exactly one real fix: stop using the shared client and bring your +own. That means a Google Cloud project, an OAuth consent screen, and a +credential. Two settings cost real time: + +- **Application type = Desktop app.** rclone's redirect is + `http://127.0.0.1:53682`, and that client type permits it without extra + configuration. +- **Publish the app.** Leave it in "Testing" and Google expires your refresh + token **every 7 days** — which turns "working" into a silent weekly + outage. + +I checked whether I could shortcut the console with the CLI that was already +authenticated here. I could not, and the reason is worth knowing: `gcloud` +was signed in only as a **service account** in an unrelated project, and +service accounts cannot create consent screens or OAuth client IDs. The +credential has to belong to a project a human owns. It was a dead end, and I +verified that instead of asserting it. + +## Three traps in the authorization flow + +1. **Publishing an unverified app can hard-block a restricted scope.** + `.../auth/drive` is restricted, and an app that is published *without* + completing Google's verification review is sometimes blocked outright + with no bypass. In my case the bypass existed — the "Google hasn't + verified this app" screen, then **Advanced → Go to rclone-backup + (unsafe)** — but that is a coin flip on scope and review state, not a + guarantee, and "unsafe" is only a label meaning the app never went + through a paid audit. +2. **The callback accepts exactly one request.** A reload, or a second tab, + hits `127.0.0.1:53682` with no code and kills the attempt. My first try + died exactly that way (`no code returned`), and it looks like a + credentials problem. One tab. Don't reload it. +3. **Finish the consent screen on the machine that runs rclone.** The + redirect goes to `localhost`, so a phone or another PC cannot complete it + — the browser has to be the one on the same host as the process waiting + on that port. + +Then: read the token, write `client_id` + `client_secret` + `token` into +**both** configs (the GUI one and the stale CLI copy), and restore the two +flags that matter for a long unattended upload: `use_trash = false` and +`stop_on_upload_limit = true`. + +## The old side: verify by reading, not by asking + +Purging the old remote reported an error — `itemNotFound` — and the space +came back anyway: + +```text +Used: 1.012 TiB -> 3.800 GiB +Free: ~0 -> 1.007 TiB +DSM folder: present -> gone Trashed: 0 B +``` + +The error was rclone complaining *after* the work finished. Also worth +recording: the tool's documentation says personal accounts cannot +`permanentDelete`, which would have meant emptying the recycle bin by hand — +in practice the trash was already empty and roughly 1 TB was reclaimed with +no browser step. Documentation describes the API; the API is not always what +you get. + +One more measured thing, because it changed my next decision: pausing the +file-sync container that shared that machine halved the load average — 16.70 +→ 8.89 — which is the difference between "the network is slow" and "the box +is busy". + +## What I would do differently + +1. **Own the OAuth client from day one.** A shared client id is a queue you + are standing in, with no status page and no one to call. +2. **Answer "what writes my backups" from a document, not from memory.** Two + rclone configs is a normal amount of drift for a system nobody has drawn + in a year. +3. **Read the destination, not the exit code.** The purge "failed" and freed + a terabyte; a successful-looking rename can land nothing. Same lesson as + [It Reported Success. Nothing Had Changed.](/posts/it-reported-success-nothing-had-changed/) + — this estate keeps teaching it. +4. **Never start a multi-day upload while the last few percent of your + attention is elsewhere.** Checking for `RATE_LIMIT_EXCEEDED` at 0.27% + costs minutes; discovering it after two days costs the window. + +## The result, honestly stated + +The plumbing works: the plans live on Google Drive, the repo identity is +unchanged, the seed is now spending **my** quota instead of a shared one, +and the old side gave back about a terabyte. What it is not, yet, is a +backup: at the time I wrote this, the first seed had uploaded 2.88 GB of +~1075 GB and the new repo had **zero snapshots** — nothing is restorable +until the first snapshot lands. That is the number I watch, because "the +transfer is running" and "you can restore" are different claims. diff --git a/src/content/posts/space-reclamation-wasnt-stuck-it-was-waiting.md b/src/content/posts/space-reclamation-wasnt-stuck-it-was-waiting.md new file mode 100644 index 0000000..11a9071 --- /dev/null +++ b/src/content/posts/space-reclamation-wasnt-stuck-it-was-waiting.md @@ -0,0 +1,124 @@ +--- +title: "Space Reclamation Wasn't Stuck. It Was Waiting for Tomorrow at 09:29." +description: "The Storage Manager card said 'running' all day and free space hadn't moved. 121 GiB had come back in two steps — and the afternoon ate 47 GiB of it." +pubDate: 2026-10-14 +updatedDate: 2026-10-16 +category: devops +tags: ["synology", "btrfs", "snapshots", "storage", "nas"] +ogImage: /og/space-reclamation-wasnt-stuck-it-was-waiting.png +banner: /banners/space-reclamation-wasnt-stuck-it-was-waiting.png +draft: false +--- + +## The question I asked was the wrong question + +I emptied a few folders, kicked off Space Reclamation on the NAS, and +waited. Hours later the Storage Manager card still read **running**, and +free space looked the way it had looked in the morning. My question was "why +didn't it reclaim anything". + +It had reclaimed — twice — and one of those reclaims was larger than +anything I had deleted. The measurement answered a different question than +the one I asked, which is the interesting part. + +## Sample free space on a timer before believing anything + +I sampled `volume1` free space every five minutes for the day instead of +looking at a progress card: + +| time | free | event | +|---|---|---| +| 08:59 → 09:24 | 1134.1 GiB | flat — nothing reclaiming | +| **09:29** | **+69.0 GiB** | Class snapshot `…10.14-09.29.02` rotated | +| **09:34** | **+46.8 GiB** | Video snapshot `…10.14-09.30.35` rotated | +| **15:28:45** | — | DSM log: `Start to empty all recycle bins.` | +| **15:34** | **+5.0 GiB** | recycle-bin empty finished — only 5 GiB was in there | +| 16:35 → 17:39 | **−39 GiB** | new data written (a burst) | +| 15:00 → 20:44 | **−47.2 GiB** | net: the volume *lost* space that afternoon | + +Two reclaims, ~121 GiB in total, and then 47 GiB of new writes swallowed the +afternoon. "Nothing came back" was a reading error on my side: I compared +the morning to the evening, and the evening had a day of writing in it. + +The recycle bin — the thing I had bothered to empty by hand — was **5 GiB**. +The snapshots, which I had not thought about at all, were **115.8 GiB**. +That ratio is the whole story. + +## Why the card said "running" and meant nothing + +```text +$ synostgreclaim --get-all-status --verbose +{"state":"running","num_subvolume_remain":1,"now_runnable":true} + schedule_table: 168 × 1 + run_time_option: run_all_time +``` + +`state: running` with a schedule table of 168 ones and `run_all_time` means +the task is configured to be "running" **24×7, by design**. It is a +schedule, not a progress indicator — so the card will read the same thing on +a day when nothing is scheduled to happen at all. Meanwhile the machine had +no `synostgreclaim` process, no btrfs balance, and md2/dm-3 I/O of roughly +zero. Nothing was being reclaimed at that moment, and nothing was stuck +either. + +Then the logs, which are mostly red herrings on DSM: + +- the reclaim task itself logs nothing; +- `/var/log/space_operation.log` is LUKS and volume-list spam; +- `/var/log/synorecycle.log` is `synounlinkd` path noise; +- Log Center for the day contained only my own recycle-bin line and a few + UPS on-battery/back-online blips. + +A UI state field that is always true and three log files that never contain +the answer: the only instrument left is free space over time. + +## The mechanism nobody tells you about + +Space Reclamation frees **deleted btrfs subvolumes**. It cannot free data +that a snapshot is still pinning. Those two shares keep **retention 1** — +one generation each — so anything I delete becomes free space only at the +**next day's rotation, around 09:29 / 09:30**, in a step of tens of +gigabytes, and then it is silent for 24 hours. + +That is exactly what the two jumps were. It also means the honest answer to +"I deleted 50 GB and nothing happened" on this box is: "wait for tomorrow +morning, then check again" — and if the deleted data lived inside a +snapshot, the answer is "it frees when that snapshot expires, and not +before". + +## What I would do differently + +1. **Sample the number, not the status.** A five-minute timer on free space + answers "is anything happening" in a way no task card can, and it made + the regrowth visible at the same time. +2. **Read the schedule field, not the state field.** `run_all_time` explains + a permanent "running" without any bug existing. +3. **Know each share's retention.** It is the clock that decides when a + delete becomes free space; without it, every deletion looks broken for a + day. +4. **Separate the two ledgers.** Recycle bin (5 GiB) and snapshots (115.8 + GiB) behave nothing alike and are two different mental models. +5. **Re-check the denominator.** The volume was losing ~1.5 GiB/h to normal + writes; a reclaim that returns 121 GiB and a day that consumes 47 GiB + look identical if you only compare two snapshots of the same number. + +## What the measurement actually says about the machine + +I went in expecting to fix a stuck job and came out with a storage trend +instead: + +```text +volume1 42 TB 98% used 1.2 TB free ~-1.5 GiB/h lately (-344 GiB over 8 days) +volume2 16 TB 100% used 70 GB free ~-19 GiB / 2 days +health btrfs corruption_errs 0 on both volumes + all md arrays [UUUU] / [UU], no balance running + read/write_io_errs identical to the September baseline +``` + +So the useful conclusion is not "the reclaim works". It is that reclaim is a +rounding error next to the write rate, deletion is not the lever I thought +it was while retention is 1, and the trend — not the task card — is what I +should be alerting on. That is the same lesson as +[Your Disk-Full Alert Is Lying](/posts/your-disk-full-alert-is-lying/), +arriving from the other direction: alert on the consequence (bytes free over +time), because the mechanism's own status is not evidence. diff --git a/src/content/posts/zh/adding-a-domain-to-someone-elses-mail-server.md b/src/content/posts/zh/adding-a-domain-to-someone-elses-mail-server.md new file mode 100644 index 0000000..d3584a6 --- /dev/null +++ b/src/content/posts/zh/adding-a-domain-to-someone-elses-mail-server.md @@ -0,0 +1,83 @@ +--- +title: "给别人家的邮件服务器加一个域名" +description: "收信只要四条 DNS 记录，一次就通了；而从它发信、以及转发它，撞上两堵任何 DNS 记录都修不了的墙。" +pubDate: 2025-05-06 +updatedDate: 2026-10-16 +category: notes +tags: ["email", "dns", "mailcow", "smtp", "deliverability"] +ogImage: /og/adding-a-domain-to-someone-elses-mail-server.png +banner: /banners/adding-a-domain-to-someone-elses-mail-server.png +draft: false +--- + +## 这件事看起来像个 DNS 活 + +有个域名要收信。邮件服务器是我自己那台跑 mailcow 的 VPS，所以这就是「加域名、加记录、建信箱」——二十分钟的活，而且我做过好几次。 + +这次花了更久，原因是我关于邮件服务器的两个假设并不成立，另外还因为互联网有一条任何服务器配置都改不了的性质。 + +## 收信：四条记录，加一次真测试 + +收信恰好只需要四样东西，而且都住在域名自己的 DNS 里（这里是 Cloudflare）： + +```text +MX ninelore.com → mail.hoelee.com （优先级 10） +TXT @ → v=spf1 mx -all +TXT dkim._domainkey → （MTA 发布的那个选择器） +TXT _dmarc → （策略 + 报告地址） +``` + +配好之后，只用一种算数的方式测：**从这套环境外面**（走第三方中继）寄进新地址，然后去读服务器自己的日志行——`rspamd` 在收下这封信时记下 `RCPT_MAILCOW_DOMAIN(ninelore.com)`。收信一次就通。**收信确实是容易的那一半**，而它之所以容易，正因为它只依赖你自己能控制的记录。 + +## 出信才是地面移动的地方 + +两堵独立的墙，各有各的错误，而且都不是改一改区域文件就能修的配置失误。 + +**第一堵墙：那条反垃圾策略，本质上是缺一条 PTR。** 一次直投尝试被对方退回： + +```text +554 5.7.1 Mail rejected due to antispam policy +``` + +在怪远端之前，我先排掉常见原因：四个主流黑名单全干净，域名自己的 SPF/DKIM/DMARC 也都正确。剩下的解释就是那条不在我控制里的记录——**发信 IP 没有反向 DNS**。一台收信服务器如果无法把连进来的 IP 映射回一个名字，就会把它当成一个它无法背书的来源；再正确的 SPF 也压不过这一点。 + +**第二堵墙：用中继绕过去也不行。** 下一个显然的动作是让所有信走那条已经能用的出信中继。结果是： + +```text +550-From header sender domain not verified (gmail.com) +``` + +这行错误要仔细读，因为它一次打死了一整类设计。中继的「已验证寄件人」策略，检查的是 **From 头里的域名**——而**转发**出去的信保留的是原寄件人的 From。所以从你转发第一封来自外部寄件人的信开始，你就是在要求中继以 `gmail.com` 的身份发信，而它永远不会允许。mailcow 在这里还有个值得先知道的坑：它按**寄件人**域名选中继，所以一封 From 是外部域的信，根本不会走到你配的那条中继上。 + +## 那个并不存在的「信箱转发」功能 + +原计划是建好信箱，再转发一份到我的主邮箱。结果连着两个意外： + +- mailcow **没有**「信箱转发」这个设置。把信箱地址写进别名的 `goto` 列表会被拒（`is_alias_or_mailbox`），因为每个信箱自带一条 self-alias 记录，代码在算目标时会跳过这个地址。 +- 想「保留信箱**并且**转发」只能走 **Sieve 过滤器**——那是另一套机制、另一种失败形态——而它最后仍然落在第一堵墙和第二堵墙上，因为这封信终究得离开这台机。 + +所以我**什么都没启用**。一条出信路径送不出去的转发，不会安静地失败：它会把退信发回给无辜的原始寄件人（backscatter），而每一封这样的退信都在花一个本来就没有余量的 IP 的信誉。对「这个新信箱要送到我的收件箱」这件事，稳定的答案是**收件侧的 POP 拉取**：我的邮件客户端从 `mail.hoelee.com:995` 拉信，VPS 什么都不用发。 + +## PTR 是一个 IP 一条，不是一个域一条 + +这是我一直需要解释、包括对我自己解释的部分： + +- 反向记录是**IP 地址**的属性。不存在「每个域一条 PTR」，下次再加域也没有任何 PTR 要动。 +- 它正确的值就是邮件服务器自己对外报的名字——这里是 `mail.hoelee.com`，也就是 postfix 用作 HELO/myhostname 的那个。正向记录（`mail.hoelee.com → 41.216.187.102`）已经存在，**这一对**才是收信方要验证的东西：**forward-confirmed rDNS**。缺任何一半都一文不值。 +- 那个 IP 上所有域名都用同一条反向记录来评判。一百个域名，一条 PTR。 +- 谁来设：跑反向区的人。这里 `216.41.in-addr.arpa` 与 `187.216.41.in-addr.arpa` 两级的 SOA 都是 VPS 商自己的名字服务器，所以它是**服务商侧**的改动——先去 VM / 网络面板里找有没有 rDNS/PTR 输入框，没有就提工单，把上面那个确切的值给他们。 + +我没有假设，而是去查了当前状态：正向解析正常（`mail.hoelee.com → 41.216.187.102`），**那个 IP 的反向解析依旧什么都不返回**。所以直投仍然会被拒，唯一能用的出信路径是中继——而它那条「已验证寄件人」的要求，新域名一样得满足。 + +## 三件要提前知道的事，免得以后被吓到 + +1. **PTR 不是信誉。** 它拆掉的是一次硬拒；它不会让你被信任。新 IP 通常需要几周的日常通信，大厂才会停止把它塞进垃圾箱——而从全新 IP 起步，就是把这口钟从零开始拨。 +2. **PTR 修的是你自己的出信，对转发毫无帮助。** 转发信带的是别人的 From；如果那个域发布了 `p=reject`（银行和大厂都会），那么不管你这边多干净，它在远端都会被挡。 +3. **中继有一个按域的前置条件。** 一个域要走中继，就必须先在中继那边注册并验证它为寄件人。这是每个域永远多出来的一步，且与你自己的 DNS 里的一切无关。 + +## 如果重来一次 + +1. **在建域之前先定：要信箱还是要转发。** 转发是「必须成立的条件」最长的那条路；收件侧 POP 拉取几乎没有条件。 +2. **在向任何人承诺邮件之前，先查发信 IP 的反向记录。** 一次反向查询就能告诉我接下来会撞上哪堵墙。 +3. **把「收件侧拉取」当一等设计，而不是权宜之计。** 「这封信根本不用离开我的服务器」一次消掉三大类故障。 +4. **把那四条收信记录当成一张清单。** 那一半很无聊、可验证，而且是唯一一次就通的部分。 diff --git a/src/content/posts/zh/embedding-fonts-so-a-pdf-looks-the-same-everywhere.md b/src/content/posts/zh/embedding-fonts-so-a-pdf-looks-the-same-everywhere.md new file mode 100644 index 0000000..b8f71ee --- /dev/null +++ b/src/content/posts/zh/embedding-fonts-so-a-pdf-looks-the-same-everywhere.md @@ -0,0 +1,82 @@ +--- +title: "把字体嵌进 PDF，让它在任何机器上都长一样" +description: "四个字族、八个字面、两个渲染引擎，外加一条 CSS 怪癖：为了不让生成的报告依赖服务器上恰好装了什么字体，我做了什么。" +pubDate: 2025-01-22 +updatedDate: 2026-10-16 +category: engineering +tags: ["pdf", "fonts", "mpdf", "typography", "php"] +ogImage: /og/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png +banner: /banners/embedding-fonts-so-a-pdf-looks-the-same-everywhere.png +draft: false +--- + +## 需求只有一句话 + +一个报告生成器必须产出在任何地方都长得一样的文档：中文标题用宋体、中文正文用黑体、英文标题与封面数字用一款展示衬线体、英文正文和表格用 Inter。字重也要对——强调用 medium、标题用 semibold——而且是三语、两个 PDF 渲染引擎都要对。 + +同时，**字体必须跟着文档走**。不是「服务器装过」，也不是「读者大概有」，而是**嵌进文件里**，这样另一个国家的印厂看到的页面是一样的。 + +老管线在最后这句上栽得很隐蔽：它依赖服务器上用包管理器装过的字体，而且**只嵌入了一款中文字体**。第二档字重根本不存在，于是「粗体」是渲染器的合成近似；而想发现这件事，唯一的办法是打开 PDF 去看里面到底装了什么。 + +## 规则一：从成品 PDF 里把字体读回来 + +「已嵌入」是一个关于**输出**的断言，所以我量的是输出。同一份文档，改前与改后： + +| | 改前 | 改后 | +|---|---|---| +| zh 页数 | 23 | **23** | +| en 页数 | 28 | **29** | +| bm 页数 | 30 | **30** | +| 真正嵌进去的字体 | 一款中文字体 + Roboto | 思源宋 Medium + SemiBold · 思源黑 Regular + Medium · Cormorant Garamond Medium + SemiBold · Inter Regular + SemiBold + Medium · 拉丁报告里的汉字走思源黑 | +| 渲染时间 / 峰值内存 | 基线 | 同一量级（±10%，属噪声） | +| 文件体积 | 基线 | 约 +3% | + +每个字面的字形数是收据：中文报告里 15,988 个字形落在正文黑体、2,247 个落在它的 medium 档、634 个落在标题宋体的 medium、222 个落在它的 semibold。这就是「字体在服务器上」与「字体在文件里」的区别——另一个国家的读者看到的是同样四个字面，而代价是某一种语言正好多一页（拉丁字体的度量比它替换掉的那个更宽）。 + +## 规则二：只出静态实例，理由写在解析器里 + +现代做法看起来该是「每个字族发一个可变字体」，但在这里是错的，有两个实测理由： + +1. **PDF 引擎的字体解析器只读 `glyf`/`loca`。** 喂它一个 PostScript/CFF 轮廓的 OpenType 文件，它直接拒绝：`Fonts with postscript outlines are not supported`。问题不是「可变」，是 CFF 轮廓。 +2. **它完全不读 `fvar` 表**，所以可变字体会**静默地**解析成它的**默认实例**——而中文字族的默认实例恰恰是最细的那一档（`name` 表里写得清清楚楚：Thin 100 与 ExtraLight 200）。整份报告会变成发丝体，而整条管线里不会报任何一个错。 + +所以构建步骤拿上游的可变字体**实例化成静态切面**：跑实例化器，然后把 `name`、`OS/2.usWeightClass` 和 `head.macStyle` 改写成它现在真实的字重（源文件的名字里还写着 Thin）。产物旁边有一份 manifest，逐文件记下 family、字重、sha256、字形数与码位数；许可证也随文件一起进仓库。 + +## 规则三：引擎只会说两种字重，那就映射到两个槽位 + +渲染引擎的 CSS 只认 `normal` 和 `bold`——它的源码自己就这么写（"Does not support: bolder, lighter, 100..900"）。所以数字字重对它毫无意义；而同一条管线里的无头 Chromium 又完全认它们。 + +可用的映射是：每个字族声明一个 **R 槽**和一个 **B 槽**（常规一档、更重一档），`font-weight: 500` 落到目标字族的 R 槽，`600` **只在 PDF 那一侧被改写成 `bold`**，落到 B 槽。Chromium 侧照写 `600`。两条测试钉住它：B 槽必须比 R 槽重（读 `usWeightClass`，不是假设），且 `font-weight: 600` 绝不允许出现在交给 PDF 引擎的样式表里。 + +有一处细节值得单写一段：在这个管线里，**medium 不是强调手段，而是一个字族成员**。中文标题用宋体的 medium 当常规、用 semibold 做强调；英文标题用展示衬线体做同样的事。这里搞错，屏幕上完全看不出来，而印出来一目了然。 + +## 规则四：不要按你控制不了输入的字集做子集化 + +第一版把中文字体按两个国家字集的并集做了子集，产物从约 50 MB 降到约 29 MB。然后我跑了姓名探针，发现这些字被裁掉了： + +```text +犇 骉 頔 祎 廸 玧 祾 赟 堃 +``` + +每一个都是真实的人名用字。报告会把客户姓名印在封面上，而姓名是自由输入——所以一套基于子集的构建，迟早会在产品最醒目的那一页上放一个豆腐块。全字形覆盖是约 31k 码位、约 50 MB 的嵌入字体；为了永远不给客户看一个空方块，这个价钱很便宜。 + +## 规则五：把「缺字」变成构建失败，而不是惊喜 + +闸门跑三项检查，任何一项不过就非零退出： + +- manifest 里声明的每个字体文件都在，且 sha256 与磁盘上的文件一致； +- 内容库（三语全部）+ 品牌字面 + 各类探针（简体、繁体、CJK 标点、全角、中英混排、数字、拉丁扩展、符号）+ 姓名用字，**每一个字在「它实际会被哪一档字体印出来」的那一档里都有字形**； +- 查字形用的是 PDF 引擎自己的解析器，而不是第三方库，所以这条检查不可能和渲染器互相矛盾。 + +已知边界写在明处，不藏起来：✅ ✔ 这类符号在这几款正文字体里都没有字形，而引擎也画不了彩色 emoji，所以它们不出现在报告正文；拉丁标题那款展示衬线体只有 974 个字形，不画 ① ★ ℃，这也是为什么内容库的检查覆盖**所有**槽位而不只是正文；而拉丁报告里的汉字必须走专门的 CJK 槽位——两条测试防止两个渲染器在这点上产生分歧。 + +## 如果重来一次 + +1. **在动样式之前先定字族与字重。** 版式做完之后再去加第二档字重，意味着每一页页数都要手工复核一遍。 +2. **选字体格式之前先读渲染器的源码。** 可变字体看着是审美问题，实际上是解析器问题——两个文件、十分钟就定了。 +3. **把要嵌进文档的一切当作许可证产物。** 字体文件连同许可证一起进仓库，否则就得进别人的法务审查。 +4. **量「到底嵌进去了什么」。** 页数、每个字面的字形数、文件体积、渲染时间：四项都动了，其中三项的方向和我猜的相反。 + +## 结果 + +改前：一款中文字体加一款界面字体、靠合成粗体、还取决于服务器装过什么。改后：四个字族、八个静态字面真正嵌在文件里，字重是各个字族真实的成员，页数与上一版相差不超过一页，而且有一条 lint 闸门：某个字没有字形时直接让构建失败——于是失败形态从「客户看到一个方块」变成了「CI 拒绝发版」。这跟这条管线里其它地方的取舍是同一个：**让错误发不出去，而不是让错误容易被看见。** diff --git a/src/content/posts/zh/my-backup-was-using-someone-elses-api-quota.md b/src/content/posts/zh/my-backup-was-using-someone-elses-api-quota.md new file mode 100644 index 0000000..609be9a --- /dev/null +++ b/src/content/posts/zh/my-backup-was-using-someone-elses-api-quota.md @@ -0,0 +1,95 @@ +--- +title: "我的备份爬得比爬虫还慢，因为我在用别人的 API 配额" +description: "1 TB 的 restic 种子传输卡在零字节。日志里点名的那个 Google 项目不是我的：那是 rclone 的共享 OAuth 客户端，配额和全世界每个 rclone 用户共池。" +pubDate: 2026-10-15 +updatedDate: 2026-10-16 +category: notes +tags: ["rclone", "backup", "google-drive", "oauth", "restic"] +ogImage: /og/my-backup-was-using-someone-elses-api-quota.png +banner: /banners/my-backup-was-using-someone-elses-api-quota.png +draft: false +--- + +## 订阅到期了，我得搬一个 TB + +这件事以最不方便的方式找上门：承载我每晚备份的那个 OneDrive 账号已经没有订阅了，而手上还躺着一个 1.5 年没用的 Google 订阅。把备份搬过去、保住历史、别丢一晚。 + +第一件撞上的不是技术问题：**我记不清这套备份到底是谁在写**。rclone？Duplicati？还是群晖自带的什么？在一套 78 个 compose 栈、三台机器的环境里（就是[《一个人跑生产基础设施》](/posts/running-production-infrastructure-solo/)里那套），「我的备份是谁写的」本该是我张口就答的问题，而它当时不是。所以第一步是去读机器。 + +## 现场到底有什么 + +**两份 rclone 配置**，正是那种会咬人的细节：Backrest（restic 的前端）用的是 GUI 那份，而另外还有一份**陈旧的 CLI 副本**，被某个 cron 或某次人手 shell 用过。改了一份，等于只改了一半系统。 + +搬迁移本身在顺序对的时候很无聊，而我是先读后动的： + +```text +rclone.conf 追加 [gdrive-*] + [gdrive-*-crypt] +rclone/rclone.conf（CLI 副本） 同样两个 remote +Portainer stack 177 env → 新的 crypt remote；配置 :ro → :rw +restic init 在 Google Drive 上建新仓库 +Backrest SetConfig dsm-repo 的 URI → 新位置 +``` + +在动计划 URI 之前，我先去源码里确认了一件一切依赖它的事：**restic 仓库的身份是 `guid = sha256(id)`** —— 一个由仓库 id 派生的稳定值，**不是**从 URL 推出来的。所以把计划指到新后端，不可能悄悄把历史分叉。六个计划事后逐字节与原版一致，仓库的 `id`/`guid` 未被动过。 + +## 然后种子停住了，而日志里点了一个陌生人的名字 + +新仓库灌到 1075 GB 里的大约 2.88 GB —— 0.27% —— 然后慢成爬行。实时日志里有答案： + +```text +'Read requests per minute' ... reason: RATE_LIMIT_EXCEEDED +consumer: projects/202264815644 <- 不是我的项目 +quota_limit_value: 840000 +``` + +`202264815644` 是 **rclone 自己的共享 OAuth 项目**。我那个 TB 级种子吃的是每分钟读取配额，和地球上每一个 rclone 用户共池——这就是为什么它从约 750 KB/s 掉到零，而我的 NAS、我的磁盘、我的线路全都闲着。等我去量的时候，种子已经完全停摆：**90 秒内 0 字节**、159 个错误、依旧 `rateLimitExceeded`；而六个夜间计划也全撞同一堵墙： + +```text +Fatal: unable to save snapshot: unexpected HTTP response (500) +googleapi: Error 403: Quota exceeded for quota metric 'Queries' + and limit 'Requests per minute' ... +``` + +在进度条里，「慢」和「被挡住」看起来一样，而它们是两个完全不同的问题。**采样 90 秒得到的零**，才是让我做出这个判断的东西。 + +## 你必须有自己的 OAuth 客户端，而控制台绕不过去 + +唯一的真修法只有一个：别再用共享客户端，用你自己的。也就是一个 Google Cloud 项目、一个 OAuth 同意屏、一个凭据。两个设置花掉了真正的时间： + +- **Application type 必须选 Desktop app。** rclone 的回调是 `http://127.0.0.1:53682`，只有这个类型能不用额外配置就允许它。 +- **要点「Publish app」。** 留在 Testing 状态，Google 会**每 7 天**让刷新令牌过期一次——那就把一个「能跑」的系统变成了每周一次的静默故障。 + +我查过能不能用机器上已经认证好的 CLI 抄近路。不能，而原因值得记下来：这里的 `gcloud` 只以一个**别人的项目里的服务账号**登录，而服务账号**无法**创建同意屏或 OAuth 客户端 ID。凭据必须属于一个真人拥有的项目。这是一条死路——我是验证出来的，不是猜的。 + +## 授权流程里的三个坑 + +1. **未经验证就发布的应用，可能会硬性堵死受限 scope。** `.../auth/drive` 是受限 scope，一个**已发布但未完成 Google 审核**的应用有时会被直接拒绝且**没有旁路**。我这一次旁路是存在的——「Google hasn't verified this app」页面，然后 **Advanced → Go to rclone-backup (unsafe)**——但那取决于 scope 与审核状态，是一枚硬币，而不是保证；而「unsafe」只是个标签，意思是从没走过付费审核。 +2. **回调只接受一次请求。** 刷新页面，或者开第二个标签，都会在**没有 code** 的情况下打到 `127.0.0.1:53682`，把这次尝试弄死。我第一次就是这么死的（`no code returned`），而它看起来像凭据错了。**一个标签，别刷新。** +3. **同意屏必须在跑 rclone 的那台机器的浏览器里完成。** 回调指向 `localhost`，所以手机或另一台电脑完不成——浏览器必须和等待那个端口的进程在同一台主机上。 + +之后就简单了：把 token 读出来，把 `client_id` + `client_secret` + `token` 写进**两份**配置（GUI 那份和陈旧的 CLI 副本），并恢复两个对长时无人值守上传很关键的开关：`use_trash = false` 与 `stop_on_upload_limit = true`。 + +## 旧的那一半：靠读，不靠问 + +清空旧 remote 时报了错——`itemNotFound`——而空间照样回来了： + +```text +Used: 1.012 TiB -> 3.800 GiB +Free: ~0 -> 1.007 TiB +DSM 文件夹: 在 -> 不在 回收站: 0 B +``` + +那个错误是 rclone 在**干完活之后**发的抱怨。另外值得记一笔：工具的文档说个人账号无法 `permanentDelete`，按它你就得手动清回收站——实际是回收站已经是空的，约 1 TB 就这么回来了，不需要任何浏览器步骤。文档描述的是 API，而 API 不总是你拿到的东西。 + +还有一条量出来的事，因为它改变了我下一个决定：把那台机器上共享的同步容器暂停之后，负载均值直接减半——16.70 → 8.89。这就是「网络慢」和「机器忙」的区别。 + +## 如果重来一次 + +1. **从第一天起就用自己的 OAuth 客户端。** 共享客户端 ID 是一条你排在里面、却看不到进度、也找不到人负责的队伍。 +2. **「我的备份是谁写的」要写在文档里，不靠记忆。** 两份 rclone 配置，对一个一年没人画过图的系统来说是很正常的漂移量。 +3. **读目的地，别读退出码。** 那次清空「失败」了，却释放了一个 TB；而一次看起来成功的改名，可能什么都没落地。跟[《它报告成功，其实什么都没变》](/posts/it-reported-success-nothing-had-changed/)是同一课——这套环境一直在教这一课。 +4. **不要在注意力只剩最后百分之几的时候启动一个要跑几天的上传。** 在 0.27% 时发现 `RATE_LIMIT_EXCEEDED` 只花几分钟；跑了两天才发现，花掉的是整个窗口。 + +## 结果，诚实地说 + +管道是通的：计划现在住在 Google Drive 上、仓库身份没变、种子花的是**我的**配额而不是共享池、旧的那一侧退回了大约 1 TB。但它现在**还不是一个备份**：写这篇的时候，第一轮种子才上传了约 1075 GB 里的 2.88 GB，而且新仓库里**快照数为零**——在第一张快照落地之前，什么都还原不了。那是我盯着的数字，因为「传输在跑」和「你能还原」是两句不同的话。 diff --git a/src/content/posts/zh/space-reclamation-wasnt-stuck-it-was-waiting.md b/src/content/posts/zh/space-reclamation-wasnt-stuck-it-was-waiting.md new file mode 100644 index 0000000..626c081 --- /dev/null +++ b/src/content/posts/zh/space-reclamation-wasnt-stuck-it-was-waiting.md @@ -0,0 +1,83 @@ +--- +title: "空间回收没卡住，它只是在等明天早上 09:29" +description: "存储管理员那张卡整天写着 running，可用空间一动不动。其实 121 GiB 已经分两步回来了——然后下午又被写掉了 47 GiB。" +pubDate: 2026-10-14 +updatedDate: 2026-10-16 +category: devops +tags: ["synology", "btrfs", "snapshots", "storage", "nas"] +ogImage: /og/space-reclamation-wasnt-stuck-it-was-waiting.png +banner: /banners/space-reclamation-wasnt-stuck-it-was-waiting.png +draft: false +--- + +## 我问错了问题 + +我清掉几个文件夹，在 NAS 上点了空间回收，然后等。几个小时后，存储管理员那张卡还写着 **running**，可用空间看起来跟早上一样。我的问题是「它为什么什么都没回收」。 + +**它回收过了——两次**，而且其中一次比我删掉的东西大得多。测量回答的是另一个问题，而这才是这件事里最有意思的地方。 + +## 在相信任何东西之前，先给可用空间上定时采样 + +我没有去看那张进度卡，而是给 `volume1` 的可用空间做了当天每五分钟一次的采样： + +| 时间 | 可用 | 事件 | +|---|---|---| +| 08:59 → 09:24 | 1134.1 GiB | 平——什么都没在回收 | +| **09:29** | **+69.0 GiB** | Class 快照 `…10.14-09.29.02` 轮转 | +| **09:34** | **+46.8 GiB** | Video 快照 `…10.14-09.30.35` 轮转 | +| **15:28:45** | — | DSM 日志：`Start to empty all recycle bins.` | +| **15:34** | **+5.0 GiB** | 清回收站结束——里面只有 5 GiB | +| 16:35 → 17:39 | **−39 GiB** | 新数据写入（一阵爆发） | +| 15:00 → 20:44 | **−47.2 GiB** | 净值：这个卷那天下午**丢**了空间 | + +两次回收，合计约 121 GiB；然后 47 GiB 的新写入把下午吃掉了。「什么都没回来」是我这边的读法错误：我拿早上和晚上比，而晚上里面多了一整天的写入。 + +我特意手动清空的回收站是 **5 GiB**；我压根没想到的快照是 **115.8 GiB**。这个比例就是全部故事。 + +## 为什么那张卡写着 running，却什么都不代表 + +```text +$ synostgreclaim --get-all-status --verbose +{"state":"running","num_subvolume_remain":1,"now_runnable":true} + schedule_table: 168 × 1 + run_time_option: run_all_time +``` + +`state: running` 加上 168 个 1 的调度表和 `run_all_time`，意思是这个任务被配置成**按设计 24×7 都处于 running**。它是调度状态，不是进度指示器——所以在某天完全无事可排的日子，这张卡照样这么写。而当时那台机器上**没有** `synostgreclaim` 进程、没有 btrfs balance、md2/dm-3 的 I/O 基本为零。那一刻没有任何东西在被回收，也没有任何东西卡住。 + +然后是日志，而 DSM 上它们大多是红鲱鱼： + +- 回收任务本身什么都不记； +- `/var/log/space_operation.log` 是 LUKS 和卷列表的噪音； +- `/var/log/synorecycle.log` 是 `synounlinkd` 的路径噪音； +- 当天的 Log Center 里只有我自己那条清回收站的记录，外加几条 UPS 断电/来电的提示。 + +一个永远为真的 UI 状态字段，加上三个永远不会给出答案的日志文件：唯一剩下的仪器，就是**随时间变化的可用空间**。 + +## 没人告诉你的那个机制 + +空间回收释放的是**已删除的 btrfs 子卷**。它释放不了仍然被快照钉住的数据。那两个共享文件夹的保留策略是 **retention 1**（各留一代），所以**我删掉的任何东西，都要等到第二天的轮转、大约 09:29 / 09:30 才会变成可用空间**，一次跳几十 GB，然后安静 24 小时。 + +那两次跳变正是这个。它也就意味着，对于「我删了 50 GB 却什么都没发生」这个问题，在这台机器上的诚实答案是：「等明天早上再看」——而如果删掉的数据本来就在某个快照里面，答案是「等那个快照过期，之前不会释放」。 + +## 如果重来一次 + +1. **采样那个数字，不要采样状态。** 每五分钟量一次可用空间，比任何任务卡都更能回答「现在有没有事情在发生」，而且它同时让「回长」变得可见。 +2. **读调度字段，不读状态字段。** `run_all_time` 解释了那个永久 running，而且不需要任何 bug 存在。 +3. **搞清楚每个共享文件夹的保留策略。** 它是决定「删除何时变成可用空间」的那口钟；不知道它，每一次删除都会看起来坏掉一整天。 +4. **把两本账分开。** 回收站（5 GiB）和快照（115.8 GiB）行为完全不同，也是两套心理模型。 +5. **重新检查分母。** 这个卷平时就在以约 1.5 GiB/时 的速度被正常写入吃掉；回收回来 121 GiB、当天又花掉 47 GiB，只要你只比同一个数字的两个快照，这两件事看起来完全一样。 + +## 测量真正告诉我的，是这台机器的趋势 + +我本来是去修一个卡住的任务，结果带回来的是一条存储趋势： + +```text +volume1 42 TB 已用 98% 剩 1.2 TB 近来约 -1.5 GiB/时（8 天 -344 GiB） +volume2 16 TB 已用 100% 剩 70 GB 约 -19 GiB / 2 天 +健康 两个卷 btrfs corruption_errs 均为 0 + 所有 md 阵列 [UUUU] / [UU]，没有在跑 balance + read/write_io_errs 与九月基线一致 +``` + +所以有用的结论不是「回收是好的」。而是：**相对于写入速率，回收只是一个舍入误差**；在保留策略为 1 的前提下，删除并不是我以为的那根杠杆；而该拿来做告警的是趋势，不是那张任务卡。这跟[《你的磁盘快满告警在撒谎》](/posts/your-disk-full-alert-is-lying/)是同一课，只是从另一个方向来的：对**后果**告警（随时间变化的可用字节），因为机制自己的状态不是证据。