Add posts: RAID silent corruption + self-hosted mem0 (en/zh, og+banner)

scripts/banner-gen/generate.mjs

@@ -133,6 +133,44 @@ const BANNERS = {
     ],
   },
 
+  'when-smart-says-healthy-but-your-raid-is-corrupting-data': {
+    titlebar: 'root@unraid — cache pool scrub',
+    lines: [
+      { t: 'dim', text: 'btrfs RAID1 · 2× Samsung PM9A3 NVMe · same batch' },
+      { t: 'prompt', text: '$' }, { t: 'cmd', text: 'btrfs scrub /mnt/cache' },
+      { t: 'err', text: 'csum 0x8941f998 recurring = CRC32C(zero block)' },
+      { t: 'prompt', text: '$' }, { t: 'cmd', text: 'self-heal rewrite → does NOT stick' },
+      { t: 'err', text: 'write-path corruption · SMART stays clean' },
+      { t: 'prompt', text: '$' }, { t: 'cmd', text: 'fix = restore VM + replace both drives' },
+    ],
+    flow: [
+      { n: '1', label: 'SMART clean' },
+      { n: '2', label: 'csum zeros', err: true },
+      { n: '3', label: 'self-heal no-stick' },
+      { n: '4', label: 'restore VM' },
+      { n: '5', label: 'replace both ✓' },
+    ],
+  },
+
+  'self-hosting-mem0-memory-stack': {
+    titlebar: 'root@dsm — mem0 memory stack',
+    lines: [
+      { t: 'prompt', text: '$' }, { t: 'cmd', text: "curl -X POST :20015/memories -H X-Api-Key" },
+      { t: 'dim', text: 'user_id + text → mem0 extracts & stores' },
+      { t: 'prompt', text: '¶' }, { t: 'ok', text: '→ remembers across sessions ✓' },
+      { t: 'prompt', text: '$' }, { t: 'cmd', text: 'stack = mem0-api + LiteLLM + pgvector(pg17)' },
+      { t: 'err', text: 'infer=true → LLM hop · slow writes' },
+      { t: 'prompt', text: '' }, { t: 'ok', text: 'self-hosted · data stays on LAN ✓' },
+    ],
+    flow: [
+      { n: '1', label: 'POST /memories' },
+      { n: '2', label: 'store fact' },
+      { n: '3', label: 'search' },
+      { n: '4', label: 'feed context' },
+      { n: '5', label: 'remembers ✓' },
+    ],
+  },
+
   'hardening-a-tor-onion-service': {
     titlebar: 'root@tor-host — onion service audit',
     lines: [

scripts/og-gen/generate.mjs

@@ -125,10 +125,20 @@ const TERMINALS = {
     <div class="line"><span class="prompt">$</span><span class="cmd">warm session · 7s pacing · sweep.py</span><span class="fix">→ 20+ listings ✓</span></div>`,
 
   'unraid-stop-array-hangs-on-swapfile': `
-    <div class="line"><span class="prompt">$</span><span class="cmd">unraid → stop array · swapfile on /mnt/cache</span></div>
-    <div class="line"><span class="prompt">&nbsp;</span><span class="err">umount: target is busy · /proc/swaps says /dev/loop0</span></div>
-    <div class="line"><span class="prompt">$</span><span class="cmd">swapoff -a + losetup -j at stopping_svcs</span><span class="fix">→ clean stop ✓</span></div>`,
-};
+      <div class="line"><span class="prompt">$</span><span class="cmd">unraid → stop array · swapfile on /mnt/cache</span></div>
+      <div class="line"><span class="prompt">&nbsp;</span><span class="err">umount: target is busy · /proc/swaps says /dev/loop0</span></div>
+      <div class="line"><span class="prompt">$</span><span class="cmd">swapoff -a + losetup -j at stopping_svcs</span><span class="fix">→ clean stop ✓</span></div>`,
+
+    'when-smart-says-healthy-but-your-raid-is-corrupting-data': `
+        <div class="line"><span class="prompt">$</span><span class="cmd">scrub → re-read vdisk1.img</span></div>
+        <div class="line"><span class="prompt">&nbsp;</span><span class="err">csum 0x8941f998 = CRC32C(zeros) · recurring</span></div>
+        <div class="line"><span class="prompt">$</span><span class="cmd">self-heal rewrite does NOT stick</span><span class="fix">→ replace both drives ✓</span></div>`,
+
+      'self-hosting-mem0-memory-stack': `
+        <div class="line"><span class="prompt">$</span><span class="cmd">curl -X POST :20015/memories · X-Api-Key</span></div>
+        <div class="line"><span class="prompt">&nbsp;</span><span class="err">infer=true → LLM hop · slow write</span></div>
+        <div class="line"><span class="prompt">$</span><span class="cmd">infer=false · pgvector · LiteLLM gateway</span><span class="fix">→ remembers across chats ✓</span></div>`,
+    };
 
 const DEFAULT_TERMINAL = `
   <div class="line"><span class="prompt">$</span><span class="cmd">engineering · devops · self-hosting</span></div>

src/content/posts/self-hosting-mem0-memory-stack.md

@@ -0,0 +1,192 @@
+---
+title: "How I Self-Hosted a Mem0 Memory Stack (So My AI Agent Remembers)"
+description: "A private, self-hosted memory layer for AI agents — mem0 API + LiteLLM + PostgreSQL with pgvector on a Synology NAS. Semantic memory under your control, not a third party's."
+pubDate: 2026-09-16
+category: case-studies
+tags: [mem0, docker, postgres, pgvector, litellm, ai, sharing]
+ogImage: /og/self-hosting-mem0-memory-stack.png
+banner: /banners/self-hosting-mem0-memory-stack.png
+---
+
+An AI agent that forgets everything between conversations is a tool you have
+to reintroduce yourself to every session. My AI assistant (the one that runs
+this very blog's workflow) used to start each chat from zero — until I gave it
+a **memory layer**: a self-hosted [mem0](https://mem0.ai) stack that stores
+facts it learns, retrieves them semantically, and survives across sessions.
+
+This is the story of how I self-hosted it on my Synology NAS — private, fully
+under my control, no data leaving my LAN.
+
+## Why this matters
+
+A memory layer changes what an agent can actually *do* for a business:
+
+- **It stops re-answering the same questions.** Remember a customer's granted
+  trial, their product, their last support issue — and the next conversation
+  picks up where the last one ended.
+- **It makes "personalization" real.** A bot that recalls your customer's
+  preferences and history is a bot that feels like a person. That's not
+  prompt-sugar; it's a stored, searchable record.
+- **It keeps control of your data.** Every "memory" I dislike about hosted
+  memory services is solved by running the stack yourself — the facts live
+  in my own PostgreSQL, on my own NAS, behind my own network.
+
+For anyone running AI agents (support bots, research assistants, personal
+automation) the question isn't *should* the agent remember — it's whether the
+memory is a private asset or a third party's.
+
+## What mem0 does
+
+Mem0 is "memory in a box" for LLM apps. Instead of you hand-rolling an
+embeddings table and a similarity search, it gives you a small API that does
+the three things memory needs:
+
+1. **Add a memory** — `POST /memories` with a fact or a conversation; it
+   extracts, dedupes, and stores it (optionally classified by an LLM).
+2. **Search** — `GET /search` with a query returns the most relevant stored
+   facts, ranked by semantic similarity.
+3. **User scoping** — memories belong to a `user_id`, so you can keep memory
+   per customer, per project, or per conversation without cross-talk.
+
+The "memory" is the extracted fact. The "retrieval" is vector similarity. You
+get the agent even if you never touch a vector database yourself.
+
+## The stack
+
+Three containers on my Synology DS1821+, as a Portainer compose stack:
+
+- **mem0 API** — the memory service itself (custom local build), listening
+  on port `20015`.
+- **LiteLLM** — the LLM gateway in front of whatever model does the fact
+  extraction. I can swap the underlying model without touching mem0's config.
+- **PostgreSQL 17 + pgvector** — the database. `pgvector/pgvector` bundles
+  Postgres with the vector extension as a first-class citizen, so semantic
+  search lives in the same DB as everything else.
+
+```text
+┌──────────────┐      /search & /memories       ┌──────────────────┐
+│ AI agent /   │ ──── X-Api-Key + JSON ───────▶ │  mem0 API :20015 │
+│ Hermes / bot │                                └────────┬─────────┘
+└──────────────┘                                         │ pgvector
+                                                         ▼
+                                             PostgreSQL 17 + pgvector
+┌──────────────┐     model calls (extract)    ┌──────────────────┐
+│ OpenAI etc.  │ ◀───────── LiteLLM ──────────│ fact-classifier  │
+└──────────────┘                              └──────────────────┘
+```
+
+A representative compose looks like this:
+
+```yaml
+services:
+  mem0-postgres:
+    image: pgvector/pgvector:pg17
+    environment:
+      POSTGRES_USER: mem0
+      POSTGRES_PASSWORD: ${MEM0_DB_PASSWORD}
+      POSTGRES_DB: mem0
+    volumes:
+      - ./data/pg:/var/lib/postgresql/data
+    restart: unless-stopped
+
+  mem0-litellm:
+    image: ghcr.io/berriai/litellm:main-stable
+    command: ["--config", "/app/config.yaml"]
+    volumes:
+      - ./litellm-config.yaml:/app/config.yaml
+    restart: unless-stopped
+
+  mem0-api:
+    build: ./mem0-api
+    environment:
+      OPENAI_API_KEY: ${OPENAI_API_KEY}
+      OPENAI_API_BASE: http://mem0-litellm:4000/v1   # go through the gateway
+      MEM0_API_KEY: ${MEM0_API_KEY}                  # X-Api-Key for /search & /memories
+      POSTGRES_URL: postgresql://mem0:${MEM0_DB_PASSWORD}@mem0-postgres/mem0
+    ports:
+      - "20015:8000"
+    restart: unless-stopped
+```
+
+(Mine is built from the mem0 repo with the model/fact-extraction directed
+through LiteLLM; the exact image tag is me maintaining a local build, and I
+keep the real API key out of the compose file — it's an env secret.)
+
+## Using it
+
+Once it's up, the API is refreshingly small. Add a fact (fast, verbatim):
+
+```bash
+curl -X POST http://192.168.1.1:20015/memories \
+  -H "Content-Type: application/json" \
+  -H "X-Api-Key: $MEM0_API_KEY" \
+  -d '{
+    "user_id": "customer-2211",
+    "text": "Customer 2211 prefers email over WhatsApp for order updates."
+  }'
+```
+
+Retrieve the most relevant memories later:
+
+```bash
+curl -X POST http://192.168.1.1:20015/memories/search \
+  -H "Content-Type: application/json" \
+  -H "X-Api-Key: $MEM0_API_KEY" \
+  -d '{
+    "user_id": "customer-2211",
+    "query": "how should I reach this customer?",
+    "limit": 5
+  }'
+```
+
+Then inject the returned facts into the agent's system prompt (or context)
+before it answers. That's the whole loop: **store facts, retrieve what's
+relevant, use it in the next reply.**
+
+### The `infer` gotcha that cost me a day
+
+- `POST /memories` with the **default `infer=true`** runs the LLM fact
+  extractor — it classifies and cleans each memory. That's smart but **slow**,
+  and every call costs a model round-trip.
+- If I just want the agent to note something verbatim and fast (user said
+  "please call me Andy"), I pass **`infer=false`** — no LLM hop, near-instant
+  storage.
+
+The route of my stack is `/search` and `/memories` **without a `/v1` prefix**
+(an early wrapper that assumed `/v1` 404'd for a while), and auth is a plain
+`X-Api-Key` header. Small details, but they cost me time to discover.
+
+## What I'd do differently
+
+- **Set `infer=false` for routine logging from day one.** Only run the LLM
+  extractor on the facts that matter; the default inference mode makes every
+  write pricey for no benefit on throwaway notes.
+- **Treat the API key as a real secret.** It gets embedded in agent configs
+  and compose envs, so I keep it out of the compose file and inject it via
+  environment secrets.
+- **Put the DB on a disk I trust.** Mem0 is only as durable as its Postgres.
+  I run it on my NAS's protected volumes, with a backup job — because a
+  memory layer that forgets is worse than none.
+
+## The result
+
+A private, self-hosted memory layer answered by two small HTTP calls. My
+agent now remembers customers, preferences, and decisions across sessions —
+and because the stack sits in my own PostgreSQL behind my own network, none
+of that leaves my control.
+
+If you run AI agents that keep forgetting their context, self-hosting mem0
+is one of the highest-value, lowest-friction upgrades you can make — and you
+can do it without sending your memory to someone else's database.
+
+---
+
+## Want a memory-enabled AI setup for your business?
+
+I build and self-host AI agents, memory stacks, websites, and infrastructure
+for businesses. If you'd like your assistant to actually *remember* your
+customers — or you want to talk about hiring me — I'd love to help:
+
+- 📱 **WhatsApp:** [+60 12-797 2969](https://wa.me/60127972969)
+- 📧 **Email:** [[email protected]](mailto:[email protected])
+- 🌐 **Website:** [hoelee.com](https://hoelee.com)

src/content/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data.md

@@ -0,0 +1,170 @@
+---
+title: "When SMART Says Healthy and Your RAID Is Silently Corrupting Data"
+description: "A btrfs RAID1 pool on two healthy-looking NVMe drives quietly corrupted a Windows VM. How I caught it, the checksum forensic that proved it, and why I replaced both drives."
+pubDate: 2026-09-16
+category: devops
+tags: [unraid, btrfs, nvme, raid, storage, data-loss, troubleshooting]
+ogImage: /og/when-smart-says-healthy-but-your-raid-is-corrupting-data.png
+banner: /banners/when-smart-says-healthy-but-your-raid-is-corrupting-data.png
+---
+
+Every storage guide tells you the same thing: trust SMART, check the event
+log, and believe your filesystem. My hardware disagreed. In September 2026 a
+btrfs RAID1 pool — two mirrored enterprise NVMe drives that said **"healthy"**
+in every tool I could run — was quietly feeding my Windows 11 VM all-zero
+bytes. Not visibly failing. Silently corrupting.
+
+This is the full story: how the corruption surfaced, the checksum that gave
+it away, and why the answer was "replace both drives, don't trust SMART".
+
+## Why this matters
+
+A corrupting disk is the worst kind of storage failure because it's the
+easiest to miss:
+
+- **SMART looked perfect.** Zero media errors, ~100% spare, on drives rated
+  for heavy enterprise write loads.
+- **The pool had redundancy.** btrfs RAID1 mirrored every block across two
+  drives. Redundancy means nothing when *both* members lie the same way.
+- **The failure was silent.** No red warning in the Unraid array status, no
+  failing drive, no hardware event — just VM reads slowly returning garbage
+  until a backup restore became the only option.
+
+If your data only gets corrupted when nothing looks wrong, "backup" is not
+paranoia — it's the entire ballgame. This incident is why a testable,
+verified restore beats any array status screen.
+
+## The setup
+
+- **unRaid 7.x** on an Intel i9-13900K, with a Windows 11 VM as the primary
+  workstation (GPU passed through).
+- **Cache pool:** two **Samsung PM9A3 3.84 TB** NVMe drives in **btrfs RAID1**.
+  Both from the **same production batch** (same controller/firmware
+  revision, `GDC7702Q`).
+- The Windows VM's virtual disks (`vdisk1.img`, `vdisk2.img`) lived on that
+  pool, in the `domains/` share — plus **Docker's data root**.
+- The VM's vdisks were **not** `NOCOW` — so every read was validated against
+  a btrfs checksum. That setting ended up being the reason I caught it at all.
+
+## The symptom
+
+I started noticing cache reads were crawling, and host commands were stalling:
+`cat`, reading `/proc/loadavg`, even `docker image ls` were timing out. The
+corrupt NVMe was being hammered at **45–75 MB/s of reads**, and every read
+saturating the drive stalled the whole host's I/O. A `docker system prune`
+that should have finished in seconds just hung.
+
+Then the real problem surfaced: the Windows VM began throwing read errors and
+data that came back was **wrong** — files that were full of zero bytes,
+checksum failures in the guest, and ultimately an unbootable primary VM.
+
+## The forensic: `csum 0x8941f998`
+
+On btrfs, when a read fails verification the kernel logs a checksum mismatch
+for the affected extent. The same value kept recurring:
+
+```text
+btrfs: checksum verify failed on <device> wanted 0x8941f998 found <other>
+```
+
+That specific value was the first real clue. **`0x8941f998` is the CRC32C of
+a block of zeros.** It has to be — it came up every time, across unrelated
+files and inodes. What was happening wasn't random bit rot; the drive was
+returning **all-zero blocks**, consistently, and those zero-filled reads were
+exactly what btrfs was failing to verify:
+
+- Check the value: `crc32c` of a zero-filled extent matches `0x8941f998`.
+- It's the same value again and again → reads were **deterministically**
+  producing empty data, not occasionally flipping bits.
+
+Two VM vdisks were affected (`vdisk1.img` inode 2358574, `vdisk2.img` inode
+2362817), which fit the picture: a storage layer was returning zeros instead
+of the data it was asked to read.
+
+## The tell: self-heal "fixes" that never stuck
+
+btrfs RAID1 is supposed to be self-healing: read a bad block on one member,
+reconstruct it from the mirror, and rewrite the good data back. But as I
+scrubbed and re-read the same (still all-zero) data, it became clear the
+**rewrites were not sticking**. Each rebuild "succeeded" and the very next
+read of the same extent produced zeros again.
+
+That's the critical distinction:
+
+- A drive that only **fails to read** is a read problem — self-heal should
+  recover it from the mirror.
+- **A drive that also accepts and silently stores garbage** is a *write* or
+  *controller* problem — the mirror can't save you, because both paths agree
+  on the wrong answer.
+
+When self-heal rewrites keep not sticking, you're no longer in "one bad
+sector" territory. You're in "the device is lying about writes" territory,
+and RAID redundancy won't help.
+
+## Why both drives had to go
+
+The data pointed at a **hardware defect in the same-batch drives**, not a
+wear or one-off event:
+
+- SMART was clean (0 media errors, ~100% spare, and `nvme0` had written
+  roughly 586 TiB — heavy, but on a drive rated for enterprise workloads).
+- The two drives were **same-batch, same controller/firmware** (`GDC7702Q`).
+- Corruption hit **both** RAID1 members in correlated fashion — the whole
+  point of mirroring (survive one drive dying) failed because both were dying
+  the same way, in lockstep.
+
+With a correlated, batch-level defect, keeping one "healthy-looking" drive as
+the surviving member is a false economy. **I restored the Windows VM from
+backup and replaced both drives.**
+
+## What I did in parallel: move the data off that pool
+
+Before the drives were replaced, anything still worth reading had to come off
+the failing pool — and it had to be **verified**, not assumed:
+
+- **Docker's data root** moved off the cache: `docker` was relocated to
+  `/mnt/user/docker` (on `disk1`, overlay2). The whole tree was verified by
+  matching **path, size, and mtime** for all **3,997,569 files** against a
+  byte-comparison.
+- The cache's `isos/`, `system/`, and `swapfile` moved to `disk1`; the
+  `isos/` set (~13 GB) was checked with `diff -rq` → **EXIT=0**, no
+  differences.
+- Everything moved was compared against the source **before** the cache copy
+  was deleted — never delete a copy you haven't verified.
+
+The read-side corruption (all-zero blocks) and the write-side corruption
+(self-heal not sticking) meant I trusted nothing from that pool without a
+mirror, a checksum, or a fresh backup to compare against.
+
+## What I'd do differently
+
+1. **Test-restore the VM backup before needing it.** I'd verified backups
+   existed, but a corruption incident is not the time to first learn your
+   restore procedure. A scheduled test-restore catches both backup failure
+   *and* silent storage corruption.
+2. **Treat same-batch drives as one drive.** Both PM9A3s came from the same
+   batch with the same firmware. For a married cache pool (or any RAID1)
+   that's a correlated-failure risk — prefer drives of different production
+   batches or at least hammer-test the pair before trusting a mirror.
+3. **Detect, don't just hope.** btrfs checksums are what surfaced this
+   through the noise. The `NOCOW`-less vdisk config meant the filesystem was
+   validating every read; a pool of checksummed data is the difference
+   between "caught it and restored" and "shipped corrupted files for months".
+4. **Don't trust SMART for flash.** For a healthy-looking NVMe that still
+   corrupts, the event count says nothing — the firmware-level defect can be
+   invisible to every SMART attribute.
+
+## The result
+
+One Windows VM restored from backup, both confirmed-failing drives replaced,
+and Docker's data root safely off the pool. The cost was a few days of noise
+and a rescue effort — but it was contained because the corruption was caught
+by checksums, not wallowed in for weeks.
+
+The lesson that sticks: **with storage, "redundant" is only as good as your
+least-trustworthy member.** Two disks that fail together are not a mirror —
+they're a single point of failure wearing two serial numbers. Verify your
+restores, checksum your data, and when a drive's *rewards* of zeros keep
+reappearing, replace the whole set.
+
+---

src/content/posts/zh/self-hosting-mem0-memory-stack.md

@@ -0,0 +1,146 @@
+---
+title: "我如何在自家 NAS 上自托管 Mem0 记忆栈（让 AI 助手有记忆）"
+description: "一套私有、自托管的 AI 助手记忆层——mem0 API + LiteLLM + PostgreSQL/pgvector，跑在 Synology NAS 上。语义记忆由你掌控，而不是第三方手里。"
+pubDate: 2026-09-16
+category: case-studies
+tags: [mem0, docker, postgres, pgvector, litellm, ai, sharing]
+ogImage: /og/self-hosting-mem0-memory-stack.png
+banner: /banners/self-hosting-mem0-memory-stack.png
+---
+
+一个每次对话后都忘光一切的 AI 助手，就是一个每轮都要你重新自我介绍的工具。我的 AI 助手（就是驱动这个博客工作流的那个）曾经每场会话都从零开始——直到我给它加了一层**记忆**：一套自托管的 [mem0](https://mem0.ai) 栈，负责存储它学到的事实、按语义检索、并跨会话存活。
+
+这就是我在自家 Synology NAS 上自托管它的故事——私有、完全由我掌控、数据绝不离开我的局域网。
+
+## 为什么这件事重要
+
+记忆层改变的是一个助手真正能为企业做的事：
+
+- **不再重复回答同样的问题。** 记住客户获批的试用、他买的产品、上次的售后问题——下一段对话从上一次结束的地方继续。
+- **让「个性化」成真。** 一个记得客户偏好和历史记录的机器人，就是一个人。这不是提示词糖衣；这是一份存储的、可检索的记录。
+- **把数据掌控权留在自己手里。** 我对托管式记忆服务不满的每一点，自托管都解决了——事实存在我自己的 PostgreSQL 里、我自己的 NAS 上、我自己网络的后面。
+
+对任何跑 AI 助手（售后机器人、研究助手、个人自动化）的人来说，问题不是助手**要不要**记住——而是这份记忆是一份私有资产，还是第三方的数据。
+
+## mem0 做了什么
+
+Mem0 是给 LLM 应用用的「开箱即用的记忆」。你不用手搓嵌入表和相似度检索，它给一个小 API，把记忆需要的三件事都做了：
+
+1. **加一条记忆** —— `POST /memories` 传一条事实或一段对话；它抽取、去重并存储（可选由 LLM 分类）。
+2. **检索** —— `GET /search` 带一个 query，返回按语义相似度排序的相关事实。
+3. **按用户隔离** —— 记忆归属 `user_id`，所以可以按客户、按项目、按会话分开，互不串扰。
+
+「记忆」就是抽取出来的事实；「检索」就是向量相似度。哪怕你从没亲手碰过向量数据库，也能让助手有记忆。
+
+## 技术栈
+
+Synology DS1821+ 上用 Portainer compose 栈跑三个容器：
+
+- **mem0 API** —— 记忆服务本身（本地自定义构建），监听 `20015` 端口。
+- **LiteLLM** —— 真正做事实抽取的模型前面的 LLM 网关。不碰 mem0 配置就能换底层模型。
+- **PostgreSQL 17 + pgvector** —— 数据库。`pgvector/pgvector` 把 Postgres 和向量扩展打包成一等公民，语义搜索和别的一切同库。
+
+```text
+┌──────────────┐      /search & /memories       ┌──────────────────┐
+│ AI agent /   │ ──── X-Api-Key + JSON ───────▶ │  mem0 API :20015 │
+│ Hermes / bot │                                └────────┬─────────┘
+└──────────────┘                                         │ pgvector
+                                                         ▼
+                                             PostgreSQL 17 + pgvector
+┌──────────────┐     model calls (extract)    ┌──────────────────┐
+│ OpenAI etc.  │ ◀───────── LiteLLM ──────────│ fact-classifier  │
+└──────────────┘                              └──────────────────┘
+```
+
+一份有代表性的 compose 长这样：
+
+```yaml
+services:
+  mem0-postgres:
+    image: pgvector/pgvector:pg17
+    environment:
+      POSTGRES_USER: mem0
+      POSTGRES_PASSWORD: ${MEM0_DB_PASSWORD}
+      POSTGRES_DB: mem0
+    volumes:
+      - ./data/pg:/var/lib/postgresql/data
+    restart: unless-stopped
+
+  mem0-litellm:
+    image: ghcr.io/berriai/litellm:main-stable
+    command: ["--config", "/app/config.yaml"]
+    volumes:
+      - ./litellm-config.yaml:/app/config.yaml
+    restart: unless-stopped
+
+  mem0-api:
+    build: ./mem0-api
+    environment:
+      OPENAI_API_KEY: ${OPENAI_API_KEY}
+      OPENAI_API_BASE: http://mem0-litellm:4000/v1   # go through the gateway
+      MEM0_API_KEY: ${MEM0_API_KEY}                  # X-Api-Key for /search & /memories
+      POSTGRES_URL: postgresql://mem0:${MEM0_DB_PASSWORD}@mem0-postgres/mem0
+    ports:
+      - "20015:8000"
+    restart: unless-stopped
+```
+
+（我是从 mem0 仓库本地构建、把模型/事实抽取指向 LiteLLM；确切的镜像版本是我自己维护本地镜像的结果。真实 API key 我放在 compose 文件外面，走环境变量密文注入。）
+
+## 使用
+
+起起来之后，这个 API 小到意外。加一条事实（快速、逐字保存）：
+
+```bash
+curl -X POST http://192.168.1.1:20015/memories \
+  -H "Content-Type: application/json" \
+  -H "X-Api-Key: $MEM0_API_KEY" \
+  -d '{
+    "user_id": "customer-2211",
+    "text": "Customer 2211 prefers email over WhatsApp for order updates."
+  }'
+```
+
+之后再取最相关的记忆：
+
+```bash
+curl -X POST http://192.168.1.1:20015/memories/search \
+  -H "Content-Type: application/json" \
+  -H "X-Api-Key: $MEM0_API_KEY" \
+  -d '{
+    "user_id": "customer-2211",
+    "query": "how should I reach this customer?",
+    "limit": 5
+  }'
+```
+
+然后把返回的事实塞进助手回答前的系统提示词（或上下文）。这就是全部循环：**存事实、召回相关的、在下一句回复里用上。**
+
+### 让我花了一天时间的 `infer` 坑
+
+- `POST /memories` 默认 **`infer=true`** 会跑 LLM 事实抽取器——分类、清洗每条记忆。聪明但**慢**，而且每次调用都要一次模型往返。
+- 如果我只是想让助手**逐字快速地**记下某件事（客户说「请叫我 Andy」），我传 **`infer=false`**——不经过 LLM，近乎瞬时存储。
+
+我栈上的路由是 `/search` 和 `/memories`，**没有 `/v1` 前缀**（早期一个自以为有 `/v1` 的包装器 404 了一阵子），认证是朴实的 `X-Api-Key` 头。都是小事，但都让我费过时间。
+
+## 如果重来我会怎么做
+
+- **从第一天起对常规记录就用 `infer=false`。** 只对重要的事实跑 LLM 抽取；默认的推理模式让每一次写入都为了一句随手记付出的成本不值。
+- **把 API key 当真正的机密。** 它会嵌进 agent 配置和 compose 环境变量里，所以我把它从 compose 文件中拿出来，用环境密文注入。
+- **把数据库放在我信任的盘上。** Mem0 的下限就是你 Postgres 的下限。我把它跑在 NAS 受保护的卷上、带备份任务——因为一个会失忆的记忆层还不如没有。
+
+## 结果
+
+一层私有、自托管的记忆，用两个小 HTTP 调用就能访问。我的助手现在跨会话记得客户、偏好和决策——而且因为整个栈都在我自己网络后面的 PostgreSQL 里，这一切都没有离开我的掌控。
+
+如果你跑的 AI 助手动不动就忘掉上下文，自托管 mem0 是最划算、最省事的升级之一——而且你能在不把记忆送进别人数据库的前提下做到。
+
+---
+
+## 想让你的生意也用上带记忆的 AI 吗？
+
+我为企业构建并自托管 AI 助手、记忆栈、网站和基础设施。如果你想让助手真正**记住**你的客户——或者你想聊聊雇佣我——我很乐意帮上忙：
+
+- 📱 **WhatsApp：** [+60 12-797 2969](https://wa.me/60127972969)
+- 📧 **邮箱：** [[email protected]](mailto:[email protected])
+- 🌐 **网站：** [hoelee.com](https://hoelee.com)

src/content/posts/zh/when-smart-says-healthy-but-your-raid-is-corrupting-data.md

@@ -0,0 +1,97 @@
+---
+title: "SMART 显示健康，RAID 却在悄悄损坏数据"
+description: "两块看起来完全健康的 NVMe 组成的 btrfs RAID1 阵列，却静默地把 Windows 虚拟机喂成了全零字节。我如何抓住它、靠校验和取证证明它，以及为什么最终要把两块盘都换掉。"
+pubDate: 2026-09-16
+category: devops
+tags: [unraid, btrfs, nvme, raid, storage, data-loss, troubleshooting]
+ogImage: /og/when-smart-says-healthy-but-your-raid-is-corrupting-data.png
+banner: /banners/when-smart-says-healthy-but-your-raid-is-corrupting-data.png
+---
+
+所有存储指南都告诉你同一件事：相信 SMART、检查事件日志、相信你的文件系统。我的硬件不这么想。2026 年 9 月，一个 btrfs RAID1 阵列——两块企业级 NVMe 盘镜像，我用每个能想到的工具查它都说**「健康」**——却在悄悄地把我的 Windows 11 虚拟机喂成全零字节。不是看得见的故障，是静默损坏。
+
+下面是完整故事：损坏是怎么浮现的、哪个校验和出卖了它、以及为什么答案必须是「两块盘都换，别信 SMART」。
+
+## 为什么这件事重要
+
+损坏状态的磁盘是最糟的存储故障，因为它最容易漏掉：
+
+- **SMART 看起来完美。** 零媒体错误、几乎满余量，而且是能扛企业级写入负载的盘。
+- **阵列有冗余。** btrfs RAID1 把每个块镜像到两块盘上。当**两个成员**用同一种方式撒谎时，冗余毫无意义。
+- **故障是静默的。** 数组状态没有红色警告、没有硬盘报错、没有硬件事件——只有虚拟机读回垃圾，直到备份恢复成了唯一的选择。
+
+如果你的数据在一切看起来都正常的时候悄悄坏掉，「备份」就不是偏执——它是全部。这次事故让我明白：一条**可测试、可验证的恢复**胜过任何阵列状态屏幕。
+
+## 环境
+
+- **unRaid 7.x**，Intel i9-13900K，Windows 11 虚拟机是主力工作站（GPU 直通）。
+- **缓存池：** 两块 **Samsung PM9A3 3.84TB** NVMe 组 **btrfs RAID1**。而且两块是**同一生产批次**（同一个控制器/固件版本 `GDC7702Q`）。
+- Windows 虚拟机的虚拟磁盘（`vdisk1.img`、`vdisk2.img`）就在这个池的 `domains/` 共享里，另外还有 **Docker 数据根目录**。
+- 虚拟磁盘**没有**设 `NOCOW`——所以每次读取都会被 btrfs 校验和验证。也正因为这个，我最后才抓得住它。
+
+## 症状
+
+我开始发现缓存读取慢得离谱，主机命令也卡住：`cat`、读 `/proc/loadavg`、甚至 `docker image ls` 都在超时。那块坏 NVMe 正以 **45–75 MB/s** 被狂读，每次读都塞满整台主机的 I/O。本该几秒跑完的 `docker system prune` 直接挂死。
+
+然后真正的麻烦来了：Windows 虚拟机开始狂报读错误，回来的是**错的**数据——满文件的全零字节、guest 里的校验和失败，最后主力 VM 直接引导不了。
+
+## 取证：`csum 0x8941f998`
+
+btrfs 在读校验失败时会往内核日志记一条校验和不匹配。同一个值反复出现：
+
+```text
+btrfs: checksum verify failed on <device> wanted 0x8941f998 found <other>
+```
+
+这个特定值是最重要的线索。**`0x8941f998` 就是一整块零字节的 CRC32C。** 它每次都出现、跨越互不相关的文件和 inode，绝不是巧合。正在发生的不是随机比特腐烂，而是盘**稳定地返回全零块**，而这些零填充的读取正是 btrfs 校验失败的对象：
+
+- 验证一下：零填充块的 `crc32c` 就等于 `0x8941f998`。
+- 同一个值反复出现 → 读取是**确定性**地吐出空数据，不是偶尔翻几个比特。
+
+两块虚拟磁盘都中招（`vdisk1.img` inode 2358574、`vdisk2.img` inode 2362817），正好对上「存储层在把零当成被请求的数据返回」的图景。
+
+## 破绽：永远不生效的「自愈」
+
+btrfs RAID1 本应自愈：一个成员坏块、从镜像重建、写回好数据。但我一遍遍 scrub、重读同一块（仍然全零的）数据后发现，**重写根本不落地**。每次重建「成功」后，立刻重读同一 extent 又吐出零。
+
+这是关键区别：
+
+- 一块**只读不出来**的盘是读问题——自愈应该能从镜像救回来。
+- 一块**连回调也无脑吞垃圾存进去**的盘是**写或控制器**问题——镜像救不了你，因为两条路径都对同一个错误答案达成一致。
+
+当自愈重写一直不生效，你就不在「坏一个扇区」的范畴了，而是「这块盘在写入上撒谎」——RAID 冗余救不了。
+
+## 为什么两块盘都得换
+
+数据指向的是**同批次盘的硬件缺陷**，而不是磨损或一次性的偶发事件：
+
+- SMART 干净（0 媒体错误、几乎满余量，`nvme0` 大约写了 586 TiB——对能扛企业级负载的盘来说算重负载但仍正常）。
+- 两块盘**同批次、同控制器/固件**（`GDC7702Q`）。
+- 损坏以**相关的方式**同时命中两个 RAID1 成员——镜像「死一块也能幸存」的整个意义，因为两块盘正以同一种方式一起死掉而完全失效。
+
+面对批次级的关联缺陷，留下那块「看起来健康」的盘当幸存成员是假省心。**我从备份恢复了 Windows 虚拟机，然后把两块盘都换掉。**
+
+## 我同时做的事：把数据迁出那个池
+
+在换盘之前，任何还值得读的数据都得搬出这个坏池——而且要**验证**，不能想当然：
+
+- **Docker 数据根目录**迁出缓存：`docker` 挪到 `/mnt/user/docker`（在 `disk1` 上，overlay2）。整棵树通过**路径、大小、mtime** 逐文件比对、并与字节对比交叉验证，共 **3,997,569** 个文件。
+- 缓存的 `isos/`、`system/`、`swapfile` 搬到 `disk1`；`isos/`（约 13 GB）用 `diff -rq` 检查 → **EXIT=0**，无差异。
+- 所有搬走的文件都在**删除缓存副本之前**先跟源比对过——没验证过的副本绝不删。
+
+因为存在读侧损坏（全零块）和写侧损坏（自愈不落地），那个池里的任何东西，没有镜像、没有校验和、没有新鲜备份可对照，我就一律不信任。
+
+## 如果重来我会怎么做
+
+1. **在需要它之前就先测试恢复虚拟机备份。** 我当时确认了备份存在，但损坏事故不是该第一次学恢复流程的时候。定时做的测试恢复既能发现备份问题，也能发现静默的存储损坏。
+2. **把同批次的两块盘当成一块盘。** 两块 PM9A3 同批次同固件。对需要长期搭档的缓存池（任何 RAID1）这是关联失效风险——优先选不同批次的盘，或者至少在信任镜像前把这对盘狠狠测一遍。
+3. **主动检测，而不是被动祈祷。** 正是 btrfs 校验和让它在噪声里浮出来。不开 `NOCOW` 意味着文件系统在验证每一次读取；一个带校验的数据池，就是「当场抓住并恢复」和「把损坏文件带出去几个月」之间的分水岭。
+4. **别用 SMART 判断闪存。** 一块看起来健康却照样损坏的 NVMe，事件计数什么也说明不了——固件级的缺陷可以躲过所有 SMART 属性。
+
+## 结果
+
+一台 Windows 虚拟机从备份恢复、两块确认故障的盘换掉、Docker 数据根目录安全迁出该池。代价是几天折腾和一次救援——但损失被控制在最小，因为损坏是被校验和抓住的，而不是在里面泡了几个星期。
+
+最该记住的一句话：**对存储来说，「冗余」只等于你最不靠谱的那个成员。** 两块一起坏掉的盘不是镜像——那是穿着两个序列号的单点故障。验证你的恢复、校验你的数据、当某块盘的零块「奖励」反复出现时，把整组都换掉。
+
+---