diff --git a/public/banners/self-hosting-mem0-memory-stack.png b/public/banners/self-hosting-mem0-memory-stack.png new file mode 100644 index 0000000..e26f88f Binary files /dev/null and b/public/banners/self-hosting-mem0-memory-stack.png differ diff --git a/public/banners/when-smart-says-healthy-but-your-raid-is-corrupting-data.png b/public/banners/when-smart-says-healthy-but-your-raid-is-corrupting-data.png new file mode 100644 index 0000000..4322db1 Binary files /dev/null and b/public/banners/when-smart-says-healthy-but-your-raid-is-corrupting-data.png differ diff --git a/public/og/self-hosting-mem0-memory-stack.png b/public/og/self-hosting-mem0-memory-stack.png new file mode 100644 index 0000000..096bdd6 Binary files /dev/null and b/public/og/self-hosting-mem0-memory-stack.png differ diff --git a/public/og/when-smart-says-healthy-but-your-raid-is-corrupting-data.png b/public/og/when-smart-says-healthy-but-your-raid-is-corrupting-data.png new file mode 100644 index 0000000..4e37ef6 Binary files /dev/null and b/public/og/when-smart-says-healthy-but-your-raid-is-corrupting-data.png differ diff --git a/scripts/banner-gen/generate.mjs b/scripts/banner-gen/generate.mjs index 7ff43c0..206ace4 100644 --- a/scripts/banner-gen/generate.mjs +++ b/scripts/banner-gen/generate.mjs @@ -133,6 +133,44 @@ const BANNERS = { ], }, + 'when-smart-says-healthy-but-your-raid-is-corrupting-data': { + titlebar: 'root@unraid — cache pool scrub', + lines: [ + { t: 'dim', text: 'btrfs RAID1 · 2× Samsung PM9A3 NVMe · same batch' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'btrfs scrub /mnt/cache' }, + { t: 'err', text: 'csum 0x8941f998 recurring = CRC32C(zero block)' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'self-heal rewrite → does NOT stick' }, + { t: 'err', text: 'write-path corruption · SMART stays clean' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'fix = restore VM + replace both drives' }, + ], + flow: [ + { n: '1', label: 'SMART clean' }, + { n: '2', label: 'csum zeros', err: true }, + { n: '3', label: 'self-heal no-stick' }, + { n: '4', label: 'restore VM' }, + { n: '5', label: 'replace both ✓' }, + ], + }, + + 'self-hosting-mem0-memory-stack': { + titlebar: 'root@dsm — mem0 memory stack', + lines: [ + { t: 'prompt', text: '$' }, { t: 'cmd', text: "curl -X POST :20015/memories -H X-Api-Key" }, + { t: 'dim', text: 'user_id + text → mem0 extracts & stores' }, + { t: 'prompt', text: '¶' }, { t: 'ok', text: '→ remembers across sessions ✓' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'stack = mem0-api + LiteLLM + pgvector(pg17)' }, + { t: 'err', text: 'infer=true → LLM hop · slow writes' }, + { t: 'prompt', text: '' }, { t: 'ok', text: 'self-hosted · data stays on LAN ✓' }, + ], + flow: [ + { n: '1', label: 'POST /memories' }, + { n: '2', label: 'store fact' }, + { n: '3', label: 'search' }, + { n: '4', label: 'feed context' }, + { n: '5', label: 'remembers ✓' }, + ], + }, + 'hardening-a-tor-onion-service': { titlebar: 'root@tor-host — onion service audit', lines: [ diff --git a/scripts/og-gen/generate.mjs b/scripts/og-gen/generate.mjs index d984b2e..1c2b4d6 100644 --- a/scripts/og-gen/generate.mjs +++ b/scripts/og-gen/generate.mjs @@ -125,10 +125,20 @@ const TERMINALS = {
$warm session · 7s pacing · sweep.py→ 20+ listings ✓
`, 'unraid-stop-array-hangs-on-swapfile': ` -
$unraid → stop array · swapfile on /mnt/cache
-
 umount: target is busy · /proc/swaps says /dev/loop0
-
$swapoff -a + losetup -j at stopping_svcs→ clean stop ✓
`, -}; +
$unraid → stop array · swapfile on /mnt/cache
+
 umount: target is busy · /proc/swaps says /dev/loop0
+
$swapoff -a + losetup -j at stopping_svcs→ clean stop ✓
`, + + 'when-smart-says-healthy-but-your-raid-is-corrupting-data': ` +
$scrub → re-read vdisk1.img
+
 csum 0x8941f998 = CRC32C(zeros) · recurring
+
$self-heal rewrite does NOT stick→ replace both drives ✓
`, + + 'self-hosting-mem0-memory-stack': ` +
$curl -X POST :20015/memories · X-Api-Key
+
 infer=true → LLM hop · slow write
+
$infer=false · pgvector · LiteLLM gateway→ remembers across chats ✓
`, + }; const DEFAULT_TERMINAL = `
$engineering · devops · self-hosting
diff --git a/src/content/posts/self-hosting-mem0-memory-stack.md b/src/content/posts/self-hosting-mem0-memory-stack.md new file mode 100644 index 0000000..176680d --- /dev/null +++ b/src/content/posts/self-hosting-mem0-memory-stack.md @@ -0,0 +1,192 @@ +--- +title: "How I Self-Hosted a Mem0 Memory Stack (So My AI Agent Remembers)" +description: "A private, self-hosted memory layer for AI agents — mem0 API + LiteLLM + PostgreSQL with pgvector on a Synology NAS. Semantic memory under your control, not a third party's." +pubDate: 2026-09-16 +category: case-studies +tags: [mem0, docker, postgres, pgvector, litellm, ai, sharing] +ogImage: /og/self-hosting-mem0-memory-stack.png +banner: /banners/self-hosting-mem0-memory-stack.png +--- + +An AI agent that forgets everything between conversations is a tool you have +to reintroduce yourself to every session. My AI assistant (the one that runs +this very blog's workflow) used to start each chat from zero — until I gave it +a **memory layer**: a self-hosted [mem0](https://mem0.ai) stack that stores +facts it learns, retrieves them semantically, and survives across sessions. + +This is the story of how I self-hosted it on my Synology NAS — private, fully +under my control, no data leaving my LAN. + +## Why this matters + +A memory layer changes what an agent can actually *do* for a business: + +- **It stops re-answering the same questions.** Remember a customer's granted + trial, their product, their last support issue — and the next conversation + picks up where the last one ended. +- **It makes "personalization" real.** A bot that recalls your customer's + preferences and history is a bot that feels like a person. That's not + prompt-sugar; it's a stored, searchable record. +- **It keeps control of your data.** Every "memory" I dislike about hosted + memory services is solved by running the stack yourself — the facts live + in my own PostgreSQL, on my own NAS, behind my own network. + +For anyone running AI agents (support bots, research assistants, personal +automation) the question isn't *should* the agent remember — it's whether the +memory is a private asset or a third party's. + +## What mem0 does + +Mem0 is "memory in a box" for LLM apps. Instead of you hand-rolling an +embeddings table and a similarity search, it gives you a small API that does +the three things memory needs: + +1. **Add a memory** — `POST /memories` with a fact or a conversation; it + extracts, dedupes, and stores it (optionally classified by an LLM). +2. **Search** — `GET /search` with a query returns the most relevant stored + facts, ranked by semantic similarity. +3. **User scoping** — memories belong to a `user_id`, so you can keep memory + per customer, per project, or per conversation without cross-talk. + +The "memory" is the extracted fact. The "retrieval" is vector similarity. You +get the agent even if you never touch a vector database yourself. + +## The stack + +Three containers on my Synology DS1821+, as a Portainer compose stack: + +- **mem0 API** — the memory service itself (custom local build), listening + on port `20015`. +- **LiteLLM** — the LLM gateway in front of whatever model does the fact + extraction. I can swap the underlying model without touching mem0's config. +- **PostgreSQL 17 + pgvector** — the database. `pgvector/pgvector` bundles + Postgres with the vector extension as a first-class citizen, so semantic + search lives in the same DB as everything else. + +```text +┌──────────────┐ /search & /memories ┌──────────────────┐ +│ AI agent / │ ──── X-Api-Key + JSON ───────▶ │ mem0 API :20015 │ +│ Hermes / bot │ └────────┬─────────┘ +└──────────────┘ │ pgvector + ▼ + PostgreSQL 17 + pgvector +┌──────────────┐ model calls (extract) ┌──────────────────┐ +│ OpenAI etc. │ ◀───────── LiteLLM ──────────│ fact-classifier │ +└──────────────┘ └──────────────────┘ +``` + +A representative compose looks like this: + +```yaml +services: + mem0-postgres: + image: pgvector/pgvector:pg17 + environment: + POSTGRES_USER: mem0 + POSTGRES_PASSWORD: ${MEM0_DB_PASSWORD} + POSTGRES_DB: mem0 + volumes: + - ./data/pg:/var/lib/postgresql/data + restart: unless-stopped + + mem0-litellm: + image: ghcr.io/berriai/litellm:main-stable + command: ["--config", "/app/config.yaml"] + volumes: + - ./litellm-config.yaml:/app/config.yaml + restart: unless-stopped + + mem0-api: + build: ./mem0-api + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY} + OPENAI_API_BASE: http://mem0-litellm:4000/v1 # go through the gateway + MEM0_API_KEY: ${MEM0_API_KEY} # X-Api-Key for /search & /memories + POSTGRES_URL: postgresql://mem0:${MEM0_DB_PASSWORD}@mem0-postgres/mem0 + ports: + - "20015:8000" + restart: unless-stopped +``` + +(Mine is built from the mem0 repo with the model/fact-extraction directed +through LiteLLM; the exact image tag is me maintaining a local build, and I +keep the real API key out of the compose file — it's an env secret.) + +## Using it + +Once it's up, the API is refreshingly small. Add a fact (fast, verbatim): + +```bash +curl -X POST http://192.168.1.1:20015/memories \ + -H "Content-Type: application/json" \ + -H "X-Api-Key: $MEM0_API_KEY" \ + -d '{ + "user_id": "customer-2211", + "text": "Customer 2211 prefers email over WhatsApp for order updates." + }' +``` + +Retrieve the most relevant memories later: + +```bash +curl -X POST http://192.168.1.1:20015/memories/search \ + -H "Content-Type: application/json" \ + -H "X-Api-Key: $MEM0_API_KEY" \ + -d '{ + "user_id": "customer-2211", + "query": "how should I reach this customer?", + "limit": 5 + }' +``` + +Then inject the returned facts into the agent's system prompt (or context) +before it answers. That's the whole loop: **store facts, retrieve what's +relevant, use it in the next reply.** + +### The `infer` gotcha that cost me a day + +- `POST /memories` with the **default `infer=true`** runs the LLM fact + extractor — it classifies and cleans each memory. That's smart but **slow**, + and every call costs a model round-trip. +- If I just want the agent to note something verbatim and fast (user said + "please call me Andy"), I pass **`infer=false`** — no LLM hop, near-instant + storage. + +The route of my stack is `/search` and `/memories` **without a `/v1` prefix** +(an early wrapper that assumed `/v1` 404'd for a while), and auth is a plain +`X-Api-Key` header. Small details, but they cost me time to discover. + +## What I'd do differently + +- **Set `infer=false` for routine logging from day one.** Only run the LLM + extractor on the facts that matter; the default inference mode makes every + write pricey for no benefit on throwaway notes. +- **Treat the API key as a real secret.** It gets embedded in agent configs + and compose envs, so I keep it out of the compose file and inject it via + environment secrets. +- **Put the DB on a disk I trust.** Mem0 is only as durable as its Postgres. + I run it on my NAS's protected volumes, with a backup job — because a + memory layer that forgets is worse than none. + +## The result + +A private, self-hosted memory layer answered by two small HTTP calls. My +agent now remembers customers, preferences, and decisions across sessions — +and because the stack sits in my own PostgreSQL behind my own network, none +of that leaves my control. + +If you run AI agents that keep forgetting their context, self-hosting mem0 +is one of the highest-value, lowest-friction upgrades you can make — and you +can do it without sending your memory to someone else's database. + +--- + +## Want a memory-enabled AI setup for your business? + +I build and self-host AI agents, memory stacks, websites, and infrastructure +for businesses. If you'd like your assistant to actually *remember* your +customers — or you want to talk about hiring me — I'd love to help: + +- 📱 **WhatsApp:** [+60 12-797 2969](https://wa.me/60127972969) +- 📧 **Email:** [me@hoelee.com](mailto:me@hoelee.com) +- 🌐 **Website:** [hoelee.com](https://hoelee.com) \ No newline at end of file diff --git a/src/content/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data.md b/src/content/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data.md new file mode 100644 index 0000000..ebf1772 --- /dev/null +++ b/src/content/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data.md @@ -0,0 +1,170 @@ +--- +title: "When SMART Says Healthy and Your RAID Is Silently Corrupting Data" +description: "A btrfs RAID1 pool on two healthy-looking NVMe drives quietly corrupted a Windows VM. How I caught it, the checksum forensic that proved it, and why I replaced both drives." +pubDate: 2026-09-16 +category: devops +tags: [unraid, btrfs, nvme, raid, storage, data-loss, troubleshooting] +ogImage: /og/when-smart-says-healthy-but-your-raid-is-corrupting-data.png +banner: /banners/when-smart-says-healthy-but-your-raid-is-corrupting-data.png +--- + +Every storage guide tells you the same thing: trust SMART, check the event +log, and believe your filesystem. My hardware disagreed. In September 2026 a +btrfs RAID1 pool — two mirrored enterprise NVMe drives that said **"healthy"** +in every tool I could run — was quietly feeding my Windows 11 VM all-zero +bytes. Not visibly failing. Silently corrupting. + +This is the full story: how the corruption surfaced, the checksum that gave +it away, and why the answer was "replace both drives, don't trust SMART". + +## Why this matters + +A corrupting disk is the worst kind of storage failure because it's the +easiest to miss: + +- **SMART looked perfect.** Zero media errors, ~100% spare, on drives rated + for heavy enterprise write loads. +- **The pool had redundancy.** btrfs RAID1 mirrored every block across two + drives. Redundancy means nothing when *both* members lie the same way. +- **The failure was silent.** No red warning in the Unraid array status, no + failing drive, no hardware event — just VM reads slowly returning garbage + until a backup restore became the only option. + +If your data only gets corrupted when nothing looks wrong, "backup" is not +paranoia — it's the entire ballgame. This incident is why a testable, +verified restore beats any array status screen. + +## The setup + +- **unRaid 7.x** on an Intel i9-13900K, with a Windows 11 VM as the primary + workstation (GPU passed through). +- **Cache pool:** two **Samsung PM9A3 3.84 TB** NVMe drives in **btrfs RAID1**. + Both from the **same production batch** (same controller/firmware + revision, `GDC7702Q`). +- The Windows VM's virtual disks (`vdisk1.img`, `vdisk2.img`) lived on that + pool, in the `domains/` share — plus **Docker's data root**. +- The VM's vdisks were **not** `NOCOW` — so every read was validated against + a btrfs checksum. That setting ended up being the reason I caught it at all. + +## The symptom + +I started noticing cache reads were crawling, and host commands were stalling: +`cat`, reading `/proc/loadavg`, even `docker image ls` were timing out. The +corrupt NVMe was being hammered at **45–75 MB/s of reads**, and every read +saturating the drive stalled the whole host's I/O. A `docker system prune` +that should have finished in seconds just hung. + +Then the real problem surfaced: the Windows VM began throwing read errors and +data that came back was **wrong** — files that were full of zero bytes, +checksum failures in the guest, and ultimately an unbootable primary VM. + +## The forensic: `csum 0x8941f998` + +On btrfs, when a read fails verification the kernel logs a checksum mismatch +for the affected extent. The same value kept recurring: + +```text +btrfs: checksum verify failed on wanted 0x8941f998 found +``` + +That specific value was the first real clue. **`0x8941f998` is the CRC32C of +a block of zeros.** It has to be — it came up every time, across unrelated +files and inodes. What was happening wasn't random bit rot; the drive was +returning **all-zero blocks**, consistently, and those zero-filled reads were +exactly what btrfs was failing to verify: + +- Check the value: `crc32c` of a zero-filled extent matches `0x8941f998`. +- It's the same value again and again → reads were **deterministically** + producing empty data, not occasionally flipping bits. + +Two VM vdisks were affected (`vdisk1.img` inode 2358574, `vdisk2.img` inode +2362817), which fit the picture: a storage layer was returning zeros instead +of the data it was asked to read. + +## The tell: self-heal "fixes" that never stuck + +btrfs RAID1 is supposed to be self-healing: read a bad block on one member, +reconstruct it from the mirror, and rewrite the good data back. But as I +scrubbed and re-read the same (still all-zero) data, it became clear the +**rewrites were not sticking**. Each rebuild "succeeded" and the very next +read of the same extent produced zeros again. + +That's the critical distinction: + +- A drive that only **fails to read** is a read problem — self-heal should + recover it from the mirror. +- **A drive that also accepts and silently stores garbage** is a *write* or + *controller* problem — the mirror can't save you, because both paths agree + on the wrong answer. + +When self-heal rewrites keep not sticking, you're no longer in "one bad +sector" territory. You're in "the device is lying about writes" territory, +and RAID redundancy won't help. + +## Why both drives had to go + +The data pointed at a **hardware defect in the same-batch drives**, not a +wear or one-off event: + +- SMART was clean (0 media errors, ~100% spare, and `nvme0` had written + roughly 586 TiB — heavy, but on a drive rated for enterprise workloads). +- The two drives were **same-batch, same controller/firmware** (`GDC7702Q`). +- Corruption hit **both** RAID1 members in correlated fashion — the whole + point of mirroring (survive one drive dying) failed because both were dying + the same way, in lockstep. + +With a correlated, batch-level defect, keeping one "healthy-looking" drive as +the surviving member is a false economy. **I restored the Windows VM from +backup and replaced both drives.** + +## What I did in parallel: move the data off that pool + +Before the drives were replaced, anything still worth reading had to come off +the failing pool — and it had to be **verified**, not assumed: + +- **Docker's data root** moved off the cache: `docker` was relocated to + `/mnt/user/docker` (on `disk1`, overlay2). The whole tree was verified by + matching **path, size, and mtime** for all **3,997,569 files** against a + byte-comparison. +- The cache's `isos/`, `system/`, and `swapfile` moved to `disk1`; the + `isos/` set (~13 GB) was checked with `diff -rq` → **EXIT=0**, no + differences. +- Everything moved was compared against the source **before** the cache copy + was deleted — never delete a copy you haven't verified. + +The read-side corruption (all-zero blocks) and the write-side corruption +(self-heal not sticking) meant I trusted nothing from that pool without a +mirror, a checksum, or a fresh backup to compare against. + +## What I'd do differently + +1. **Test-restore the VM backup before needing it.** I'd verified backups + existed, but a corruption incident is not the time to first learn your + restore procedure. A scheduled test-restore catches both backup failure + *and* silent storage corruption. +2. **Treat same-batch drives as one drive.** Both PM9A3s came from the same + batch with the same firmware. For a married cache pool (or any RAID1) + that's a correlated-failure risk — prefer drives of different production + batches or at least hammer-test the pair before trusting a mirror. +3. **Detect, don't just hope.** btrfs checksums are what surfaced this + through the noise. The `NOCOW`-less vdisk config meant the filesystem was + validating every read; a pool of checksummed data is the difference + between "caught it and restored" and "shipped corrupted files for months". +4. **Don't trust SMART for flash.** For a healthy-looking NVMe that still + corrupts, the event count says nothing — the firmware-level defect can be + invisible to every SMART attribute. + +## The result + +One Windows VM restored from backup, both confirmed-failing drives replaced, +and Docker's data root safely off the pool. The cost was a few days of noise +and a rescue effort — but it was contained because the corruption was caught +by checksums, not wallowed in for weeks. + +The lesson that sticks: **with storage, "redundant" is only as good as your +least-trustworthy member.** Two disks that fail together are not a mirror — +they're a single point of failure wearing two serial numbers. Verify your +restores, checksum your data, and when a drive's *rewards* of zeros keep +reappearing, replace the whole set. + +--- \ No newline at end of file diff --git a/src/content/posts/zh/self-hosting-mem0-memory-stack.md b/src/content/posts/zh/self-hosting-mem0-memory-stack.md new file mode 100644 index 0000000..05d4afc --- /dev/null +++ b/src/content/posts/zh/self-hosting-mem0-memory-stack.md @@ -0,0 +1,146 @@ +--- +title: "我如何在自家 NAS 上自托管 Mem0 记忆栈（让 AI 助手有记忆）" +description: "一套私有、自托管的 AI 助手记忆层——mem0 API + LiteLLM + PostgreSQL/pgvector，跑在 Synology NAS 上。语义记忆由你掌控，而不是第三方手里。" +pubDate: 2026-09-16 +category: case-studies +tags: [mem0, docker, postgres, pgvector, litellm, ai, sharing] +ogImage: /og/self-hosting-mem0-memory-stack.png +banner: /banners/self-hosting-mem0-memory-stack.png +--- + +一个每次对话后都忘光一切的 AI 助手，就是一个每轮都要你重新自我介绍的工具。我的 AI 助手（就是驱动这个博客工作流的那个）曾经每场会话都从零开始——直到我给它加了一层**记忆**：一套自托管的 [mem0](https://mem0.ai) 栈，负责存储它学到的事实、按语义检索、并跨会话存活。 + +这就是我在自家 Synology NAS 上自托管它的故事——私有、完全由我掌控、数据绝不离开我的局域网。 + +## 为什么这件事重要 + +记忆层改变的是一个助手真正能为企业做的事： + +- **不再重复回答同样的问题。** 记住客户获批的试用、他买的产品、上次的售后问题——下一段对话从上一次结束的地方继续。 +- **让「个性化」成真。** 一个记得客户偏好和历史记录的机器人，就是一个人。这不是提示词糖衣；这是一份存储的、可检索的记录。 +- **把数据掌控权留在自己手里。** 我对托管式记忆服务不满的每一点，自托管都解决了——事实存在我自己的 PostgreSQL 里、我自己的 NAS 上、我自己网络的后面。 + +对任何跑 AI 助手（售后机器人、研究助手、个人自动化）的人来说，问题不是助手**要不要**记住——而是这份记忆是一份私有资产，还是第三方的数据。 + +## mem0 做了什么 + +Mem0 是给 LLM 应用用的「开箱即用的记忆」。你不用手搓嵌入表和相似度检索，它给一个小 API，把记忆需要的三件事都做了： + +1. **加一条记忆** —— `POST /memories` 传一条事实或一段对话；它抽取、去重并存储（可选由 LLM 分类）。 +2. **检索** —— `GET /search` 带一个 query，返回按语义相似度排序的相关事实。 +3. **按用户隔离** —— 记忆归属 `user_id`，所以可以按客户、按项目、按会话分开，互不串扰。 + +「记忆」就是抽取出来的事实；「检索」就是向量相似度。哪怕你从没亲手碰过向量数据库，也能让助手有记忆。 + +## 技术栈 + +Synology DS1821+ 上用 Portainer compose 栈跑三个容器： + +- **mem0 API** —— 记忆服务本身（本地自定义构建），监听 `20015` 端口。 +- **LiteLLM** —— 真正做事实抽取的模型前面的 LLM 网关。不碰 mem0 配置就能换底层模型。 +- **PostgreSQL 17 + pgvector** —— 数据库。`pgvector/pgvector` 把 Postgres 和向量扩展打包成一等公民，语义搜索和别的一切同库。 + +```text +┌──────────────┐ /search & /memories ┌──────────────────┐ +│ AI agent / │ ──── X-Api-Key + JSON ───────▶ │ mem0 API :20015 │ +│ Hermes / bot │ └────────┬─────────┘ +└──────────────┘ │ pgvector + ▼ + PostgreSQL 17 + pgvector +┌──────────────┐ model calls (extract) ┌──────────────────┐ +│ OpenAI etc. │ ◀───────── LiteLLM ──────────│ fact-classifier │ +└──────────────┘ └──────────────────┘ +``` + +一份有代表性的 compose 长这样： + +```yaml +services: + mem0-postgres: + image: pgvector/pgvector:pg17 + environment: + POSTGRES_USER: mem0 + POSTGRES_PASSWORD: ${MEM0_DB_PASSWORD} + POSTGRES_DB: mem0 + volumes: + - ./data/pg:/var/lib/postgresql/data + restart: unless-stopped + + mem0-litellm: + image: ghcr.io/berriai/litellm:main-stable + command: ["--config", "/app/config.yaml"] + volumes: + - ./litellm-config.yaml:/app/config.yaml + restart: unless-stopped + + mem0-api: + build: ./mem0-api + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY} + OPENAI_API_BASE: http://mem0-litellm:4000/v1 # go through the gateway + MEM0_API_KEY: ${MEM0_API_KEY} # X-Api-Key for /search & /memories + POSTGRES_URL: postgresql://mem0:${MEM0_DB_PASSWORD}@mem0-postgres/mem0 + ports: + - "20015:8000" + restart: unless-stopped +``` + +（我是从 mem0 仓库本地构建、把模型/事实抽取指向 LiteLLM；确切的镜像版本是我自己维护本地镜像的结果。真实 API key 我放在 compose 文件外面，走环境变量密文注入。） + +## 使用 + +起起来之后，这个 API 小到意外。加一条事实（快速、逐字保存）： + +```bash +curl -X POST http://192.168.1.1:20015/memories \ + -H "Content-Type: application/json" \ + -H "X-Api-Key: $MEM0_API_KEY" \ + -d '{ + "user_id": "customer-2211", + "text": "Customer 2211 prefers email over WhatsApp for order updates." + }' +``` + +之后再取最相关的记忆： + +```bash +curl -X POST http://192.168.1.1:20015/memories/search \ + -H "Content-Type: application/json" \ + -H "X-Api-Key: $MEM0_API_KEY" \ + -d '{ + "user_id": "customer-2211", + "query": "how should I reach this customer?", + "limit": 5 + }' +``` + +然后把返回的事实塞进助手回答前的系统提示词（或上下文）。这就是全部循环：**存事实、召回相关的、在下一句回复里用上。** + +### 让我花了一天时间的 `infer` 坑 + +- `POST /memories` 默认 **`infer=true`** 会跑 LLM 事实抽取器——分类、清洗每条记忆。聪明但**慢**，而且每次调用都要一次模型往返。 +- 如果我只是想让助手**逐字快速地**记下某件事（客户说「请叫我 Andy」），我传 **`infer=false`**——不经过 LLM，近乎瞬时存储。 + +我栈上的路由是 `/search` 和 `/memories`，**没有 `/v1` 前缀**（早期一个自以为有 `/v1` 的包装器 404 了一阵子），认证是朴实的 `X-Api-Key` 头。都是小事，但都让我费过时间。 + +## 如果重来我会怎么做 + +- **从第一天起对常规记录就用 `infer=false`。** 只对重要的事实跑 LLM 抽取；默认的推理模式让每一次写入都为了一句随手记付出的成本不值。 +- **把 API key 当真正的机密。** 它会嵌进 agent 配置和 compose 环境变量里，所以我把它从 compose 文件中拿出来，用环境密文注入。 +- **把数据库放在我信任的盘上。** Mem0 的下限就是你 Postgres 的下限。我把它跑在 NAS 受保护的卷上、带备份任务——因为一个会失忆的记忆层还不如没有。 + +## 结果 + +一层私有、自托管的记忆，用两个小 HTTP 调用就能访问。我的助手现在跨会话记得客户、偏好和决策——而且因为整个栈都在我自己网络后面的 PostgreSQL 里，这一切都没有离开我的掌控。 + +如果你跑的 AI 助手动不动就忘掉上下文，自托管 mem0 是最划算、最省事的升级之一——而且你能在不把记忆送进别人数据库的前提下做到。 + +--- + +## 想让你的生意也用上带记忆的 AI 吗？ + +我为企业构建并自托管 AI 助手、记忆栈、网站和基础设施。如果你想让助手真正**记住**你的客户——或者你想聊聊雇佣我——我很乐意帮上忙： + +- 📱 **WhatsApp：** [+60 12-797 2969](https://wa.me/60127972969) +- 📧 **邮箱：** [me@hoelee.com](mailto:me@hoelee.com) +- 🌐 **网站：** [hoelee.com](https://hoelee.com) \ No newline at end of file diff --git a/src/content/posts/zh/when-smart-says-healthy-but-your-raid-is-corrupting-data.md b/src/content/posts/zh/when-smart-says-healthy-but-your-raid-is-corrupting-data.md new file mode 100644 index 0000000..f68c899 --- /dev/null +++ b/src/content/posts/zh/when-smart-says-healthy-but-your-raid-is-corrupting-data.md @@ -0,0 +1,97 @@ +--- +title: "SMART 显示健康，RAID 却在悄悄损坏数据" +description: "两块看起来完全健康的 NVMe 组成的 btrfs RAID1 阵列，却静默地把 Windows 虚拟机喂成了全零字节。我如何抓住它、靠校验和取证证明它，以及为什么最终要把两块盘都换掉。" +pubDate: 2026-09-16 +category: devops +tags: [unraid, btrfs, nvme, raid, storage, data-loss, troubleshooting] +ogImage: /og/when-smart-says-healthy-but-your-raid-is-corrupting-data.png +banner: /banners/when-smart-says-healthy-but-your-raid-is-corrupting-data.png +--- + +所有存储指南都告诉你同一件事：相信 SMART、检查事件日志、相信你的文件系统。我的硬件不这么想。2026 年 9 月，一个 btrfs RAID1 阵列——两块企业级 NVMe 盘镜像，我用每个能想到的工具查它都说**「健康」**——却在悄悄地把我的 Windows 11 虚拟机喂成全零字节。不是看得见的故障，是静默损坏。 + +下面是完整故事：损坏是怎么浮现的、哪个校验和出卖了它、以及为什么答案必须是「两块盘都换，别信 SMART」。 + +## 为什么这件事重要 + +损坏状态的磁盘是最糟的存储故障，因为它最容易漏掉： + +- **SMART 看起来完美。** 零媒体错误、几乎满余量，而且是能扛企业级写入负载的盘。 +- **阵列有冗余。** btrfs RAID1 把每个块镜像到两块盘上。当**两个成员**用同一种方式撒谎时，冗余毫无意义。 +- **故障是静默的。** 数组状态没有红色警告、没有硬盘报错、没有硬件事件——只有虚拟机读回垃圾，直到备份恢复成了唯一的选择。 + +如果你的数据在一切看起来都正常的时候悄悄坏掉，「备份」就不是偏执——它是全部。这次事故让我明白：一条**可测试、可验证的恢复**胜过任何阵列状态屏幕。 + +## 环境 + +- **unRaid 7.x**，Intel i9-13900K，Windows 11 虚拟机是主力工作站（GPU 直通）。 +- **缓存池：** 两块 **Samsung PM9A3 3.84TB** NVMe 组 **btrfs RAID1**。而且两块是**同一生产批次**（同一个控制器/固件版本 `GDC7702Q`）。 +- Windows 虚拟机的虚拟磁盘（`vdisk1.img`、`vdisk2.img`）就在这个池的 `domains/` 共享里，另外还有 **Docker 数据根目录**。 +- 虚拟磁盘**没有**设 `NOCOW`——所以每次读取都会被 btrfs 校验和验证。也正因为这个，我最后才抓得住它。 + +## 症状 + +我开始发现缓存读取慢得离谱，主机命令也卡住：`cat`、读 `/proc/loadavg`、甚至 `docker image ls` 都在超时。那块坏 NVMe 正以 **45–75 MB/s** 被狂读，每次读都塞满整台主机的 I/O。本该几秒跑完的 `docker system prune` 直接挂死。 + +然后真正的麻烦来了：Windows 虚拟机开始狂报读错误，回来的是**错的**数据——满文件的全零字节、guest 里的校验和失败，最后主力 VM 直接引导不了。 + +## 取证：`csum 0x8941f998` + +btrfs 在读校验失败时会往内核日志记一条校验和不匹配。同一个值反复出现： + +```text +btrfs: checksum verify failed on wanted 0x8941f998 found +``` + +这个特定值是最重要的线索。**`0x8941f998` 就是一整块零字节的 CRC32C。** 它每次都出现、跨越互不相关的文件和 inode，绝不是巧合。正在发生的不是随机比特腐烂，而是盘**稳定地返回全零块**，而这些零填充的读取正是 btrfs 校验失败的对象： + +- 验证一下：零填充块的 `crc32c` 就等于 `0x8941f998`。 +- 同一个值反复出现 → 读取是**确定性**地吐出空数据，不是偶尔翻几个比特。 + +两块虚拟磁盘都中招（`vdisk1.img` inode 2358574、`vdisk2.img` inode 2362817），正好对上「存储层在把零当成被请求的数据返回」的图景。 + +## 破绽：永远不生效的「自愈」 + +btrfs RAID1 本应自愈：一个成员坏块、从镜像重建、写回好数据。但我一遍遍 scrub、重读同一块（仍然全零的）数据后发现，**重写根本不落地**。每次重建「成功」后，立刻重读同一 extent 又吐出零。 + +这是关键区别： + +- 一块**只读不出来**的盘是读问题——自愈应该能从镜像救回来。 +- 一块**连回调也无脑吞垃圾存进去**的盘是**写或控制器**问题——镜像救不了你，因为两条路径都对同一个错误答案达成一致。 + +当自愈重写一直不生效，你就不在「坏一个扇区」的范畴了，而是「这块盘在写入上撒谎」——RAID 冗余救不了。 + +## 为什么两块盘都得换 + +数据指向的是**同批次盘的硬件缺陷**，而不是磨损或一次性的偶发事件： + +- SMART 干净（0 媒体错误、几乎满余量，`nvme0` 大约写了 586 TiB——对能扛企业级负载的盘来说算重负载但仍正常）。 +- 两块盘**同批次、同控制器/固件**（`GDC7702Q`）。 +- 损坏以**相关的方式**同时命中两个 RAID1 成员——镜像「死一块也能幸存」的整个意义，因为两块盘正以同一种方式一起死掉而完全失效。 + +面对批次级的关联缺陷，留下那块「看起来健康」的盘当幸存成员是假省心。**我从备份恢复了 Windows 虚拟机，然后把两块盘都换掉。** + +## 我同时做的事：把数据迁出那个池 + +在换盘之前，任何还值得读的数据都得搬出这个坏池——而且要**验证**，不能想当然： + +- **Docker 数据根目录**迁出缓存：`docker` 挪到 `/mnt/user/docker`（在 `disk1` 上，overlay2）。整棵树通过**路径、大小、mtime** 逐文件比对、并与字节对比交叉验证，共 **3,997,569** 个文件。 +- 缓存的 `isos/`、`system/`、`swapfile` 搬到 `disk1`；`isos/`（约 13 GB）用 `diff -rq` 检查 → **EXIT=0**，无差异。 +- 所有搬走的文件都在**删除缓存副本之前**先跟源比对过——没验证过的副本绝不删。 + +因为存在读侧损坏（全零块）和写侧损坏（自愈不落地），那个池里的任何东西，没有镜像、没有校验和、没有新鲜备份可对照，我就一律不信任。 + +## 如果重来我会怎么做 + +1. **在需要它之前就先测试恢复虚拟机备份。** 我当时确认了备份存在，但损坏事故不是该第一次学恢复流程的时候。定时做的测试恢复既能发现备份问题，也能发现静默的存储损坏。 +2. **把同批次的两块盘当成一块盘。** 两块 PM9A3 同批次同固件。对需要长期搭档的缓存池（任何 RAID1）这是关联失效风险——优先选不同批次的盘，或者至少在信任镜像前把这对盘狠狠测一遍。 +3. **主动检测，而不是被动祈祷。** 正是 btrfs 校验和让它在噪声里浮出来。不开 `NOCOW` 意味着文件系统在验证每一次读取；一个带校验的数据池，就是「当场抓住并恢复」和「把损坏文件带出去几个月」之间的分水岭。 +4. **别用 SMART 判断闪存。** 一块看起来健康却照样损坏的 NVMe，事件计数什么也说明不了——固件级的缺陷可以躲过所有 SMART 属性。 + +## 结果 + +一台 Windows 虚拟机从备份恢复、两块确认故障的盘换掉、Docker 数据根目录安全迁出该池。代价是几天折腾和一次救援——但损失被控制在最小，因为损坏是被校验和抓住的，而不是在里面泡了几个星期。 + +最该记住的一句话：**对存储来说，「冗余」只等于你最不靠谱的那个成员。** 两块一起坏掉的盘不是镜像——那是穿着两个序列号的单点故障。验证你的恢复、校验你的数据、当某块盘的零块「奖励」反复出现时，把整组都换掉。 + +--- \ No newline at end of file