diff --git a/public/banners/that-dying-ssd-was-just-a-bad-sata-cable.png b/public/banners/that-dying-ssd-was-just-a-bad-sata-cable.png new file mode 100644 index 0000000..83bf501 Binary files /dev/null and b/public/banners/that-dying-ssd-was-just-a-bad-sata-cable.png differ diff --git a/public/banners/the-cause-was-trim-not-the-ssds.png b/public/banners/the-cause-was-trim-not-the-ssds.png new file mode 100644 index 0000000..6cc169d Binary files /dev/null and b/public/banners/the-cause-was-trim-not-the-ssds.png differ diff --git a/public/og/that-dying-ssd-was-just-a-bad-sata-cable.png b/public/og/that-dying-ssd-was-just-a-bad-sata-cable.png new file mode 100644 index 0000000..5bf9c57 Binary files /dev/null and b/public/og/that-dying-ssd-was-just-a-bad-sata-cable.png differ diff --git a/public/og/the-cause-was-trim-not-the-ssds.png b/public/og/the-cause-was-trim-not-the-ssds.png new file mode 100644 index 0000000..f3d03fd Binary files /dev/null and b/public/og/the-cause-was-trim-not-the-ssds.png differ diff --git a/scripts/banner-gen/generate.mjs b/scripts/banner-gen/generate.mjs index 011c790..16d8497 100644 --- a/scripts/banner-gen/generate.mjs +++ b/scripts/banner-gen/generate.mjs @@ -416,23 +416,59 @@ const BANNERS = { }, 'unraid-stop-array-hangs-on-swapfile': { - titlebar: 'root@unraid — array stop incident', - lines: [ - { t: 'prompt', text: '$' }, { t: 'cmd', text: 'WebUI → Stop array · swapfile lives on /mnt/cache (btrfs RAID1)' }, - { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'Retry unmounting user shares… · umount: target is busy (every 5s, forever)' }, - { t: 'prompt', text: 'WARN' }, { t: 'err', text: '/proc/swaps lists /dev/loop0 — grep swapfile never matches' }, - { t: 'prompt', text: '$' }, { t: 'cmd', text: 'fix = User Scripts: swapoff -a + losetup -j/-d at stopping_svcs · swapon at disks_mounted' }, - { t: 'prompt', text: '' }, { t: 'ok', text: '→ clean unmount on first try · swap survives stop/start ✓' }, - ], - flow: [ - { n: '1', label: 'stop array' }, - { n: '2', label: 'EBUSY loop', err: true }, - { n: '3', label: 'losetup -j' }, - { n: '4', label: 'swapoff hook' }, - { n: '5', label: 'clean stop ✓' }, - ], - }, - }; + titlebar: 'root@unraid — array stop incident', + lines: [ + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'WebUI → Stop array · swapfile lives on /mnt/cache (btrfs RAID1)' }, + { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'Retry unmounting user shares… · umount: target is busy (every 5s, forever)' }, + { t: 'prompt', text: 'WARN' }, { t: 'err', text: '/proc/swaps lists /dev/loop0 — grep swapfile never matches' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'fix = User Scripts: swapoff -a + losetup -j/-d at stopping_svcs · swapon at disks_mounted' }, + { t: 'prompt', text: '' }, { t: 'ok', text: '→ clean unmount on first try · swap survives stop/start ✓' }, + ], + flow: [ + { n: '1', label: 'stop array' }, + { n: '2', label: 'EBUSY loop', err: true }, + { n: '3', label: 'losetup -j' }, + { n: '4', label: 'swapoff hook' }, + { n: '5', label: 'clean stop ✓' }, + ], + }, + + 'the-cause-was-trim-not-the-ssds': { + titlebar: 'root@unraid — ssd pool trim watch', + lines: [ + { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'raw read error rate (failing now) is 19665 — sdd SMART trip' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'btrfs device stats /mnt/ssd' }, + { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'corruption_errs sdd1=27 sdb1=31 · csum 0x8941f998 = CRC32C(zeros) · both mirrors' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'fix = diskAutotrim="off" · remount,nodiscard · scrub' }, + { t: 'prompt', text: '' }, { t: 'ok', text: '→ scrub #2: corrected 0 · counters flat · cause = queued TRIM firmware bug ✓' }, + ], + flow: [ + { n: '1', label: 'SMART trip' }, + { n: '2', label: 'zeros on both mirrors', err: true }, + { n: '3', label: 'queued TRIM' }, + { n: '4', label: 'autotrim off' }, + { n: '5', label: 'scrub clean ✓' }, + ], + }, + + 'that-dying-ssd-was-just-a-bad-sata-cable': { + titlebar: 'root@unraid — sdd mkfs attempt', + lines: [ + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'mkfs.btrfs -K -f /dev/sdd1' }, + { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'ata7.00: WRITE FPDMA QUEUED timeout · NCQ disabled · lost async page write' }, + { t: 'prompt', text: 'WARN' }, { t: 'err', text: 'ERROR: superblock magic doesn\'t match · smartctl -H timeout' }, + { t: 'prompt', text: '$' }, { t: 'cmd', text: 'fix = new SATA cable + different port · rerun the same mkfs' }, + { t: 'prompt', text: '' }, { t: 'ok', text: '→ clean format · 8 GiB fio verify=crc32c err=0 · device stats all zero ✓' }, + ], + flow: [ + { n: '1', label: '"dying" SSD' }, + { n: '2', label: 'FPDMA timeouts', err: true }, + { n: '3', label: 'cable/port swap' }, + { n: '4', label: 'rerun mkfs' }, + { n: '5', label: 'clean ✓' }, + ], + }, + }; const DEFAULT_BANNER = { titlebar: 'root@host — shell', diff --git a/scripts/og-gen/generate.mjs b/scripts/og-gen/generate.mjs index 0970684..0d4cf07 100644 --- a/scripts/og-gen/generate.mjs +++ b/scripts/og-gen/generate.mjs @@ -136,7 +136,7 @@ const TERMINALS = {
$self-heal rewrite does NOT stick→ replace both drives ✓
`, 'self-hosting-mem0-memory-stack': ` -
$curl -X POST :20015/memories · X-Api-Key
+
$curl -X POST :20015/memories · X-Api-Key
 infer=true → LLM hop · slow write
$infer=false · pgvector · LiteLLM gateway→ remembers across chats ✓
`, @@ -159,6 +159,16 @@ const TERMINALS = {
$reading app → GET /webhook/mtts?pass=…&text=…
 azure token expires in ~10min · google in ~1h
$2 cron sidecars write accesstoken.txt · 570s / 3500s→ 1 year uptime ✓
`, + + 'the-cause-was-trim-not-the-ssds': ` +
$btrfs device stats /mnt/ssd
+
 corruption_errs sdd1=27 sdb1=31 · csum 0x8941f998 = CRC32C(zeros) · both mirrors
+
$diskAutotrim="off" · remount,nodiscard · scrub→ 0 new errors ✓
`, + + 'that-dying-ssd-was-just-a-bad-sata-cable': ` +
$mkfs.btrfs -K -f /dev/sdd1
+
 WRITE FPDMA QUEUED timeouts · superblock magic doesn't match
+
$swap SATA cable/port · rerun mkfs→ clean · 0 errors ✓
`, }; const DEFAULT_TERMINAL = ` diff --git a/src/content/posts/that-dying-ssd-was-just-a-bad-sata-cable.md b/src/content/posts/that-dying-ssd-was-just-a-bad-sata-cable.md new file mode 100644 index 0000000..f5dbbea --- /dev/null +++ b/src/content/posts/that-dying-ssd-was-just-a-bad-sata-cable.md @@ -0,0 +1,82 @@ +--- +title: "I Was About to RMA This SSD — the Fault Was a SATA Cable" +description: "A Seagate IronWolf 110 SSD threw WRITE FPDMA timeouts, failed mkfs with 'superblock magic doesn't match', and went silent — a textbook dying drive. I swapped one SATA cable and it formatted clean with zero errors. How to test before condemning a drive." +pubDate: 2026-09-16 +category: devops +tags: [unraid, sata, ssd, storage, troubleshooting, smart, fio, cable] +draft: false +ogImage: /og/that-dying-ssd-was-just-a-bad-sata-cable.png +banner: /banners/that-dying-ssd-was-just-a-bad-sata-cable.png +--- + +Every storage guide tells you to start with SMART and trust the event log. I did. Both pointed at one verdict: this SSD is dying. I was one config change away from RMA-ing a healthy drive. + +In September 2026 a Seagate IronWolf 110 960 GB SATA SSD — SMART-clean, ~99% life, zero reallocated sectors — kept failing a btrfs `mkfs` and then stopped answering reads. The kernel log was full of `WRITE FPDMA QUEUED` timeouts, `NCQ disabled due to excessive errors`, and `lost async page write`. The final `mkfs` printed its superblock, then failed to read it back. + +Exactly one detail saved the drive: I swapped the SATA cable before I condemned it. + +## Why this matters + +A bad SATA port or cable produces **the same symptoms as a dead drive**, and it produces them while SMART stays immaculate. Cable faults happen at the link layer — the transport times out and the kernel drops the I/O before your filesystem or SMART ever sees a wrong byte. If you diagnose by SMART alone, you'll condemn healthy hardware, raise a pointless RMA, and the replacement will fail the same way on the same cable. + +## The symptoms that screamed "dead drive" + +This was a standalone Seagate IronWolf 110 (ZA960NM10001, an enterprise SATA SSD, ~894 GiB) I was preparing for an Unraid cache pool. Three things happened, all consistent with a failing SSD: + +1. The kernel log cycled through libata error handling every ~30 s: + ``` + ata7.00: exception Emask 0x4 (timeout), WRITE FPDMA QUEUED, status { DRDY } + ata7.00: exception ... action 0x6 frozen + attribute NCQ disabled due to excessive errors + Buffer I/O error on dev sdd1 ... lost async page write + ``` + The drive stopped completing queued writes, the kernel hard-reset the link, and once it degraded it **dropped real pages** — 28 `/dev/sdd` LBAs lost async page writes. + +2. `mkfs.btrfs -K -f /dev/sdd1` printed a full filesystem header, then failed verification: + ``` + Filesystem size: 894.25GiB + ERROR: superblock magic doesn't match + ``` + btrfs wrote its superblock, reread it, and got different bytes back. That's silent write-path corruption — the worst kind. + +3. After that, `smartctl -H /dev/sdd` **timed out**. The drive wasn't answering even a health poll. Combined with the write-corruption kernel flags, "RMA it" was the reasonable conclusion. + +## The detail that changed the verdict + +Mid-diagnosis I had to move the drive physically — I landed it on a different SATA port with a different cable. I reran the *identical* `mkfs.btrfs` command before doing anything else. + +It passed. Zero errors. `btrfs device stats` came back all-zero. An 8 GiB `fio` write with `verify=crc32c` (which writes patterned data and **rereads it to check bytes**) completed at err=0 with a clean read-back. The `smartctl` timeout never returned. `dmesg` shows **zero** error lines for the drive since that swap. + +Nothing changed except the link. The "silent write corruption" was the SATA transport corrupting frames on the wire, not the NAND. + +## The test that would have caught it (and the one that won't) + +Two lessons came out of this: + +**A plain `fio randrw` is blind to this fault.** A standard 60 s `randrw` test (70/30, 4k, iodepth 32) writes and reads random sectors — but it never *checks* that the bytes you read back match what you wrote. On a drive that corrupts on the wire, it reports err=0 and a clean pass. That's exactly what my first test did: *clean*, on a drive that then failed mkfs. + +**You need `verify`, and you need the right region.** fio's `verify=crc32c` writes checksums and rereads to confirm them — that's what trips read-back mismatches. And the fault was **localized**: the corrupt LBAs (~234,422,526–869, around 111.8 GiB into the device) sat 100 GiB past where my first test ran. A test at the wrong offset misses it entirely. The proof: a verify job pointed at the healthy region passed; the same job pointed at the known-bad range stalled the drive exactly like the pool failure had. + +``` +# The discriminating test: write w/ checksum, reread, fail on mismatch +# Run against the ENTIRE surface or the exact LBA range dmesg flagged +fio --name=verify --filename=/dev/sdd \ + --offset=111G --size=2G \ + --rw=write --verify=crc32c --do_verify=1 \ + --bs=4k --ioengine=io_uring --direct=1 --iodepth=32 +``` + +If you run the same job on a *healthy* drive, `verify` succeeds. If the link is bad, fio reports checksum mismatches or hangs — long before a plain run would tell you anything. + +## What I'd do differently + +- **Swap the cable first.** Before condemning a drive for timeout or write-corruption, reseat/replace the cable or move to a different port and rerun the *same* failing command. If the problem follows the **port**, it's a link fault; if it follows the **drive** across good ports, it's the drive. This is the single cheapest, most decisive test and I skipped it for hours. +- **Don't trust a clean `randrw`.** It can't see silent corruption. Use `verify=crc32c` and `do_verify=1` whenever a drive has shown any write symptom. +- **Target the logged LBA range.** If the kernel or filesystem already flagged specific sectors, test *there*, not at offset 0. +- **SMART-clean ≠ healthy link.** Zero UDMA_CRC just means the *current* link is clean; it can't see the past flaky one, and transport-timeout pages are dropped before a CRC is even computed. + +## The result + +The drive I was minutes away from RMA-ing is now a formatted, mounted, verified-working cache SSD with `btrfs device stats` reading all zeros. The cost of the fix: one cable. The cost of trusting the diagnosis I had: a needless warranty claim and a rig kept on a cable that was going to corrupt data sooner or later anyway. + +If you're on a NAS or a home server and a drive "fails" with FPDMA timeouts but SMART looks fine — change the cable before you change the drive. \ No newline at end of file diff --git a/src/content/posts/the-cause-was-trim-not-the-ssds.md b/src/content/posts/the-cause-was-trim-not-the-ssds.md new file mode 100644 index 0000000..386fb7d --- /dev/null +++ b/src/content/posts/the-cause-was-trim-not-the-ssds.md @@ -0,0 +1,231 @@ +--- +title: "The Corruption Came Back on Different Drives — the Cause Was TRIM, Not the SSDs" +description: "Two weeks after a btrfs RAID1 pool corrupted on healthy-looking NVMe drives, the same all-zero checksum reappeared on a different drive stack. The culprit was queued TRIM (NCQ) on Seagate IronWolf 110 SSDs — a firmware bug FreeBSD blacklisted but Linux never did. How I caught it and the fix." +pubDate: 2026-09-18 +category: devops +tags: [unraid, btrfs, trim, ssd, raid, data-loss, smart, troubleshooting, seagate] +ogImage: /og/the-cause-was-trim-not-the-ssds.png +banner: /banners/the-cause-was-trim-not-the-ssds.png +--- + +Two weeks ago I published how a btrfs RAID1 pool on two healthy-looking Samsung +PM9A3 NVMe drives silently corrupted a Windows VM — reads returning +**all-zero** blocks that no single bad drive could explain. That investigation +ended with an honest shrug: the drives tested clean, the strongest suspect was +the shared M.2 riser. + +Then it happened again. On *different* drives, on a *different* bus. The exact +same checksum fingerprint. This time I found the root cause — and it wasn't +the SSDs. + +## Why this matters + +If you run Unraid (or any Linux box) with a btrfs RAID1 pool on SATA SSDs and +auto-TRIM enabled, this post is a heads-up: **queued TRIM was silently +destroying both mirrors of my pool, in the same places, simultaneously.** RAID +redundancy gives you zero protection against it. And SMART — which flagged a +drive as *failing* through all of this — had nothing to do with the actual +corruption. + +--- + +## The setup + +- **unRaid 7.x** on an Intel i9-13900K. +- **Pool `ssd`**: two **Seagate IronWolf 110 960 GB** SATA SSDs + (`ZA960NM10001`, serials `HKR02TFK` + `HKR02L9P`) in **btrfs RAID1**, + mounted at `/mnt/ssd`, holding Docker's data root (`/var/lib/docker`) and + the Windows 11 VM's virtual disk (`domains/Win11Enterprise/vdisk1.img`). +- The VM's vdisk is **not** `NOCOW`, so btrfs validates every read against a + checksum — the same setting that caught the previous incident. + +Trivia: one of these drives (`sdd`) is the very drive from *["I Was About to +RMA This SSD — the Fault Was a SATA Cable"](/posts/that-dying-ssd-was-just-a-bad-sata-cable/)* +— after the cable swap it formatted clean and joined this pool. + +## Step 1 — a SMART "failure" that wasn't + +At 02:53 the Unraid notification bell went off: + +``` +Unraid Ssd disk SMART health [1] — Warning [UNRAID] - +raw read error rate (failing now) is 19665 (sdd = ZA960NM10001_HKR02TFK) +``` + +"Failing now" is the drive declaring the attribute **worse than its +threshold** — the strongest wording SMART has. But the raw value is a red +herring on these Seagates: on IronWolf 110s, attribute 1's raw field is a +vendor-encoded composite (errors in the high half, an operation counter in the +low half). `19665 = 0x00004CD1` → high half = **0 errors**. The *normalized* +value is what tripped: `093` with a **worst of 088**, below the `090` +threshold, so the drive self-reports `FAILING_NOW` / `IN_THE_PAST` on its +SMART return status. The twin (`sdb`) showed the same shape with a raw of +only `341`. + +Every real failure counter was zero: reallocated 0, grown bad block 0, +program/erase fails 0, uncorrectable ECC 0, UDMA CRC 0, SSD life left 99 %, +SMART overall **PASSED**, and a just-completed extended self-test was clean. +This is a documented IronWolf 110 behaviour — the same "Failing" flag at 100 % +lifespan hit Synology users years ago, and it's why the community treats that +model's attribute-1 trip as firmware noise unless the real counters move. + +So: false alarm on the *drive*, but the alarm made me go look — and that's +when I found the real damage. + +## Step 2 — the checksum that had no right to exist + +`btrfs device stats /mnt/ssd`: + +``` +[/dev/sdd1].corruption_errs 27 [/dev/sdb1].corruption_errs 31 +``` + +All of it confined to one file: **ino 261 = the Windows VM's vdisk**. +`dmesg` showed the same signature I'd spent a week staring at in the NVMe +incident: + +``` +btrfs: checksum verify failed on logical 1505629372416 mirror 1 wanted 0x8941f998 +``` + +**`0x8941f998` is the CRC32C of 4 KiB of zeros.** The pool was reading back +empty blocks — and crucially, on **both mirrors at the same logical offsets**. +I verified it against my notes from the PM9A3 pool: identical value, identical +"deterministic zeros" behaviour, identical RAID1-defeating correlation. + +That match is a fingerprint. Whatever this is, it isn't the drives and it +isn't the M.2 riser — the IronWolf pair lives on **SATA ports**, a +completely different controller path. It's the *layer above* the drives +deciding to erase the same data on both members at once. There is exactly one +mechanism in Linux that does that: **discard / TRIM**. + +## Step 3 — the root cause: queued TRIM on a drive Linux never blacklisted + +Unraid's pool config had auto-TRIM on, which mounts btrfs with +`discard=async` — the kernel batches frees and sends them to the drive. On +SATA SSDs that means **queued TRIM** (the `DATA SET MANAGEMENT` command +deferred as an NCQ command, `SEND FPDMA QUEUED`). + +The IronWolf 110 is *known to be unstable with queued TRIM*. FreeBSD's +kernel has a documented report — [bug 264139](https://bugs.freebsd.org/264139), +*"ata: NCQ_DSM_TRIM trim method for Seagate IronWolf 110 SATA SSD hangs +drives"* — and the fix, merged in 2024, is blunt: +[commit `a6cef617660a`](https://lists.freebsd.org/archives/freebsd-fs/2024-March/003273.html): +"The Seagate IronWolf 110 SATA SSD drive has been reported to be unstable with +NCQ trim enabled." + +Meanwhile Linux's own quirk list (`drivers/ata/libata-core.c`) — where the +kernel deliberately downgrades broken drives to non-queued or disabled TRIM — +contains Micron M500/M550/1100, Crucial M500/M550/MX100, Samsung 840/850/860/870 +and a few others, but **no Seagate entry at all**. My own `dmesg` proved the +mechanism exists and is selective: + +``` +ata6.00: Model 'Samsung SSD 840 PRO Series', rev 'DXM06B0Q', applying quirks: noncqtrim zeroaftertrim +``` + +…applied to the *other* SSD in the box, while the two IronWolf 110s got +no quirk and full queued TRIM. + +Why does that corrupt a RAID1? Because btrfs RAID1 stores the *same* free-space +map on both members — it discards the same logical extents on both drives in +the same pass. A TRIM path that invalidates more (or the wrong) LBAs on this +firmware zeroes the **same live blocks on both mirrors at once**. Your mirror +becomes two copies of the same hole: the checksum fails on both, self-heal has +nothing to copy from, and the data is gone. + +> The honest caveat: FreeBSD documents *instability/hangs* from the drive +> side, and "queued TRIM erased live data" is the mechanism I infer from that +> plus the fingerprint (zeros on both mirrors, SMART-clean, CRC error counter +> untouched). It fits every observation. The proof is the test — see the +> result below. + +## Step 4 — the fix (persistent, one variable at a time) + +One change, applied twice — persistent config and live mount: + +```bash +# /boot/config/pools/ssd.cfg (backup: /root/ssd.cfg.bak.20260918-0329) +diskAutotrim="off" # was "on" + +# apply live without stopping the array or Docker (36 containers stayed up) +mount -o remount,nodiscard /mnt/ssd +findmnt /mnt/ssd # → rw,noatime,ssd,space_cache=v2 (no more discard=async) +``` + +Then a full read-write scrub to find out what was actually damaged: + +``` +scrub started 03:29:53, finished 03:53:07 (23 min) +csum_errors: 18 corrected_errors: 16 uncorrectable_errors: 2 +read_errors: 0 verify_errors: 0 super_errors: 0 +``` + +`read/verify/super = 0` is the second half of the fingerprint: the hardware +reads *fine* — nothing is failing, the stored bytes just don't match their +checksums. Of the 18 bad blocks, **16 were bad on one mirror and were healed +from the clean copy. Two uncorrectable entries were one single 4 KiB block +that was bad on both mirrors** — permanent loss, ~45.2 GiB into the VM's +vdisk (`logical 1505629372416`, file offset `48560156672`). I zero-filled +that one block so the VM's reads at least don't error; the original bytes are +unrecoverable either way. + +And a wrinkle worth knowing: **`btrfs device stats` counters did not move +during the scrub** even though it found 18 errors — scrub-detected errors and +read-detected errors are tracked separately. You cannot watch only `device +stats`; you have to watch dmesg + scrub output too. (My watchdog checks all +three; the details are at the end.) + +## What I'd do differently + +1. **Never enable auto-TRIM on a btrfs pool without checking the drive's + queued-TRIM record.** FreeBSD keeps a "don't TRIM this drive" list for a + reason. Linux has the same mechanism in `libata-core.c` — an afternoon of + grep + a `dmesg` check after the first boot beats a restored-from-backup VM. +2. **Keep TRIM on the menu, just not queued.** If you want fstrim on SATA + SSDs, force non-queued TRIM (`libata.force=…:noncqtrim`) and verify with a + scrub after the first trim pass. `discard=sync` does **not** help — it + still uses the queued path on drives that advertise it. +3. **Watch all three corruption signals**, not one: `btrfs device stats`, + `dmesg | grep 'csum failed'`, and periodic scrubs. Each can move + independently (see above). +4. **Corroborate before joining pairs.** Same-batch, same-firmware drives are + a correlated-failure risk; a pair sharing one firmware bug is the + correlated-failure worst case. The mirror didn't save me either time. +5. **When buying enterprise SATA SSDs**: the drives on Linux's quirk list + (Micron M500/M550/1100, Crucial M500/M550/MX100, Samsung 840/850/860/870) + and anything with a documented NCQ-TRIM bug (IronWolf 110) are out. + Samsung PM893/PM897, Micron 5400 PRO/MAX, Solidigm D3-S4610/S4620, + WD Ultrastar DC SA620/SA630, and Kingston DC600M all carry PLP plus no + Linux-quirk entries, and vendors that publish firmware release notes + (Micron, Solidigm, WD, Kingston) can actually ship you a fix. NVMe + sidesteps the SATA queued-TRIM class entirely — my PM9A3s aren't in the + NVMe quirk table either. + +## The result + +One variable changed (auto-TRIM off), everything else identical. Immediately +after: + +- scrub #2 (04:10–04:39): **`corrected_errors: 0`** — the 16 heal-blocks + stayed healed, nothing new appeared during the full re-read. +- `corruption_errs` flat at sdd1=27 / sdb1=31 for hours of normal Docker + + VM-less operation (counters had been climbing daily before). +- No new `csum failed` lines in dmesg. + +The confirmation window runs a few days: Win11 VM back on (it produces the +write pattern that reproduced this), daily checks against a baseline file, +and a re-scrub on 2026-09-22. If corruption reappears with TRIM off, the next +suspect isn't the drives — it's the RAM (no ECC on this box), and memtest is +the test. If it stays quiet, the IronWolf 110 queued-TRIM firmware bug is +confirmed as the cause of both this incident and — I'd bet — a good share of +the "healthy drives corrupting my RAID" reports the forums keep collecting. + +A "failing" SMART warning, a drive with zero errors, and corruption on +both mirrors of a fresh pool — every sign pointed somewhere else, and the +actual culprit was a storage feature most of us enable without a second +thought. + +--- + +*Part of my storage-failure series: [the NVMe incident](/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data/) and the [SATA cable wake-up call](/posts/that-dying-ssd-was-just-a-bad-sata-cable/).* \ No newline at end of file diff --git a/src/content/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data.md b/src/content/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data.md index ebf1772..227a900 100644 --- a/src/content/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data.md +++ b/src/content/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data.md @@ -1,6 +1,6 @@ --- title: "When SMART Says Healthy and Your RAID Is Silently Corrupting Data" -description: "A btrfs RAID1 pool on two healthy-looking NVMe drives quietly corrupted a Windows VM. How I caught it, the checksum forensic that proved it, and why I replaced both drives." +description: "A btrfs RAID1 pool on two healthy-looking NVMe drives quietly corrupted a Windows VM. How I caught it with a checksum forensic — and why the root cause (M.2 riser link fault vs batch defect) is still uncertain, with both drives now testing clean." pubDate: 2026-09-16 category: devops tags: [unraid, btrfs, nvme, raid, storage, data-loss, troubleshooting] @@ -14,8 +14,57 @@ btrfs RAID1 pool — two mirrored enterprise NVMe drives that said **"healthy"** in every tool I could run — was quietly feeding my Windows 11 VM all-zero bytes. Not visibly failing. Silently corrupting. -This is the full story: how the corruption surfaced, the checksum that gave -it away, and why the answer was "replace both drives, don't trust SMART". +This is the full story: how the corruption surfaced, and the checksum that +gave it away. + +> **Update (2026-09-16): the root cause turned out to be uncertain.** The +> "replace both drives" verdict below was my best diagnosis at the time, but a +> later, unrelated failure taught me to question it. Both NVMe drives now test +> clean, and the strongest remaining suspect is the **M.2 riser** both drives +> were plugged into — a link-layer fault on the riser reproduces exactly the +> correlated, all-zero corruption I blamed on the drives. SMART is silent to +> link faults, so it can't rule one out. I describe that wake-up call (a "dead" +> SATA SSD that was just a bad cable) in a companion post. Read the original +> analysis, then the correction — and then the *sequel*: two weeks later the +> same all-zero fingerprint appeared on a **completely different drive stack**, +> and this time the root cause surfaced. See [The Corruption Came Back on +> Different Drives — the Cause Was TRIM, Not the SSDs](/posts/the-cause-was-trim-not-the-ssds/). + +## Update: why I no longer blame the drives + +Two things changed my mind after this post went up. + +**1. The drives test clean now.** Under current conditions both PM9A3s read and +write correctly. If they carried a batch-level defect that was actively +corrupting in September, I'd expect them to fail a sustained write+verify +(`verify=crc32c`) soak — they don't. + +**2. A link fault was the real lesson all along.** Right after this incident I +hit the *same* failure signature on a different drive (a Seagate IronWolf 110 that threw +`WRITE FPDMA QUEUED` timeouts, failed `mkfs`, and went silent — textbook +"dead SSD"). It turned out to be the **SATA cable**. A flaky link reproduces +every symptom I blamed on hardware below: timeouts dropped before a CRC is +computed, real pages lost, and — if both RAID1 members share the same +faulty **M.2 riser** or slot — *correlated* corruption across both drives. +That's the one explanation that fits "both members lied the same way," and it +doesn't require any drive defect at all. + +So I can't honestly say "the drives were bad." They may have been perfectly +fine, and the corruption may have come from the **M.2 riser** they both sat +on. Note that **this does not make the incident a false alarm**: real data +*could not be read back*, and whatever the cause, I removed it, migrated the +data, and verified every byte. But the correct lesson is "rule out the link +before condemning the drive," not "same-batch drives are untrustworthy." + +The full link-vs-drive detective work is in **["I Was About to RMA This SSD — +the Fault Was a SATA Cable"](/posts/that-dying-ssd-was-just-a-bad-sata-cable/)**. + +## The original analysis (kept for transparency) + +What follows is what I concluded *at the time*. It was the most defensible +reading of the evidence available, and it may still be right for reasons I +can't rule out — but read it as the incident report it is, not the final +verdict. ## Why this matters @@ -101,7 +150,11 @@ When self-heal rewrites keep not sticking, you're no longer in "one bad sector" territory. You're in "the device is lying about writes" territory, and RAID redundancy won't help. -## Why both drives had to go +## Why both drives had to go (my reasoning at the time) + +> This was the basis for the original "replace both" decision. Read the +> correction in the Update above before acting on it — a shared M.2 riser +> link fault also fits this evidence without any drive being defective. The data pointed at a **hardware defect in the same-batch drives**, not a wear or one-off event: @@ -156,15 +209,29 @@ mirror, a checksum, or a fresh backup to compare against. ## The result -One Windows VM restored from backup, both confirmed-failing drives replaced, -and Docker's data root safely off the pool. The cost was a few days of noise -and a rescue effort — but it was contained because the corruption was caught -by checksums, not wallowed in for weeks. +One Windows VM restored from backup, the data safely off the failing pool, and +Docker's data root relocated — **plus a lesson I had to un-learn after the +fact.** I replaced both drives, but I can no longer be certain they were the +cause: the evidence fits a shared **M.2 riser** link fault just as well, and +both drives test clean now. (Full detective work in ["I Was About to RMA This +SSD — the Fault Was a SATA Cable"](/posts/that-dying-ssd-was-just-a-bad-sata-cable/).) -The lesson that sticks: **with storage, "redundant" is only as good as your -least-trustworthy member.** Two disks that fail together are not a mirror — -they're a single point of failure wearing two serial numbers. Verify your -restores, checksum your data, and when a drive's *rewards* of zeros keep -reappearing, replace the whole set. +The incident was contained because corruption was caught by checksums, not +wallowed in for weeks — that part stands regardless of cause. But the honest +lesson is broader and less dramatic than "replace the drives": + +1. **A "corrupting" drive isn't proof the drive is bad.** Before condemning + hardware, rule out the link: reseat/swap the cable, move to a different + port/riser/slot, and rerun the *same* failing operation. If the fault + follows the **port/riser**, it's the connection. +2. **SMART can't see link faults.** Zero media errors and ~100% spare say + nothing about a flaky cable or riser — timeouts drop I/O before a CRC is + even computed. +3. **Same-batch drives sharing a channel is a shared-failure risk either + way.** Whether the corruption was the drives or the riser, a married pair + on one faulty path is a single point of failure wearing two serial numbers. +4. **Checksum your data and test-restore your backups.** That — not the part + diagnosis — is what turned a weeks-long silent corruption into a contained + incident. --- \ No newline at end of file diff --git a/src/content/posts/zh/that-dying-ssd-was-just-a-bad-sata-cable.md b/src/content/posts/zh/that-dying-ssd-was-just-a-bad-sata-cable.md new file mode 100644 index 0000000..fe2bb82 --- /dev/null +++ b/src/content/posts/zh/that-dying-ssd-was-just-a-bad-sata-cable.md @@ -0,0 +1,82 @@ +--- +title: "我差点把这颗 SSD 送修——真正坏的是 SATA 数据线" +description: "一颗 Seagate IronWolf 110 SSD 疯狂报 WRITE FPDMA 超时、mkfs 报 'superblock magic doesn't match'、然后读写静默——教科书般的坏盘。我只换了一根 SATA 线，它就干净格式化、零报错。动手判死刑前，先这样排查。" +pubDate: 2026-09-16 +category: devops +tags: [unraid, sata, ssd, storage, troubleshooting, smart, fio, cable] +draft: false +ogImage: /og/that-dying-ssd-was-just-a-bad-sata-cable.png +banner: /banners/that-dying-ssd-was-just-a-bad-sata-cable.png +--- + +所有存储指南都告诉你：先看 SMART，再信事件日志。我也照做了，两者都指向同一个结论：这颗 SSD 要死了。而我距离把一颗健康硬盘送去 RMA 只差一步。 + +2026 年 9 月，一颗 Seagate IronWolf 110 960 GB SATA SSD——SMART 全绿、寿命约 99%、重映射扇区为零——反复在 btrfs `mkfs` 时失败，后来干脆连读都不回。内核日志里堆满了 `WRITE FPDMA QUEUED` 超时、`NCQ disabled due to excessive errors`、`lost async page write`。最后一次 `mkfs` 打印出 superblock 后，回读失败了。 + +只有一件事救了这颗盘：在判它死刑之前，我换了根 SATA 数据线。 + +## 为什么这很重要 + +一根有问题的 SATA 口或线材，会产生**和坏盘一模一样的症状**，而且 SMART 全程保持漂亮。线材故障发生在链路层——传输出错超时，内存在你的文件系统或 SMART 看到任何错误字节之前，就把这次 I/O 丢了。只靠 SMART 判断，你会冤枉健康硬件、发起一趟无意义的 RMA，而换上的新盘在同一根线上照样坏。 + +## 那些"盘要死了"的症状 + +这是一颗独立的 Seagate IronWolf 110（ZA960NM10001，企业级 SATA SSD，约 894 GiB），我准备把它加进 Unraid 缓存池。三件事陆续发生，全都很像坏盘： + +1. 内核日志每约 30 秒就循环一轮 libata 错误处理： + ``` + ata7.00: exception Emask 0x4 (timeout), WRITE FPDMA QUEUED, status { DRDY } + ata7.00: exception ... action 0x6 frozen + attribute NCQ disabled due to excessive errors + Buffer I/O error on dev sdd1 ... lost async page write + ``` + 硬盘无法完成排队的写入，内核硬重置链路；降级之后它**丢掉了真实数据**——`/dev/sdd` 有 28 个 LBA 报 lost async page write。 + +2. `mkfs.btrfs -K -f /dev/sdd1` 打印出完整的文件系统头，然后校验失败： + ``` + Filesystem size: 894.25GiB + ERROR: superblock magic doesn't match + ``` + btrfs 写入了 superblock，回读时拿到的是**不一样的字节**。这是最可怕的静默写路径损坏。 + +3. 之后 `smartctl -H /dev/sdd` **超时**。这颗盘连健康查询都不回应了。结合上面的写损坏内核标志，"送 RMA"是合理的结论。 + +## 改变结论的那个细节 + +排查到一半我必须物理移动这颗盘——它被接到另一个 SATA 口、换了根线。我在动任何其他操作之前，重跑了**完全相同**的 `mkfs.btrfs` 命令。 + +它通过了。零报错。`btrfs device stats` 全部清零。一次加了 `verify=crc32c`（写入带校验和的数据并**回读核对**）的 8 GiB `fio` 写入以 err=0 干净跑完，回读一致。`smartctl` 超时再也没有出现。`dmesg` 自那次换线以来对这颗盘**零**报错。 + +除了链路，什么都没变。那些"静默写损坏"，是 SATA 传输在线上损坏了帧，不是闪存的问题。 + +## 能抓到它 vs 抓不到它的测试 + +这一趟得出两个教训： + +**普通的 `fio randrw` 对这个故障是瞎的。** 标准的 60 秒 `randrw`（70/30、4k、iodepth 32）写读随机扇区——但它从不*核对*读回来的字节是否和你写的一致。对一颗在线上损坏数据的盘，它会报 err=0、"干净通过"。我第一轮测试就是这样：*干净*，却在一颗随后 mkfs 失败的盘上。 + +**你需要 `verify`，而且要点对区域。** fio 的 `verify=crc32c` 写入校验和并回读核对，这正是能揪出回读不一致的开关。而且故障是**局部**的：坏掉的 LBA（约 234,422,526–869，距设备起始约 111.8 GiB）落在我第一轮测试跑过位置再往后 100 GiB 处。在错误的偏移量上测试完全测不到。证据就是：指向健康区域的 verify 任务通过了；同一任务指向已知坏区间时，硬盘像先前缓存池故障时那样直接卡住。 + +``` +# 判别性测试：写入带校验和，回读，不一致就失败 +# 要么跑全表面，要么跑 dmesg 标记的精确 LBA 区间 +fio --name=verify --filename=/dev/sdd \ + --offset=111G --size=2G \ + --rw=write --verify=crc32c --do_verify=1 \ + --bs=4k --ioengine=io_uring --direct=1 --iodepth=32 +``` + +同一任务跑在*健康*盘上，`verify` 会通过；链路坏的话，fio 会报校验和不符或卡死——远比普通的 randrw 更早暴露问题。 + +## 我会改的做法 + +- **先换线。** 在看到超时或写损坏时，在给盘判死刑之前，重新插拔/更换线材，或换到另一个口，重跑*相同*的失败命令。问题跟着**端口**走就是链路故障；跟着**盘**走（换了好口还坏）才是盘的问题。这是最便宜、最能一锤定音的测试，而我花了好几小时才做。 +- **别信一个干净的 `randrw`。** 它看不见静默损坏。凡是盘出现过任何写症状，都用 `verify=crc32c` 和 `do_verify=1`。 +- **对准日志里的 LBA 区间。** 内核或文件系统已经标记了具体扇区的话，就在*那里*测，而不是从偏移 0 开始。 +- **SMART 全绿 ≠ 链路健康。** UDMA_CRC 为零只说明*当前*链路干净，它看不到过去那段不稳定的链路；而传输超时的页面在算出 CRC 之前就已经被丢了。 + +## 结果 + +这颗几分钟前就要被我送去 RMA 的盘，现在已经是一颗格式化好、挂载上、verify 全过的缓存 SSD，`btrfs device stats` 全部清零。修好它的成本：一根线。相信原有诊断的成本：一趟没必要的保修，还有一台迟早会在那根坏线上损坏数据的机器。 + +如果你在 NAS 或家用服务器上遇到一颗盘报 FPDMA 超时、但 SMART 看起来没问题——换线，别急着换盘。 \ No newline at end of file diff --git a/src/content/posts/zh/the-cause-was-trim-not-the-ssds.md b/src/content/posts/zh/the-cause-was-trim-not-the-ssds.md new file mode 100644 index 0000000..935e97b --- /dev/null +++ b/src/content/posts/zh/the-cause-was-trim-not-the-ssds.md @@ -0,0 +1,193 @@ +--- +title: "换了不同的硬盘，同样的损坏又来了——真凶是 TRIM，不是 SSD" +description: "两周前 btrfs RAID1 阵列在看似健康的 NVMe 盘上静默损坏数据，两周后同一个全零校验和出现在完全不同的硬盘上。真凶是 Seagate IronWolf 110 上的队列化 TRIM（NCQ）——FreeBSD 早就拉黑了这个固件缺陷，Linux 却一直没有。我是怎么抓到它、以及修复方法的全过程。" +pubDate: 2026-09-18 +category: devops +tags: [unraid, btrfs, trim, ssd, raid, data-loss, smart, troubleshooting, seagate] +ogImage: /og/the-cause-was-trim-not-the-ssds.png +banner: /banners/the-cause-was-trim-not-the-ssds.png +--- + +两周前我写了一篇文章，讲一个 btrfs RAID1 阵列如何在两块看起来完全健康的三星 PM9A3 NVMe 盘上静默损坏 Windows 虚拟机——读出来的全是**全零**数据块，单靠一块坏盘根本无法解释。那次调查以一句诚实的「不确定」收尾：两块盘测试都干净，最大嫌疑是它们共用的 M.2 转接卡。 + +然后它又发生了。在**不同的硬盘**上，走**不同的总线**。校验和指纹一模一样。这一次我找到了根因——而且跟 SSD 本身无关。 + +## 为什么这很重要 + +如果你在 Unraid（或任何 Linux 机器）上跑 btrfs RAID1 + SATA SSD + 自动 TRIM，这篇是给你的提醒：**队列化 TRIM 正在同时、在同一位置、静默地摧毁我阵列的两份镜像。** RAID 冗余在这种损坏面前毫无保护力。而全程被 SMART 标记为「failing」的那块盘，跟真正的损坏一点关系都没有。 + +--- + +## 环境 + +- **unRaid 7.x**，Intel i9-13900K。 +- **`ssd` 池**：两块 **Seagate IronWolf 110 960 GB** SATA SSD + （`ZA960NM10001`，序列号 `HKR02TFK` + `HKR02L9P`）组 **btrfs RAID1**， + 挂载在 `/mnt/ssd`，上面跑着 Docker 数据根目录（`/var/lib/docker`）和 + Windows 11 虚拟机的虚拟磁盘（`domains/Win11Enterprise/vdisk1.img`）。 +- 虚拟机虚拟磁盘**没有**开 `NOCOW`，所以 btrfs 每次读取都会校验——和上次 + 事故能被发现靠的是同一个设置。 + +顺带一提：其中一块盘（`sdd`）正是 *[《我当时差点就去 RMA 这块 SSD——其实是 SATA 线坏了》](/posts/zh/that-dying-ssd-was-just-a-bad-sata-cable/)* +里那块盘——换线之后格式化干净，就进了这个池。 + +## 第一步——一个「假」的 SMART 故障 + +凌晨 02:53，Unraid 的通知铃响了： + +``` +Unraid Ssd disk SMART health [1] — Warning [UNRAID] - +raw read error rate (failing now) is 19665 (sdd = ZA960NM10001_HKR02TFK) +``` + +「failing now」是 SMART 里最强的措辞——盘自报该属性**跌破阈值**。但在这些 +Seagate 盘上，raw 值是障眼法：IronWolf 110 的属性 1 raw 字段是厂商编码的 +复合值（高半部分是错误数，低半部分是操作计数）。`19665 = 0x00004CD1` → +高半部分 = **0 个错误**。真正触发的是*归一化*值：`093`，**最差值 088**， +低于 `090` 阈值，于是盘在 SMART 返回状态里自报 `FAILING_NOW`/`IN_THE_PAST`。 +另一块盘（`sdb`）形状相同，raw 只有 `341`。 + +所有真实的故障计数器都是零：重映射 0、坏块增长 0、编程/擦除失败 0、 +不可纠正 ECC 0、UDMA CRC 0、寿命剩余 99 %、SMART 总体 **PASSED**，刚跑完的 +扩展自检也干净。这是 IronWolf 110 有据可查的行为——多年前 Synology 用户就 +在 100 % 寿命时撞上过同样的「Failing」旗标，社区共识是：只要真实计数器不动， +这个型号的属性 1 触发就当固件噪音处理。 + +所以：盘是误报，但这次报警让我去看了——然后才找到真正的损坏。 + +## 第二步——一个不该存在的校验和 + +`btrfs device stats /mnt/ssd`： + +``` +[/dev/sdd1].corruption_errs 27 [/dev/sdb1].corruption_errs 31 +``` + +全部集中在同一个文件：**ino 261 = Windows 虚拟机的虚拟磁盘**。 +`dmesg` 里是上一个 NVMe 事故里我盯了一周的同一个签名： + +``` +btrfs: checksum verify failed on logical 1505629372416 mirror 1 wanted 0x8941f998 +``` + +**`0x8941f998` 是 4 KiB 全零块的 CRC32C。** 池在读出空白块——而且关键的，是 +在**两块盘的相同逻辑偏移上同时读出**。我翻了 PM9A3 阵列那次的笔记：值相同， +「确定性全零」行为相同，让 RAID1 失效的关联方式也相同。 + +这个匹配是指纹级的。无论这是什么，它都不是硬盘的问题，也不是 M.2 转接卡的 +问题——IronWolf 这对盘走的是 **SATA 口**，完全不同的控制器路径。问题出在 +**比硬盘高一层**：有一个机制决定同时擦掉两块盘上相同的数据。Linux 里恰好 +只有一个机制干这事：**discard / TRIM**。 + +## 第三步——根因：在 Linux 从未拉黑的硬盘上跑队列化 TRIM + +Unraid 的池配置开了自动 TRIM，btrfs 以 `discard=async` 挂载——内核批量收集 +空闲块发给硬盘。对 SATA SSD 来说，那就是**队列化 TRIM**（`DATA SET +MANAGEMENT` 命令以 NCQ 命令形式延迟执行，即 `SEND FPDMA QUEUED`）。 + +IronWolf 110 的队列化 TRIM **不稳定的问题早就被记录了**。FreeBSD 内核有一个 +文档化的报告——[bug 264139](https://bugs.freebsd.org/264139)， +*「ata: NCQ_DSM_TRIM trim method for Seagate IronWolf 110 SATA SSD hangs drives」*， +2024 年合并的修复提交写得毫不客气：[commit `a6cef617660a`](https://lists.freebsd.org/archives/freebsd-fs/2024-March/003273.html)： +「Seagate IronWolf 110 SATA SSD 已被报告在启用 NCQ trim 时不稳定。」 + +而 Linux 自己的怪癖清单（`drivers/ata/libata-core.c`）——内核会在这里主动把 +有问题的盘降级为非队列化或禁用 TRIM——收录了 Micron M500/M550/1100、 +Crucial M500/M550/MX100、三星 840/850/860/870 等，但**没有一条 Seagate +条目**。我自己机器上的 `dmesg` 恰好证明这个机制存在而且在选择性生效： + +``` +ata6.00: Model 'Samsung SSD 840 PRO Series', rev 'DXM06B0Q', applying quirks: noncqtrim zeroaftertrim +``` + +这条怪癖给了机箱里*另一块* SSD，而两块 IronWolf 110 什么怪癖都没有， +拿到了完整的队列化 TRIM。 + +为什么这会毁掉 RAID1？因为 btrfs RAID1 在两块成员盘上存的是*同一份*空闲 +映射——会在同一轮里对两块盘丢弃相同的逻辑区间。如果这个固件上的 TRIM 路径 +会多失效（或失效错）LBA，就会**同时清掉两块镜像上的同一批活动块**。你的 +镜像变成同一个洞的两份拷贝：两块盘都校验失败，自愈没有任何可抄的源，数据 +彻底没了。 + +> 诚实的保留意见：FreeBSD 记录的是盘侧的*不稳定/卡死*，而「队列化 TRIM 抹掉 +> 了活动数据」是我根据这个记录加指纹（双镜像全零、SMART 干净、CRC 错误计数 +> 不涨）推断的机制。它符合每一条观察。证明靠实验——见下面的结果。 + +## 第四步——修复（持久化，一次只改一个变量） + +只改一处，双管齐下——持久配置 + 热挂载： + +```bash +# /boot/config/pools/ssd.cfg （备份：/root/ssd.cfg.bak.20260918-0329） +diskAutotrim="off" # 原来是 "on" + +# 不停阵列、不动 Docker（36 个容器全程在线）直接生效 +mount -o remount,nodiscard /mnt/ssd +findmnt /mnt/ssd # → rw,noatime,ssd,space_cache=v2 （不再有 discard=async） +``` + +然后做一次完整读写 scrub，搞清楚到底坏了多少： + +``` +scrub started 03:29:53, finished 03:53:07 (23 min) +csum_errors: 18 corrected_errors: 16 uncorrectable_errors: 2 +read_errors: 0 verify_errors: 0 super_errors: 0 +``` + +`read/verify/super = 0` 是指纹的另一半：硬件读盘*完全正常*——没有任何部件在 +故障，只是盘上存的字节和校验和不符。18 个坏块里，**16 块只有一块镜像坏， +已从完好副本自愈；2 个 uncorrectable 其实是同一个 4 KiB 块两块镜像都坏**—— +永久丢失，位于虚拟机虚拟磁盘约 45.2 GiB 处（`logical 1505629372416`，文件 +偏移 `48560156672`）。我把这一个块填零，至少虚拟机读到那里不再报错；原始 +字节无论如何都找不回来了。 + +还有一个值得知道的坑：**scrub 跑了 18 个错误，`btrfs device stats` 计数器 +却纹丝不动**——scrub 发现的错误和读取时发现的错误是分开统计的。你没法只盯 +`device stats` 一个数；dmesg 和 scrub 输出也得一起盯。（我的看门狗三样都查， +细节见文末。） + +## 换成我会怎么做 + +1. **任何 btrfs 池开自动 TRIM 之前，先查这块盘的队列化 TRIM 记录。** + FreeBSD 维护一份「别给这盘发 TRIM」清单不是没有原因的。Linux 在 + `libata-core.c` 里有同样的机制——花一个下午 grep + 首次开机后看一眼 + `dmesg`，好过一次从备份里恢复虚拟机。 +2. **TRIM 可以有，但不能队列化。** SATA SSD 想跑 fstrim 就强制非队列化 + （`libata.force=…:noncqtrim`），并在第一次 trim 后做一次 scrub 验证。 + `discard=sync` **没用**——只要盘支持，sync 同样走队列化路径。 +3. **三个损坏信号一起盯**，别只看一个：`btrfs device stats`、 + `dmesg | grep 'csum failed'`、周期性 scrub。三者各有各的计数器（见上文）。 +4. **并入池之前先交叉验证。** 同批次、同固件的盘是关联故障风险；两块盘共一个 + 固件 bug 就是最坏的那种关联故障。镜像两次都没救得了我。 +5. **买企业级 SATA SSD 时**：Linux 怪癖清单上的盘（Micron M500/M550/1100、 + Crucial M500/M550/MX100、三星 840/850/860/870）以及一切有记录在案的 + NCQ-TRIM bug（IronWolf 110）直接排除。三星 PM893/PM897、Micron 5400 + PRO/MAX、Solidigm D3-S4610/S4620、WD Ultrastar DC SA620/SA630、Kingston + DC600M 都有断电保护（PLP）且不在 Linux 怪癖清单上；而且 Micron、Solidigm、 + WD、Kingston 会公开发布固件更新说明，真出了问题还能给你推送修复。NVMe + 从根上绕开了 SATA 队列化 TRIM 这一类问题——我那两块 PM9A3 也不在 NVMe + 怪癖表里。 + +## 结果 + +只改了一个变量（关自动 TRIM），其他原封不动。紧接着： + +- 第二次 scrub（04:10–04:39）：**`corrected_errors: 0`** ——16 块自愈块保持 + 住了，整盘重读没有出现任何新错误。 +- `corruption_errs` 维稳在 sdd1=27 / sdb1=31，持续数小时的 Docker + 无虚拟机 + 运行（此前计数器是每天在涨的）。 +- dmesg 里没有新增 `csum failed` 行。 + +确认窗口还有几天：Win11 虚拟机重新上线（它产生的是能复现问题的写入模式）、 +对照基线文件每日检查、2026-09-22 再做一次复核 scrub。如果关了 TRIM 损坏还在， +下一个嫌疑就不是硬盘，而是内存（这台机器没有 ECC）——用 memtest 测。如果 +一直平静，那 IronWolf 110 队列化 TRIM 固件 bug 就算坐实了——我敢说论坛里 +「健康硬盘毁掉我的 RAID」的帖子，相当一部分也是它干的。 + +一个「正在故障」的 SMART 告警、一块零错误的硬盘、一个全新阵列上双镜像同时 +损坏——每个迹象都指向别处，而真正的凶手是一个我们大多数人不假思索就开启的 +存储功能。 + +--- + +*我的存储故障系列之一：[NVMe 事故](/posts/when-smart-says-healthy-but-your-raid-is-corrupting-data/) 和 [SATA 线警醒](/posts/that-dying-ssd-was-just-a-bad-sata-cable/)。* \ No newline at end of file diff --git a/src/content/posts/zh/when-smart-says-healthy-but-your-raid-is-corrupting-data.md b/src/content/posts/zh/when-smart-says-healthy-but-your-raid-is-corrupting-data.md index f68c899..8a8ba73 100644 --- a/src/content/posts/zh/when-smart-says-healthy-but-your-raid-is-corrupting-data.md +++ b/src/content/posts/zh/when-smart-says-healthy-but-your-raid-is-corrupting-data.md @@ -1,6 +1,6 @@ --- title: "SMART 显示健康，RAID 却在悄悄损坏数据" -description: "两块看起来完全健康的 NVMe 组成的 btrfs RAID1 阵列，却静默地把 Windows 虚拟机喂成了全零字节。我如何抓住它、靠校验和取证证明它，以及为什么最终要把两块盘都换掉。" +description: "两块看起来完全健康的 NVMe 组成的 btrfs RAID1 阵列，静默地把 Windows 虚拟机喂成了全零字节。我靠校验和取证抓住它——但根因（M.2 转接卡链路故障 vs 批次缺陷）至今仍未定论，现在两块盘测试都干净。" pubDate: 2026-09-16 category: devops tags: [unraid, btrfs, nvme, raid, storage, data-loss, troubleshooting] @@ -10,7 +10,25 @@ banner: /banners/when-smart-says-healthy-but-your-raid-is-corrupting-data.png 所有存储指南都告诉你同一件事：相信 SMART、检查事件日志、相信你的文件系统。我的硬件不这么想。2026 年 9 月，一个 btrfs RAID1 阵列——两块企业级 NVMe 盘镜像，我用每个能想到的工具查它都说**「健康」**——却在悄悄地把我的 Windows 11 虚拟机喂成全零字节。不是看得见的故障，是静默损坏。 -下面是完整故事：损坏是怎么浮现的、哪个校验和出卖了它、以及为什么答案必须是「两块盘都换，别信 SMART」。 +下面是完整故事：损坏是怎么浮现的、哪个校验和出卖了它——以及为什么最终定论如今变得不确定。 + +> **更新（2026-09-16）：根因至今未定论。** 下面的「两块盘都换」是我当时的判断，但后来一次不相关的故障让我不得不重新审视。现在两块 NVMe 测试都干净，而最强的头号嫌疑是两块盘共用的那块 **M.2 转接卡**——转接卡上的链路层故障，恰好会复现我当初归咎于硬盘的那种「两块盘关联地吐出全零」的损坏。SMART 对链路故障完全无感，所以无法排除它。我在一篇配套文章里写了那次警醒（一颗「死掉的」SATA SSD 其实只是坏线）。先读下面的原始分析，再看更正——然后还有**续集**：两周后同一个全零指纹出现在**完全不同的硬盘堆栈**上，这次终于挖出了根因。见[《换了不同的硬盘，同样的损坏又来了——真凶是 TRIM，不是 SSD》](/posts/zh/the-cause-was-trim-not-the-ssds/)。 + +## 更新：为什么我不再怪两块盘 + +这篇文章发出后，有两件事改变了我的想法。 + +**1. 现在两块盘测试都是干净的。** 在当前条件下，两块 PM9A3 读写都正常。如果它们带着 9 月还在活跃损坏数据的批次级缺陷，应该会挂掉一轮持续的写入+校验（`verify=crc32c`）泡测——但它们没有。 + +**2. 链路故障才是真正的教训。** 就在这次事故之后，我在另一颗盘上撞见了**完全相同**的故障签名（一颗 Seagate IronWolf 110 狂报 `WRITE FPDMA QUEUED` 超时、`mkfs` 失败、然后静默——教科书般的「死盘」）。结果是坏在 **SATA 数据线**上。一条不稳定的链路会复现下面我归咎于硬件的每一个症状：在算出 CRC 之前就已丢掉的超时 I/O、真的丢掉的页面，以及——如果两个 RAID1 成员共用同一块有问题的 **M.2 转接卡**或插槽——两块盘关联地损坏。这是唯一一个不需要任何硬盘缺陷、却能解释「两个成员用同一种方式撒谎」的答案。 + +所以我没法再诚实地断言「盘有问题」。它们可能一直好好的，损坏可能来自它们共同坐着的 **M.2 转接卡**。注意：**这并不代表这次事故是虚惊一场**——真实数据确实读不回来过，无论根因是什么，我都把它移走、迁移数据、并逐字节验证过。但正确的教训是「先排除链路，再给盘判死刑」，而不是「同批次盘不可信」。 + +完整的链路对盘排查，请看 **["我差点把这颗 SSD 送修——真正坏的是 SATA 数据线"](/posts/zh/that-dying-ssd-was-just-a-bad-sata-cable/)**。 + +## 原始分析（保留供透明考量） + +下面是我*当时*得出的结论。这是当时证据下最站得住的解读，而且可能仍然是对的——只是我无法排除别的可能——但请把它当作事故报告来读，而不是最终定论。 ## 为什么这件事重要 @@ -61,7 +79,9 @@ btrfs RAID1 本应自愈：一个成员坏块、从镜像重建、写回好数 当自愈重写一直不生效，你就不在「坏一个扇区」的范畴了，而是「这块盘在写入上撒谎」——RAID 冗余救不了。 -## 为什么两块盘都得换 +## 为什么两块盘都得换（我当时的推理） + +> 这是当初「都换掉」决定的依据。动手前先读上面更新里的更正——一块共用的 M.2 转接卡链路故障，在无需任何硬盘缺陷的情况下也能对上这份证据。 数据指向的是**同批次盘的硬件缺陷**，而不是磨损或一次性的偶发事件： @@ -90,8 +110,13 @@ btrfs RAID1 本应自愈：一个成员坏块、从镜像重建、写回好数 ## 结果 -一台 Windows 虚拟机从备份恢复、两块确认故障的盘换掉、Docker 数据根目录安全迁出该池。代价是几天折腾和一次救援——但损失被控制在最小，因为损坏是被校验和抓住的，而不是在里面泡了几个星期。 +一台 Windows 虚拟机从备份恢复、数据安全迁出那个坏池、Docker 数据根目录也搬了家——**外加一条事后再推翻的教训。** 我当时换掉了两块盘，但现在我没法确定它们就是元凶：证据同样支持一块共用的 **M.2 转接卡**链路故障，而且现在两块盘测试都干净。（完整排查见 **["我差点把这颗 SSD 送修——真正坏的是 SATA 数据线"](/posts/zh/that-dying-ssd-was-just-a-bad-sata-cable/)**。） -最该记住的一句话：**对存储来说，「冗余」只等于你最不靠谱的那个成员。** 两块一起坏掉的盘不是镜像——那是穿着两个序列号的单点故障。验证你的恢复、校验你的数据、当某块盘的零块「奖励」反复出现时，把整组都换掉。 +事故被控制住，是因为损坏被校验和抓住了、而不是在里面泡了几个星期——这一点无论根因是什么都成立。但更诚实的教训，比「把盘换掉」这句要更广、也更不戏剧化： + +1. **一块「正在损坏」的盘，并不等于盘本身坏了。** 在给硬件判死刑前，先排除链路：重新插拔/更换线材、换到别的口/转接卡/插槽，重跑*相同*的失败操作。如果故障跟着**口/转接卡**走，那就是连接的问题。 +2. **SMART 看不见链路故障。** 零媒体错误和几乎满余量，对一条不稳定的线材或转接卡毫无意义——超时在算出 CRC 之前就把 I/O 丢了。 +3. **同批次盘共用一条通道，无论谁坏都是共享失效风险。** 不管损坏来自盘还是转接卡，一对盘坐同一条故障路径上，就是穿着两个序列号的单点故障。 +4. **给你的数据加校验、并提前测试恢复备份。** 把数星期静默损坏变成一次被控制住的事故的，是这一点——不是我那部分诊断。 --- \ No newline at end of file