SEO: sitemap lastmod + hreflang alternates; robots.txt allows all crawlers

robots.txt carried a UTF-8 em dash in its header comment, which renders as
mojibake ("鈥�") in clients that read text/plain as a legacy codepage. The
file is now pure ASCII (verified with `LC_ALL=C grep '[^ -~]'`), and the
encoding trap is documented in the endpoint so it does not come back.

Policy change to allow-all: the previous version blocked CCBot, Bytespider,
Amazonbot and Applebot-Extended. Training crawlers are now welcome too - the
blog benefits from being read, and robots.txt is advisory anyway (Cloudflare
documents it as unenforced; AI Crawl Control is the enforcement layer).
Drops the Content-Signal directives along with the blocklist.

Sitemap gains both fields Google actually uses:
- lastmod on all 58 post URLs, from updatedDate ?? pubDate, read straight
  from the markdown frontmatter at config-eval time (the sitemap runs in
  astro:build:done, after the content collection is gone).
- xhtml:link hreflang alternates on all 77 URLs, pairing EN/ZH twins.
  /posts/ is special-cased: it has no /zh/posts/ twin, the Chinese post
  listing IS the /zh/ homepage, so the pair is declared rather than derived.
  Verified programmatically that every emitted alternate resolves to a page
  that is actually built and present in the sitemap.

astro.config.mjs

@@ -1,5 +1,60 @@
 import { defineConfig } from 'astro/config';
 import sitemap from '@astrojs/sitemap';
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// ---------------------------------------------------------------------------
+// Post metadata, read straight from the markdown at config-eval time.
+// The sitemap is generated in `astro:build:done`, so the config layer resolves
+// BEFORE the content collection exists - reading the files directly is the
+// supported way to get post dates into `serialize()`.
+// ---------------------------------------------------------------------------
+const POSTS_DIR = fileURLToPath(new URL('./src/content/posts', import.meta.url));
+
+/** Pull `key: value` pairs out of a markdown frontmatter block. */
+function frontmatter(file) {
+  const raw = readFileSync(file, 'utf-8');
+  const end = raw.indexOf('---', 3);
+  const block = end === -1 ? raw : raw.slice(0, end);
+  const out = {};
+  for (const line of block.split(/\r?\n/)) {
+    const m = line.match(/^([A-Za-z][A-Za-z0-9_]*):\s*(.+)$/);
+    if (m) out[m[1]] = m[2].trim().replace(/^["']|["']$/g, '');
+  }
+  return out;
+}
+
+/** Normalize a frontmatter date (2026-09-04) to a sitemap lastmod (W3C). */
+function isoDate(d) {
+  if (!d) return undefined;
+  const t = Date.parse(d);
+  return Number.isNaN(t) ? undefined : new Date(t).toISOString();
+}
+
+// slug -> { en, zh } ISO dates. EN and ZH twins share a slug, so one entry
+// serves both localized URLs; lastmod = the newest of the two.
+const postDates = new Map();
+{
+  const collect = (dir, key) => {
+    for (const file of readdirSync(dir, { withFileTypes: true })) {
+      if (!file.isFile() || !file.name.endsWith('.md')) continue;
+      const slug = file.name.slice(0, -3);
+      const fm = frontmatter(join(dir, file.name));
+      const date = isoDate(fm.updatedDate ?? fm.pubDate);
+      if (!date) continue;
+      const entry = postDates.get(slug) ?? {};
+      entry[key] = date;
+      postDates.set(slug, entry);
+    }
+  };
+  try {
+    collect(POSTS_DIR, 'en');
+    collect(join(POSTS_DIR, 'zh'), 'zh');
+  } catch (e) {
+    console.warn('[sitemap] could not read post dates:', e.message);
+  }
+}
 
 // https://astro.build/config
 export default defineConfig({
@@ -12,5 +67,47 @@ export default defineConfig({
       prefixDefaultLocale: false,
     },
   },
-  integrations: [sitemap()],
+  integrations: [
+    sitemap({
+      // lastmod lets Google re-crawl content that actually changed instead of
+      // falling back to its own heuristics. hreflang alternates in the sitemap
+      // are treated as more reliable than the in-page <link rel="alternate">
+      // tags when the two disagree, so both are emitted.
+      serialize(item) {
+        const path = new URL(item.url).pathname;
+        const post = path.match(/^\/posts\/(?:zh\/)?([^/]+)\/$/);
+
+        if (post) {
+          // Post pages get lastmod; their alternates are the EN/ZH twins.
+          const dates = postDates.get(post[1]);
+          if (dates) item.lastmod = dates.en > dates.zh ? dates.en : dates.zh ?? dates.en;
+
+          item.links = [
+            { lang: 'en', url: `https://blog.hoelee.com/posts/${post[1]}/` },
+            { lang: 'zh', url: `https://blog.hoelee.com/posts/zh/${post[1]}/` },
+          ];
+          return item;
+        }
+
+        // Non-post routes: pair the two landing/section pages by pathname.
+        // Exception: /posts/ has no /zh/posts/ twin - the Chinese post listing
+        // IS the /zh/ homepage (the ZH nav "Posts" points there), so that pair
+        // is declared by hand rather than derived.
+        const MANUAL = {
+          '/posts/': { en: '/posts/', zh: '/zh/' },
+          '/zh/': { en: '/posts/', zh: '/zh/' },
+        };
+
+        const src = /^\/zh\/(.*)$/.exec(path);
+        const pair = MANUAL[path]
+          ?? (src ? { en: `/${src[1]}`, zh: path } : { en: path, zh: `/zh/${path.slice(1)}` });
+
+        item.links = [
+          { lang: 'en', url: `https://blog.hoelee.com${pair.en}` },
+          { lang: 'zh', url: `https://blog.hoelee.com${pair.zh}` },
+        ];
+        return item;
+      },
+    }),
+  ],
 });

src/pages/robots.txt.ts

@@ -2,18 +2,20 @@ import type { APIRoute } from 'astro';
 import { SITE } from '../config';
 
 /**
- * robots.txt — served from the origin so Cloudflare merges its managed
+ * robots.txt - served from the origin so Cloudflare merges its managed
  * block instead of substituting the placeholder Content Signals Policy.
  *
- * Policy here: allow classic search (this blog is a name-search / GEO play)
- * and allow the citation crawlers that power AI Overviews / ChatGPT Search /
- * Perplexity, because being cited is the point of the two-lane strategy in
- * docs/seo-reference.md. Block only the training-only harvesters that give
- * no referral traffic back.
+ * Policy: allow everything. This blog is a name-search / GEO play, so
+ * classic search, citation engines AND AI training crawlers are all
+ * welcome - being cited and being read are both wins.
+ * See docs/seo-reference.md (two-lane strategy).
+ *
+ * ASCII-ONLY on purpose: a UTF-8 em dash in a text/plain file renders as
+ * mojibake (e.g. "鈥�") in clients that read it as a legacy codepage.
  */
 export const GET: APIRoute = () => {
-  const body = `# robots.txt — blog.hoelee.com
-# Policy: index freely. Allow citation/answer engines, block training-only harvesters.
+  const body = `# robots.txt - blog.hoelee.com
+# Policy: allow all crawlers. Search, citation and AI training welcome.
 
 User-agent: *
 Allow: /
@@ -21,27 +23,6 @@ Allow: /
 # Pagefind's search shards are build artifacts, not content.
 Disallow: /pagefind/
 
-# AI trainers that send no traffic back (blocked).
-User-agent: CCBot
-Disallow: /
-
-User-agent: Bytespider
-Disallow: /
-
-User-agent: Amazonbot
-Disallow: /
-
-User-agent: Applebot-Extended
-Disallow: /
-
-# Deliberately ALLOWED (cite us, don't train on us — enforced by Cloudflare
-# Content-Signal below): Googlebot / Google-Extended (AI Overviews, Gemini),
-# OAI-SearchBot / GPTBot (ChatGPT Search), PerplexityBot, ClaudeBot, Bingbot.
-
-# Content signals: machine-readable preference statements (contentsignals.org).
-User-Agent: *
-Content-Signal: search=yes, ai-input=yes, ai-train=no
-
 Sitemap: ${SITE.url}/sitemap-index.xml
 `;