diff --git a/astro.config.mjs b/astro.config.mjs index 5cd6211..271cf20 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -1,5 +1,60 @@ import { defineConfig } from 'astro/config'; import sitemap from '@astrojs/sitemap'; +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +// --------------------------------------------------------------------------- +// Post metadata, read straight from the markdown at config-eval time. +// The sitemap is generated in `astro:build:done`, so the config layer resolves +// BEFORE the content collection exists - reading the files directly is the +// supported way to get post dates into `serialize()`. +// --------------------------------------------------------------------------- +const POSTS_DIR = fileURLToPath(new URL('./src/content/posts', import.meta.url)); + +/** Pull `key: value` pairs out of a markdown frontmatter block. */ +function frontmatter(file) { + const raw = readFileSync(file, 'utf-8'); + const end = raw.indexOf('---', 3); + const block = end === -1 ? raw : raw.slice(0, end); + const out = {}; + for (const line of block.split(/\r?\n/)) { + const m = line.match(/^([A-Za-z][A-Za-z0-9_]*):\s*(.+)$/); + if (m) out[m[1]] = m[2].trim().replace(/^["']|["']$/g, ''); + } + return out; +} + +/** Normalize a frontmatter date (2026-09-04) to a sitemap lastmod (W3C). */ +function isoDate(d) { + if (!d) return undefined; + const t = Date.parse(d); + return Number.isNaN(t) ? undefined : new Date(t).toISOString(); +} + +// slug -> { en, zh } ISO dates. EN and ZH twins share a slug, so one entry +// serves both localized URLs; lastmod = the newest of the two. +const postDates = new Map(); +{ + const collect = (dir, key) => { + for (const file of readdirSync(dir, { withFileTypes: true })) { + if (!file.isFile() || !file.name.endsWith('.md')) continue; + const slug = file.name.slice(0, -3); + const fm = frontmatter(join(dir, file.name)); + const date = isoDate(fm.updatedDate ?? fm.pubDate); + if (!date) continue; + const entry = postDates.get(slug) ?? {}; + entry[key] = date; + postDates.set(slug, entry); + } + }; + try { + collect(POSTS_DIR, 'en'); + collect(join(POSTS_DIR, 'zh'), 'zh'); + } catch (e) { + console.warn('[sitemap] could not read post dates:', e.message); + } +} // https://astro.build/config export default defineConfig({ @@ -12,5 +67,47 @@ export default defineConfig({ prefixDefaultLocale: false, }, }, - integrations: [sitemap()], + integrations: [ + sitemap({ + // lastmod lets Google re-crawl content that actually changed instead of + // falling back to its own heuristics. hreflang alternates in the sitemap + // are treated as more reliable than the in-page + // tags when the two disagree, so both are emitted. + serialize(item) { + const path = new URL(item.url).pathname; + const post = path.match(/^\/posts\/(?:zh\/)?([^/]+)\/$/); + + if (post) { + // Post pages get lastmod; their alternates are the EN/ZH twins. + const dates = postDates.get(post[1]); + if (dates) item.lastmod = dates.en > dates.zh ? dates.en : dates.zh ?? dates.en; + + item.links = [ + { lang: 'en', url: `https://blog.hoelee.com/posts/${post[1]}/` }, + { lang: 'zh', url: `https://blog.hoelee.com/posts/zh/${post[1]}/` }, + ]; + return item; + } + + // Non-post routes: pair the two landing/section pages by pathname. + // Exception: /posts/ has no /zh/posts/ twin - the Chinese post listing + // IS the /zh/ homepage (the ZH nav "Posts" points there), so that pair + // is declared by hand rather than derived. + const MANUAL = { + '/posts/': { en: '/posts/', zh: '/zh/' }, + '/zh/': { en: '/posts/', zh: '/zh/' }, + }; + + const src = /^\/zh\/(.*)$/.exec(path); + const pair = MANUAL[path] + ?? (src ? { en: `/${src[1]}`, zh: path } : { en: path, zh: `/zh/${path.slice(1)}` }); + + item.links = [ + { lang: 'en', url: `https://blog.hoelee.com${pair.en}` }, + { lang: 'zh', url: `https://blog.hoelee.com${pair.zh}` }, + ]; + return item; + }, + }), + ], }); diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts index 03a1cf9..819f985 100644 --- a/src/pages/robots.txt.ts +++ b/src/pages/robots.txt.ts @@ -2,18 +2,20 @@ import type { APIRoute } from 'astro'; import { SITE } from '../config'; /** - * robots.txt — served from the origin so Cloudflare merges its managed + * robots.txt - served from the origin so Cloudflare merges its managed * block instead of substituting the placeholder Content Signals Policy. * - * Policy here: allow classic search (this blog is a name-search / GEO play) - * and allow the citation crawlers that power AI Overviews / ChatGPT Search / - * Perplexity, because being cited is the point of the two-lane strategy in - * docs/seo-reference.md. Block only the training-only harvesters that give - * no referral traffic back. + * Policy: allow everything. This blog is a name-search / GEO play, so + * classic search, citation engines AND AI training crawlers are all + * welcome - being cited and being read are both wins. + * See docs/seo-reference.md (two-lane strategy). + * + * ASCII-ONLY on purpose: a UTF-8 em dash in a text/plain file renders as + * mojibake (e.g. "鈥�") in clients that read it as a legacy codepage. */ export const GET: APIRoute = () => { - const body = `# robots.txt — blog.hoelee.com -# Policy: index freely. Allow citation/answer engines, block training-only harvesters. + const body = `# robots.txt - blog.hoelee.com +# Policy: allow all crawlers. Search, citation and AI training welcome. User-agent: * Allow: / @@ -21,27 +23,6 @@ Allow: / # Pagefind's search shards are build artifacts, not content. Disallow: /pagefind/ -# AI trainers that send no traffic back (blocked). -User-agent: CCBot -Disallow: / - -User-agent: Bytespider -Disallow: / - -User-agent: Amazonbot -Disallow: / - -User-agent: Applebot-Extended -Disallow: / - -# Deliberately ALLOWED (cite us, don't train on us — enforced by Cloudflare -# Content-Signal below): Googlebot / Google-Extended (AI Overviews, Gemini), -# OAI-SearchBot / GPTBot (ChatGPT Search), PerplexityBot, ClaudeBot, Bingbot. - -# Content signals: machine-readable preference statements (contentsignals.org). -User-Agent: * -Content-Signal: search=yes, ai-input=yes, ai-train=no - Sitemap: ${SITE.url}/sitemap-index.xml `;