feat: initial release of llm-fingerprint-detector

TypeScript library + CLI implementing single-token behavioral fingerprinting of LLMs behind OpenAI-compatible APIs, per 'One Token Is Enough' (Bruckner, arXiv:2607.10252). - Zero runtime dependencies (Node >= 18.17 built-in fetch) - fingerprint / compare / verify API + llm-fingerprint CLI - Jensen-Shannon divergence over per-cell answer distributions - Reasoning-model aware (auto-disables hidden thinking channels) - Bundled sample references for 11 popular models (CC-BY-4.0, derived from the paper's public Zenodo dataset) - Unit + integration tests via node --test Co-authored-by: Cursor <[email protected]>
2026-07-21 16:58:00 +08:00
commit 232fbc6dd8
32 changed files with 5676 additions and 0 deletions
@@ -0,0 +1,169 @@
+#!/usr/bin/env node
+/**
+ * Build data/reference-fingerprints.sample.json from the paper's public
+ * Zenodo dataset (DOI 10.5281/zenodo.21278557, CC-BY-4.0).
+ *
+ * Input: the dataset's aggregated `distributions.json` — an object with a
+ * `distributions` array of per-cell records:
+ *   { model, task_id, lang, temperature, n_valid, dist: { answer: probability }, ... }
+ *
+ * Counts are reconstructed as round(probability × n_valid) and answers are
+ * re-normalized with this package's normalizer so vocabularies line up
+ * (e.g. the dataset's coin answers h/t → heads/tails, 蓝色 → 蓝). Cells that
+ * lose more than 20% of their probability mass in re-normalization are
+ * skipped.
+ *
+ * Usage:
+ *   npm run build   # the script imports the compiled normalizer from dist/
+ *   node scripts/build-sample-references.mjs <path-to-distributions.json> \
+ *     [--models comma,separated,slugs] [--out data/reference-fingerprints.sample.json]
+ */
+
+import { readFileSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import process from 'node:process'
+import { fileURLToPath } from 'node:url'
+
+import { PROBE_TASKS } from '../dist/battery.js'
+import { normalizeAnswer } from '../dist/normalizer.js'
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+
+/** Dataset task_id → this package's battery task id. */
+const TASK_MAP = {
+  'num100-random': 'random-number-1-100',
+  'num10-random': 'random-number-1-10',
+  'letter-random': 'random-letter',
+  'color-random': 'random-color',
+  'coin-flip': 'coin-flip',
+  'animal-random': 'random-animal',
+  'city-random': 'random-city',
+  'num-favorite': 'favorite-number',
+}
+
+const LANGS = new Set(['en', 'zh'])
+
+/** The dataset folds coin answers to single letters; pre-expand before normalizing. */
+const COIN_PREMAP = { h: 'heads', t: 'tails' }
+
+const DEFAULT_MODELS = [
+  'openai/gpt-4o-mini',
+  'openai/gpt-4o',
+  'openai/gpt-4.1-mini',
+  'anthropic/claude-sonnet-4.5',
+  'google/gemini-2.5-flash',
+  'deepseek/deepseek-chat',
+  'meta-llama/llama-3.1-8b-instruct',
+  'qwen/qwen3-30b-a3b-instruct-2507',
+  'mistralai/mistral-small-3.2-24b-instruct',
+  'z-ai/glm-4.5',
+  'moonshotai/kimi-k2',
+]
+
+function parseArgs(argv) {
+  const args = { input: null, models: DEFAULT_MODELS, out: join(HERE, '..', 'data', 'reference-fingerprints.sample.json') }
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i]
+    if (token === '--models') args.models = argv[++i].split(',').map((m) => m.trim())
+    else if (token === '--out') args.out = argv[++i]
+    else if (!token.startsWith('--') && !args.input) args.input = token
+    else throw new Error(`Unknown argument: ${token}`)
+  }
+  if (!args.input) {
+    console.error('usage: node scripts/build-sample-references.mjs <distributions.json> [--models a,b] [--out file]')
+    process.exit(1)
+  }
+  return args
+}
+
+function convertCell(record, taskId) {
+  const domain = PROBE_TASKS[taskId].domain
+  const counts = {}
+  let keptMass = 0
+  let totalMass = 0
+  for (const [rawAnswer, probability] of Object.entries(record.dist)) {
+    totalMass += probability
+    const premapped = taskId === 'coin-flip' ? (COIN_PREMAP[rawAnswer] ?? rawAnswer) : rawAnswer
+    const { normalized, category } = normalizeAnswer(premapped, domain)
+    if (category !== 'valid' || normalized === null) continue
+    const count = Math.round(probability * record.n_valid)
+    if (count <= 0) continue
+    counts[normalized] = (counts[normalized] ?? 0) + count
+    keptMass += probability
+  }
+  if (totalMass <= 0 || keptMass / totalMass < 0.8) return null
+  const n = Object.values(counts).reduce((sum, c) => sum + c, 0)
+  if (n < 10) return null
+  return { n, counts }
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2))
+  const payload = JSON.parse(readFileSync(args.input, 'utf8'))
+  const records = payload.distributions
+  if (!Array.isArray(records)) throw new Error('Input has no "distributions" array')
+
+  const collectedAt = (payload.generated_utc ?? '').slice(0, 10) || 'unknown'
+  const wanted = new Set(args.models)
+  const models = {}
+  let skippedCells = 0
+
+  for (const record of records) {
+    if (!wanted.has(record.model)) continue
+    if (!LANGS.has(record.lang)) continue
+    const taskId = TASK_MAP[record.task_id]
+    if (!taskId) continue
+    if (record.temperature !== 1) continue
+
+    const cell = convertCell(record, taskId)
+    if (!cell) {
+      skippedCells += 1
+      continue
+    }
+    const cellId = `${taskId}:${record.lang}`
+    models[record.model] ??= {
+      model: record.model,
+      collectedAt,
+      channel: 'openrouter',
+      cells: {},
+    }
+    models[record.model].cells[cellId] = cell
+  }
+
+  for (const model of args.models) {
+    const entry = models[model]
+    if (!entry) {
+      console.warn(`warn: model not found in dataset: ${model}`)
+    } else if (Object.keys(entry.cells).length < 8) {
+      console.warn(`warn: ${model} only has ${Object.keys(entry.cells).length} usable cells`)
+    }
+  }
+
+  const output = {
+    formatVersion: 1,
+    protocol: 'bruckner-zenodo-2026',
+    samplesPerCell: 30,
+    source: {
+      dataset:
+        'Single-token output distributions as behavioral fingerprints of large language models',
+      author: 'Tomáš Bruckner (Prague University of Economics and Business)',
+      datasetDoi: '10.5281/zenodo.21278557',
+      paper: 'arXiv:2607.10252',
+      license: 'CC-BY-4.0',
+      note:
+        'Counts reconstructed as round(probability × n_valid) from the published per-cell distributions; ' +
+        'answers re-normalized with the llm-fingerprint-detector normalizer. Collected via OpenRouter by ' +
+        "the paper's harness under the paper's prompt protocol (not this package's battery), so comparisons " +
+        'against these samples are indicative rather than strict.',
+    },
+    models,
+  }
+
+  writeFileSync(args.out, `${JSON.stringify(output, null, 2)}\n`, 'utf8')
+  const cellTotal = Object.values(models).reduce((sum, m) => sum + Object.keys(m.cells).length, 0)
+  console.log(
+    `wrote ${args.out}: ${Object.keys(models).length} models, ${cellTotal} cells (${skippedCells} cells skipped in re-normalization)`,
+  )
+}
+
+main()