feat: initial release of llm-fingerprint-detector

TypeScript library + CLI implementing single-token behavioral fingerprinting of LLMs behind OpenAI-compatible APIs, per 'One Token Is Enough' (Bruckner, arXiv:2607.10252). - Zero runtime dependencies (Node >= 18.17 built-in fetch) - fingerprint / compare / verify API + llm-fingerprint CLI - Jensen-Shannon divergence over per-cell answer distributions - Reasoning-model aware (auto-disables hidden thinking channels) - Bundled sample references for 11 popular models (CC-BY-4.0, derived from the paper's public Zenodo dataset) - Unit + integration tests via node --test Co-authored-by: Cursor <[email protected]>
2026-07-21 16:58:00 +08:00
commit 232fbc6dd8
32 changed files with 5676 additions and 0 deletions
@@ -0,0 +1,48 @@
+/**
+ * Example 1 — collect a fingerprint from an endpoint you trust and save it
+ * as a reference for later verification.
+ *
+ * Run (from the package root, after `npm run build`):
+ *   OPENAI_API_KEY=sk-... node examples/01-fingerprint-endpoint.mjs
+ *
+ * With the published package, import from 'llm-fingerprint-detector' instead
+ * of '../dist/index.js'.
+ */
+
+import { writeFileSync } from 'node:fs'
+
+import { fingerprint } from '../dist/index.js'
+
+const apiKey = process.env.OPENAI_API_KEY
+if (!apiKey) {
+  console.error('Set OPENAI_API_KEY (the key is only sent to the endpoint you probe).')
+  process.exit(1)
+}
+
+const run = await fingerprint(
+  {
+    baseUrl: 'https://api.openai.com/v1', // any OpenAI-compatible endpoint
+    model: 'gpt-4o-mini',
+    apiKey,
+  },
+  {
+    cells: 8, // top-8 most discriminative cells (= "standard" preset)
+    samplesPerCell: 25,
+    concurrency: 4,
+    onProgress: (e) => {
+      if (e.stage === 'sampling') process.stderr.write(`\r${e.done}/${e.total}`)
+    },
+  },
+)
+process.stderr.write('\n')
+
+for (const warning of run.warnings) console.warn(`warning: ${warning}`)
+
+console.log(`model:            ${run.fingerprint.model}`)
+console.log(`cells:            ${Object.keys(run.fingerprint.cells).length}`)
+console.log(`errors:           ${run.errorCount}`)
+console.log(`split-half JSD:   ${run.splitHalfJsd?.toFixed(3) ?? 'n/a'} (same-model baseline ≈ 0.14)`)
+console.log(`reasoning adapter: ${run.adapter.strategy}`)
+
+writeFileSync('gpt-4o-mini.fingerprint.json', JSON.stringify(run.fingerprint, null, 2))
+console.log('\nsaved to gpt-4o-mini.fingerprint.json — use it later with verify()')
@@ -0,0 +1,52 @@
+/**
+ * Example 2 — verify that a cheap/unknown endpoint really serves the model it
+ * claims, by comparing its live fingerprint against a reference.
+ *
+ * Run (from the package root, after `npm run build`):
+ *   SUSPECT_API_KEY=sk-... node examples/02-verify-endpoint.mjs
+ *
+ * The reference here is a bundled sample derived from the paper's public
+ * dataset (CC-BY-4.0) — handy for a demo. For real audits, collect your own
+ * reference from the official API with example 01 and load that file instead.
+ */
+
+import { verify } from '../dist/index.js'
+import { loadBundledReference } from '../dist/reference.js'
+
+const apiKey = process.env.SUSPECT_API_KEY
+if (!apiKey) {
+  console.error('Set SUSPECT_API_KEY for the endpoint you want to test.')
+  process.exit(1)
+}
+
+// Endpoint under test: does it really serve gpt-4o-mini?
+const suspectEndpoint = {
+  baseUrl: 'https://openrouter.ai/api/v1', // ← put the reseller/aggregator URL here
+  model: 'openai/gpt-4o-mini',
+  apiKey,
+}
+
+const reference = loadBundledReference('openai/gpt-4o-mini')
+
+const result = await verify(suspectEndpoint, reference, {
+  cells: 8,
+  samplesPerCell: 25,
+  onProgress: (e) => {
+    if (e.stage === 'sampling') process.stderr.write(`\r${e.done}/${e.total}`)
+  },
+})
+process.stderr.write('\n')
+
+for (const warning of result.warnings) console.warn(`warning: ${warning}`)
+
+console.log(`verdict:  ${result.verdict}`)
+console.log(`mean JSD: ${result.meanJsd?.toFixed(3)} over ${result.comparison.comparableCellCount} cells`)
+console.log('baselines: same model ≈ 0.14 · cross-provider ≈ 0.227 · different model ≈ 0.463')
+
+console.log('\nmost divergent cells:')
+for (const cell of result.comparison.cells.slice(0, 5)) {
+  console.log(`  ${cell.cellId.padEnd(26)} JSD ${cell.jsd.toFixed(3)}`)
+}
+
+// CI-style decision:
+process.exitCode = result.verdict === 'match' ? 0 : 1
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# CLI usage examples. Keys always come from environment variables — never
+# paste keys into files or shell history.
+set -euo pipefail
+
+# After `npm install -g llm-fingerprint-detector` use `llm-fingerprint ...`;
+# inside this repo use `node dist/cli.js ...` (shown here via npx).
+LLM_FP="npx llm-fingerprint"
+
+# --- 0. Explore ---------------------------------------------------------------
+$LLM_FP --help
+$LLM_FP references                      # bundled sample reference fingerprints
+
+# --- 1. Fingerprint the official API and save a trusted reference --------------
+# export OPENAI_API_KEY=sk-...          # ← set in your shell / CI secret store
+$LLM_FP fingerprint \
+  --base-url https://api.openai.com/v1 \
+  --model gpt-4o-mini \
+  --preset standard \
+  --out reference.gpt-4o-mini.json
+
+# --- 2. Verify a cheap reseller claims honestly --------------------------------
+# export LLM_FINGERPRINT_API_KEY=sk-...  # key for the endpoint under test
+$LLM_FP verify \
+  --base-url https://cheap-llm-reseller.example.com/v1 \
+  --model gpt-4o-mini \
+  --reference reference.gpt-4o-mini.json
+# exit codes: 0 match · 2 mismatch · 3 uncertain · 4 insufficient · 1 error
+
+# --- 3. Quick demo against a bundled sample reference --------------------------
+$LLM_FP verify \
+  --base-url https://openrouter.ai/api/v1 \
+  --model openai/gpt-4o-mini \
+  --reference openai/gpt-4o-mini \
+  --preset quick --json
+
+# --- 4. Offline: compare two saved fingerprints --------------------------------
+$LLM_FP compare reference.gpt-4o-mini.json some-other.fingerprint.json
+
+# --- 5. Custom cell selection ---------------------------------------------------
+$LLM_FP fingerprint \
+  --base-url https://api.deepseek.com/v1 \
+  --model deepseek-chat \
+  --api-key-env DEEPSEEK_API_KEY \
+  --cells random-number-1-100:en,random-number-1-100:zh,random-color:en,coin-flip:en \
+  --samples 30 --json