feat: initial release of llm-fingerprint-detector

TypeScript library + CLI implementing single-token behavioral
fingerprinting of LLMs behind OpenAI-compatible APIs, per
'One Token Is Enough' (Bruckner, arXiv:2607.10252).

- Zero runtime dependencies (Node >= 18.17 built-in fetch)
- fingerprint / compare / verify API + llm-fingerprint CLI
- Jensen-Shannon divergence over per-cell answer distributions
- Reasoning-model aware (auto-disables hidden thinking channels)
- Bundled sample references for 11 popular models (CC-BY-4.0,
  derived from the paper's public Zenodo dataset)
- Unit + integration tests via node --test

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Deng-m1
2026-07-21 16:58:00 +08:00
co-authored by Cursor
commit 232fbc6dd8
32 changed files with 5676 additions and 0 deletions
+67
View File
@@ -0,0 +1,67 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { compare } from '../dist/api.js'
import {
getBundledAttribution,
listBundledReferences,
loadBundledReference,
parseFingerprintJson,
} from '../dist/reference.js'
test('bundled references load and carry attribution', () => {
const references = listBundledReferences()
assert.ok(references.length >= 5, 'expected several bundled sample references')
const attribution = getBundledAttribution()
assert.match(attribution.datasetDoi, /10\.5281\/zenodo\.21278557/)
assert.match(attribution.license, /CC-BY-4\.0/)
assert.match(attribution.author, /Bruckner/)
})
test('a bundled reference converts to a well-formed fingerprint', () => {
const fp = loadBundledReference('openai/gpt-4o-mini')
assert.equal(fp.model, 'openai/gpt-4o-mini')
assert.equal(fp.protocol, 'bruckner-zenodo-2026')
const cells = Object.values(fp.cells)
assert.ok(cells.length >= 8)
for (const cell of cells) {
assert.ok(cell.validCount >= 10)
const sum = Object.values(cell.counts).reduce((s, n) => s + n, 0)
assert.ok(Math.abs(sum - cell.validCount) <= 2, `${cell.cellId}: counts ≈ validCount`)
assert.ok(cell.entropyBits >= 0)
assert.ok(cell.normalizedEntropy >= 0 && cell.normalizedEntropy <= 1)
}
})
test('bundled references: self-compare → 0 distance, cross-model → clearly separated', () => {
const gpt = loadBundledReference('openai/gpt-4o-mini')
const self = compare(gpt, loadBundledReference('openai/gpt-4o-mini'))
assert.equal(self.meanJsd, 0)
assert.equal(self.verdict, 'match')
const claude = loadBundledReference('anthropic/claude-sonnet-4.5')
const cross = compare(gpt, claude)
assert.ok(
cross.meanJsd > 0.3,
`different models should diverge (got ${cross.meanJsd}) — paper median is 0.463`,
)
assert.notEqual(cross.verdict, 'match')
})
test('unknown bundled id throws with the available list', () => {
assert.throws(() => loadBundledReference('no/such-model'), /Available:/)
})
test('parseFingerprintJson validates structure', () => {
const fp = loadBundledReference('openai/gpt-4o-mini')
const roundTripped = parseFingerprintJson(JSON.stringify(fp))
assert.equal(roundTripped.model, fp.model)
assert.throws(() => parseFingerprintJson('not json'), /not valid JSON/)
assert.throws(() => parseFingerprintJson('{}'), /formatVersion/)
assert.throws(
() => parseFingerprintJson(JSON.stringify({ formatVersion: 1, model: 'x' })),
/cells/,
)
})