commit 232fbc6dd87c3231715b6a2643a52fd00fc09960 Author: Deng-m1 <1264440273@qq.com> Date: Tue Jul 21 16:58:00 2026 +0800 feat: initial release of llm-fingerprint-detector TypeScript library + CLI implementing single-token behavioral fingerprinting of LLMs behind OpenAI-compatible APIs, per 'One Token Is Enough' (Bruckner, arXiv:2607.10252). - Zero runtime dependencies (Node >= 18.17 built-in fetch) - fingerprint / compare / verify API + llm-fingerprint CLI - Jensen-Shannon divergence over per-cell answer distributions - Reasoning-model aware (auto-disables hidden thinking channels) - Bundled sample references for 11 popular models (CC-BY-4.0, derived from the paper's public Zenodo dataset) - Unit + integration tests via node --test Co-authored-by: Cursor diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d227fc4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +node_modules/ +dist/ +*.tsbuildinfo +.env +.env.* +*.log +.DS_Store +coverage/ +my-fingerprints/ +*.fingerprint.json diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..d537d18 --- /dev/null +++ b/.npmignore @@ -0,0 +1,13 @@ +# Published files are whitelisted via the "files" field in package.json; +# this file is a safety net in case "files" is ever removed. +node_modules/ +test/ +examples/ +scripts/ +src/ +tsconfig.json +*.tsbuildinfo +.env +.env.* +*.log +coverage/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7d1ca90 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Tosea.ai and llm-fingerprint-detector contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0a87170 --- /dev/null +++ b/README.md @@ -0,0 +1,253 @@ +# llm-fingerprint-detector + +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) +[![Node >= 18.17](https://img.shields.io/badge/node-%E2%89%A5%2018.17-brightgreen.svg)](package.json) +[![Runtime dependencies: 0](https://img.shields.io/badge/runtime%20deps-0-success.svg)](package.json) +[![Paper: arXiv:2607.10252](https://img.shields.io/badge/paper-arXiv%3A2607.10252-b31b1b.svg)](https://arxiv.org/abs/2607.10252) + +**Is that API really serving the model it claims?** Fingerprint and verify any OpenAI-compatible LLM endpoint from single-token output distributions — no logits, no weights, no privileged access. Just ~100–400 cheap one-word completions. Catches **model substitution**, silent quantization and server-side prompt injection by API resellers, gateways and aggregators. + +An independent, open-source TypeScript implementation of: + +> Tomáš Bruckner, **"One Token Is Enough: Fingerprinting and Verifying Large Language Models from Single-Token Output Distributions"**, [arXiv:2607.10252](https://arxiv.org/abs/2607.10252). +> Dataset: [DOI 10.5281/zenodo.21278557](https://doi.org/10.5281/zenodo.21278557) (CC-BY-4.0) · Paper code: [DOI 10.5281/zenodo.21278793](https://doi.org/10.5281/zenodo.21278793) (MIT) + +This package is **not affiliated with the paper's author** — it is a from-scratch implementation of the published method, engineered as a reusable library + CLI. If you use the method in research, cite the paper. + +- **Zero runtime dependencies** — Node ≥ 18 built-in `fetch`, nothing else +- **Library + CLI** — embed it, or run `llm-fingerprint verify` in CI +- **Reasoning-model aware** — auto-detects how to disable hidden "thinking" (OpenRouter / Zhipu / OpenAI-style), with a graceful fallback +- **Filter-resistant probes** — every probe is a plain semantic question drawn from a paraphrase pool; there is no magic string a gateway can special-case +- **Bundled sample references** for 11 popular models, derived from the paper's public dataset + +> Prefer a no-install web version? Try the free online checker: **[tosea.ai/free-tools/llm-api-fingerprint-checker](https://tosea.ai/free-tools/llm-api-fingerprint-checker)** — same method, runs entirely in your browser. + +--- + +## How it works + +LLMs answer "*Name a random number between 1 and 100*" with model-specific, surprisingly stable biases (GPT-family models love 42 and 73; other families prefer 57, 37, 7…). The paper's key result: the **empirical distribution of one-word answers** across a small battery of such tasks is a reliable *behavioral fingerprint* — stable across time, load and providers for the same model, and clearly different between models. + +``` + probe battery (task × language cells) collect at temperature 1.0 + ┌──────────────────────────────┐ ┌─────────────────────────┐ + │ random number 1-100 (en/zh) │ │ "42" ×19 "73" ×4 ... │ + │ random color / letter / city │ ──25×──▶ │ per-cell answer │ + │ coin flip / animal / fav-num │ │ distributions │ + └──────────────────────────────┘ └───────────┬─────────────┘ + ▼ + reference fingerprint ──── mean per-cell Jensen-Shannon + (trusted endpoint) divergence (base 2) ──▶ verdict +``` + +1. **Probe** — ask one-word questions (random numbers, colors, letters, coin flips…) in English and Chinese, `temperature=1.0`, `max_tokens=16`, a fixed minimal system prompt, hidden reasoning disabled. Requests are shuffled and paraphrased per call. +2. **Normalize** — NFC, punctuation/emoji stripping, case folding, first word, digit unification (`seven`/`七`/`٧`/`７` → `7`), color and coin-word canonicalization; answers are classified valid / invalid / refusal / empty. +3. **Compare** — per-cell Jensen-Shannon divergence (base 2, range 0–1 bit), averaged over cells where both sides have ≥10 valid samples. +4. **Verdict** — three bands calibrated against the paper's baselines (see [Interpreting results](#interpreting-results)). + +## Install + +```bash +npm install llm-fingerprint-detector # library + `llm-fingerprint` CLI +# or run the CLI ad hoc: +npx llm-fingerprint-detector --help +``` + +Requires Node ≥ 18.17 (built-in `fetch`). The core library is runtime-agnostic and also works in browsers/edge runtimes; only the CLI and the bundled-reference loader touch the filesystem. + +## Quick start — CLI + +```bash +# 1. Fingerprint the endpoint you trust (key read from OPENAI_API_KEY) +export OPENAI_API_KEY=sk-... +llm-fingerprint fingerprint \ + --base-url https://api.openai.com/v1 \ + --model gpt-4o-mini \ + --out reference.gpt-4o-mini.json + +# 2. Verify the endpoint you don't +export LLM_FINGERPRINT_API_KEY=sk-... # key for the endpoint under test +llm-fingerprint verify \ + --base-url https://cheap-llm-reseller.example.com/v1 \ + --model gpt-4o-mini \ + --reference reference.gpt-4o-mini.json +``` + +``` +Verdict: MISMATCH — behavior differs from the reference +Mean JSD: 0.481 over 8 comparable cell(s) + +Interpretation scale (paper baselines, arXiv:2607.10252): + same model ≈ 0.14 · same model, other provider ≈ 0.227 · different model ≈ 0.463 + thresholds: match ≤ 0.25 < uncertain ≤ 0.35 < mismatch + +Per-cell JSD (most divergent first): + random-number-1-100:en 0.712 (24 vs 25 valid) + ... +``` + +Exit codes are CI-friendly: `0` match · `2` mismatch · `3` uncertain · `4` insufficient · `1` error. API keys are only ever read from environment variables (`--api-key-env NAME`, defaulting to `LLM_FINGERPRINT_API_KEY` then `OPENAI_API_KEY`) and are never logged. + +More: `llm-fingerprint --help`, [`examples/cli-examples.sh`](examples/cli-examples.sh). + +## Quick start — library + +```ts +import { fingerprint, compare, verify } from 'llm-fingerprint-detector' + +// Collect a fingerprint +const run = await fingerprint( + { baseUrl: 'https://api.openai.com/v1', model: 'gpt-4o-mini', apiKey: process.env.OPENAI_API_KEY }, + { cells: 8, samplesPerCell: 25, onProgress: (e) => console.log(e.done, '/', e.total) }, +) +console.log(run.fingerprint) // JSON-serializable artifact +console.log(run.splitHalfJsd) // self-consistency (≈0.14 is normal) + +// Verify another endpoint against it +const result = await verify( + { baseUrl: 'https://suspect.example.com/v1', model: 'gpt-4o-mini', apiKey: process.env.SUSPECT_KEY }, + run.fingerprint, +) +console.log(result.verdict, result.meanJsd) // 'match' | 'uncertain' | 'mismatch' | 'insufficient' + +// Or compare two saved fingerprints offline +const distance = compare(fingerprintA, fingerprintB) +``` + +Bundled sample references (Node only): + +```ts +import { listBundledReferences, loadBundledReference } from 'llm-fingerprint-detector/references' + +const reference = loadBundledReference('openai/gpt-4o-mini') +const result = await verify(suspectEndpoint, reference) +``` + +Runnable examples: [`examples/01-fingerprint-endpoint.mjs`](examples/01-fingerprint-endpoint.mjs), [`examples/02-verify-endpoint.mjs`](examples/02-verify-endpoint.mjs). + +## Tutorial: "Is my cheap API really GPT-4o / Claude / DeepSeek?" + +You bought API access from a reseller/aggregator at half price. Are you getting the real model, a cheaper substitute, or a quantized clone? Ten minutes: + +1. **Collect a trusted reference.** Fingerprint the *official* API (or any endpoint you fully trust) for the model in question: + + ```bash + llm-fingerprint fingerprint --base-url https://api.openai.com/v1 \ + --model gpt-4o-mini --api-key-env OFFICIAL_KEY --out ref.json + ``` + + No official access? Start with a bundled sample (`llm-fingerprint references`) — good enough for a first signal, with the caveats below. + +2. **Verify the suspect endpoint** with the *same* model id: + + ```bash + llm-fingerprint verify --base-url https://reseller.example.com/v1 \ + --model gpt-4o-mini --api-key-env RESELLER_KEY --reference ref.json + ``` + +3. **Read the verdict.** + - `match` — the endpoint's one-token behavior is statistically consistent with your reference. That is strong (not absolute) evidence it's the same model. + - `mismatch` — the behavior is as far from the reference as *different* models typically are. Common causes, in practice: a substituted cheaper model, a heavily quantized deployment, or an injected system prompt. + - `uncertain` — the gray zone. Raise `--samples` (e.g. 40), use `--preset strict` (all 16 cells), or refresh your reference — models drift when providers ship updates. + - Also watch the warnings: a high **split-half JSD** means the endpoint disagrees *with itself* between the first and second half of your own run — a classic sign of an aggregator rotating several backends. + +4. **Re-run before you conclude anything.** Two independent mismatch runs on different days are a much stronger signal than one. + +## Interpreting results + +Distances are mean Jensen-Shannon divergence (base 2), so 0 = identical behavior, 1 = disjoint answer sets. Paper baselines: + +| meanJsd | reading | +|---|---| +| ≈ 0.14 | same model, same endpoint (sampling noise floor) | +| ≈ 0.227 | same model, different provider (median) | +| **≤ 0.25** | → verdict **match** | +| 0.25 – 0.35 | → verdict **uncertain** | +| **> 0.35** | → verdict **mismatch** | +| ≈ 0.463 | different models (median) | + +**Error rates (paper):** distinguishing same-model vs different-model pairs achieves an equal error rate of ≈ **10.6% with 8 cells** and ≈ **7.3% with 40 cells**. A single run is *evidence*, not proof — treat verdicts accordingly. + +### Limitations you must know + +- **Fingerprints drift.** Providers silently update models; a 3-month-old reference can legitimately mismatch today's deployment. Always check `collectedAt`, refresh references regularly. +- **You need a trusted reference.** The method compares two endpoints; it cannot conjure ground truth. If your reference is wrong, your verdict is wrong. +- **The system prompt must be identical on both sides.** Swapping only the system prompt shifts fingerprints by JSD ≈ 0.44–0.46 — the magnitude of a model swap. This tool pins the same minimal system prompt on both sides automatically; if an endpoint *injects* its own server-side prompt, that will (correctly) surface as divergence. +- **Reasoning fallback lowers confidence.** When hidden reasoning can't be disabled, the run is flagged `postReasoning` — distributions shift measurably in that channel. +- **Quantization/serving stack changes** the same weights can move distances into the uncertain band. +- **A mismatch is a statistical observation, not an accusation.** Do not treat any single verdict as proof of fraud by a provider; investigate, re-run, and compare notes before drawing conclusions. + +## Bundled sample references + +`data/reference-fingerprints.sample.json` ships fingerprints of 11 popular models (GPT-4o, GPT-4o-mini, GPT-4.1-mini, Claude Sonnet 4.5, Gemini 2.5 Flash, DeepSeek-chat, Llama-3.1-8B, Qwen3-30B-A3B, Mistral Small 3.2, GLM-4.5, Kimi K2), derived from the paper's public dataset: + +> Bruckner, T. (2026). *Single-token output distributions as behavioral fingerprints of large language models* [Data set]. Zenodo. [https://doi.org/10.5281/zenodo.21278557](https://doi.org/10.5281/zenodo.21278557) — CC-BY-4.0. Counts reconstructed from the published per-cell distributions and re-normalized with this package's normalizer (see [`scripts/build-sample-references.mjs`](scripts/build-sample-references.mjs)). + +Those samples were collected by the paper's harness (via OpenRouter) under the paper's prompt protocol — close to, but not identical to, this package's battery. `compare()` flags such pairs with `protocolMismatch: true` and the verdict should be read as *indicative*. For anything that matters, collect your own reference: + +```bash +llm-fingerprint fingerprint --base-url --model --out my-reference.json +``` + +To rebuild or extend the bundled samples from the Zenodo dataset, download the dataset, then: + +```bash +npm run build +node scripts/build-sample-references.mjs path/to/distributions.json --models openai/gpt-4o,another/model +``` + +## API surface (TypeScript, fully typed) + +| export | what it does | +|---|---| +| `fingerprint(endpoint, options?)` | probe an endpoint → `FingerprintRun` (fingerprint, adapter, split-half, warnings) | +| `compare(a, b)` | two fingerprints → `ComparisonResult` (meanJsd, per-cell JSD, verdict, baselines) | +| `verify(endpoint, reference, options?)` | fingerprint + compare in one call → `VerifyResult` | +| `normalizeAnswer(raw, domain)` | the full normalization pipeline (pure, unit-tested) | +| `jensenShannonDivergence(p, q)` | base-2 JSD over count maps | +| `splitHalfJsd(samplesByCell)` | endpoint self-consistency check | +| `detectReasoningAdapter(endpoint)` | which reasoning-disable field the endpoint accepts | +| `PROBE_TASKS`, `CELL_PRIORITY_ORDER`, `SYSTEM_PROMPTS` | the battery itself | +| `llm-fingerprint-detector/references` | bundled sample loader (Node only) | + +All options (`cells`, `samplesPerCell`, `concurrency`, `timeoutMs`, `maxRetries`, `signal`, `onProgress`, …) are documented in [`src/types.ts`](src/types.ts). + +## Development + +```bash +npm install +npm run build # tsc → dist/ +npm test # builds, then runs node --test against the built output +``` + +No test framework, no bundler — TypeScript and the Node built-in test runner only. + +## Contributing + +Issues and PRs are welcome. Especially valuable: + +- **More languages** in the probe battery (the paper also used Arabic and Russian) — requires matching normalizer support, see `src/normalizer.ts` +- **Reference fingerprints** for more models/providers, collected with this tool's protocol (`one-token/v1`) and a documented date/channel +- **Threshold calibration data**: pairs of same-model / different-model runs to sharpen the match/mismatch cut points + +Please keep changes dependency-free and covered by `node --test` tests. + +## Citation & license + +Method: cite the paper — + +```bibtex +@article{bruckner2026onetoken, + title = {One Token Is Enough: Fingerprinting and Verifying Large Language Models + from Single-Token Output Distributions}, + author = {Bruckner, Tom{\'a}{\v s}}, + journal = {arXiv preprint arXiv:2607.10252}, + year = {2026} +} +``` + +This package: [MIT](LICENSE). Bundled sample data: CC-BY-4.0, © Tomáš Bruckner (see above). + +--- + +*Built and maintained by [Tosea.ai](https://tosea.ai). Want the zero-setup version? → [LLM API Fingerprint Checker](https://tosea.ai/free-tools/llm-api-fingerprint-checker) (runs in your browser, your key never leaves it).* diff --git a/data/reference-fingerprints.sample.json b/data/reference-fingerprints.sample.json new file mode 100644 index 0000000..c68b081 --- /dev/null +++ b/data/reference-fingerprints.sample.json @@ -0,0 +1,1684 @@ +{ + "formatVersion": 1, + "protocol": "bruckner-zenodo-2026", + "samplesPerCell": 30, + "source": { + "dataset": "Single-token output distributions as behavioral fingerprints of large language models", + "author": "Tomáš Bruckner (Prague University of Economics and Business)", + "datasetDoi": "10.5281/zenodo.21278557", + "paper": "arXiv:2607.10252", + "license": "CC-BY-4.0", + "note": "Counts reconstructed as round(probability × n_valid) from the published per-cell distributions; answers re-normalized with the llm-fingerprint-detector normalizer. Collected via OpenRouter by the paper's harness under the paper's prompt protocol (not this package's battery), so comparisons against these samples are indicative rather than strict." + }, + "models": { + "mistralai/mistral-small-3.2-24b-instruct": { + "model": "mistralai/mistral-small-3.2-24b-instruct", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "random-city:en": { + "n": 30, + "counts": { + "paris": 18, + "tokyo": 9, + "chicago": 2, + "boston": 1 + } + }, + "random-number-1-10:zh": { + "n": 30, + "counts": { + "3": 2, + "4": 1, + "5": 1, + "7": 26 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "37": 2, + "42": 27, + "47": 1 + } + }, + "random-number-1-10:en": { + "n": 30, + "counts": { + "3": 2, + "7": 28 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "42": 30 + } + }, + "random-city:zh": { + "n": 30, + "counts": { + "巴黎": 15, + "东京": 7, + "柏林": 1, + "伦敦": 1, + "多伦多": 1, + "帕萨迪纳": 1, + "巴塞罗那": 1, + "纽约": 1, + "不来梅": 1, + "伯明翰": 1 + } + }, + "random-animal:en": { + "n": 30, + "counts": { + "elephant": 17, + "lion": 4, + "giraffe": 3, + "eagle": 2, + "dog": 2, + "lemur": 1, + "kangaroo": 1 + } + }, + "coin-flip:en": { + "n": 30, + "counts": { + "heads": 20, + "tails": 10 + } + }, + "favorite-number:zh": { + "n": 28, + "counts": { + "3": 10, + "6": 2, + "7": 8, + "8": 2, + "42": 6 + } + }, + "coin-flip:zh": { + "n": 30, + "counts": { + "heads": 22, + "tails": 8 + } + }, + "random-animal:zh": { + "n": 30, + "counts": { + "企鹅": 14, + "狮子": 4, + "鼠": 2, + "老鹰": 1, + "蜜蜂": 1, + "鲸": 1, + "獴": 1, + "北极熊": 1, + "鸭子": 1, + "鼹鼠": 1, + "鲨鱼": 1, + "熊猫": 1, + "大象": 1 + } + }, + "random-color:zh": { + "n": 30, + "counts": { + "蓝": 19, + "绿": 4, + "红": 3, + "洋红": 1, + "粉红": 1, + "灰": 1, + "紫": 1 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "q": 8, + "a": 4, + "e": 3, + "f": 3, + "g": 3, + "b": 2, + "x": 2, + "m": 2, + "d": 2, + "p": 1 + } + }, + "favorite-number:en": { + "n": 30, + "counts": { + "3": 1, + "4": 1, + "42": 28 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "blue": 22, + "green": 4, + "purple": 2, + "red": 1, + "cyan": 1 + } + } + } + }, + "moonshotai/kimi-k2": { + "model": "moonshotai/kimi-k2", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "coin-flip:en": { + "n": 25, + "counts": { + "heads": 24, + "tails": 1 + } + }, + "random-number-1-100:zh": { + "n": 27, + "counts": { + "37": 2, + "42": 6, + "47": 8, + "73": 11 + } + }, + "random-number-1-100:en": { + "n": 28, + "counts": { + "47": 10, + "57": 1, + "73": 17 + } + }, + "favorite-number:en": { + "n": 30, + "counts": { + "7": 24, + "42": 6 + } + }, + "coin-flip:zh": { + "n": 29, + "counts": { + "heads": 28, + "tails": 1 + } + }, + "random-city:zh": { + "n": 29, + "counts": { + "成都": 11, + "杭州": 9, + "上海": 7, + "巴塞罗那": 2 + } + }, + "random-animal:zh": { + "n": 29, + "counts": { + "企鹅": 7, + "猫头鹰": 5, + "老虎": 5, + "猫": 3, + "虎": 2, + "长颈鹿": 2, + "海豚": 1, + "熊猫": 1, + "猫鼬": 1, + "穿山甲": 1, + "雪豹": 1 + } + }, + "random-color:zh": { + "n": 27, + "counts": { + "靛蓝": 17, + "青": 6, + "蓝": 2, + "紫": 2 + } + }, + "favorite-number:zh": { + "n": 28, + "counts": { + "7": 22, + "23": 1, + "42": 5 + } + }, + "random-animal:en": { + "n": 29, + "counts": { + "octopus": 9, + "axolotl": 7, + "platypus": 6, + "pangolin": 4, + "elephant": 2, + "okapi": 1 + } + }, + "random-number-1-10:zh": { + "n": 29, + "counts": { + "5": 1, + "7": 28 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "teal": 8, + "azure": 7, + "cyan": 5, + "magenta": 3, + "purple": 2, + "blue": 2, + "crimson": 1, + "maroon": 1, + "amber": 1 + } + }, + "random-city:en": { + "n": 27, + "counts": { + "lisbon": 9, + "kyoto": 6, + "tallinn": 3, + "lagos": 2, + "timbuktu": 2, + "osaka": 1, + "lviv": 1, + "bishkek": 1, + "tirana": 1, + "budapest": 1 + } + }, + "random-letter:en": { + "n": 29, + "counts": { + "q": 17, + "k": 6, + "x": 3, + "w": 1, + "j": 1, + "r": 1 + } + }, + "random-number-1-10:en": { + "n": 29, + "counts": { + "5": 1, + "7": 28 + } + } + } + }, + "deepseek/deepseek-chat": { + "model": "deepseek/deepseek-chat", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "random-number-1-10:zh": { + "n": 30, + "counts": { + "3": 1, + "4": 1, + "5": 5, + "7": 23 + } + }, + "random-animal:zh": { + "n": 30, + "counts": { + "狮子": 13, + "河马": 8, + "猫": 3, + "大象": 1, + "老虎": 1, + "猎豹": 1, + "长颈鹿": 1, + "斑马": 1, + "熊猫": 1 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "blue": 30 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "b": 17, + "k": 5, + "m": 5, + "a": 1, + "h": 1, + "z": 1 + } + }, + "coin-flip:en": { + "n": 30, + "counts": { + "heads": 27, + "tails": 3 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "7": 1, + "42": 28, + "47": 1 + } + }, + "random-color:zh": { + "n": 30, + "counts": { + "蓝": 29, + "blue": 1 + } + }, + "coin-flip:zh": { + "n": 30, + "counts": { + "heads": 30 + } + }, + "random-city:zh": { + "n": 30, + "counts": { + "东京": 28, + "柏林": 1, + "纽约": 1 + } + }, + "random-city:en": { + "n": 30, + "counts": { + "tokyo": 28, + "berlin": 1, + "toronto": 1 + } + }, + "favorite-number:en": { + "n": 30, + "counts": { + "7": 26, + "42": 4 + } + }, + "random-animal:en": { + "n": 30, + "counts": { + "elephant": 27, + "lion": 3 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "42": 27, + "47": 1, + "50": 1, + "73": 1 + } + }, + "random-number-1-10:en": { + "n": 30, + "counts": { + "7": 30 + } + }, + "favorite-number:zh": { + "n": 30, + "counts": { + "7": 29, + "42": 1 + } + } + } + }, + "meta-llama/llama-3.1-8b-instruct": { + "model": "meta-llama/llama-3.1-8b-instruct", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "random-city:zh": { + "n": 28, + "counts": { + "曼彻斯特": 3, + "洛杉矶": 2, + "都灵": 1, + "布达佩斯": 1, + "芝加哥": 1, + "巴黎": 1, + "布鲁塞尔": 1, + "新德里": 1, + "柏林": 1, + "布里斯本": 1, + "法兰克福": 1, + "芒特市": 1, + "孟": 1, + "海得拉姆": 1, + "拿骚": 1, + "帕贡": 1, + "明斯克": 1, + "鹿特丹": 1, + "纽约": 1, + "马尼拉": 1, + "圣地亚哥": 1, + "小城镇": 1, + "雅加达": 1, + "紐約": 1, + "津巴布韦": 1 + } + }, + "random-city:en": { + "n": 30, + "counts": { + "perth": 5, + "prague": 2, + "marseille": 2, + "rome": 1, + "edinburgh": 1, + "grenoble": 1, + "paris": 1, + "rouen": 1, + "kuala": 1, + "montreal": 1, + "kiev": 1, + "osaka": 1, + "portland": 1, + "albany": 1, + "vancouver": 1, + "marrakech": 1, + "bratislava": 1, + "bordeaux": 1, + "calgary": 1, + "auckland": 1, + "minsk": 1, + "kolkata": 1, + "kazan": 1, + "bologna": 1 + } + }, + "random-animal:zh": { + "n": 29, + "counts": { + "狮子": 7, + "河狸": 3, + "河马": 3, + "猴子": 2, + "鲸鱼": 2, + "狼": 1, + "狐狸": 1, + "豹子": 1, + "象": 1, + "雄鹿": 1, + "老鹰": 1, + "袋鼠": 1, + "猫": 1, + "企鹅": 1, + "鸵鸟": 1, + "章鱼": 1, + "大象": 1 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "k": 15, + "f": 4, + "q": 4, + "n": 3, + "m": 1, + "r": 1, + "g": 1, + "j": 1 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "34": 3, + "42": 1, + "45": 3, + "47": 1, + "53": 1, + "61": 1, + "63": 1, + "67": 8, + "73": 1, + "75": 1, + "83": 2, + "84": 3, + "85": 1, + "87": 3 + } + }, + "random-animal:en": { + "n": 30, + "counts": { + "kangaroo": 10, + "koala": 9, + "quail": 3, + "narwhal": 2, + "tiger": 2, + "giraffe": 1, + "mongoose": 1, + "hedgehog": 1, + "jaguar": 1 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "38": 1, + "42": 1, + "43": 3, + "51": 1, + "53": 6, + "67": 3, + "73": 4, + "74": 1, + "75": 1, + "83": 2, + "85": 1, + "87": 6 + } + }, + "coin-flip:en": { + "n": 30, + "counts": { + "heads": 17, + "tails": 13 + } + }, + "favorite-number:en": { + "n": 20, + "counts": { + "3": 3, + "5": 1, + "7": 7, + "8": 1, + "13": 5, + "38": 1, + "42": 2 + } + }, + "random-number-1-10:zh": { + "n": 30, + "counts": { + "3": 1, + "4": 3, + "5": 6, + "6": 2, + "7": 11, + "8": 5, + "9": 2 + } + }, + "coin-flip:zh": { + "n": 28, + "counts": { + "heads": 20, + "tails": 8 + } + }, + "random-number-1-10:en": { + "n": 30, + "counts": { + "5": 3, + "7": 27 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "turquoise": 20, + "indigo": 4, + "purple": 3, + "blue": 2, + "magenta": 1 + } + }, + "favorite-number:zh": { + "n": 30, + "counts": { + "0": 2, + "3": 3, + "6": 1, + "7": 11, + "8": 3, + "13": 5, + "17": 1, + "42": 4 + } + }, + "random-color:zh": { + "n": 29, + "counts": { + "紫": 10, + "蓝": 4, + "绿": 4, + "青": 2, + "橙": 2, + "咖啡": 1, + "碧蓝": 1, + "棕": 1, + "紫红": 1, + "黄": 1, + "粉": 1, + "兰蓝": 1 + } + } + } + }, + "qwen/qwen3-30b-a3b-instruct-2507": { + "model": "qwen/qwen3-30b-a3b-instruct-2507", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "random-color:en": { + "n": 30, + "counts": { + "blue": 29, + "green": 1 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "42": 29, + "67": 1 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "42": 30 + } + }, + "favorite-number:zh": { + "n": 29, + "counts": { + "7": 29 + } + }, + "random-number-1-10:zh": { + "n": 28, + "counts": { + "7": 28 + } + }, + "random-animal:zh": { + "n": 30, + "counts": { + "老虎": 17, + "狮子": 7, + "猫": 3, + "tiger": 1, + "熊猫": 1, + "豹": 1 + } + }, + "random-city:zh": { + "n": 29, + "counts": { + "上海": 14, + "巴黎": 9, + "北京": 2, + "纽约": 2, + "柏林": 1, + "杭州": 1 + } + }, + "random-animal:en": { + "n": 29, + "counts": { + "lion": 17, + "elephant": 5, + "tiger": 3, + "dog": 3, + "penguin": 1 + } + }, + "random-color:zh": { + "n": 29, + "counts": { + "蓝": 27, + "绿": 2 + } + }, + "favorite-number:en": { + "n": 28, + "counts": { + "7": 28 + } + }, + "random-number-1-10:en": { + "n": 29, + "counts": { + "7": 29 + } + }, + "random-city:en": { + "n": 30, + "counts": { + "paris": 14, + "sydney": 7, + "tokyo": 3, + "chicago": 1, + "melbourne": 1, + "london": 1, + "amsterdam": 1, + "lisbon": 1, + "budapest": 1 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "m": 18, + "k": 3, + "a": 2, + "b": 2, + "g": 2, + "d": 1, + "e": 1, + "c": 1 + } + }, + "coin-flip:zh": { + "n": 29, + "counts": { + "heads": 29 + } + }, + "coin-flip:en": { + "n": 29, + "counts": { + "heads": 21, + "tails": 8 + } + } + } + }, + "openai/gpt-4o": { + "model": "openai/gpt-4o", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "random-animal:en": { + "n": 30, + "counts": { + "elephant": 22, + "tiger": 5, + "giraffe": 2, + "penguin": 1 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "27": 2, + "37": 7, + "42": 13, + "47": 2, + "56": 1, + "57": 5 + } + }, + "coin-flip:en": { + "n": 30, + "counts": { + "heads": 20, + "tails": 10 + } + }, + "random-city:zh": { + "n": 30, + "counts": { + "东京": 13, + "巴黎": 10, + "北京": 2, + "上海": 2, + "伦敦": 1, + "柏林": 1, + "悉尼": 1 + } + }, + "favorite-number:zh": { + "n": 30, + "counts": { + "7": 26, + "42": 4 + } + }, + "random-number-1-10:zh": { + "n": 30, + "counts": { + "3": 1, + "5": 3, + "7": 26 + } + }, + "coin-flip:zh": { + "n": 30, + "counts": { + "heads": 27, + "tails": 3 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "37": 2, + "42": 17, + "47": 2, + "57": 9 + } + }, + "random-number-1-10:en": { + "n": 30, + "counts": { + "4": 2, + "5": 2, + "7": 26 + } + }, + "random-city:en": { + "n": 30, + "counts": { + "tokyo": 14, + "paris": 10, + "berlin": 3, + "toronto": 2, + "denver": 1 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "g": 8, + "m": 6, + "k": 6, + "r": 3, + "q": 2, + "z": 2, + "c": 1, + "j": 1, + "h": 1 + } + }, + "favorite-number:en": { + "n": 20, + "counts": { + "3": 1, + "7": 12, + "42": 7 + } + }, + "random-animal:zh": { + "n": 30, + "counts": { + "狮子": 7, + "象": 4, + "熊猫": 3, + "猫": 3, + "大象": 2, + "狐貍": 1, + "豹": 1, + "海豚": 1, + "狐狸": 1, + "狐猴": 1, + "长颈鹿": 1, + "鹰": 1, + "狐": 1, + "鲸": 1, + "豹子": 1, + "鲸鱼": 1 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "blue": 19, + "azure": 3, + "turquoise": 2, + "cyan": 2, + "cerulean": 2, + "teal": 1, + "azurite": 1 + } + }, + "random-color:zh": { + "n": 30, + "counts": { + "蓝": 29, + "紫": 1 + } + } + } + }, + "google/gemini-2.5-flash": { + "model": "google/gemini-2.5-flash", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "random-city:zh": { + "n": 30, + "counts": { + "巴黎": 8, + "上海": 6, + "伦敦": 6, + "东京": 6, + "北京": 3, + "开罗": 1 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "red": 13, + "blue": 12, + "green": 2, + "chartreuse": 1, + "purple": 1, + "yellow": 1 + } + }, + "random-number-1-10:en": { + "n": 30, + "counts": { + "7": 30 + } + }, + "random-color:zh": { + "n": 30, + "counts": { + "蓝": 17, + "绿": 3, + "红": 6, + "黄": 2, + "橙": 2 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "42": 14, + "57": 2, + "67": 1, + "73": 12, + "87": 1 + } + }, + "favorite-number:zh": { + "n": 29, + "counts": { + "3": 2, + "7": 17, + "8": 3, + "9": 1, + "10": 1, + "24": 1, + "42": 4 + } + }, + "coin-flip:en": { + "n": 30, + "counts": { + "heads": 21, + "tails": 9 + } + }, + "favorite-number:en": { + "n": 30, + "counts": { + "6": 1, + "42": 28, + "1729": 1 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "15": 1, + "42": 20, + "67": 1, + "72": 2, + "73": 3, + "76": 1, + "82": 2 + } + }, + "random-number-1-10:zh": { + "n": 30, + "counts": { + "7": 30 + } + }, + "random-city:en": { + "n": 30, + "counts": { + "london": 15, + "paris": 8, + "tokyo": 4, + "lisbon": 1, + "cairo": 1, + "beijing": 1 + } + }, + "coin-flip:zh": { + "n": 30, + "counts": { + "tails": 15, + "heads": 15 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "r": 6, + "x": 4, + "q": 3, + "g": 3, + "z": 3, + "l": 2, + "m": 2, + "p": 2, + "e": 2, + "w": 1, + "h": 1, + "s": 1 + } + }, + "random-animal:en": { + "n": 30, + "counts": { + "tiger": 7, + "dog": 6, + "lion": 5, + "cat": 5, + "camel": 1, + "penguin": 1, + "giraffe": 1, + "elephant": 1, + "sloth": 1, + "okapi": 1, + "bear": 1 + } + }, + "random-animal:zh": { + "n": 30, + "counts": { + "猫": 13, + "狗": 8, + "老虎": 3, + "狮子": 2, + "海鸥": 1, + "虎": 1, + "蛇": 1, + "蜂鸟": 1 + } + } + } + }, + "z-ai/glm-4.5": { + "model": "z-ai/glm-4.5", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "random-city:en": { + "n": 30, + "counts": { + "stockholm": 4, + "chicago": 4, + "barcelona": 3, + "tokyo": 3, + "london": 2, + "paris": 2, + "montreal": 2, + "denver": 2, + "oslo": 1, + "sydney": 1, + "shanghai": 1, + "phoenix": 1, + "kathmandu": 1, + "vienna": 1, + "amsterdam": 1, + "kyoto": 1 + } + }, + "random-color:zh": { + "n": 30, + "counts": { + "蓝": 29, + "红": 1 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "k": 10, + "q": 7, + "p": 6, + "g": 2, + "r": 2, + "x": 1, + "e": 1, + "h": 1 + } + }, + "favorite-number:en": { + "n": 30, + "counts": { + "3": 2, + "4": 1, + "7": 5, + "8": 1, + "17": 1, + "42": 20 + } + }, + "coin-flip:zh": { + "n": 30, + "counts": { + "heads": 30 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "blue": 25, + "turquoise": 3, + "maroon": 1, + "yellow": 1 + } + }, + "random-animal:zh": { + "n": 30, + "counts": { + "大象": 20, + "长颈鹿": 5, + "狮子": 3, + "狗": 1, + "熊猫": 1 + } + }, + "favorite-number:zh": { + "n": 30, + "counts": { + "1": 1, + "3": 2, + "7": 20, + "42": 7 + } + }, + "random-number-1-10:en": { + "n": 30, + "counts": { + "7": 30 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "37": 2, + "42": 23, + "47": 1, + "57": 1, + "73": 3 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "37": 1, + "42": 22, + "47": 3, + "57": 1, + "67": 1, + "73": 2 + } + }, + "random-animal:en": { + "n": 30, + "counts": { + "giraffe": 16, + "elephant": 13, + "zebra": 1 + } + }, + "random-number-1-10:zh": { + "n": 30, + "counts": { + "3": 1, + "4": 1, + "7": 28 + } + }, + "coin-flip:en": { + "n": 30, + "counts": { + "heads": 26, + "tails": 4 + } + }, + "random-city:zh": { + "n": 30, + "counts": { + "巴黎": 8, + "东京": 7, + "上海": 5, + "柏林": 2, + "伦敦": 2, + "汉堡": 1, + "开罗": 1, + "马德里": 1, + "京都": 1, + "开普敦": 1, + "北京": 1 + } + } + } + }, + "anthropic/claude-sonnet-4.5": { + "model": "anthropic/claude-sonnet-4.5", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "random-number-1-10:zh": { + "n": 30, + "counts": { + "7": 30 + } + }, + "coin-flip:zh": { + "n": 30, + "counts": { + "heads": 30 + } + }, + "favorite-number:en": { + "n": 30, + "counts": { + "7": 30 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "42": 19, + "47": 11 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "blue": 30 + } + }, + "favorite-number:zh": { + "n": 30, + "counts": { + "7": 30 + } + }, + "random-animal:en": { + "n": 30, + "counts": { + "elephant": 30 + } + }, + "coin-flip:en": { + "n": 30, + "counts": { + "heads": 30 + } + }, + "random-number-1-10:en": { + "n": 30, + "counts": { + "7": 30 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "m": 30 + } + }, + "random-color:zh": { + "n": 30, + "counts": { + "蓝": 30 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "42": 2, + "47": 28 + } + }, + "random-city:zh": { + "n": 30, + "counts": { + "京都": 11, + "巴塞罗那": 5, + "布拉格": 4, + "巴黎": 3, + "柏林": 2, + "悉尼": 2, + "墨尔本": 2, + "东京": 1 + } + }, + "random-animal:zh": { + "n": 30, + "counts": { + "企鹅": 17, + "长颈鹿": 9, + "大象": 1, + "熊猫": 1, + "海豚": 1, + "袋鼠": 1 + } + }, + "random-city:en": { + "n": 30, + "counts": { + "toronto": 22, + "tokyo": 7, + "berlin": 1 + } + } + } + }, + "openai/gpt-4.1-mini": { + "model": "openai/gpt-4.1-mini", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "random-animal:zh": { + "n": 30, + "counts": { + "猫": 12, + "大象": 4, + "袋鼠": 3, + "企鹅": 3, + "熊猫": 2, + "考拉": 2, + "海豚": 1, + "章鱼": 1, + "鲸鱼": 1, + "猴子": 1 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "blue": 28, + "azure": 1, + "turquoise": 1 + } + }, + "favorite-number:zh": { + "n": 30, + "counts": { + "7": 27, + "8": 3 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "37": 1, + "47": 9, + "57": 17, + "67": 1, + "73": 2 + } + }, + "random-number-1-10:zh": { + "n": 30, + "counts": { + "7": 30 + } + }, + "random-number-1-10:en": { + "n": 30, + "counts": { + "7": 30 + } + }, + "coin-flip:en": { + "n": 30, + "counts": { + "heads": 22, + "tails": 8 + } + }, + "coin-flip:zh": { + "n": 30, + "counts": { + "heads": 29, + "tails": 1 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "42": 1, + "47": 6, + "57": 23 + } + }, + "random-animal:en": { + "n": 30, + "counts": { + "elephant": 29, + "tiger": 1 + } + }, + "random-city:zh": { + "n": 30, + "counts": { + "巴黎": 18, + "东京": 11, + "杭州": 1 + } + }, + "favorite-number:en": { + "n": 28, + "counts": { + "7": 27, + "42": 1 + } + }, + "random-city:en": { + "n": 30, + "counts": { + "tokyo": 18, + "paris": 8, + "berlin": 1, + "zurich": 1, + "madrid": 1, + "kyoto": 1 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "k": 8, + "m": 7, + "g": 7, + "q": 6, + "x": 1, + "r": 1 + } + }, + "random-color:zh": { + "n": 30, + "counts": { + "蓝": 30 + } + } + } + }, + "openai/gpt-4o-mini": { + "model": "openai/gpt-4o-mini", + "collectedAt": "2026-07-08", + "channel": "openrouter", + "cells": { + "favorite-number:en": { + "n": 30, + "counts": { + "7": 29, + "42": 1 + } + }, + "coin-flip:en": { + "n": 30, + "counts": { + "heads": 21, + "tails": 9 + } + }, + "random-number-1-100:en": { + "n": 30, + "counts": { + "37": 3, + "42": 5, + "47": 4, + "57": 18 + } + }, + "coin-flip:zh": { + "n": 30, + "counts": { + "heads": 30 + } + }, + "random-city:en": { + "n": 30, + "counts": { + "tokyo": 18, + "berlin": 5, + "oslo": 4, + "paris": 1, + "madrid": 1, + "zurich": 1 + } + }, + "random-animal:zh": { + "n": 30, + "counts": { + "海豚": 11, + "猫": 6, + "大象": 3, + "猩猩": 3, + "企鹅": 1, + "蛇": 1, + "虎": 1, + "松鼠": 1, + "狮子": 1, + "鹦鹉": 1, + "豹": 1 + } + }, + "random-city:zh": { + "n": 30, + "counts": { + "东京": 26, + "巴黎": 3, + "巴塞罗那": 1 + } + }, + "random-animal:en": { + "n": 30, + "counts": { + "giraffe": 14, + "elephant": 8, + "kangaroo": 4, + "octopus": 1, + "penguin": 1, + "panda": 1, + "zebra": 1 + } + }, + "random-color:zh": { + "n": 30, + "counts": { + "蓝": 30 + } + }, + "random-color:en": { + "n": 30, + "counts": { + "turquoise": 18, + "cerulean": 7, + "cyan": 1, + "azure": 1, + "blue": 1, + "magenta": 1, + "chartreuse": 1 + } + }, + "random-letter:en": { + "n": 30, + "counts": { + "g": 10, + "q": 7, + "k": 7, + "m": 4, + "r": 1, + "j": 1 + } + }, + "random-number-1-10:en": { + "n": 30, + "counts": { + "7": 30 + } + }, + "favorite-number:zh": { + "n": 30, + "counts": { + "7": 29, + "8": 1 + } + }, + "random-number-1-10:zh": { + "n": 30, + "counts": { + "7": 30 + } + }, + "random-number-1-100:zh": { + "n": 30, + "counts": { + "37": 2, + "42": 3, + "47": 1, + "57": 24 + } + } + } + } + } +} diff --git a/examples/01-fingerprint-endpoint.mjs b/examples/01-fingerprint-endpoint.mjs new file mode 100644 index 0000000..65d989b --- /dev/null +++ b/examples/01-fingerprint-endpoint.mjs @@ -0,0 +1,48 @@ +/** + * Example 1 — collect a fingerprint from an endpoint you trust and save it + * as a reference for later verification. + * + * Run (from the package root, after `npm run build`): + * OPENAI_API_KEY=sk-... node examples/01-fingerprint-endpoint.mjs + * + * With the published package, import from 'llm-fingerprint-detector' instead + * of '../dist/index.js'. + */ + +import { writeFileSync } from 'node:fs' + +import { fingerprint } from '../dist/index.js' + +const apiKey = process.env.OPENAI_API_KEY +if (!apiKey) { + console.error('Set OPENAI_API_KEY (the key is only sent to the endpoint you probe).') + process.exit(1) +} + +const run = await fingerprint( + { + baseUrl: 'https://api.openai.com/v1', // any OpenAI-compatible endpoint + model: 'gpt-4o-mini', + apiKey, + }, + { + cells: 8, // top-8 most discriminative cells (= "standard" preset) + samplesPerCell: 25, + concurrency: 4, + onProgress: (e) => { + if (e.stage === 'sampling') process.stderr.write(`\r${e.done}/${e.total}`) + }, + }, +) +process.stderr.write('\n') + +for (const warning of run.warnings) console.warn(`warning: ${warning}`) + +console.log(`model: ${run.fingerprint.model}`) +console.log(`cells: ${Object.keys(run.fingerprint.cells).length}`) +console.log(`errors: ${run.errorCount}`) +console.log(`split-half JSD: ${run.splitHalfJsd?.toFixed(3) ?? 'n/a'} (same-model baseline ≈ 0.14)`) +console.log(`reasoning adapter: ${run.adapter.strategy}`) + +writeFileSync('gpt-4o-mini.fingerprint.json', JSON.stringify(run.fingerprint, null, 2)) +console.log('\nsaved to gpt-4o-mini.fingerprint.json — use it later with verify()') diff --git a/examples/02-verify-endpoint.mjs b/examples/02-verify-endpoint.mjs new file mode 100644 index 0000000..216cd12 --- /dev/null +++ b/examples/02-verify-endpoint.mjs @@ -0,0 +1,52 @@ +/** + * Example 2 — verify that a cheap/unknown endpoint really serves the model it + * claims, by comparing its live fingerprint against a reference. + * + * Run (from the package root, after `npm run build`): + * SUSPECT_API_KEY=sk-... node examples/02-verify-endpoint.mjs + * + * The reference here is a bundled sample derived from the paper's public + * dataset (CC-BY-4.0) — handy for a demo. For real audits, collect your own + * reference from the official API with example 01 and load that file instead. + */ + +import { verify } from '../dist/index.js' +import { loadBundledReference } from '../dist/reference.js' + +const apiKey = process.env.SUSPECT_API_KEY +if (!apiKey) { + console.error('Set SUSPECT_API_KEY for the endpoint you want to test.') + process.exit(1) +} + +// Endpoint under test: does it really serve gpt-4o-mini? +const suspectEndpoint = { + baseUrl: 'https://openrouter.ai/api/v1', // ← put the reseller/aggregator URL here + model: 'openai/gpt-4o-mini', + apiKey, +} + +const reference = loadBundledReference('openai/gpt-4o-mini') + +const result = await verify(suspectEndpoint, reference, { + cells: 8, + samplesPerCell: 25, + onProgress: (e) => { + if (e.stage === 'sampling') process.stderr.write(`\r${e.done}/${e.total}`) + }, +}) +process.stderr.write('\n') + +for (const warning of result.warnings) console.warn(`warning: ${warning}`) + +console.log(`verdict: ${result.verdict}`) +console.log(`mean JSD: ${result.meanJsd?.toFixed(3)} over ${result.comparison.comparableCellCount} cells`) +console.log('baselines: same model ≈ 0.14 · cross-provider ≈ 0.227 · different model ≈ 0.463') + +console.log('\nmost divergent cells:') +for (const cell of result.comparison.cells.slice(0, 5)) { + console.log(` ${cell.cellId.padEnd(26)} JSD ${cell.jsd.toFixed(3)}`) +} + +// CI-style decision: +process.exitCode = result.verdict === 'match' ? 0 : 1 diff --git a/examples/cli-examples.sh b/examples/cli-examples.sh new file mode 100755 index 0000000..ed01c8a --- /dev/null +++ b/examples/cli-examples.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# CLI usage examples. Keys always come from environment variables — never +# paste keys into files or shell history. +set -euo pipefail + +# After `npm install -g llm-fingerprint-detector` use `llm-fingerprint ...`; +# inside this repo use `node dist/cli.js ...` (shown here via npx). +LLM_FP="npx llm-fingerprint" + +# --- 0. Explore --------------------------------------------------------------- +$LLM_FP --help +$LLM_FP references # bundled sample reference fingerprints + +# --- 1. Fingerprint the official API and save a trusted reference -------------- +# export OPENAI_API_KEY=sk-... # ← set in your shell / CI secret store +$LLM_FP fingerprint \ + --base-url https://api.openai.com/v1 \ + --model gpt-4o-mini \ + --preset standard \ + --out reference.gpt-4o-mini.json + +# --- 2. Verify a cheap reseller claims honestly -------------------------------- +# export LLM_FINGERPRINT_API_KEY=sk-... # key for the endpoint under test +$LLM_FP verify \ + --base-url https://cheap-llm-reseller.example.com/v1 \ + --model gpt-4o-mini \ + --reference reference.gpt-4o-mini.json +# exit codes: 0 match · 2 mismatch · 3 uncertain · 4 insufficient · 1 error + +# --- 3. Quick demo against a bundled sample reference -------------------------- +$LLM_FP verify \ + --base-url https://openrouter.ai/api/v1 \ + --model openai/gpt-4o-mini \ + --reference openai/gpt-4o-mini \ + --preset quick --json + +# --- 4. Offline: compare two saved fingerprints -------------------------------- +$LLM_FP compare reference.gpt-4o-mini.json some-other.fingerprint.json + +# --- 5. Custom cell selection --------------------------------------------------- +$LLM_FP fingerprint \ + --base-url https://api.deepseek.com/v1 \ + --model deepseek-chat \ + --api-key-env DEEPSEEK_API_KEY \ + --cells random-number-1-100:en,random-number-1-100:zh,random-color:en,coin-flip:en \ + --samples 30 --json diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..e40ab16 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,54 @@ +{ + "name": "llm-fingerprint-detector", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "llm-fingerprint-detector", + "version": "0.1.0", + "license": "MIT", + "bin": { + "llm-fingerprint": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^22.10.0", + "typescript": "^5.7.0" + }, + "engines": { + "node": ">=18.17" + } + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..f07c475 --- /dev/null +++ b/package.json @@ -0,0 +1,69 @@ +{ + "name": "llm-fingerprint-detector", + "version": "0.1.0", + "description": "Fingerprint and verify LLMs behind OpenAI-compatible APIs from single-token output distributions. Independent open-source implementation of \"One Token Is Enough\" (Bruckner, arXiv:2607.10252).", + "type": "module", + "license": "MIT", + "engines": { + "node": ">=18.17" + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./references": { + "types": "./dist/reference.d.ts", + "import": "./dist/reference.js" + }, + "./package.json": "./package.json" + }, + "bin": { + "llm-fingerprint": "./dist/cli.js" + }, + "files": [ + "dist", + "data", + "README.md", + "LICENSE" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test", + "prepublishOnly": "npm run build && node --test" + }, + "keywords": [ + "llm", + "fingerprint", + "fingerprinting", + "model-verification", + "openai-compatible", + "api-verification", + "jensen-shannon", + "behavioral-fingerprint", + "one-token-is-enough", + "llm-security", + "model-identity", + "model-substitution", + "api-audit", + "llm-verification", + "openrouter", + "gpt", + "claude", + "cli" + ], + "homepage": "https://tosea.ai/free-tools/llm-api-fingerprint-checker", + "repository": { + "type": "git", + "url": "git+https://github.com/ToseaAI/llm-fingerprint-detector.git" + }, + "bugs": { + "url": "https://github.com/ToseaAI/llm-fingerprint-detector/issues" + }, + "devDependencies": { + "@types/node": "^22.10.0", + "typescript": "^5.7.0" + } +} diff --git a/scripts/build-sample-references.mjs b/scripts/build-sample-references.mjs new file mode 100755 index 0000000..d8ac434 --- /dev/null +++ b/scripts/build-sample-references.mjs @@ -0,0 +1,169 @@ +#!/usr/bin/env node +/** + * Build data/reference-fingerprints.sample.json from the paper's public + * Zenodo dataset (DOI 10.5281/zenodo.21278557, CC-BY-4.0). + * + * Input: the dataset's aggregated `distributions.json` — an object with a + * `distributions` array of per-cell records: + * { model, task_id, lang, temperature, n_valid, dist: { answer: probability }, ... } + * + * Counts are reconstructed as round(probability × n_valid) and answers are + * re-normalized with this package's normalizer so vocabularies line up + * (e.g. the dataset's coin answers h/t → heads/tails, 蓝色 → 蓝). Cells that + * lose more than 20% of their probability mass in re-normalization are + * skipped. + * + * Usage: + * npm run build # the script imports the compiled normalizer from dist/ + * node scripts/build-sample-references.mjs \ + * [--models comma,separated,slugs] [--out data/reference-fingerprints.sample.json] + */ + +import { readFileSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' + +import { PROBE_TASKS } from '../dist/battery.js' +import { normalizeAnswer } from '../dist/normalizer.js' + +const HERE = dirname(fileURLToPath(import.meta.url)) + +/** Dataset task_id → this package's battery task id. */ +const TASK_MAP = { + 'num100-random': 'random-number-1-100', + 'num10-random': 'random-number-1-10', + 'letter-random': 'random-letter', + 'color-random': 'random-color', + 'coin-flip': 'coin-flip', + 'animal-random': 'random-animal', + 'city-random': 'random-city', + 'num-favorite': 'favorite-number', +} + +const LANGS = new Set(['en', 'zh']) + +/** The dataset folds coin answers to single letters; pre-expand before normalizing. */ +const COIN_PREMAP = { h: 'heads', t: 'tails' } + +const DEFAULT_MODELS = [ + 'openai/gpt-4o-mini', + 'openai/gpt-4o', + 'openai/gpt-4.1-mini', + 'anthropic/claude-sonnet-4.5', + 'google/gemini-2.5-flash', + 'deepseek/deepseek-chat', + 'meta-llama/llama-3.1-8b-instruct', + 'qwen/qwen3-30b-a3b-instruct-2507', + 'mistralai/mistral-small-3.2-24b-instruct', + 'z-ai/glm-4.5', + 'moonshotai/kimi-k2', +] + +function parseArgs(argv) { + const args = { input: null, models: DEFAULT_MODELS, out: join(HERE, '..', 'data', 'reference-fingerprints.sample.json') } + for (let i = 0; i < argv.length; i++) { + const token = argv[i] + if (token === '--models') args.models = argv[++i].split(',').map((m) => m.trim()) + else if (token === '--out') args.out = argv[++i] + else if (!token.startsWith('--') && !args.input) args.input = token + else throw new Error(`Unknown argument: ${token}`) + } + if (!args.input) { + console.error('usage: node scripts/build-sample-references.mjs [--models a,b] [--out file]') + process.exit(1) + } + return args +} + +function convertCell(record, taskId) { + const domain = PROBE_TASKS[taskId].domain + const counts = {} + let keptMass = 0 + let totalMass = 0 + for (const [rawAnswer, probability] of Object.entries(record.dist)) { + totalMass += probability + const premapped = taskId === 'coin-flip' ? (COIN_PREMAP[rawAnswer] ?? rawAnswer) : rawAnswer + const { normalized, category } = normalizeAnswer(premapped, domain) + if (category !== 'valid' || normalized === null) continue + const count = Math.round(probability * record.n_valid) + if (count <= 0) continue + counts[normalized] = (counts[normalized] ?? 0) + count + keptMass += probability + } + if (totalMass <= 0 || keptMass / totalMass < 0.8) return null + const n = Object.values(counts).reduce((sum, c) => sum + c, 0) + if (n < 10) return null + return { n, counts } +} + +function main() { + const args = parseArgs(process.argv.slice(2)) + const payload = JSON.parse(readFileSync(args.input, 'utf8')) + const records = payload.distributions + if (!Array.isArray(records)) throw new Error('Input has no "distributions" array') + + const collectedAt = (payload.generated_utc ?? '').slice(0, 10) || 'unknown' + const wanted = new Set(args.models) + const models = {} + let skippedCells = 0 + + for (const record of records) { + if (!wanted.has(record.model)) continue + if (!LANGS.has(record.lang)) continue + const taskId = TASK_MAP[record.task_id] + if (!taskId) continue + if (record.temperature !== 1) continue + + const cell = convertCell(record, taskId) + if (!cell) { + skippedCells += 1 + continue + } + const cellId = `${taskId}:${record.lang}` + models[record.model] ??= { + model: record.model, + collectedAt, + channel: 'openrouter', + cells: {}, + } + models[record.model].cells[cellId] = cell + } + + for (const model of args.models) { + const entry = models[model] + if (!entry) { + console.warn(`warn: model not found in dataset: ${model}`) + } else if (Object.keys(entry.cells).length < 8) { + console.warn(`warn: ${model} only has ${Object.keys(entry.cells).length} usable cells`) + } + } + + const output = { + formatVersion: 1, + protocol: 'bruckner-zenodo-2026', + samplesPerCell: 30, + source: { + dataset: + 'Single-token output distributions as behavioral fingerprints of large language models', + author: 'Tomáš Bruckner (Prague University of Economics and Business)', + datasetDoi: '10.5281/zenodo.21278557', + paper: 'arXiv:2607.10252', + license: 'CC-BY-4.0', + note: + 'Counts reconstructed as round(probability × n_valid) from the published per-cell distributions; ' + + 'answers re-normalized with the llm-fingerprint-detector normalizer. Collected via OpenRouter by ' + + "the paper's harness under the paper's prompt protocol (not this package's battery), so comparisons " + + 'against these samples are indicative rather than strict.', + }, + models, + } + + writeFileSync(args.out, `${JSON.stringify(output, null, 2)}\n`, 'utf8') + const cellTotal = Object.values(models).reduce((sum, m) => sum + Object.keys(m.cells).length, 0) + console.log( + `wrote ${args.out}: ${Object.keys(models).length} models, ${cellTotal} cells (${skippedCells} cells skipped in re-normalization)`, + ) +} + +main() diff --git a/src/adapter.ts b/src/adapter.ts new file mode 100644 index 0000000..fa54cc4 --- /dev/null +++ b/src/adapter.ts @@ -0,0 +1,134 @@ +/** + * Reasoning-disable adapter layer. Hidden "thinking" must be turned off: + * it burns the max_tokens budget before any visible answer appears, and it + * shifts the sampled distribution. Three request-body variants are known to + * work across OpenAI-compatible providers: + * + * 1. `reasoning: { enabled: false }` — OpenRouter style + * 2. `thinking: { type: "disabled" }` — Zhipu BigModel style + * 3. `reasoning_effort: "none"` — OpenAI style + * + * Before a run, each variant is probed in a provider-specific order; the + * first one that returns a non-empty visible answer without a 4xx wins. If + * every variant fails, a bare request is tried; if even that yields no + * visible text, the run falls back to a "post-reasoning" channel + * (max_tokens=1024) and the resulting fingerprint is flagged as lower + * confidence. + */ + +import { getSystemPrompt, pickParaphrase } from './battery.js' +import { + POST_REASONING_MAX_TOKENS, + PROBE_MAX_TOKENS, + PROBE_TEMPERATURE, +} from './constants.js' +import { guessAdapterHint } from './endpoint.js' +import { fetchChatCompletion, ProbeRequestError } from './http.js' +import type { + CellId, + ReasoningAdapter, + ReasoningStrategyId, + ResolvedEndpoint, +} from './types.js' + +export const STRATEGY_BODIES: Record< + Exclude, + Record +> = { + 'openrouter-reasoning': { reasoning: { enabled: false } }, + 'zhipu-thinking': { thinking: { type: 'disabled' } }, + 'openai-effort': { reasoning_effort: 'none' }, +} + +const ADAPTER_PROBE_CELL: CellId = 'random-number-1-100:en' + +export interface AdapterDetectionOptions { + signal?: AbortSignal + timeoutMs?: number + onProbe?: (strategy: ReasoningStrategyId) => void +} + +/** + * Detect which reasoning-disable field the endpoint accepts. + * Auth (401/403) and transport errors abort detection immediately — they are + * unrelated to the strategy and would fail the whole run anyway. + */ +export async function detectReasoningAdapter( + endpoint: ResolvedEndpoint, + options: AdapterDetectionOptions = {}, +): Promise { + const hint = guessAdapterHint(endpoint.baseUrl) + const systemPrompt = getSystemPrompt(ADAPTER_PROBE_CELL) + + for (const strategy of hint) { + if (strategy === 'none') continue + options.onProbe?.(strategy) + try { + const result = await fetchChatCompletion({ + endpoint, + systemPrompt, + userPrompt: pickParaphrase(ADAPTER_PROBE_CELL), + temperature: PROBE_TEMPERATURE, + maxTokens: PROBE_MAX_TOKENS, + extraBody: STRATEGY_BODIES[strategy], + signal: options.signal, + timeoutMs: options.timeoutMs, + retries: 0, + }) + if (result.content.trim().length > 0) { + return { + strategy, + extraBody: STRATEGY_BODIES[strategy], + maxTokens: PROBE_MAX_TOKENS, + postReasoning: false, + } + } + } catch (error) { + if (error instanceof ProbeRequestError) { + if (error.kind === 'auth' || error.kind === 'network' || error.kind === 'aborted') { + throw error + } + // 4xx parameter rejection or timeout: try the next strategy. + continue + } + throw error + } + } + + // No disable field accepted → probe with a bare request. + options.onProbe?.('none') + try { + const bare = await fetchChatCompletion({ + endpoint, + systemPrompt, + userPrompt: pickParaphrase(ADAPTER_PROBE_CELL), + temperature: PROBE_TEMPERATURE, + maxTokens: PROBE_MAX_TOKENS, + extraBody: {}, + signal: options.signal, + timeoutMs: options.timeoutMs, + retries: 0, + }) + if (bare.content.trim().length > 0) { + // Non-reasoning model (or reasoning is free): bare requests are fine. + return { strategy: 'none', extraBody: {}, maxTokens: PROBE_MAX_TOKENS, postReasoning: false } + } + } catch (error) { + if ( + error instanceof ProbeRequestError && + (error.kind === 'auth' || error.kind === 'network' || error.kind === 'aborted') + ) { + throw error + } + // Anything else falls through to the post-reasoning channel. + } + + // Fallback: reasoning cannot be disabled. Raise max_tokens so a visible + // answer survives after the hidden reasoning; flag reduced confidence. + return { + strategy: 'none', + extraBody: {}, + maxTokens: POST_REASONING_MAX_TOKENS, + postReasoning: true, + } +} diff --git a/src/api.ts b/src/api.ts new file mode 100644 index 0000000..524ef27 --- /dev/null +++ b/src/api.ts @@ -0,0 +1,182 @@ +/** + * Public high-level API: + * + * fingerprint(endpoint, options?) → collect a behavioral fingerprint + * compare(fingerprintA, fingerprintB) → distance + verdict + * verify(endpoint, reference, options?) → fingerprint + compare in one call + */ + +import { detectReasoningAdapter } from './adapter.js' +import { CELL_PRIORITY_ORDER, getTaskSpec } from './battery.js' +import { + DEFAULT_CELL_COUNT, + DEFAULT_CONCURRENCY, + DEFAULT_SAMPLES_PER_CELL, + FINGERPRINT_FORMAT_VERSION, + PROBE_PROTOCOL, + SPLIT_HALF_WARN_THRESHOLD, +} from './constants.js' +import { resolveEndpoint } from './endpoint.js' +import { runProbeBattery } from './sampler.js' +import { buildCellDistribution, compareCellSets, splitHalfJsd } from './stats.js' +import { buildComparisonResult } from './verdict.js' +import type { + CellDistribution, + CellId, + ComparisonResult, + Endpoint, + Fingerprint, + FingerprintOptions, + FingerprintRun, + VerifyResult, +} from './types.js' + +function resolveCells(cells: FingerprintOptions['cells']): CellId[] { + if (cells === undefined) return CELL_PRIORITY_ORDER.slice(0, DEFAULT_CELL_COUNT) + if (typeof cells === 'number') { + const count = Math.max(1, Math.min(CELL_PRIORITY_ORDER.length, Math.floor(cells))) + return CELL_PRIORITY_ORDER.slice(0, count) + } + if (cells.length === 0) throw new Error('options.cells must not be empty') + return cells +} + +/** + * Probe an OpenAI-compatible endpoint and collect its behavioral fingerprint. + * + * Steps: normalize the endpoint → detect a working reasoning-disable strategy + * → run the probe battery (shuffled, concurrent) → aggregate per-cell answer + * distributions. + */ +export async function fingerprint( + endpoint: Endpoint, + options: FingerprintOptions = {}, +): Promise { + const startedAt = Date.now() + const { resolved, warnings } = resolveEndpoint(endpoint) + const cells = resolveCells(options.cells) + const samplesPerCell = options.samplesPerCell ?? DEFAULT_SAMPLES_PER_CELL + const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY + + const adapter = + options.adapter ?? + (await detectReasoningAdapter(resolved, { + signal: options.signal, + timeoutMs: options.timeoutMs, + onProbe: (strategy) => + options.onProgress?.({ stage: 'adapter', done: 0, total: 1, errors: 0, strategy }), + })) + + if (adapter.postReasoning) { + warnings.push( + 'Reasoning could not be disabled; fell back to the post-reasoning channel (max_tokens=1024). Fingerprint confidence is reduced.', + ) + } + + const { samples, samplesByCell, errorCount } = await runProbeBattery({ + endpoint: resolved, + adapter, + cells, + samplesPerCell, + concurrency, + timeoutMs: options.timeoutMs, + maxRetries: options.maxRetries, + signal: options.signal, + onProgress: options.onProgress, + }) + + const cellDistributions: Partial> = {} + for (const cellId of cells) { + cellDistributions[cellId] = buildCellDistribution( + cellId, + samplesByCell.get(cellId) ?? [], + getTaskSpec(cellId).domain, + ) + } + + const selfJsd = splitHalfJsd(samplesByCell) + if (selfJsd !== null && selfJsd > SPLIT_HALF_WARN_THRESHOLD) { + warnings.push( + `Split-half self distance is high (${selfJsd.toFixed(3)} > ${SPLIT_HALF_WARN_THRESHOLD}); the endpoint may be routing across multiple backends.`, + ) + } + if (errorCount > 0) { + warnings.push(`${errorCount} of ${samples.length} requests failed and were excluded.`) + } + + const result: FingerprintRun = { + fingerprint: { + formatVersion: FINGERPRINT_FORMAT_VERSION, + protocol: PROBE_PROTOCOL, + model: resolved.model, + collectedAt: new Date().toISOString(), + samplesPerCell, + postReasoning: adapter.postReasoning, + cells: cellDistributions, + meta: { + tool: 'llm-fingerprint-detector', + ...options.meta, + }, + }, + adapter, + errorCount, + splitHalfJsd: selfJsd, + durationMs: Date.now() - startedAt, + warnings, + } + if (options.keepSamples) result.samples = samples + return result +} + +/** + * Compare two fingerprints: mean per-cell Jensen-Shannon divergence (base 2) + * over cells where both sides have enough valid samples, plus a three-way + * verdict against the paper-derived thresholds. + */ +export function compare(a: Fingerprint, b: Fingerprint): ComparisonResult { + const { entries, meanJsd } = compareCellSets(a.cells, b.cells) + const protocolMismatch = a.protocol !== b.protocol + return buildComparisonResult(entries, meanJsd, protocolMismatch) +} + +/** + * Verify that an endpoint behaves like a reference fingerprint: collect a + * fresh fingerprint from the endpoint, then compare against the reference. + */ +export async function verify( + endpoint: Endpoint, + reference: Fingerprint, + options: FingerprintOptions = {}, +): Promise { + const referenceCells = Object.keys(reference.cells) as CellId[] + const cells = + options.cells !== undefined + ? resolveCells(options.cells) + : CELL_PRIORITY_ORDER.filter((cellId) => referenceCells.includes(cellId)) + if (cells.length === 0) { + throw new Error('Reference fingerprint has no cells overlapping the probe battery') + } + + const target = await fingerprint(endpoint, { ...options, cells }) + const comparison = compare(target.fingerprint, reference) + + const warnings = [...target.warnings] + if (comparison.protocolMismatch) { + warnings.push( + `Protocol mismatch: target "${target.fingerprint.protocol}" vs reference "${reference.protocol}". ` + + 'Fingerprints collected under different prompts/batteries are only loosely comparable; treat the verdict as indicative.', + ) + } + if (reference.postReasoning) { + warnings.push('Reference fingerprint was collected over the post-reasoning channel (reduced confidence).') + } + + return { + verdict: comparison.verdict, + meanJsd: comparison.meanJsd, + comparison, + target, + reference, + warnings, + } +} diff --git a/src/battery.ts b/src/battery.ts new file mode 100644 index 0000000..839fcca --- /dev/null +++ b/src/battery.ts @@ -0,0 +1,234 @@ +/** + * Probe battery: 8 tasks × 2 languages = 16 cells. + * + * Design (arXiv:2607.10252): + * - ≥3 paraphrases per task per language, drawn at random per request — every + * probe is a plain semantic question, so gateways cannot keyword-filter them; + * - a fixed minimal system prompt forces a one-word answer; max_tokens=16 + * truncates stragglers; + * - cells are ordered by empirical discriminativeness (random-number tasks + * separate models best); presets take a prefix of that order. + */ + +import type { + CellId, + ProbeLang, + ProbePreset, + ProbePresetId, + ProbeTaskId, + ProbeTaskSpec, +} from './types.js' + +/** + * Fixed minimal system prompt — a critical invariant. + * + * The reference fingerprint and the endpoint under test MUST be collected + * under the *same* system prompt. Swapping only the system prompt (all else + * equal) can shift a model's fingerprint by JSD ≈ 0.44–0.46 — the same + * magnitude as swapping the model itself — which would turn a prompt + * difference into a false "different model" verdict. + */ +export const SYSTEM_PROMPTS: Record = { + en: 'Answer with exactly one word. No punctuation, no explanation.', + zh: '只回答一个词，不要标点，不要解释。', +} + +export const PROBE_TASKS: Record = { + 'random-number-1-100': { + id: 'random-number-1-100', + domain: { kind: 'int', min: 1, max: 100 }, + paraphrases: { + en: [ + 'Name a random number between 1 and 100.', + 'Pick a random number from 1 to 100.', + 'Give me a random number between 1 and 100.', + 'Choose any number between 1 and 100 at random.', + ], + zh: [ + '说一个 1 到 100 之间的随机数。', + '随机挑一个 1 到 100 的数字。', + '给我一个 1 至 100 之间的随机数字。', + '从 1 到 100 里随便选一个数。', + ], + }, + }, + 'random-number-1-10': { + id: 'random-number-1-10', + domain: { kind: 'int', min: 1, max: 10 }, + paraphrases: { + en: [ + 'Name a random number between 1 and 10.', + 'Pick a random number from 1 to 10.', + 'Give me a random number between 1 and 10.', + ], + zh: [ + '说一个 1 到 10 之间的随机数。', + '随机挑一个 1 到 10 的数字。', + '从 1 到 10 里随便选一个数。', + ], + }, + }, + 'random-letter': { + id: 'random-letter', + domain: { kind: 'letter' }, + paraphrases: { + en: [ + 'Name a random letter of the alphabet.', + 'Pick a random letter from A to Z.', + 'Give me one random letter.', + ], + zh: [ + '说一个随机的英文字母。', + '从 A 到 Z 里随便挑一个字母。', + '随机给我一个英文字母。', + ], + }, + }, + 'random-color': { + id: 'random-color', + domain: { kind: 'color' }, + paraphrases: { + en: [ + 'Name a random color.', + 'Pick a color at random.', + 'Give me one random color.', + ], + zh: [ + '说一个随机的颜色。', + '随便说一种颜色。', + '随机挑一个颜色告诉我。', + ], + }, + }, + 'coin-flip': { + id: 'coin-flip', + domain: { kind: 'coin' }, + paraphrases: { + en: [ + 'Flip a coin. Answer heads or tails.', + 'Toss a coin and tell me the result: heads or tails.', + 'Imagine flipping a coin. Which side came up, heads or tails?', + ], + zh: [ + '抛一枚硬币，回答正面还是反面。', + '掷一次硬币，告诉我结果：正面或反面。', + '想象抛硬币，落地是正面还是反面？', + ], + }, + }, + 'random-animal': { + id: 'random-animal', + domain: { kind: 'word' }, + paraphrases: { + en: [ + 'Name a random animal.', + 'Pick an animal at random.', + 'Give me one random animal.', + ], + zh: [ + '说一个随机的动物。', + '随便说一种动物。', + '随机挑一个动物告诉我。', + ], + }, + }, + 'random-city': { + id: 'random-city', + domain: { kind: 'word' }, + paraphrases: { + en: [ + 'Name a random city.', + 'Pick a city at random.', + 'Give me the name of one random city.', + ], + zh: [ + '说一个随机的城市。', + '随便说一座城市。', + '随机挑一个城市告诉我。', + ], + }, + }, + 'favorite-number': { + id: 'favorite-number', + domain: { kind: 'int', min: 0, max: 10_000 }, + paraphrases: { + en: [ + 'What is your favorite number?', + 'Tell me your favourite number.', + 'If you had to pick a favorite number, what would it be?', + ], + zh: [ + '你最喜欢的数字是什么？', + '说说你最爱的数字。', + '如果必须选一个最喜欢的数字，你选哪个？', + ], + }, + }, +} + +export function makeCellId(task: ProbeTaskId, lang: ProbeLang): CellId { + return `${task}:${lang}` +} + +export function parseCellId(cellId: CellId): { task: ProbeTaskId; lang: ProbeLang } { + const idx = cellId.lastIndexOf(':') + return { + task: cellId.slice(0, idx) as ProbeTaskId, + lang: cellId.slice(idx + 1) as ProbeLang, + } +} + +export function isCellId(value: string): value is CellId { + const idx = value.lastIndexOf(':') + if (idx <= 0) return false + const task = value.slice(0, idx) + const lang = value.slice(idx + 1) + return task in PROBE_TASKS && (lang === 'en' || lang === 'zh') +} + +/** + * All 16 cells, ordered by discriminativeness (random-number tasks first, + * per the paper and our own cross-model measurements). Presets take a prefix. + */ +export const CELL_PRIORITY_ORDER: CellId[] = [ + 'random-number-1-100:en', + 'random-number-1-100:zh', + 'random-color:en', + 'random-animal:en', + 'random-number-1-10:en', + 'random-letter:en', + 'random-color:zh', + 'coin-flip:en', + 'favorite-number:en', + 'random-city:en', + 'random-number-1-10:zh', + 'coin-flip:zh', + 'random-letter:zh', + 'random-animal:zh', + 'random-city:zh', + 'favorite-number:zh', +] + +export const PROBE_PRESETS: Record = { + quick: { id: 'quick', cellCount: 4, samplesPerCell: 15 }, + standard: { id: 'standard', cellCount: 8, samplesPerCell: 25 }, + strict: { id: 'strict', cellCount: 16, samplesPerCell: 25 }, +} + +export function getCellsForPreset(preset: ProbePresetId): CellId[] { + return CELL_PRIORITY_ORDER.slice(0, PROBE_PRESETS[preset].cellCount) +} + +export function getTaskSpec(cellId: CellId): ProbeTaskSpec { + return PROBE_TASKS[parseCellId(cellId).task] +} + +export function getSystemPrompt(cellId: CellId): string { + return SYSTEM_PROMPTS[parseCellId(cellId).lang] +} + +export function pickParaphrase(cellId: CellId, random: () => number = Math.random): string { + const { task, lang } = parseCellId(cellId) + const pool = PROBE_TASKS[task].paraphrases[lang] + return pool[Math.floor(random() * pool.length)] ?? pool[0] +} diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..95bea9b --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,527 @@ +#!/usr/bin/env node +/** + * llm-fingerprint — CLI for fingerprinting and verifying LLM endpoints. + * + * The API key is read from an environment variable (never from a file, never + * logged). Verify exit codes are CI-friendly: + * 0 match · 2 mismatch · 3 uncertain · 4 insufficient · 1 error + */ + +import { readFileSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' + +import { compare, fingerprint, verify } from './api.js' +import { CELL_PRIORITY_ORDER, isCellId } from './battery.js' +import { + DEFAULT_CELL_COUNT, + DEFAULT_CONCURRENCY, + DEFAULT_SAMPLES_PER_CELL, +} from './constants.js' +import { + getBundledAttribution, + listBundledReferences, + loadBundledReference, + parseFingerprintJson, +} from './reference.js' +import { ProbeRunError } from './sampler.js' +import type { + CellId, + ComparisonResult, + Endpoint, + Fingerprint, + FingerprintOptions, + ProgressEvent, + VerdictLevel, +} from './types.js' + +const DEFAULT_KEY_ENV_VARS = ['LLM_FINGERPRINT_API_KEY', 'OPENAI_API_KEY'] + +const VALUE_OPTIONS = new Set([ + '--base-url', + '--model', + '--api-key', + '--api-key-env', + '--cells', + '--samples', + '--concurrency', + '--timeout', + '--preset', + '--reference', + '--out', +]) +const BOOLEAN_OPTIONS = new Set(['--json', '--quiet', '--help', '-h', '--version', '-V']) + +interface ParsedArgs { + positionals: string[] + options: Map +} + +function parseArgs(argv: string[]): ParsedArgs { + const positionals: string[] = [] + const options = new Map() + for (let i = 0; i < argv.length; i++) { + const token = argv[i] + if (!token.startsWith('-')) { + positionals.push(token) + continue + } + const eq = token.indexOf('=') + if (eq > 0) { + options.set(token.slice(0, eq), token.slice(eq + 1)) + continue + } + if (BOOLEAN_OPTIONS.has(token)) { + options.set(token, true) + continue + } + if (VALUE_OPTIONS.has(token)) { + const value = argv[i + 1] + if (value === undefined || value.startsWith('--')) { + fail(`Option ${token} expects a value`) + } + options.set(token, value) + i += 1 + continue + } + fail(`Unknown option: ${token} (see --help)`) + } + return { positionals, options } +} + +function fail(message: string): never { + process.stderr.write(`error: ${message}\n`) + process.exit(1) +} + +function packageVersion(): string { + try { + const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json') + return (JSON.parse(readFileSync(pkgPath, 'utf8')) as { version: string }).version + } catch { + return 'unknown' + } +} + +const HELP = `llm-fingerprint — fingerprint & verify LLMs behind OpenAI-compatible APIs +Method: "One Token Is Enough" (Bruckner, arXiv:2607.10252) + +USAGE + llm-fingerprint [options] + +COMMANDS + fingerprint Probe an endpoint and print/save its behavioral fingerprint + verify Fingerprint an endpoint and compare it to a reference + compare Compare two saved fingerprints (files or bundled ids) + references List bundled sample reference fingerprints + +ENDPOINT OPTIONS + --base-url OpenAI-compatible base URL, e.g. https://api.openai.com/v1 + --model Model id to request, e.g. gpt-4o-mini + --api-key-env Env var holding the API key + (default: tries ${DEFAULT_KEY_ENV_VARS.join(', ')}) + --api-key API key literal — avoid; prefer --api-key-env + +SAMPLING OPTIONS + --cells Cell count 1-16 (top-N most discriminative) or a + comma-separated list of cell ids (default: ${DEFAULT_CELL_COUNT}) + --samples Samples per cell (default: ${DEFAULT_SAMPLES_PER_CELL}) + --preset quick (4×15) | standard (8×25) | strict (16×25) + --concurrency Concurrent requests (default: ${DEFAULT_CONCURRENCY}) + --timeout Per-request timeout (default: 30000) + +VERIFY / COMPARE + --reference Reference fingerprint: a JSON file produced by + 'fingerprint --out', or a bundled id (see 'references') + +OUTPUT + --json Machine-readable JSON on stdout + --out Write the fingerprint JSON to a file + --quiet No progress output + --help, -h Show this help + --version, -V Show version + +EXIT CODES (verify) + 0 match · 2 mismatch · 3 uncertain · 4 insufficient · 1 error + +EXAMPLES + # Fingerprint an endpoint (key read from OPENAI_API_KEY) + llm-fingerprint fingerprint --base-url https://api.openai.com/v1 \\ + --model gpt-4o-mini --out gpt-4o-mini.fingerprint.json + + # Is this cheap reseller really serving gpt-4o-mini? + LLM_FINGERPRINT_API_KEY=sk-... llm-fingerprint verify \\ + --base-url https://cheap-api.example.com/v1 --model gpt-4o-mini \\ + --reference gpt-4o-mini.fingerprint.json + + # Quick demo against a bundled sample reference + llm-fingerprint verify --base-url https://openrouter.ai/api/v1 \\ + --model openai/gpt-4o-mini --reference openai/gpt-4o-mini --preset quick + + # Compare two saved fingerprints offline + llm-fingerprint compare a.fingerprint.json b.fingerprint.json + +Web version (no install): https://tosea.ai/free-tools/llm-api-fingerprint-checker +` + +function readEndpoint(args: ParsedArgs): Endpoint { + const baseUrl = args.options.get('--base-url') + const model = args.options.get('--model') + if (typeof baseUrl !== 'string') fail('--base-url is required') + if (typeof model !== 'string') fail('--model is required') + return { baseUrl, model, apiKey: resolveApiKey(args) } +} + +function resolveApiKey(args: ParsedArgs): string | undefined { + const literal = args.options.get('--api-key') + if (typeof literal === 'string' && literal.trim()) return literal.trim() + + const envName = args.options.get('--api-key-env') + if (typeof envName === 'string') { + const value = process.env[envName] + if (!value) fail(`Environment variable ${envName} is empty or not set`) + return value + } + + for (const name of DEFAULT_KEY_ENV_VARS) { + const value = process.env[name] + if (value) return value + } + process.stderr.write( + `note: no API key found (checked ${DEFAULT_KEY_ENV_VARS.join(', ')}); ` + + 'sending requests without Authorization header\n', + ) + return undefined +} + +function readSamplingOptions(args: ParsedArgs): FingerprintOptions { + const options: FingerprintOptions = {} + + const preset = args.options.get('--preset') + if (typeof preset === 'string') { + const presets: Record = { + quick: { cells: 4, samples: 15 }, + standard: { cells: 8, samples: 25 }, + strict: { cells: 16, samples: 25 }, + } + const found = presets[preset] + if (!found) fail(`Unknown preset "${preset}" (quick | standard | strict)`) + options.cells = found.cells + options.samplesPerCell = found.samples + } + + const cells = args.options.get('--cells') + if (typeof cells === 'string') { + if (/^\d+$/.test(cells)) { + const n = Number(cells) + if (n < 1 || n > CELL_PRIORITY_ORDER.length) { + fail(`--cells must be 1-${CELL_PRIORITY_ORDER.length} or a comma-separated cell list`) + } + options.cells = n + } else { + const list = cells.split(',').map((cell) => cell.trim()) + for (const cell of list) { + if (!isCellId(cell)) { + fail(`Unknown cell id "${cell}". Valid cells:\n ${CELL_PRIORITY_ORDER.join('\n ')}`) + } + } + options.cells = list as CellId[] + } + } + + const samples = args.options.get('--samples') + if (typeof samples === 'string') { + const n = Number(samples) + if (!Number.isInteger(n) || n < 1) fail('--samples must be a positive integer') + options.samplesPerCell = n + } + + const concurrency = args.options.get('--concurrency') + if (typeof concurrency === 'string') { + const n = Number(concurrency) + if (!Number.isInteger(n) || n < 1) fail('--concurrency must be a positive integer') + options.concurrency = n + } + + const timeout = args.options.get('--timeout') + if (typeof timeout === 'string') { + const n = Number(timeout) + if (!Number.isFinite(n) || n < 100) fail('--timeout must be ≥ 100 (milliseconds)') + options.timeoutMs = n + } + + return options +} + +function makeProgressRenderer(args: ParsedArgs): ((event: ProgressEvent) => void) | undefined { + if (args.options.get('--quiet')) return undefined + const isTty = process.stderr.isTTY === true + let lastPercent = -1 + return (event) => { + if (event.stage === 'adapter') { + process.stderr.write( + isTty + ? `\rprobing reasoning adapter (${event.strategy})... ` + : `probing reasoning adapter (${event.strategy})...\n`, + ) + return + } + if (isTty) { + const errs = event.errors > 0 ? `, errors: ${event.errors}` : '' + process.stderr.write(`\rsampling ${event.done}/${event.total}${errs} `) + if (event.done === event.total) process.stderr.write('\n') + } else { + const percent = Math.floor((event.done / event.total) * 10) * 10 + if (percent > lastPercent) { + lastPercent = percent + process.stderr.write(`sampling ${event.done}/${event.total} (${percent}%)\n`) + } + } + } +} + +/** Load a reference: a JSON file path first, then a bundled sample id. */ +function loadReference(source: string): Fingerprint { + let fileText: string | null = null + try { + fileText = readFileSync(source, 'utf8') + } catch { + fileText = null + } + if (fileText !== null) return parseFingerprintJson(fileText, source) + try { + return loadBundledReference(source) + } catch (error) { + fail( + `"${source}" is neither a readable file nor a bundled reference id.\n${(error as Error).message}`, + ) + } +} + +function verdictLabel(verdict: VerdictLevel): string { + switch (verdict) { + case 'match': + return 'MATCH — behavior is consistent with the reference' + case 'uncertain': + return 'UNCERTAIN — in the gray zone; collect more samples or a fresh reference' + case 'mismatch': + return 'MISMATCH — behavior differs from the reference' + case 'insufficient': + return 'INSUFFICIENT — not enough comparable cells for a verdict' + } +} + +function verdictExitCode(verdict: VerdictLevel): number { + switch (verdict) { + case 'match': + return 0 + case 'mismatch': + return 2 + case 'uncertain': + return 3 + case 'insufficient': + return 4 + } +} + +function renderComparison(result: ComparisonResult): string { + const lines: string[] = [] + const mean = result.meanJsd === null ? 'n/a' : result.meanJsd.toFixed(3) + lines.push(`Verdict: ${verdictLabel(result.verdict)}`) + lines.push(`Mean JSD: ${mean} over ${result.comparableCellCount} comparable cell(s)`) + lines.push('') + lines.push('Interpretation scale (paper baselines, arXiv:2607.10252):') + lines.push( + ` same model ≈ ${result.baselines.sameModelSelf} · same model, other provider ≈ ${result.baselines.sameModelCrossProvider} · different model ≈ ${result.baselines.differentModel}`, + ) + lines.push( + ` thresholds: match ≤ ${result.thresholds.match} < uncertain ≤ ${result.thresholds.mismatch} < mismatch`, + ) + if (result.cells.length > 0) { + lines.push('') + lines.push('Per-cell JSD (most divergent first):') + for (const cell of result.cells) { + lines.push( + ` ${cell.cellId.padEnd(26)} ${cell.jsd.toFixed(3)} (${cell.validA} vs ${cell.validB} valid)`, + ) + } + } + if (result.protocolMismatch) { + lines.push('') + lines.push( + 'note: the fingerprints were collected under different probe protocols; treat the verdict as indicative only.', + ) + } + return lines.join('\n') +} + +function summarizeFingerprint(fp: Fingerprint): string { + const lines: string[] = [] + lines.push(`Model: ${fp.model}`) + lines.push(`Protocol: ${fp.protocol} · collected ${fp.collectedAt}`) + lines.push(`Cells: ${Object.keys(fp.cells).length} × ${fp.samplesPerCell} samples`) + if (fp.postReasoning) lines.push('warning: collected via post-reasoning fallback (reduced confidence)') + lines.push('') + lines.push('Top answers per cell:') + for (const [cellId, cell] of Object.entries(fp.cells)) { + if (!cell) continue + const top = Object.entries(cell.counts) + .sort((a, b) => b[1] - a[1]) + .slice(0, 3) + .map(([answer, count]) => `${answer}×${count}`) + .join(', ') + lines.push( + ` ${cellId.padEnd(26)} valid ${String(cell.validCount).padStart(3)} H=${cell.entropyBits.toFixed(2)}b ${top || '(no valid answers)'}`, + ) + } + return lines.join('\n') +} + +function writeWarnings(warnings: string[]): void { + for (const warning of warnings) process.stderr.write(`warning: ${warning}\n`) +} + +async function cmdFingerprint(args: ParsedArgs): Promise { + const endpoint = readEndpoint(args) + const options = readSamplingOptions(args) + options.onProgress = makeProgressRenderer(args) + + const run = await fingerprint(endpoint, options) + writeWarnings(run.warnings) + + const out = args.options.get('--out') + if (typeof out === 'string') { + writeFileSync(out, `${JSON.stringify(run.fingerprint, null, 2)}\n`, 'utf8') + process.stderr.write(`fingerprint written to ${out}\n`) + } + + if (args.options.get('--json')) { + const { fingerprint: fp, adapter, errorCount, splitHalfJsd, durationMs, warnings } = run + process.stdout.write( + `${JSON.stringify({ fingerprint: fp, run: { adapter, errorCount, splitHalfJsd, durationMs, warnings } }, null, 2)}\n`, + ) + } else if (typeof out !== 'string') { + process.stdout.write(`${summarizeFingerprint(run.fingerprint)}\n`) + } else { + process.stderr.write(`${summarizeFingerprint(run.fingerprint)}\n`) + } + return 0 +} + +async function cmdVerify(args: ParsedArgs): Promise { + const referenceSource = args.options.get('--reference') + if (typeof referenceSource !== 'string') { + fail('--reference is required (see `llm-fingerprint references`)') + } + const reference = loadReference(referenceSource) + const endpoint = readEndpoint(args) + const options = readSamplingOptions(args) + options.onProgress = makeProgressRenderer(args) + + const result = await verify(endpoint, reference, options) + writeWarnings(result.warnings) + + const out = args.options.get('--out') + if (typeof out === 'string') { + writeFileSync(out, `${JSON.stringify(result.target.fingerprint, null, 2)}\n`, 'utf8') + process.stderr.write(`target fingerprint written to ${out}\n`) + } + + if (args.options.get('--json')) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`) + } else { + process.stdout.write(`${renderComparison(result.comparison)}\n`) + process.stdout.write( + `\nReference: ${reference.model} (collected ${reference.collectedAt}, protocol ${reference.protocol})\n`, + ) + } + return verdictExitCode(result.verdict) +} + +async function cmdCompare(args: ParsedArgs): Promise { + if (args.positionals.length !== 2) { + fail('compare expects exactly two arguments: ') + } + const a = loadReference(args.positionals[0]) + const b = loadReference(args.positionals[1]) + const result = compare(a, b) + + if (args.options.get('--json')) { + process.stdout.write(`${JSON.stringify({ a: a.model, b: b.model, ...result }, null, 2)}\n`) + } else { + process.stdout.write(`A: ${a.model} (${a.collectedAt})\nB: ${b.model} (${b.collectedAt})\n\n`) + process.stdout.write(`${renderComparison(result)}\n`) + } + return verdictExitCode(result.verdict) +} + +function cmdReferences(args: ParsedArgs): number { + const references = listBundledReferences() + const attribution = getBundledAttribution() + if (args.options.get('--json')) { + process.stdout.write(`${JSON.stringify({ source: attribution, references }, null, 2)}\n`) + return 0 + } + process.stdout.write('Bundled sample reference fingerprints:\n\n') + for (const ref of references) { + process.stdout.write( + ` ${ref.id.padEnd(36)} ${String(ref.cellCount).padStart(2)} cells collected ${ref.collectedAt}\n`, + ) + } + process.stdout.write( + `\nSource: ${attribution.dataset}\n` + + `by ${attribution.author} — DOI ${attribution.datasetDoi} (${attribution.license})\n` + + 'Collected under the paper\'s protocol: fine for demos; for high-stakes checks,\n' + + 'collect your own reference with `llm-fingerprint fingerprint --out ...`.\n', + ) + return 0 +} + +async function main(): Promise { + const argv = process.argv.slice(2) + const args = parseArgs(argv) + + if (args.options.get('--version') || args.options.get('-V')) { + process.stdout.write(`${packageVersion()}\n`) + process.exit(0) + } + const command = args.positionals.shift() + if (!command || args.options.get('--help') || args.options.get('-h') || command === 'help') { + process.stdout.write(HELP) + process.exit(0) + } + + try { + let exitCode: number + switch (command) { + case 'fingerprint': + exitCode = await cmdFingerprint(args) + break + case 'verify': + exitCode = await cmdVerify(args) + break + case 'compare': + exitCode = await cmdCompare(args) + break + case 'references': + exitCode = cmdReferences(args) + break + default: + fail(`Unknown command: ${command} (see --help)`) + } + process.exit(exitCode) + } catch (error) { + if (error instanceof ProbeRunError) { + const hints: Record = { + auth: 'The endpoint rejected the API key (401/403).', + network: 'The endpoint is unreachable — check the base URL and your network.', + aborted: 'Run cancelled.', + } + fail(`${hints[error.reason] ?? ''} ${error.message}`.trim()) + } + fail(error instanceof Error ? error.message : String(error)) + } +} + +void main() diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 0000000..5e7c1d1 --- /dev/null +++ b/src/constants.ts @@ -0,0 +1,60 @@ +/** + * Thresholds and statistical constants, centralized for calibration. + * + * Baselines from Bruckner, "One Token Is Enough" (arXiv:2607.10252): + * - same model, split-half distance (median) ≈ 0.140 + * - same model served by different providers (median) ≈ 0.227 + * - different models (median) ≈ 0.463 + * - equal error rate: ≈ 10.6% with 8 cells, ≈ 7.3% with 40 cells + * + * The match/mismatch cut points below sit between those baselines and leave a + * deliberate "uncertain" band; they are heuristics, not proofs. + */ + +/** meanJsd ≤ this → `match` (paper same-model cross-provider median 0.227, plus margin). */ +export const JSD_MATCH_THRESHOLD = 0.25 + +/** + * meanJsd > this → `mismatch` (paper different-model median 0.463; impostor + * distances rarely fall below ≈ 0.3). Between the two thresholds → `uncertain`. + */ +export const JSD_MISMATCH_THRESHOLD = 0.35 + +/** Paper baseline anchors, exposed for result interpretation. */ +export const JSD_BASELINE_SELF = 0.14 +export const JSD_BASELINE_CROSS_PROVIDER = 0.227 +export const JSD_BASELINE_DIFFERENT_MODEL = 0.463 + +/** A cell participates in the distance only when both sides have ≥ this many valid samples. */ +export const MIN_VALID_SAMPLES_PER_CELL = 10 + +/** Fewer comparable cells than this → verdict `insufficient`. */ +export const MIN_COMPARABLE_CELLS = 4 + +/** Split-half self check: a cell participates only when each half has ≥ this many valid samples. */ +export const MIN_SPLIT_HALF_SAMPLES = 5 + +/** Split-half JSD above this suggests unstable routing (multi-backend aggregator). */ +export const SPLIT_HALF_WARN_THRESHOLD = 0.25 + +/** Probe request parameters (paper protocol). */ +export const PROBE_TEMPERATURE = 1.0 +export const PROBE_MAX_TOKENS = 16 +/** Fallback max_tokens when reasoning cannot be disabled (post-reasoning channel). */ +export const POST_REASONING_MAX_TOKENS = 1024 + +/** Sampler defaults. */ +export const DEFAULT_SAMPLES_PER_CELL = 25 +export const DEFAULT_CELL_COUNT = 8 +export const DEFAULT_CONCURRENCY = 4 +export const DEFAULT_REQUEST_TIMEOUT_MS = 30_000 +export const DEFAULT_MAX_RETRIES = 2 +/** Abort the run after this many consecutive transport-level failures. */ +export const CONSECUTIVE_NETWORK_ERROR_LIMIT = 8 + +/** Fingerprint artifact identifiers. */ +export const FINGERPRINT_FORMAT_VERSION = 1 as const +/** Protocol id for fingerprints collected by this package's battery. */ +export const PROBE_PROTOCOL = 'one-token/v1' +/** Protocol id for samples derived from the paper's Zenodo dataset. */ +export const ZENODO_PROTOCOL = 'bruckner-zenodo-2026' diff --git a/src/endpoint.ts b/src/endpoint.ts new file mode 100644 index 0000000..f7fa09e --- /dev/null +++ b/src/endpoint.ts @@ -0,0 +1,97 @@ +/** + * Endpoint normalization and per-provider reasoning-strategy hints. + */ + +import type { Endpoint, ReasoningStrategyId, ResolvedEndpoint } from './types.js' + +export interface BaseUrlNormalization { + ok: boolean + normalized: string + reason?: 'empty' | 'invalid' + /** Non-fatal notes, e.g. plain-http endpoints. */ + warnings: string[] +} + +/** + * Base URL cleanup: + * - trim whitespace and trailing slashes; drop an accidentally pasted + * `/chat/completions` suffix; + * - bare domains (no path) get `/v1` appended; + * - plain `http://` is allowed (local vLLM/Ollama/LM Studio) but flagged + * with a warning for non-local hosts. + */ +export function normalizeBaseUrl(input: string): BaseUrlNormalization { + const trimmed = (input ?? '').trim() + if (!trimmed) return { ok: false, normalized: '', reason: 'empty', warnings: [] } + + const withScheme = /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}` + let url: URL + try { + url = new URL(withScheme) + } catch { + return { ok: false, normalized: trimmed, reason: 'invalid', warnings: [] } + } + + const warnings: string[] = [] + const isLocal = + url.hostname === 'localhost' || url.hostname === '127.0.0.1' || url.hostname === '::1' + if (url.protocol === 'http:' && !isLocal) { + warnings.push(`Plain http:// endpoint (${url.host}): the API key is sent unencrypted.`) + } + + let path = url.pathname.replace(/\/+$/, '') + path = path.replace(/\/chat\/completions$/, '') + if (path === '' || path === '/') path = '/v1' + + return { ok: true, normalized: `${url.protocol}//${url.host}${path}`, warnings } +} + +/** Order in which reasoning-disable strategies are probed for a given host. */ +const HOST_ADAPTER_HINTS: Array<{ hostIncludes: string; hint: ReasoningStrategyId[] }> = [ + { hostIncludes: 'openrouter.ai', hint: ['openrouter-reasoning', 'openai-effort', 'zhipu-thinking'] }, + { hostIncludes: 'api.openai.com', hint: ['openai-effort', 'openrouter-reasoning', 'zhipu-thinking'] }, + { hostIncludes: 'api.deepseek.com', hint: ['openai-effort', 'zhipu-thinking', 'openrouter-reasoning'] }, + { hostIncludes: 'bigmodel.cn', hint: ['zhipu-thinking', 'openai-effort', 'openrouter-reasoning'] }, +] + +export const DEFAULT_ADAPTER_HINT: ReasoningStrategyId[] = [ + 'openrouter-reasoning', + 'zhipu-thinking', + 'openai-effort', +] + +export function guessAdapterHint(baseUrl: string): ReasoningStrategyId[] { + const lower = baseUrl.toLowerCase() + for (const { hostIncludes, hint } of HOST_ADAPTER_HINTS) { + if (lower.includes(hostIncludes)) return hint + } + return DEFAULT_ADAPTER_HINT +} + +/** + * Validate and normalize a user-supplied endpoint. Throws on empty/invalid + * base URL; returns the resolved endpoint plus non-fatal warnings. + */ +export function resolveEndpoint(endpoint: Endpoint): { + resolved: ResolvedEndpoint + warnings: string[] +} { + const { ok, normalized, reason, warnings } = normalizeBaseUrl(endpoint.baseUrl) + if (!ok) { + throw new Error( + reason === 'empty' ? 'Endpoint baseUrl is empty' : `Invalid baseUrl: ${endpoint.baseUrl}`, + ) + } + if (!endpoint.model || !endpoint.model.trim()) { + throw new Error('Endpoint model is empty') + } + return { + resolved: { + baseUrl: normalized, + model: endpoint.model.trim(), + apiKey: endpoint.apiKey?.trim() || null, + headers: endpoint.headers ?? {}, + }, + warnings, + } +} diff --git a/src/http.ts b/src/http.ts new file mode 100644 index 0000000..c4bdb34 --- /dev/null +++ b/src/http.ts @@ -0,0 +1,215 @@ +/** + * Minimal HTTP layer over `POST {baseUrl}/chat/completions`: + * timeout + exponential backoff on 429/5xx/timeouts + AbortSignal + * pass-through + error classification. Uses the global `fetch` + * (Node ≥ 18 built-in, or any browser). + * + * The API key only ever appears in the Authorization header of the request + * to the endpoint under test; it is never logged or sent anywhere else. + */ + +import { DEFAULT_MAX_RETRIES, DEFAULT_REQUEST_TIMEOUT_MS } from './constants.js' +import type { ResolvedEndpoint, SampleUsage } from './types.js' + +export type ProbeErrorKind = 'network' | 'auth' | 'http' | 'timeout' | 'aborted' + +export class ProbeRequestError extends Error { + readonly kind: ProbeErrorKind + readonly status: number | null + + constructor(kind: ProbeErrorKind, message: string, status: number | null = null) { + super(message) + this.name = 'ProbeRequestError' + this.kind = kind + this.status = status + } +} + +export interface ChatCompletionResult { + content: string + usage: SampleUsage | null + latencyMs: number + status: number +} + +interface ChatCompletionUsagePayload { + prompt_tokens?: number + completion_tokens?: number + reasoning_tokens?: number + completion_tokens_details?: { reasoning_tokens?: number } +} + +function parseUsage(payload: ChatCompletionUsagePayload | undefined | null): SampleUsage | null { + if (!payload) return null + return { + promptTokens: typeof payload.prompt_tokens === 'number' ? payload.prompt_tokens : null, + completionTokens: + typeof payload.completion_tokens === 'number' ? payload.completion_tokens : null, + reasoningTokens: + typeof payload.completion_tokens_details?.reasoning_tokens === 'number' + ? payload.completion_tokens_details.reasoning_tokens + : typeof payload.reasoning_tokens === 'number' + ? payload.reasoning_tokens + : null, + } +} + +/** `content` is a string in the OpenAI schema, but some gateways send part arrays. */ +function extractContent(message: unknown): string { + if (!message || typeof message !== 'object') return '' + const content = (message as { content?: unknown }).content + if (typeof content === 'string') return content + if (Array.isArray(content)) { + return content + .map((part) => + typeof part === 'string' + ? part + : typeof (part as { text?: unknown })?.text === 'string' + ? (part as { text: string }).text + : '', + ) + .join('') + } + return '' +} + +export interface ChatCompletionRequest { + endpoint: ResolvedEndpoint + systemPrompt: string + userPrompt: string + temperature: number + maxTokens: number + extraBody: Record + signal?: AbortSignal + timeoutMs?: number + retries?: number +} + +async function delay(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + const onAbort = () => { + clearTimeout(timer) + reject(new ProbeRequestError('aborted', 'Aborted')) + } + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} + +/** + * One chat/completions call with retries. Retried: 429, 5xx, timeouts. + * Not retried: transport errors and 401/403 — those are classified and + * re-thrown for the caller to decide. + */ +export async function fetchChatCompletion( + request: ChatCompletionRequest, +): Promise { + const retries = request.retries ?? DEFAULT_MAX_RETRIES + const timeoutMs = request.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS + let lastError: ProbeRequestError | null = null + + for (let attempt = 0; attempt <= retries; attempt++) { + if (request.signal?.aborted) throw new ProbeRequestError('aborted', 'Aborted') + + const timeoutController = new AbortController() + const timer = setTimeout(() => timeoutController.abort(), timeoutMs) + const onOuterAbort = () => timeoutController.abort() + request.signal?.addEventListener('abort', onOuterAbort, { once: true }) + + const startedAt = performance.now() + try { + const headers: Record = { + 'Content-Type': 'application/json', + ...request.endpoint.headers, + } + if (request.endpoint.apiKey) { + headers.Authorization = `Bearer ${request.endpoint.apiKey}` + } + const response = await fetch(`${request.endpoint.baseUrl}/chat/completions`, { + method: 'POST', + headers, + body: JSON.stringify({ + model: request.endpoint.model, + temperature: request.temperature, + max_tokens: request.maxTokens, + stream: false, + messages: [ + { role: 'system', content: request.systemPrompt }, + { role: 'user', content: request.userPrompt }, + ], + ...request.extraBody, + }), + signal: timeoutController.signal, + }) + const latencyMs = performance.now() - startedAt + + if (response.status === 401 || response.status === 403) { + throw new ProbeRequestError( + 'auth', + `HTTP ${response.status} — API key rejected`, + response.status, + ) + } + if (response.status === 429 || response.status >= 500) { + const retryAfterHeader = response.headers.get('retry-after') + const retryAfterMs = retryAfterHeader ? Number(retryAfterHeader) * 1000 : NaN + lastError = new ProbeRequestError('http', `HTTP ${response.status}`, response.status) + if (attempt < retries) { + const backoff = Number.isFinite(retryAfterMs) + ? retryAfterMs + : 800 * 2 ** attempt + Math.random() * 400 + await delay(backoff, request.signal) + continue + } + throw lastError + } + if (!response.ok) { + let detail = '' + try { + detail = (await response.text()).slice(0, 300) + } catch { + // Ignore body read failures. + } + throw new ProbeRequestError( + 'http', + `HTTP ${response.status} ${detail}`.trim(), + response.status, + ) + } + + const payload = (await response.json()) as { + choices?: Array<{ message?: unknown }> + usage?: ChatCompletionUsagePayload + } + return { + content: extractContent(payload?.choices?.[0]?.message), + usage: parseUsage(payload?.usage), + latencyMs, + status: response.status, + } + } catch (error) { + if (error instanceof ProbeRequestError) throw error + if (request.signal?.aborted) throw new ProbeRequestError('aborted', 'Aborted') + if (timeoutController.signal.aborted) { + lastError = new ProbeRequestError('timeout', `Request timed out after ${timeoutMs}ms`) + } else { + lastError = new ProbeRequestError( + 'network', + error instanceof Error ? error.message : 'Network error', + ) + } + if (lastError.kind === 'timeout' && attempt < retries) { + continue + } + throw lastError + } finally { + clearTimeout(timer) + request.signal?.removeEventListener('abort', onOuterAbort) + } + } + + throw lastError ?? new ProbeRequestError('network', 'Unknown error') +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..031f3b5 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,90 @@ +/** + * llm-fingerprint-detector — fingerprint and verify LLMs behind + * OpenAI-compatible APIs from single-token output distributions. + * + * Independent open-source implementation of Tomáš Bruckner, + * "One Token Is Enough: Fingerprinting and Verifying Large Language Models + * from Single-Token Output Distributions" (arXiv:2607.10252). + * + * Everything exported here is runtime-agnostic (Node ≥ 18 or browsers with + * fetch). Bundled sample references are Node-only and live in the + * `llm-fingerprint-detector/references` subpath export. + */ + +export { fingerprint, compare, verify } from './api.js' + +export { + CELL_PRIORITY_ORDER, + PROBE_PRESETS, + PROBE_TASKS, + SYSTEM_PROMPTS, + getCellsForPreset, + getSystemPrompt, + getTaskSpec, + isCellId, + makeCellId, + parseCellId, + pickParaphrase, +} from './battery.js' + +export { + normalizeAnswer, + parseAnyNumber, + parseChineseNumeral, + parseEnglishNumberWord, +} from './normalizer.js' +export type { NormalizedAnswer } from './normalizer.js' + +export { + buildCellDistribution, + compareCellSets, + domainSize, + jensenShannonDivergence, + median, + shannonEntropyBits, + splitHalfJsd, +} from './stats.js' +export type { CellJsdEntry, CountMap } from './stats.js' + +export { decideVerdict, buildComparisonResult } from './verdict.js' + +export { detectReasoningAdapter, STRATEGY_BODIES } from './adapter.js' +export type { AdapterDetectionOptions } from './adapter.js' + +export { runProbeBattery, ProbeRunError } from './sampler.js' +export type { SamplerOptions, SamplerResult } from './sampler.js' + +export { fetchChatCompletion, ProbeRequestError } from './http.js' +export type { ChatCompletionRequest, ChatCompletionResult, ProbeErrorKind } from './http.js' + +export { normalizeBaseUrl, resolveEndpoint, guessAdapterHint } from './endpoint.js' +export type { BaseUrlNormalization } from './endpoint.js' + +export * from './constants.js' + +export type { + AnswerDomain, + CellComparison, + CellDistribution, + CellId, + ComparisonBaselines, + ComparisonResult, + Endpoint, + Fingerprint, + FingerprintOptions, + FingerprintRun, + ProbeLang, + ProbePreset, + ProbePresetId, + ProbeTaskId, + ProbeTaskSpec, + ProgressEvent, + ReasoningAdapter, + ReasoningStrategyId, + ResolvedEndpoint, + SampleCategory, + SampleResult, + SampleUsage, + VerdictLevel, + VerifyResult, +} from './types.js' diff --git a/src/normalizer.ts b/src/normalizer.ts new file mode 100644 index 0000000..148a140 --- /dev/null +++ b/src/normalizer.ts @@ -0,0 +1,203 @@ +/** + * Answer normalization pipeline (pure functions): + * + * NFC → trim → refusal detection → strip punctuation/quotes/emoji + * → case fold → take first word + * → digit unification (Chinese numerals / English number words / + * full-width and Arabic-Indic digits → Latin digits) + * → color canonicalization (蓝色→蓝, grey→gray) + * → coin folding (正/正面/heads → heads) + * + * Categories: valid (inside the cell's answer domain) / invalid / refusal / + * empty. Models frequently answer "seven", "四十二" or "forty-two" instead of + * "7"/"42", so variants must be folded before distributions are compared. + */ + +import type { AnswerDomain, SampleCategory } from './types.js' + +const REFUSAL_PATTERNS: RegExp[] = [ + /\bas an ai\b/i, + /\bi (?:cannot|can't|can not|won't|will not)\b/i, + /\bi'?m (?:unable|not able|sorry)\b/i, + /\bsorry,? (?:i|but)\b/i, + /\bcannot (?:comply|assist|help)\b/i, + /我不能/, + /我无法/, + /无法回答/, + /不能回答/, + /抱歉/, + /对不起/, + /作为(?:一个)?(?:AI|人工智能)/i, +] + +/** + * Remove everything that is not a letter, digit or whitespace (quotes, + * punctuation, emoji). In-word hyphens are removed too: forty-seven → fortyseven. + */ +function stripPunctuation(value: string): string { + return value.replace(/[^\p{L}\p{N}\s]/gu, '') +} + +/** Full-width and Arabic-Indic digits → Latin digits. */ +function normalizeDigitScript(value: string): string { + return value.replace(/[\uFF10-\uFF19\u0660-\u0669\u06F0-\u06F9]/g, (ch) => { + const code = ch.charCodeAt(0) + if (code >= 0xff10 && code <= 0xff19) return String(code - 0xff10) + if (code >= 0x0660 && code <= 0x0669) return String(code - 0x0660) + return String(code - 0x06f0) + }) +} + +const CN_DIGITS: Record = { + 零: 0, 〇: 0, 一: 1, 二: 2, 两: 2, 三: 3, 四: 4, + 五: 5, 六: 6, 七: 7, 八: 8, 九: 9, +} +const CN_UNITS: Record = { 十: 10, 百: 100, 千: 1000 } + +/** Chinese numerals (一二三…百/千, incl. 两) → number; null when not a pure numeral. */ +export function parseChineseNumeral(value: string): number | null { + if (!value || !/^[零〇一二两三四五六七八九十百千]+$/.test(value)) return null + let total = 0 + let current = 0 + for (const ch of value) { + if (ch in CN_DIGITS) { + current = CN_DIGITS[ch] + } else { + const unit = CN_UNITS[ch] + // A leading 十 (十, 十五) counts as 1 × 10. + total += (current === 0 ? 1 : current) * unit + current = 0 + } + } + return total + current +} + +const EN_ONES: Record = { + zero: 0, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, + eight: 8, nine: 9, ten: 10, eleven: 11, twelve: 12, thirteen: 13, + fourteen: 14, fifteen: 15, sixteen: 16, seventeen: 17, eighteen: 18, + nineteen: 19, +} +const EN_TENS: Record = { + twenty: 20, thirty: 30, forty: 40, fifty: 50, sixty: 60, seventy: 70, + eighty: 80, ninety: 90, +} + +/** + * English number words → number, including de-hyphenated compounds + * (fortyseven, onehundred). Null when unparseable. + */ +export function parseEnglishNumberWord(value: string): number | null { + const word = value.toLowerCase() + if (word in EN_ONES) return EN_ONES[word] + if (word in EN_TENS) return EN_TENS[word] + if (word === 'hundred' || word === 'onehundred') return 100 + for (const [tens, tensValue] of Object.entries(EN_TENS)) { + if (word.startsWith(tens)) { + const rest = word.slice(tens.length) + if (rest in EN_ONES && EN_ONES[rest] >= 1 && EN_ONES[rest] <= 9) { + return tensValue + EN_ONES[rest] + } + } + } + return null +} + +/** Any-format number parsing: Latin digits / Chinese numerals / English words. */ +export function parseAnyNumber(value: string): number | null { + if (/^\d+$/.test(value)) return Number(value) + const cn = parseChineseNumeral(value) + if (cn !== null) return cn + return parseEnglishNumberWord(value) +} + +/** English letter names → letter (zee/kay/queue and friends show up in the wild). */ +const EN_LETTER_NAMES: Record = { + bee: 'b', cee: 'c', dee: 'd', gee: 'g', jay: 'j', kay: 'k', + el: 'l', ell: 'l', em: 'm', en: 'n', oh: 'o', pee: 'p', + cue: 'q', queue: 'q', ar: 'r', es: 's', ess: 's', tee: 't', + vee: 'v', ex: 'x', why: 'y', zee: 'z', zed: 'z', +} + +/** Color variant folding (within a language). */ +const EN_COLOR_ALIASES: Record = { + grey: 'gray', + aqua: 'cyan', +} + +function normalizeColorWord(word: string): string { + if (EN_COLOR_ALIASES[word]) return EN_COLOR_ALIASES[word] + // Chinese: 蓝色→蓝, 青色→青 (multi-character names keep their stem; 靛蓝/蔚蓝 unchanged). + if (/^[\u4e00-\u9fff]{2,}$/.test(word) && word.endsWith('色')) { + return word.slice(0, -1) + } + return word +} + +const COIN_HEADS = new Set(['heads', 'head', '正', '正面', '字']) +const COIN_TAILS = new Set(['tails', 'tail', '反', '反面', '花']) + +function normalizeCoinWord(word: string): string | null { + if (COIN_HEADS.has(word)) return 'heads' + if (COIN_TAILS.has(word)) return 'tails' + if (/^正面?/.test(word)) return 'heads' + if (/^反面?/.test(word)) return 'tails' + return null +} + +export interface NormalizedAnswer { + normalized: string | null + category: Exclude +} + +/** Main entry point: raw completion text + the cell's answer domain → normalized answer + category. */ +export function normalizeAnswer(raw: string, domain: AnswerDomain): NormalizedAnswer { + const nfc = (raw ?? '').normalize('NFC').trim() + if (!nfc) return { normalized: null, category: 'empty' } + + if (REFUSAL_PATTERNS.some((pattern) => pattern.test(nfc))) { + return { normalized: null, category: 'refusal' } + } + + const cleaned = normalizeDigitScript(stripPunctuation(nfc)).toLowerCase().trim() + if (!cleaned) return { normalized: null, category: 'empty' } + + // First whitespace-separated word (Chinese text without spaces is one word). + const firstWord = cleaned.split(/\s+/)[0] + if (!firstWord) return { normalized: null, category: 'empty' } + + switch (domain.kind) { + case 'int': { + const num = parseAnyNumber(firstWord) + if (num === null) return { normalized: firstWord, category: 'invalid' } + if (num < domain.min || num > domain.max) { + return { normalized: String(num), category: 'invalid' } + } + return { normalized: String(num), category: 'valid' } + } + case 'letter': { + const mapped = EN_LETTER_NAMES[firstWord] ?? firstWord + if (/^[a-z]$/.test(mapped)) return { normalized: mapped, category: 'valid' } + return { normalized: firstWord, category: 'invalid' } + } + case 'color': { + const color = normalizeColorWord(firstWord) + if (/^(?:[a-z]+|[\u4e00-\u9fff]{1,4})$/.test(color)) { + return { normalized: color, category: 'valid' } + } + return { normalized: firstWord, category: 'invalid' } + } + case 'coin': { + const coin = normalizeCoinWord(firstWord) + if (coin) return { normalized: coin, category: 'valid' } + return { normalized: firstWord, category: 'invalid' } + } + case 'word': { + // Word tasks (animal/city): a single Latin word or a CJK word of ≤6 chars. + if (/^(?:[a-z]+|[\u4e00-\u9fff]{1,6})$/.test(firstWord)) { + return { normalized: firstWord, category: 'valid' } + } + return { normalized: firstWord, category: 'invalid' } + } + } +} diff --git a/src/reference.ts b/src/reference.ts new file mode 100644 index 0000000..276c1c7 --- /dev/null +++ b/src/reference.ts @@ -0,0 +1,164 @@ +/** + * Bundled sample reference fingerprints (Node-only module — uses the + * filesystem; import via `llm-fingerprint-detector/references`). + * + * The samples in `data/reference-fingerprints.sample.json` are derived from + * the paper's public dataset (Bruckner, "Single-token output distributions as + * behavioral fingerprints of large language models", Zenodo, + * DOI 10.5281/zenodo.21278557, CC-BY-4.0). They were collected by the paper's + * own harness — a slightly different prompt protocol than this package's + * battery — so `compare()` flags them with `protocolMismatch: true`. They are + * great for demos and exploration; for high-stakes verification, collect your + * own reference from a trusted endpoint with this tool. + */ + +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { getTaskSpec, isCellId } from './battery.js' +import { FINGERPRINT_FORMAT_VERSION } from './constants.js' +import { domainSize, shannonEntropyBits } from './stats.js' +import type { CellDistribution, CellId, Fingerprint } from './types.js' + +export interface SampleReferenceSource { + dataset: string + author: string + datasetDoi: string + paper: string + license: string + note?: string +} + +export interface SampleReferenceEntry { + model: string + collectedAt: string + channel?: string + cells: Record }> +} + +export interface SampleReferenceFile { + formatVersion: number + protocol: string + samplesPerCell: number + source: SampleReferenceSource + models: Record +} + +const DATA_PATH = join( + dirname(fileURLToPath(import.meta.url)), + '..', + 'data', + 'reference-fingerprints.sample.json', +) + +let cached: SampleReferenceFile | null = null + +function loadFile(): SampleReferenceFile { + if (cached) return cached + let text: string + try { + text = readFileSync(DATA_PATH, 'utf8') + } catch { + throw new Error( + `Bundled reference data not found at ${DATA_PATH}. ` + + 'Reinstall the package, or build your own references (see README "Building your own reference fingerprints").', + ) + } + cached = JSON.parse(text) as SampleReferenceFile + return cached +} + +export interface BundledReferenceInfo { + id: string + model: string + collectedAt: string + channel?: string + cellCount: number +} + +export function listBundledReferences(): BundledReferenceInfo[] { + const file = loadFile() + return Object.entries(file.models).map(([id, entry]) => ({ + id, + model: entry.model, + collectedAt: entry.collectedAt, + channel: entry.channel, + cellCount: Object.keys(entry.cells).length, + })) +} + +export function getBundledAttribution(): SampleReferenceSource { + return loadFile().source +} + +function entryToFingerprint(id: string, entry: SampleReferenceEntry, file: SampleReferenceFile): Fingerprint { + const cells: Partial> = {} + for (const [cellKey, cell] of Object.entries(entry.cells)) { + if (!isCellId(cellKey)) continue + const cellId = cellKey + const entropyBits = shannonEntropyBits(cell.counts) + const size = domainSize(getTaskSpec(cellId).domain) + cells[cellId] = { + cellId, + counts: cell.counts, + validCount: cell.n, + invalidCount: 0, + refusalCount: 0, + emptyCount: 0, + errorCount: 0, + totalCount: cell.n, + entropyBits, + normalizedEntropy: size > 1 ? Math.min(1, entropyBits / Math.log2(size)) : 0, + medianLatencyMs: null, + meanCompletionTokens: null, + meanReasoningTokens: null, + } + } + return { + formatVersion: FINGERPRINT_FORMAT_VERSION, + protocol: file.protocol, + model: entry.model, + collectedAt: entry.collectedAt, + samplesPerCell: file.samplesPerCell, + postReasoning: false, + cells, + meta: { + source: `${file.source.dataset} (DOI ${file.source.datasetDoi}, ${file.source.license})`, + channel: entry.channel, + note: `Bundled sample reference "${id}"`, + }, + } +} + +/** Load one bundled reference as a runtime Fingerprint. */ +export function loadBundledReference(id: string): Fingerprint { + const file = loadFile() + const entry = file.models[id] + if (!entry) { + const available = Object.keys(file.models).sort().join(', ') + throw new Error(`Unknown bundled reference "${id}". Available: ${available}`) + } + return entryToFingerprint(id, entry, file) +} + +/** Parse and minimally validate a fingerprint JSON file produced by this tool. */ +export function parseFingerprintJson(text: string, sourceLabel = 'fingerprint file'): Fingerprint { + let parsed: unknown + try { + parsed = JSON.parse(text) + } catch (error) { + throw new Error(`${sourceLabel} is not valid JSON: ${(error as Error).message}`) + } + const fp = parsed as Partial + if (fp.formatVersion !== FINGERPRINT_FORMAT_VERSION) { + throw new Error(`${sourceLabel}: unsupported formatVersion ${String(fp.formatVersion)}`) + } + if (!fp.model || typeof fp.model !== 'string') { + throw new Error(`${sourceLabel}: missing "model"`) + } + if (!fp.cells || typeof fp.cells !== 'object') { + throw new Error(`${sourceLabel}: missing "cells"`) + } + return fp as Fingerprint +} diff --git a/src/sampler.ts b/src/sampler.ts new file mode 100644 index 0000000..5c4188b --- /dev/null +++ b/src/sampler.ts @@ -0,0 +1,199 @@ +/** + * Sampling engine: a worker pool over a shuffled request queue with retries, + * live progress callbacks and AbortSignal cancellation. + * + * - Requests are shuffled so a single cell is never hammered in a burst, + * which would bias endpoint-side caching/rate-limiting systematically; + * - a request that still fails after retries is recorded as an `error` + * sample (excluded from distributions); + * - ≥N consecutive transport errors abort the run (endpoint unreachable); + * - 401/403 aborts immediately (bad key). + */ + +import { getSystemPrompt, getTaskSpec, pickParaphrase } from './battery.js' +import { CONSECUTIVE_NETWORK_ERROR_LIMIT, PROBE_TEMPERATURE } from './constants.js' +import { fetchChatCompletion, ProbeRequestError } from './http.js' +import { normalizeAnswer } from './normalizer.js' +import type { + CellId, + ProgressEvent, + ReasoningAdapter, + ResolvedEndpoint, + SampleResult, +} from './types.js' + +export class ProbeRunError extends Error { + readonly reason: 'network' | 'auth' | 'aborted' + + constructor(reason: 'network' | 'auth' | 'aborted', message: string) { + super(message) + this.name = 'ProbeRunError' + this.reason = reason + } +} + +interface ProbeJob { + cellId: CellId + paraphrase: string +} + +export interface SamplerOptions { + endpoint: ResolvedEndpoint + adapter: ReasoningAdapter + cells: CellId[] + samplesPerCell: number + concurrency: number + timeoutMs?: number + maxRetries?: number + signal?: AbortSignal + onSample?: (sample: SampleResult) => void + onProgress?: (event: ProgressEvent) => void +} + +export interface SamplerResult { + samples: SampleResult[] + samplesByCell: Map + errorCount: number +} + +function shuffle(items: T[]): T[] { + const arr = [...items] + for (let i = arr.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)) + ;[arr[i], arr[j]] = [arr[j], arr[i]] + } + return arr +} + +/** + * Run the full probe battery against one endpoint. Throws ProbeRunError when + * the whole run must stop (unreachable endpoint / invalid key / cancelled). + */ +export async function runProbeBattery(options: SamplerOptions): Promise { + const jobs: ProbeJob[] = [] + for (const cellId of options.cells) { + for (let i = 0; i < options.samplesPerCell; i++) { + jobs.push({ cellId, paraphrase: pickParaphrase(cellId) }) + } + } + const queue = shuffle(jobs) + + const samples: SampleResult[] = [] + const samplesByCell = new Map() + for (const cellId of options.cells) samplesByCell.set(cellId, []) + + let arrivalIndex = 0 + let errorCount = 0 + let consecutiveNetworkErrors = 0 + let fatalError: ProbeRunError | null = null + let cursor = 0 + + const recordSample = (sample: SampleResult) => { + samples.push(sample) + samplesByCell.get(sample.cellId)?.push(sample) + options.onSample?.(sample) + options.onProgress?.({ + stage: 'sampling', + done: samples.length, + total: queue.length, + errors: errorCount, + cellId: sample.cellId, + }) + } + + async function worker(): Promise { + while (true) { + if (fatalError || options.signal?.aborted) return + const index = cursor + cursor += 1 + if (index >= queue.length) return + const job = queue[index] + + const startedAt = performance.now() + const systemPrompt = getSystemPrompt(job.cellId) + try { + const result = await fetchChatCompletion({ + endpoint: options.endpoint, + systemPrompt, + userPrompt: job.paraphrase, + temperature: PROBE_TEMPERATURE, + maxTokens: options.adapter.maxTokens, + extraBody: options.adapter.extraBody, + signal: options.signal, + timeoutMs: options.timeoutMs, + retries: options.maxRetries, + }) + consecutiveNetworkErrors = 0 + const domain = getTaskSpec(job.cellId).domain + const { normalized, category } = normalizeAnswer(result.content, domain) + recordSample({ + cellId: job.cellId, + raw: result.content, + normalized, + category, + latencyMs: result.latencyMs, + usage: result.usage, + arrivalIndex: arrivalIndex++, + }) + } catch (error) { + if (options.signal?.aborted) { + fatalError = fatalError ?? new ProbeRunError('aborted', 'Run cancelled') + return + } + if (error instanceof ProbeRequestError) { + if (error.kind === 'aborted') { + fatalError = fatalError ?? new ProbeRunError('aborted', 'Run cancelled') + return + } + if (error.kind === 'auth') { + fatalError = new ProbeRunError('auth', error.message) + return + } + if (error.kind === 'network') { + consecutiveNetworkErrors += 1 + if (consecutiveNetworkErrors >= CONSECUTIVE_NETWORK_ERROR_LIMIT) { + fatalError = new ProbeRunError( + 'network', + `Endpoint unreachable (${consecutiveNetworkErrors} consecutive transport errors): ${error.message}`, + ) + return + } + } else { + consecutiveNetworkErrors = 0 + } + errorCount += 1 + recordSample({ + cellId: job.cellId, + raw: '', + normalized: null, + category: 'error', + latencyMs: performance.now() - startedAt, + usage: null, + arrivalIndex: arrivalIndex++, + errorMessage: error.message, + }) + } else { + errorCount += 1 + recordSample({ + cellId: job.cellId, + raw: '', + normalized: null, + category: 'error', + latencyMs: performance.now() - startedAt, + usage: null, + arrivalIndex: arrivalIndex++, + errorMessage: error instanceof Error ? error.message : String(error), + }) + } + } + } + } + + const workerCount = Math.max(1, Math.min(options.concurrency, queue.length)) + await Promise.all(Array.from({ length: workerCount }, () => worker())) + + if (fatalError) throw fatalError + if (options.signal?.aborted) throw new ProbeRunError('aborted', 'Run cancelled') + + return { samples, samplesByCell, errorCount } +} diff --git a/src/stats.ts b/src/stats.ts new file mode 100644 index 0000000..053acbe --- /dev/null +++ b/src/stats.ts @@ -0,0 +1,219 @@ +/** + * Statistics: Shannon entropy, Jensen-Shannon divergence (base 2, not + * square-rooted, so per-cell values live in [0, 1] bit), distribution + * aggregation and the split-half self check. + * + * Distance between two fingerprints (arXiv:2607.10252): the mean of per-cell + * JSD over all cells where both sides have enough valid samples. + */ + +import { MIN_SPLIT_HALF_SAMPLES, MIN_VALID_SAMPLES_PER_CELL } from './constants.js' +import type { AnswerDomain, CellDistribution, CellId, SampleResult } from './types.js' + +export type CountMap = Record + +/** Shannon entropy in bits. */ +export function shannonEntropyBits(counts: CountMap): number { + const total = Object.values(counts).reduce((sum, n) => sum + n, 0) + if (total <= 0) return 0 + let entropy = 0 + for (const n of Object.values(counts)) { + if (n <= 0) continue + const p = n / total + entropy -= p * Math.log2(p) + } + return entropy +} + +/** Nominal domain size (denominator log2(size) for normalized entropy). */ +export function domainSize(domain: AnswerDomain): number { + switch (domain.kind) { + case 'int': + return Math.max(2, domain.max - domain.min + 1) + case 'letter': + return 26 + case 'coin': + return 2 + case 'color': + return 30 + case 'word': + return 50 + } +} + +/** + * Jensen-Shannon divergence, base 2: JSD(P,Q) = H(M) − (H(P)+H(Q))/2 with + * M = (P+Q)/2, over the union of both supports. Range [0, 1] bit. + */ +export function jensenShannonDivergence(countsP: CountMap, countsQ: CountMap): number { + const totalP = Object.values(countsP).reduce((sum, n) => sum + n, 0) + const totalQ = Object.values(countsQ).reduce((sum, n) => sum + n, 0) + if (totalP <= 0 || totalQ <= 0) return 0 + + const support = new Set([...Object.keys(countsP), ...Object.keys(countsQ)]) + let hM = 0 + let hP = 0 + let hQ = 0 + for (const key of support) { + const p = (countsP[key] ?? 0) / totalP + const q = (countsQ[key] ?? 0) / totalQ + const m = (p + q) / 2 + if (m > 0) hM -= m * Math.log2(m) + if (p > 0) hP -= p * Math.log2(p) + if (q > 0) hQ -= q * Math.log2(q) + } + const jsd = hM - (hP + hQ) / 2 + // Clamp numerical noise. + return Math.min(1, Math.max(0, jsd)) +} + +export function median(values: number[]): number | null { + if (values.length === 0) return null + const sorted = [...values].sort((a, b) => a - b) + const mid = Math.floor(sorted.length / 2) + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid] +} + +/** Aggregate raw samples into one cell's distribution. */ +export function buildCellDistribution( + cellId: CellId, + samples: SampleResult[], + domain: AnswerDomain, +): CellDistribution { + const counts: CountMap = {} + let validCount = 0 + let invalidCount = 0 + let refusalCount = 0 + let emptyCount = 0 + let errorCount = 0 + const latenciesMs: number[] = [] + let completionTokensSum = 0 + let completionTokensN = 0 + let reasoningTokensSum = 0 + let reasoningTokensN = 0 + + for (const sample of samples) { + switch (sample.category) { + case 'valid': + validCount += 1 + if (sample.normalized !== null) { + counts[sample.normalized] = (counts[sample.normalized] ?? 0) + 1 + } + break + case 'invalid': + invalidCount += 1 + break + case 'refusal': + refusalCount += 1 + break + case 'empty': + emptyCount += 1 + break + case 'error': + errorCount += 1 + break + } + if (sample.category !== 'error') { + latenciesMs.push(sample.latencyMs) + if (sample.usage?.completionTokens != null) { + completionTokensSum += sample.usage.completionTokens + completionTokensN += 1 + } + if (sample.usage?.reasoningTokens != null) { + reasoningTokensSum += sample.usage.reasoningTokens + reasoningTokensN += 1 + } + } + } + + const entropyBits = shannonEntropyBits(counts) + const size = domainSize(domain) + + return { + cellId, + counts, + validCount, + invalidCount, + refusalCount, + emptyCount, + errorCount, + totalCount: samples.length, + entropyBits, + normalizedEntropy: size > 1 ? Math.min(1, entropyBits / Math.log2(size)) : 0, + medianLatencyMs: median(latenciesMs), + meanCompletionTokens: completionTokensN > 0 ? completionTokensSum / completionTokensN : null, + meanReasoningTokens: reasoningTokensN > 0 ? reasoningTokensSum / reasoningTokensN : null, + } +} + +export interface CellJsdEntry { + cellId: CellId + jsd: number + validA: number + validB: number +} + +/** + * Distance between two fingerprint cell sets: mean JSD over cells where both + * sides have ≥ minValidSamples valid samples. Returns per-cell details sorted + * by descending JSD. + */ +export function compareCellSets( + cellsA: Partial>, + cellsB: Partial>, + minValidSamples: number = MIN_VALID_SAMPLES_PER_CELL, +): { entries: CellJsdEntry[]; meanJsd: number | null } { + const entries: CellJsdEntry[] = [] + for (const cellId of Object.keys(cellsA) as CellId[]) { + const a = cellsA[cellId] + const b = cellsB[cellId] + if (!a || !b) continue + if (a.validCount < minValidSamples || b.validCount < minValidSamples) continue + entries.push({ + cellId, + jsd: jensenShannonDivergence(a.counts, b.counts), + validA: a.validCount, + validB: b.validCount, + }) + } + entries.sort((x, y) => y.jsd - x.jsd) + const meanJsd = + entries.length > 0 + ? entries.reduce((sum, entry) => sum + entry.jsd, 0) / entries.length + : null + return { entries, meanJsd } +} + +/** + * Split-half self check: split each cell's valid samples by arrival parity + * and measure the JSD between the halves. Values far above the same-model + * baseline (≈ 0.14) indicate the endpoint itself is unstable — a hint that an + * aggregator is rotating between different backends. + */ +export function splitHalfJsd( + samplesByCell: Map, + minPerHalf: number = MIN_SPLIT_HALF_SAMPLES, +): number | null { + const cellJsds: number[] = [] + for (const samples of samplesByCell.values()) { + const even: CountMap = {} + const odd: CountMap = {} + let evenN = 0 + let oddN = 0 + for (const sample of samples) { + if (sample.category !== 'valid' || sample.normalized === null) continue + if (sample.arrivalIndex % 2 === 0) { + even[sample.normalized] = (even[sample.normalized] ?? 0) + 1 + evenN += 1 + } else { + odd[sample.normalized] = (odd[sample.normalized] ?? 0) + 1 + oddN += 1 + } + } + if (evenN >= minPerHalf && oddN >= minPerHalf) { + cellJsds.push(jensenShannonDivergence(even, odd)) + } + } + if (cellJsds.length === 0) return null + return cellJsds.reduce((sum, jsd) => sum + jsd, 0) / cellJsds.length +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..f72a330 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,260 @@ +/** + * Core type definitions. + * + * Method: Tomáš Bruckner, "One Token Is Enough: Fingerprinting and Verifying + * Large Language Models from Single-Token Output Distributions" + * (arXiv:2607.10252). A probe battery of task × language "cells" is sampled + * repeatedly at temperature 1.0 with a one-word answer constraint; the + * empirical distribution of normalized answers is the model's behavioral + * fingerprint. Two fingerprints are compared with the mean per-cell + * Jensen-Shannon divergence (base 2, so each cell's JSD lies in [0, 1] bit). + */ + +export type ProbeTaskId = + | 'random-number-1-100' + | 'random-number-1-10' + | 'random-letter' + | 'random-color' + | 'coin-flip' + | 'random-animal' + | 'random-city' + | 'favorite-number' + +export type ProbeLang = 'en' | 'zh' + +/** + * A cell is one task in one language. Distributions are only ever compared + * within the same cell; there is no cross-language pooling. + */ +export type CellId = `${ProbeTaskId}:${ProbeLang}` + +export type AnswerDomain = + | { kind: 'int'; min: number; max: number } + | { kind: 'letter' } + | { kind: 'color' } + | { kind: 'coin' } + | { kind: 'word' } + +export interface ProbeTaskSpec { + id: ProbeTaskId + domain: AnswerDomain + /** + * At least 3 paraphrases per language. One is drawn at random per request, + * so the probes are plain semantic questions with no fixed magic string a + * gateway could keyword-filter. + */ + paraphrases: Record +} + +export type ProbePresetId = 'quick' | 'standard' | 'strict' + +export interface ProbePreset { + id: ProbePresetId + cellCount: number + samplesPerCell: number +} + +/** An OpenAI-compatible chat-completions endpoint to probe. */ +export interface Endpoint { + /** Base URL, e.g. `https://api.openai.com/v1`. A bare domain gets `/v1` appended. */ + baseUrl: string + /** Model id to request, e.g. `gpt-4o-mini`. */ + model: string + /** API key sent as `Authorization: Bearer `. Omit for keyless local servers. */ + apiKey?: string + /** Extra HTTP headers merged into every request. */ + headers?: Record +} + +/** Internal, normalized endpoint (base URL cleaned up, key resolved). */ +export interface ResolvedEndpoint { + baseUrl: string + model: string + apiKey: string | null + headers: Record +} + +/** Strategy used to disable hidden reasoning ("thinking") on the endpoint. */ +export type ReasoningStrategyId = + | 'openrouter-reasoning' + | 'zhipu-thinking' + | 'openai-effort' + | 'none' + +export interface ReasoningAdapter { + strategy: ReasoningStrategyId + /** Extra fields merged into the request body. */ + extraBody: Record + /** max_tokens used for probe requests. */ + maxTokens: number + /** + * True when no disabling strategy produced visible output and the run fell + * back to a large max_tokens "post-reasoning" channel. Fingerprints + * collected this way are lower confidence (reasoning shifts sampling). + */ + postReasoning: boolean +} + +export type SampleCategory = 'valid' | 'invalid' | 'refusal' | 'empty' | 'error' + +export interface SampleUsage { + promptTokens: number | null + completionTokens: number | null + reasoningTokens: number | null +} + +export interface SampleResult { + cellId: CellId + /** Verbatim completion text. */ + raw: string + /** Normalized answer; non-null only when `category === 'valid'`. */ + normalized: string | null + category: SampleCategory + latencyMs: number + usage: SampleUsage | null + /** Arrival order across the whole run (used for the split-half self check). */ + arrivalIndex: number + errorMessage?: string +} + +/** Aggregated answer distribution for one cell. */ +export interface CellDistribution { + cellId: CellId + /** Normalized answer → count (valid samples only). */ + counts: Record + validCount: number + invalidCount: number + refusalCount: number + emptyCount: number + errorCount: number + totalCount: number + /** Shannon entropy of the valid-answer distribution, in bits. */ + entropyBits: number + /** entropyBits / log2(nominal domain size), clamped to [0, 1]. */ + normalizedEntropy: number + medianLatencyMs: number | null + meanCompletionTokens: number | null + meanReasoningTokens: number | null +} + +/** + * A behavioral fingerprint: per-cell answer distributions plus collection + * metadata. This is the JSON artifact written/read by the CLI. + */ +export interface Fingerprint { + formatVersion: 1 + /** + * Probe protocol identifier. Fingerprints are only strictly comparable when + * both sides used the same protocol (same battery, same system prompt). + * This package emits `one-token/v1`; bundled Zenodo-derived samples use + * `bruckner-zenodo-2026`. + */ + protocol: string + model: string + /** ISO timestamp. Fingerprints drift when models are updated, so age matters. */ + collectedAt: string + samplesPerCell: number + postReasoning: boolean + cells: Partial> + meta?: { + tool?: string + channel?: string + source?: string + note?: string + [key: string]: unknown + } +} + +export type VerdictLevel = 'match' | 'uncertain' | 'mismatch' | 'insufficient' + +export interface CellComparison { + cellId: CellId + /** Jensen-Shannon divergence, base 2, in [0, 1] bit. */ + jsd: number + validA: number + validB: number +} + +export interface ComparisonBaselines { + /** Median split-half distance of a model against itself (paper): ≈ 0.140. */ + sameModelSelf: number + /** Median distance, same model served by different providers (paper): ≈ 0.227. */ + sameModelCrossProvider: number + /** Median distance between different models (paper): ≈ 0.463. */ + differentModel: number +} + +export interface ComparisonResult { + /** Mean per-cell JSD across comparable cells; null when none are comparable. */ + meanJsd: number | null + verdict: VerdictLevel + /** Per-cell details, sorted by descending JSD. */ + cells: CellComparison[] + comparableCellCount: number + /** True when the two fingerprints were collected under different probe protocols. */ + protocolMismatch: boolean + thresholds: { match: number; mismatch: number } + baselines: ComparisonBaselines +} + +export interface ProgressEvent { + stage: 'adapter' | 'sampling' + /** Completed requests (sampling stage) or probes attempted (adapter stage). */ + done: number + total: number + errors: number + cellId?: CellId + strategy?: ReasoningStrategyId +} + +export interface FingerprintOptions { + /** + * Cells to probe: an explicit list, or a number N meaning the top-N cells + * from the discriminativeness-ordered battery. Default: 8 (standard preset). + */ + cells?: CellId[] | number + /** Samples per cell. Default: 25. */ + samplesPerCell?: number + /** Concurrent in-flight requests. Default: 4. */ + concurrency?: number + /** Per-request timeout in milliseconds. Default: 30000. */ + timeoutMs?: number + /** Retries per request on 429/5xx/timeout. Default: 2. */ + maxRetries?: number + /** Abort the whole run (in-flight requests are cancelled). */ + signal?: AbortSignal + onProgress?: (event: ProgressEvent) => void + /** Skip reasoning-adapter detection and use this adapter directly. */ + adapter?: ReasoningAdapter + /** Keep raw per-sample results on the run result (off by default). */ + keepSamples?: boolean + /** Free-form metadata merged into `fingerprint.meta`. */ + meta?: Fingerprint['meta'] +} + +export interface FingerprintRun { + fingerprint: Fingerprint + adapter: ReasoningAdapter + /** Requests that errored out after retries (excluded from distributions). */ + errorCount: number + /** + * Mean JSD between odd/even arrival halves of the run itself. + * Values far above the same-model baseline (≈ 0.14) suggest the endpoint + * routes across multiple backends (aggregator behavior). + */ + splitHalfJsd: number | null + durationMs: number + /** Present only when `keepSamples: true`. */ + samples?: SampleResult[] + /** Human-readable caveats collected during the run. */ + warnings: string[] +} + +export interface VerifyResult { + verdict: VerdictLevel + meanJsd: number | null + comparison: ComparisonResult + target: FingerprintRun + reference: Fingerprint + warnings: string[] +} diff --git a/src/verdict.ts b/src/verdict.ts new file mode 100644 index 0000000..a17ae8a --- /dev/null +++ b/src/verdict.ts @@ -0,0 +1,53 @@ +/** + * Verdict: meanJsd → three-way conclusion (match / uncertain / mismatch), + * or `insufficient` when too few cells are comparable. + * + * Threshold provenance (see constants.ts): the paper reports a same-model + * cross-provider median distance of ≈ 0.227 and a different-model median of + * ≈ 0.463; the cut points 0.25 / 0.35 sit between those with a deliberate + * uncertainty band. Verdicts are statistical evidence, not proof. + */ + +import { + JSD_BASELINE_CROSS_PROVIDER, + JSD_BASELINE_DIFFERENT_MODEL, + JSD_BASELINE_SELF, + JSD_MATCH_THRESHOLD, + JSD_MISMATCH_THRESHOLD, + MIN_COMPARABLE_CELLS, +} from './constants.js' +import type { CellJsdEntry } from './stats.js' +import type { CellComparison, ComparisonResult, VerdictLevel } from './types.js' + +export function decideVerdict(meanJsd: number | null, comparableCellCount: number): VerdictLevel { + if (meanJsd === null || comparableCellCount < MIN_COMPARABLE_CELLS) return 'insufficient' + if (meanJsd <= JSD_MATCH_THRESHOLD) return 'match' + if (meanJsd <= JSD_MISMATCH_THRESHOLD) return 'uncertain' + return 'mismatch' +} + +export function buildComparisonResult( + entries: CellJsdEntry[], + meanJsd: number | null, + protocolMismatch: boolean, +): ComparisonResult { + const cells: CellComparison[] = entries.map((entry) => ({ + cellId: entry.cellId, + jsd: entry.jsd, + validA: entry.validA, + validB: entry.validB, + })) + return { + meanJsd, + verdict: decideVerdict(meanJsd, entries.length), + cells, + comparableCellCount: entries.length, + protocolMismatch, + thresholds: { match: JSD_MATCH_THRESHOLD, mismatch: JSD_MISMATCH_THRESHOLD }, + baselines: { + sameModelSelf: JSD_BASELINE_SELF, + sameModelCrossProvider: JSD_BASELINE_CROSS_PROVIDER, + differentModel: JSD_BASELINE_DIFFERENT_MODEL, + }, + } +} diff --git a/test/api.integration.test.mjs b/test/api.integration.test.mjs new file mode 100644 index 0000000..1918870 --- /dev/null +++ b/test/api.integration.test.mjs @@ -0,0 +1,199 @@ +/** + * End-to-end test against a local mock OpenAI-compatible server: adapter + * detection, concurrent sampling, normalization, aggregation and verify(). + */ +import assert from 'node:assert/strict' +import { createServer } from 'node:http' +import { after, before, test } from 'node:test' + +import { compare, fingerprint, verify } from '../dist/api.js' + +/** Two simulated "models" with different answer distributions. */ +const MODEL_BEHAVIOR = { + 'mock-alpha': { + 'random-number-1-100': () => (Math.random() < 0.8 ? '42' : '73'), + 'random-number-1-10': () => '7', + 'random-letter': () => 'k', + 'random-color': () => (Math.random() < 0.6 ? 'blue' : 'teal'), + 'coin-flip': () => (Math.random() < 0.7 ? 'Heads' : 'Tails'), + 'random-animal': () => 'octopus', + 'random-city': () => 'Tokyo', + 'favorite-number': () => '42', + }, + 'mock-beta': { + 'random-number-1-100': () => (Math.random() < 0.8 ? '57' : '3'), + 'random-number-1-10': () => '3', + 'random-letter': () => 'm', + 'random-color': () => 'crimson', + 'coin-flip': () => 'Tails', + 'random-animal': () => 'giraffe', + 'random-city': () => 'Paris', + 'favorite-number': () => '9', + }, +} + +/** Map a user prompt back to the battery task (rough but sufficient for the mock). */ +function taskFromPrompt(prompt) { + const p = prompt.toLowerCase() + if (p.includes('100') || p.includes('1 到 100') || p.includes('1 至 100') || p.includes('1 到 100')) + return 'random-number-1-100' + if (p.includes('10') || p.includes('1 到 10')) return 'random-number-1-10' + if (p.includes('letter') || p.includes('字母')) return 'random-letter' + if (p.includes('color') || p.includes('颜色')) return 'random-color' + if (p.includes('coin') || p.includes('硬币')) return 'coin-flip' + if (p.includes('animal') || p.includes('动物')) return 'random-animal' + if (p.includes('city') || p.includes('城市')) return 'random-city' + if (p.includes('favorite') || p.includes('favourite') || p.includes('喜欢') || p.includes('最爱')) + return 'favorite-number' + return 'random-number-1-100' +} + +let server +let baseUrl +let requestCount = 0 +let sawReasoningField = 0 + +before(async () => { + server = createServer((req, res) => { + let body = '' + req.on('data', (chunk) => (body += chunk)) + req.on('end', () => { + requestCount += 1 + const payload = JSON.parse(body) + + if (req.url !== '/v1/chat/completions') { + res.writeHead(404).end('not found') + return + } + if (req.headers.authorization !== 'Bearer test-key') { + res.writeHead(401, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: { message: 'bad key' } })) + return + } + // The mock accepts the OpenRouter-style reasoning field (counts it for assertions). + if (payload.reasoning !== undefined) sawReasoningField += 1 + + const behavior = MODEL_BEHAVIOR[payload.model] + if (!behavior) { + res.writeHead(404, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: { message: `unknown model ${payload.model}` } })) + return + } + const userPrompt = payload.messages.find((m) => m.role === 'user')?.content ?? '' + const answer = behavior[taskFromPrompt(userPrompt)]() + res.writeHead(200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + choices: [{ message: { role: 'assistant', content: answer } }], + usage: { prompt_tokens: 25, completion_tokens: 2 }, + }), + ) + }) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + baseUrl = `http://127.0.0.1:${server.address().port}/v1` +}) + +after(() => server.close()) + +test('fingerprint(): collects distributions from a live endpoint', async () => { + const run = await fingerprint( + { baseUrl, model: 'mock-alpha', apiKey: 'test-key' }, + { cells: 4, samplesPerCell: 12, concurrency: 6 }, + ) + assert.equal(run.errorCount, 0) + assert.equal(run.fingerprint.model, 'mock-alpha') + assert.equal(run.fingerprint.protocol, 'one-token/v1') + assert.equal(Object.keys(run.fingerprint.cells).length, 4) + + const rn100 = run.fingerprint.cells['random-number-1-100:en'] + assert.ok(rn100, 'expected the top-priority cell to be probed') + assert.equal(rn100.validCount, 12) + assert.ok(rn100.counts['42'] > 0, 'mock-alpha answers 42 most of the time') + assert.ok(run.adapter.postReasoning === false) + assert.ok(sawReasoningField > 0, 'adapter probe should have tried the reasoning field') +}) + +test('fingerprint(): progress callback covers all requests', async () => { + const events = [] + const run = await fingerprint( + { baseUrl, model: 'mock-alpha', apiKey: 'test-key' }, + { cells: 2, samplesPerCell: 5, onProgress: (e) => events.push(e) }, + ) + const sampling = events.filter((e) => e.stage === 'sampling') + assert.equal(sampling.length, 10) + assert.equal(sampling.at(-1).done, 10) + assert.equal(sampling.at(-1).total, 10) + assert.equal(run.durationMs >= 0, true) +}) + +test('verify(): same mock model → match, different mock model → mismatch', async () => { + const options = { cells: 6, samplesPerCell: 20, concurrency: 8 } + const referenceRun = await fingerprint({ baseUrl, model: 'mock-alpha', apiKey: 'test-key' }, options) + + const same = await verify( + { baseUrl, model: 'mock-alpha', apiKey: 'test-key' }, + referenceRun.fingerprint, + options, + ) + assert.equal(same.verdict, 'match') + assert.ok(same.meanJsd < 0.25, `same model meanJsd should be small, got ${same.meanJsd}`) + + const different = await verify( + { baseUrl, model: 'mock-beta', apiKey: 'test-key' }, + referenceRun.fingerprint, + options, + ) + assert.equal(different.verdict, 'mismatch') + assert.ok(different.meanJsd > 0.35, `different model meanJsd should be large, got ${different.meanJsd}`) +}) + +test('compare(): flags protocol mismatch', async () => { + const runA = await fingerprint( + { baseUrl, model: 'mock-alpha', apiKey: 'test-key' }, + { cells: 4, samplesPerCell: 12 }, + ) + const foreign = { ...runA.fingerprint, protocol: 'someone-elses-protocol' } + const result = compare(runA.fingerprint, foreign) + assert.equal(result.protocolMismatch, true) + assert.equal(result.verdict, 'match') // identical distributions still match +}) + +test('fingerprint(): invalid API key aborts the run with an auth error', async () => { + await assert.rejects( + fingerprint( + { baseUrl, model: 'mock-alpha', apiKey: 'wrong-key' }, + { cells: 2, samplesPerCell: 3 }, + ), + (error) => { + assert.match(error.message, /401/) + return true + }, + ) +}) + +test('fingerprint(): AbortSignal cancels the run', async () => { + const controller = new AbortController() + const promise = fingerprint( + { baseUrl, model: 'mock-alpha', apiKey: 'test-key' }, + { + cells: 8, + samplesPerCell: 50, + onProgress: (e) => { + if (e.stage === 'sampling' && e.done >= 5) controller.abort() + }, + signal: controller.signal, + }, + ) + await assert.rejects(promise, (error) => { + assert.equal(error.name, 'ProbeRunError') + assert.equal(error.reason, 'aborted') + return true + }) +}) + +test('endpoint validation: bad base URL rejects before any request', async () => { + await assert.rejects(fingerprint({ baseUrl: '', model: 'x' }), /baseUrl is empty/) + await assert.rejects(fingerprint({ baseUrl: 'https://ok.example/v1', model: ' ' }), /model is empty/) + assert.ok(requestCount > 0) +}) diff --git a/test/battery.test.mjs b/test/battery.test.mjs new file mode 100644 index 0000000..987ba9a --- /dev/null +++ b/test/battery.test.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + CELL_PRIORITY_ORDER, + PROBE_TASKS, + SYSTEM_PROMPTS, + getCellsForPreset, + getSystemPrompt, + getTaskSpec, + isCellId, + makeCellId, + parseCellId, + pickParaphrase, +} from '../dist/battery.js' + +test('battery covers 8 tasks × 2 languages = 16 unique cells', () => { + assert.equal(Object.keys(PROBE_TASKS).length, 8) + assert.equal(CELL_PRIORITY_ORDER.length, 16) + assert.equal(new Set(CELL_PRIORITY_ORDER).size, 16) + for (const cellId of CELL_PRIORITY_ORDER) { + assert.ok(isCellId(cellId), `${cellId} should be a valid cell id`) + } +}) + +test('every task has ≥3 paraphrases per language (anti-filter design)', () => { + for (const task of Object.values(PROBE_TASKS)) { + for (const lang of ['en', 'zh']) { + assert.ok( + task.paraphrases[lang].length >= 3, + `${task.id}:${lang} has only ${task.paraphrases[lang].length} paraphrases`, + ) + } + } +}) + +test('cell id round-trip', () => { + const cellId = makeCellId('random-number-1-100', 'zh') + assert.equal(cellId, 'random-number-1-100:zh') + assert.deepEqual(parseCellId(cellId), { task: 'random-number-1-100', lang: 'zh' }) + assert.equal(isCellId('not-a-task:en'), false) + assert.equal(isCellId('random-color:fr'), false) +}) + +test('system prompt matches the cell language', () => { + assert.equal(getSystemPrompt('random-color:en'), SYSTEM_PROMPTS.en) + assert.equal(getSystemPrompt('random-color:zh'), SYSTEM_PROMPTS.zh) +}) + +test('presets slice the priority order', () => { + assert.deepEqual(getCellsForPreset('quick'), CELL_PRIORITY_ORDER.slice(0, 4)) + assert.deepEqual(getCellsForPreset('standard'), CELL_PRIORITY_ORDER.slice(0, 8)) + assert.deepEqual(getCellsForPreset('strict'), CELL_PRIORITY_ORDER) +}) + +test('pickParaphrase draws from the cell language pool', () => { + const spec = getTaskSpec('coin-flip:zh') + for (let i = 0; i < 20; i++) { + const paraphrase = pickParaphrase('coin-flip:zh') + assert.ok(spec.paraphrases.zh.includes(paraphrase)) + } + // Deterministic draw with an injected RNG. + assert.equal(pickParaphrase('coin-flip:en', () => 0), PROBE_TASKS['coin-flip'].paraphrases.en[0]) +}) diff --git a/test/normalizer.test.mjs b/test/normalizer.test.mjs new file mode 100644 index 0000000..e54e658 --- /dev/null +++ b/test/normalizer.test.mjs @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { normalizeAnswer, parseAnyNumber } from '../dist/normalizer.js' + +const INT_100 = { kind: 'int', min: 1, max: 100 } +const INT_10 = { kind: 'int', min: 1, max: 10 } + +test('digits pass through', () => { + assert.deepEqual(normalizeAnswer('42', INT_100), { normalized: '42', category: 'valid' }) + assert.deepEqual(normalizeAnswer(' 7.\n', INT_100), { normalized: '7', category: 'valid' }) +}) + +test('English number words fold to digits', () => { + assert.deepEqual(normalizeAnswer('Seven!', INT_100), { normalized: '7', category: 'valid' }) + assert.deepEqual(normalizeAnswer('forty-two', INT_100), { normalized: '42', category: 'valid' }) + assert.deepEqual(normalizeAnswer('Twelve', INT_100), { normalized: '12', category: 'valid' }) + assert.equal(parseAnyNumber('onehundred'), 100) +}) + +test('Chinese numerals fold to digits', () => { + assert.deepEqual(normalizeAnswer('四十二', INT_100), { normalized: '42', category: 'valid' }) + assert.deepEqual(normalizeAnswer('十五', INT_100), { normalized: '15', category: 'valid' }) + assert.deepEqual(normalizeAnswer('两', INT_10), { normalized: '2', category: 'valid' }) + assert.deepEqual(normalizeAnswer('一百', INT_100), { normalized: '100', category: 'valid' }) +}) + +test('full-width and Arabic-Indic digits fold to Latin', () => { + assert.deepEqual(normalizeAnswer('４２', INT_100), { normalized: '42', category: 'valid' }) + assert.deepEqual(normalizeAnswer('٤٢', INT_100), { normalized: '42', category: 'valid' }) +}) + +test('out-of-range numbers are invalid', () => { + assert.equal(normalizeAnswer('101', INT_100).category, 'invalid') + assert.equal(normalizeAnswer('0', INT_100).category, 'invalid') + assert.equal(normalizeAnswer('42', INT_10).category, 'invalid') +}) + +test('refusals are detected in both languages', () => { + assert.equal(normalizeAnswer('I cannot help with that request.', INT_100).category, 'refusal') + assert.equal(normalizeAnswer("I'm sorry, but I can't do that", INT_100).category, 'refusal') + assert.equal(normalizeAnswer('抱歉，我不能这样做。', INT_100).category, 'refusal') + assert.equal(normalizeAnswer('作为一个AI，我没有偏好。', INT_100).category, 'refusal') +}) + +test('empty and whitespace-only answers are empty', () => { + assert.equal(normalizeAnswer('', INT_100).category, 'empty') + assert.equal(normalizeAnswer(' \n ', INT_100).category, 'empty') + assert.equal(normalizeAnswer('"…"', INT_100).category, 'empty') +}) + +test('colors: quotes stripped, aliases folded, Chinese 色 suffix dropped', () => { + const COLOR = { kind: 'color' } + assert.deepEqual(normalizeAnswer('"Blue".', COLOR), { normalized: 'blue', category: 'valid' }) + assert.deepEqual(normalizeAnswer('Grey', COLOR), { normalized: 'gray', category: 'valid' }) + assert.deepEqual(normalizeAnswer('蓝色', COLOR), { normalized: '蓝', category: 'valid' }) + assert.deepEqual(normalizeAnswer('青', COLOR), { normalized: '青', category: 'valid' }) +}) + +test('letters: single letters and letter names', () => { + const LETTER = { kind: 'letter' } + assert.deepEqual(normalizeAnswer('Q', LETTER), { normalized: 'q', category: 'valid' }) + assert.deepEqual(normalizeAnswer('zee', LETTER), { normalized: 'z', category: 'valid' }) + assert.deepEqual(normalizeAnswer('queue', LETTER), { normalized: 'q', category: 'valid' }) + assert.equal(normalizeAnswer('hello', LETTER).category, 'invalid') +}) + +test('coin: heads/tails variants in both languages', () => { + const COIN = { kind: 'coin' } + assert.deepEqual(normalizeAnswer('Heads!', COIN), { normalized: 'heads', category: 'valid' }) + assert.deepEqual(normalizeAnswer('tail', COIN), { normalized: 'tails', category: 'valid' }) + assert.deepEqual(normalizeAnswer('正面', COIN), { normalized: 'heads', category: 'valid' }) + assert.deepEqual(normalizeAnswer('反', COIN), { normalized: 'tails', category: 'valid' }) + assert.equal(normalizeAnswer('maybe', COIN).category, 'invalid') +}) + +test('word tasks: first word, Latin or CJK', () => { + const WORD = { kind: 'word' } + assert.deepEqual(normalizeAnswer('Tokyo', WORD), { normalized: 'tokyo', category: 'valid' }) + assert.deepEqual(normalizeAnswer('New York City', WORD), { normalized: 'new', category: 'valid' }) + assert.deepEqual(normalizeAnswer('大象', WORD), { normalized: '大象', category: 'valid' }) +}) + +test('emoji and punctuation are stripped before classification', () => { + assert.deepEqual(normalizeAnswer('🎲 42 🎲', INT_100), { normalized: '42', category: 'valid' }) +}) diff --git a/test/reference.test.mjs b/test/reference.test.mjs new file mode 100644 index 0000000..37ff98c --- /dev/null +++ b/test/reference.test.mjs @@ -0,0 +1,67 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { compare } from '../dist/api.js' +import { + getBundledAttribution, + listBundledReferences, + loadBundledReference, + parseFingerprintJson, +} from '../dist/reference.js' + +test('bundled references load and carry attribution', () => { + const references = listBundledReferences() + assert.ok(references.length >= 5, 'expected several bundled sample references') + + const attribution = getBundledAttribution() + assert.match(attribution.datasetDoi, /10\.5281\/zenodo\.21278557/) + assert.match(attribution.license, /CC-BY-4\.0/) + assert.match(attribution.author, /Bruckner/) +}) + +test('a bundled reference converts to a well-formed fingerprint', () => { + const fp = loadBundledReference('openai/gpt-4o-mini') + assert.equal(fp.model, 'openai/gpt-4o-mini') + assert.equal(fp.protocol, 'bruckner-zenodo-2026') + const cells = Object.values(fp.cells) + assert.ok(cells.length >= 8) + for (const cell of cells) { + assert.ok(cell.validCount >= 10) + const sum = Object.values(cell.counts).reduce((s, n) => s + n, 0) + assert.ok(Math.abs(sum - cell.validCount) <= 2, `${cell.cellId}: counts ≈ validCount`) + assert.ok(cell.entropyBits >= 0) + assert.ok(cell.normalizedEntropy >= 0 && cell.normalizedEntropy <= 1) + } +}) + +test('bundled references: self-compare → 0 distance, cross-model → clearly separated', () => { + const gpt = loadBundledReference('openai/gpt-4o-mini') + const self = compare(gpt, loadBundledReference('openai/gpt-4o-mini')) + assert.equal(self.meanJsd, 0) + assert.equal(self.verdict, 'match') + + const claude = loadBundledReference('anthropic/claude-sonnet-4.5') + const cross = compare(gpt, claude) + assert.ok( + cross.meanJsd > 0.3, + `different models should diverge (got ${cross.meanJsd}) — paper median is 0.463`, + ) + assert.notEqual(cross.verdict, 'match') +}) + +test('unknown bundled id throws with the available list', () => { + assert.throws(() => loadBundledReference('no/such-model'), /Available:/) +}) + +test('parseFingerprintJson validates structure', () => { + const fp = loadBundledReference('openai/gpt-4o-mini') + const roundTripped = parseFingerprintJson(JSON.stringify(fp)) + assert.equal(roundTripped.model, fp.model) + + assert.throws(() => parseFingerprintJson('not json'), /not valid JSON/) + assert.throws(() => parseFingerprintJson('{}'), /formatVersion/) + assert.throws( + () => parseFingerprintJson(JSON.stringify({ formatVersion: 1, model: 'x' })), + /cells/, + ) +}) diff --git a/test/stats.test.mjs b/test/stats.test.mjs new file mode 100644 index 0000000..6670f21 --- /dev/null +++ b/test/stats.test.mjs @@ -0,0 +1,141 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + buildCellDistribution, + compareCellSets, + jensenShannonDivergence, + median, + shannonEntropyBits, + splitHalfJsd, +} from '../dist/stats.js' + +function almostEqual(actual, expected, epsilon = 1e-9) { + assert.ok( + Math.abs(actual - expected) < epsilon, + `expected ${actual} ≈ ${expected} (±${epsilon})`, + ) +} + +test('entropy: uniform over 4 outcomes is 2 bits, point mass is 0', () => { + almostEqual(shannonEntropyBits({ a: 1, b: 1, c: 1, d: 1 }), 2) + almostEqual(shannonEntropyBits({ a: 10 }), 0) + almostEqual(shannonEntropyBits({}), 0) +}) + +test('JSD: identical distributions → 0', () => { + almostEqual(jensenShannonDivergence({ a: 3, b: 1 }, { a: 6, b: 2 }), 0) +}) + +test('JSD: disjoint distributions → 1 bit', () => { + almostEqual(jensenShannonDivergence({ a: 5 }, { b: 7 }), 1) +}) + +test('JSD: known hand-computed value', () => { + // P = (1, 0), Q = (0.5, 0.5), M = (0.75, 0.25) + // JSD = H(M) − (H(P)+H(Q))/2 = 0.8112781245 − 0.5 = 0.3112781245 + almostEqual(jensenShannonDivergence({ a: 4 }, { a: 2, b: 2 }), 0.31127812445913294, 1e-12) +}) + +test('JSD is symmetric and count-scale invariant', () => { + const p = { x: 3, y: 9, z: 1 } + const q = { x: 5, y: 2 } + almostEqual(jensenShannonDivergence(p, q), jensenShannonDivergence(q, p)) + almostEqual( + jensenShannonDivergence(p, q), + jensenShannonDivergence({ x: 30, y: 90, z: 10 }, q), + ) +}) + +test('median of even/odd lists', () => { + assert.equal(median([3, 1, 2]), 2) + assert.equal(median([4, 1, 2, 3]), 2.5) + assert.equal(median([]), null) +}) + +function sample(cellId, normalized, category, arrivalIndex, latencyMs = 100) { + return { + cellId, + raw: normalized ?? '', + normalized, + category, + latencyMs, + usage: { promptTokens: 20, completionTokens: 2, reasoningTokens: null }, + arrivalIndex, + } +} + +test('buildCellDistribution aggregates categories and entropy', () => { + const cellId = 'random-number-1-100:en' + const samples = [ + sample(cellId, '42', 'valid', 0, 100), + sample(cellId, '42', 'valid', 1, 200), + sample(cellId, '7', 'valid', 2, 300), + sample(cellId, 'banana', 'invalid', 3, 400), + sample(cellId, null, 'refusal', 4, 500), + sample(cellId, null, 'empty', 5, 600), + sample(cellId, null, 'error', 6, 9999), // error latency is excluded + ] + const dist = buildCellDistribution(cellId, samples, { kind: 'int', min: 1, max: 100 }) + assert.deepEqual(dist.counts, { 42: 2, 7: 1 }) + assert.equal(dist.validCount, 3) + assert.equal(dist.invalidCount, 1) + assert.equal(dist.refusalCount, 1) + assert.equal(dist.emptyCount, 1) + assert.equal(dist.errorCount, 1) + assert.equal(dist.totalCount, 7) + almostEqual(dist.entropyBits, shannonEntropyBits({ a: 2, b: 1 })) + assert.ok(dist.normalizedEntropy > 0 && dist.normalizedEntropy <= 1) + assert.equal(dist.medianLatencyMs, 350) // median of [100..600]; the error sample is excluded + assert.equal(dist.meanCompletionTokens, 2) +}) + +test('compareCellSets: skips thin cells, averages the rest', () => { + const mk = (counts, validCount) => ({ counts, validCount }) + const a = { + 'random-number-1-100:en': mk({ 42: 20 }, 20), + 'random-color:en': mk({ blue: 15 }, 15), + 'coin-flip:en': mk({ heads: 3 }, 3), // below the 10-valid minimum + } + const b = { + 'random-number-1-100:en': mk({ 42: 20 }, 20), + 'random-color:en': mk({ red: 15 }, 15), + 'coin-flip:en': mk({ heads: 30 }, 30), + } + const { entries, meanJsd } = compareCellSets(a, b) + assert.equal(entries.length, 2) + assert.equal(entries[0].cellId, 'random-color:en') // sorted by descending JSD + almostEqual(entries[0].jsd, 1) + almostEqual(entries[1].jsd, 0) + almostEqual(meanJsd, 0.5) +}) + +test('compareCellSets: nothing comparable → meanJsd null', () => { + const { entries, meanJsd } = compareCellSets({}, {}) + assert.equal(entries.length, 0) + assert.equal(meanJsd, null) +}) + +test('splitHalfJsd: stable endpoint → 0, alternating endpoint → 1', () => { + const cellId = 'random-number-1-100:en' + const stable = new Map([ + [cellId, Array.from({ length: 20 }, (_, i) => sample(cellId, '42', 'valid', i))], + ]) + almostEqual(splitHalfJsd(stable), 0) + + const alternating = new Map([ + [ + cellId, + Array.from({ length: 20 }, (_, i) => sample(cellId, i % 2 === 0 ? '1' : '2', 'valid', i)), + ], + ]) + almostEqual(splitHalfJsd(alternating), 1) +}) + +test('splitHalfJsd: too few samples → null', () => { + const cellId = 'random-number-1-100:en' + const thin = new Map([ + [cellId, Array.from({ length: 6 }, (_, i) => sample(cellId, '42', 'valid', i))], + ]) + assert.equal(splitHalfJsd(thin), null) +}) diff --git a/test/verdict.test.mjs b/test/verdict.test.mjs new file mode 100644 index 0000000..d26919d --- /dev/null +++ b/test/verdict.test.mjs @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + JSD_MATCH_THRESHOLD, + JSD_MISMATCH_THRESHOLD, + MIN_COMPARABLE_CELLS, +} from '../dist/constants.js' +import { buildComparisonResult, decideVerdict } from '../dist/verdict.js' + +test('verdict thresholds: match / uncertain / mismatch bands', () => { + const cells = MIN_COMPARABLE_CELLS + assert.equal(decideVerdict(0.0, cells), 'match') + assert.equal(decideVerdict(JSD_MATCH_THRESHOLD, cells), 'match') + assert.equal(decideVerdict(JSD_MATCH_THRESHOLD + 1e-9, cells), 'uncertain') + assert.equal(decideVerdict(JSD_MISMATCH_THRESHOLD, cells), 'uncertain') + assert.equal(decideVerdict(JSD_MISMATCH_THRESHOLD + 1e-9, cells), 'mismatch') + assert.equal(decideVerdict(0.9, cells), 'mismatch') +}) + +test('verdict: too few comparable cells → insufficient', () => { + assert.equal(decideVerdict(0.1, MIN_COMPARABLE_CELLS - 1), 'insufficient') + assert.equal(decideVerdict(null, 10), 'insufficient') +}) + +test('buildComparisonResult carries thresholds, baselines and per-cell details', () => { + const entries = [ + { cellId: 'random-number-1-100:en', jsd: 0.5, validA: 25, validB: 30 }, + { cellId: 'random-color:en', jsd: 0.1, validA: 25, validB: 30 }, + { cellId: 'coin-flip:en', jsd: 0.2, validA: 25, validB: 30 }, + { cellId: 'random-animal:en', jsd: 0.2, validA: 25, validB: 30 }, + ] + const result = buildComparisonResult(entries, 0.25, false) + assert.equal(result.verdict, 'match') + assert.equal(result.comparableCellCount, 4) + assert.equal(result.cells.length, 4) + assert.equal(result.protocolMismatch, false) + assert.equal(result.thresholds.match, JSD_MATCH_THRESHOLD) + assert.equal(result.baselines.differentModel, 0.463) +}) diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..b0c763c --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "noUncheckedIndexedAccess": false, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src"] +}