What are hallucinations and what causes them?

// Why a next-token model hallucinates, in three small experiments:
// (1) it answers an unknown question with the same fluency as a known one,
// (2) accuracy-only grading rewards guessing over "I don't know",
// (3) a cheap grounding check catches claims the source does not support.

// (1) "Training data": birth years the model saw, and how often.
const training = [
  ["Ada Lovelace", 1815, 9], ["Ada Lovelace", 1816, 1],
  ["Alan Turing", 1912, 10],
  ["Grace Hopper", 1906, 7], ["Grace Hopper", 1907, 1],
  ["Dana Ruiz", 1958, 1], // an invented person, a "singleton": seen once in all of training
];
// A model has limited capacity, so it does not store each fact exactly: its belief for a
// person blends what it saw about them with what birth years look like in general (K = 5).
const K = 5;
const years = [...new Set(training.map((t) => t[1]))].sort((a, b) => a - b);
const all = training.reduce((s, t) => s + t[2], 0);
const prior = (y) => training.filter((t) => t[1] === y).reduce((s, t) => s + t[2], 0) / all;
function answer(person) {
  const seen = training.filter((t) => t[0] === person);
  const n = seen.reduce((s, t) => s + t[2], 0);
  let best = null;
  for (const y of years) {
    const c = seen.filter((t) => t[1] === y).reduce((s, t) => s + t[2], 0);
    const p = (c + K * prior(y)) / (n + K);
    if (!best || p > best.p) best = { year: y, p };
  }
  return { ...best, n };
}
console.log("(1) prompt: '<person> was born in'");
// Dana Ruiz and Zelda Quimby are invented names.
for (const person of ["Alan Turing", "Ada Lovelace", "Dana Ruiz", "Zelda Quimby"]) {
  const a = answer(person);
  const truth = training.filter((t) => t[0] === person).sort((x, y) => y[2] - x[2])[0];
  const verdict = !truth ? "no answer in training: fabricated" : truth[1] === a.year ? "correct" : `WRONG, truth ${truth[1]}`;
  console.log(`  ${person.padEnd(16)} seen ${String(a.n).padStart(2)}x -> "${person} was born in ${a.year}." p=${a.p.toFixed(2)} ${verdict}`);
}

// (2) 20 questions; the model's probability of being right on each.
const conf = [0.95, 0.9, 0.9, 0.85, 0.8, 0.8, 0.75, 0.7, 0.6, 0.55, 0.45, 0.4, 0.35, 0.3, 0.3, 0.25, 0.2, 0.15, 0.1, 0.05];
function expected(strategyThreshold, wrongPenalty) {
  let score = 0, wrong = 0, abstain = 0;
  for (const p of conf) {
    if (p < strategyThreshold) { abstain++; continue; } // says "I don't know": 0 points
    score += p * 1 - (1 - p) * wrongPenalty;
    wrong += 1 - p;
  }
  return `score=${score.toFixed(2).padStart(6)} expected wrong answers=${wrong.toFixed(2).padStart(5)} abstained=${String(abstain).padStart(2)}`;
}
console.log("(2) expected results on 20 questions");
for (const [grading, penalty] of [["accuracy only (wrong = 0)", 0], ["wrong = -1, abstain = 0", 1]]) {
  console.log(`  grading: ${grading}`);
  console.log(`    always guess            ${expected(0, penalty)}`);
  console.log(`    abstain when p < 0.5    ${expected(0.5, penalty)}`);
}

// (3) Grounding check: every number and capitalised name in the answer must appear in the source.
const source = "Refunds are accepted within 14 days of delivery. Contact Support via the Help Center.";
const generated = [
  "Refunds are accepted within 14 days of delivery.",
  "Refunds are processed by Stripe within 5 business days.",
  "You can contact Support through the Help Center.",
];
const claims = (s) => s.match(/\b(\d+|[A-Z][a-z]+)\b/g).filter((w, i) => i > 0 || /\d/.test(w)); // skip sentence-initial word
console.log("(3) grounding check against the retrieved source");
for (const sentence of generated) {
  const missing = claims(sentence).filter((c) => !source.includes(c));
  console.log(`  ${missing.length ? "UNSUPPORTED" : "supported  "} ${sentence}${missing.length ? "  [not in source: " + missing.join(", ") + "]" : ""}`);
}