How do you reduce hallucinations in a production LLM application?

// Four layers against hallucination, scored on 8 questions. The "model" is a stub with
// canned answers; each canned sentence carries a hidden truth label that only the grader
// reads. The verifier in layer 4 sees only text, like a real one.
const DOCS = [
  "Returns are accepted within 30 days of delivery for unused items.",
  "Standard shipping takes 5 business days and costs 4.99 euros.",
  "Express shipping takes 2 business days and costs 12.99 euros.",
  "The warranty covers hardware defects for 1 year from purchase.",
  "Gift cards cannot be refunded or exchanged for cash.",
];
// [question, gold phrase or null if the docs do not answer it, closed-book answer, grounded answer]
// An answer is a list of [sentence, cited doc number or 0, true?]
const QS = [
  ["How long do I have to return an item?", "30 days",
    [["You can return items within 60 days.", 0, false]],
    [["Returns are accepted within 30 days of delivery for unused items.", 1, true]]],
  ["How much is standard shipping?", "4.99",
    [["Standard shipping is free on all orders.", 0, false]],
    [["Standard shipping costs 4.99 euros and takes 5 business days.", 2, true], ["Orders over 50 euros ship free.", 2, false]]],
  ["How fast is express shipping?", "2 business days",
    [["Express shipping takes 2 business days.", 0, true]],
    [["Express shipping takes 2 business days.", 3, true]]],
  ["What does the warranty cover?", "hardware defects",
    [["The warranty covers accidental damage for 2 years.", 0, false]],
    [["The warranty covers hardware defects for 1 year from purchase.", 4, true]]],
  ["Can a gift card be refunded for cash?", "cannot",
    [["Yes, gift cards can be refunded within 14 days.", 0, false]],
    [["Gift cards cannot be refunded or exchanged for cash.", 5, true]]],
  ["How many days after delivery is my refund paid?", null,
    [["Refunds are paid within 3 days.", 0, false]],
    [["Refunds are paid within 30 days of delivery.", 1, false]]],
  ["Do you ship to Canada?", null,
    [["Yes, we ship to Canada in 7 days.", 0, false]],
    [["Yes, standard shipping to Canada takes 5 business days.", 2, false]]],
  ["Is the hardware warranty transferable?", null,
    [["Yes, the warranty is transferable to a new owner.", 0, false]],
    [["Yes, the warranty is transferable to a new owner.", 4, false]]],
];

const STOP = new Set("a an the is are do does i my you we to of for and or be can how what much long many is in on after from have".split(" "));
const words = s => s.toLowerCase().replace(/[^a-z0-9. ]/g, " ").replace(/\.(?!\d)/g, " ").split(/\s+/)
  .filter(w => w && !STOP.has(w)).map(w => w.replace(/s$/, ""));
function retrieve(q) {                         // best doc by shared content words
  let best = 0, score = -1;
  DOCS.forEach((d, i) => { const dw = new Set(words(d)); const s = words(q).filter(w => dw.has(w)).length; if (s > score) { score = s; best = i + 1; } });
  return { doc: best, score };
}
function supported(sentence, doc) {            // lexical check: 60% of content words appear in the cited doc
  if (!doc) return false;
  const sw = words(sentence), dw = new Set(words(DOCS[doc - 1]));
  return sw.filter(w => dw.has(w)).length / sw.length >= 0.6;
}

const ABSTAIN = [["I do not know based on our policy documents.", 0, true]];
const LAYERS = {
  "1 closed book": q => q[2],
  "2 + retrieval": q => q[3],
  "3 + abstain on weak retrieval": q => (retrieve(q[0]).score >= 2 ? q[3] : ABSTAIN),
  "4 + verify each cited claim": (q, min = 2) => {
    if (retrieve(q[0]).score < min) return ABSTAIN;
    const kept = q[3].filter(([s, doc]) => supported(s, doc));
    return kept.length ? kept : ABSTAIN;
  },
};

function grade(q, ans) {
  if (ans === ABSTAIN) return q[1] === null ? "abstain_ok" : "abstain_miss";
  if (ans.some(([, , ok]) => !ok)) return "hallucinated";
  return ans.map(a => a[0]).join(" ").includes(q[1]) ? "correct" : "wrong";
}

console.log("retrieval scores: " + QS.map((q, i) => `Q${i + 1}=${retrieve(q[0]).score}`).join(" "));
console.log("layer                          correct  halluc  abstain_ok  abstain_miss");
let last;
for (const [name, fn] of Object.entries(LAYERS)) {
  const res = QS.map(q => grade(q, fn(q)));
  const n = k => res.filter(r => r === k).length;
  console.log(`${name.padEnd(30)} ${String(n("correct")).padStart(5)}  ${String(n("hallucinated")).padStart(6)}  ${String(n("abstain_ok")).padStart(10)}  ${String(n("abstain_miss")).padStart(12)}`);
  last = res;
}
console.log("\nlayer 4, per question:");
QS.forEach((q, i) => console.log(`  Q${i + 1} ${last[i].padEnd(12)} ${q[0]}`));
console.log("\nlayer 4 abstain threshold sweep (minimum shared words):");
for (const min of [1, 2, 3]) {
  const res = QS.map(q => grade(q, LAYERS["4 + verify each cited claim"](q, min)));
  const n = k => res.filter(r => r === k).length;
  console.log(`  min ${min}: correct ${n("correct")}, hallucinated ${n("hallucinated")}, abstain_ok ${n("abstain_ok")}, abstain_miss ${n("abstain_miss")}`);
}