Why are BLEU and ROUGE poor metrics for LLM outputs, and what replaced them?

// BLEU and ROUGE from scratch, on answers to one question, next to a key-fact check
// and a human verdict. Reference: "Refunds take 5-7 business days."
const tok = (s) => s.toLowerCase().match(/[a-z0-9]+(?:-[a-z0-9]+)*/g) || [];
function ngrams(t, n) {
  const m = new Map();
  for (let i = 0; i + n <= t.length; i++) { const k = t.slice(i, i + n).join(" "); m.set(k, (m.get(k) || 0) + 1); }
  return m;
}
function bleu(cand, refs) {
  const c = tok(cand), rs = refs.map(tok);
  let logSum = 0;
  for (let n = 1; n <= 4; n++) {
    const cg = ngrams(c, n);
    let match = 0, total = 0;
    for (const [g, cnt] of cg) {
      total += cnt;
      match += Math.min(cnt, Math.max(...rs.map((r) => ngrams(r, n).get(g) || 0)));
    }
    const p = n === 1 ? match / total : (match + 1) / (total + 1); // add-one smoothing for n >= 2
    if (p === 0) return 0;
    logSum += Math.log(p) / 4;
  }
  const r = rs.map((x) => x.length).sort((a, b) => Math.abs(a - c.length) - Math.abs(b - c.length) || a - b)[0];
  const bp = c.length > r ? 1 : Math.exp(1 - r / c.length);
  return bp * Math.exp(logSum);
}
function f1(match, cLen, rLen) { if (!match) return 0; const p = match / cLen, r = match / rLen; return (2 * p * r) / (p + r); }
function rouge1(cand, ref) {
  const c = ngrams(tok(cand), 1), r = ngrams(tok(ref), 1);
  let m = 0;
  for (const [g, cnt] of c) m += Math.min(cnt, r.get(g) || 0);
  return f1(m, tok(cand).length, tok(ref).length);
}
function rougeL(cand, ref) {
  const a = tok(cand), b = tok(ref);
  const dp = Array.from({ length: a.length + 1 }, () => new Array(b.length + 1).fill(0));
  for (let i = 1; i <= a.length; i++) for (let j = 1; j <= b.length; j++)
    dp[i][j] = a[i - 1] === b[j - 1] ? dp[i - 1][j - 1] + 1 : Math.max(dp[i - 1][j], dp[i][j - 1]);
  return f1(dp[a.length][b.length], a.length, b.length);
}
// task-specific code check: normalise a few equivalent phrasings, then require the fact and no negation
function factCheck(s) {
  const t = " " + tok(s).join(" ") + " ";
  const n = t.replace(/ five to seven /g, " 5-7 ").replace(/ working /g, " business ");
  return n.includes(" 5-7 business days ") && !/ (not|never|no) /.test(n);
}
const REF = "Refunds take 5-7 business days.";
const answers = [
  ["copy of reference", "Refunds take 5-7 business days.", "correct"],
  ["paraphrase", "Expect the money back within five to seven working days.", "correct"],
  ["friendly, longer", "Thanks for asking! Refunds usually take 5-7 business days to reach your account.", "correct"],
  ["wrong unit", "Refunds take 5-7 business months.", "wrong"],
  ["negated", "Refunds do not take 5-7 business days.", "wrong"],
  ["wrong number", "Refunds take 15-20 business days.", "wrong"],
];
console.log(`reference: "${REF}"`);
console.log("answer              BLEU   ROUGE-1  ROUGE-L  fact check  human");
const rows = answers.map(([name, text, human]) => {
  const r = { name, human, bleu: bleu(text, [REF]), r1: rouge1(text, REF), rl: rougeL(text, REF), fact: factCheck(text) };
  console.log(`${name.padEnd(18)}  ${r.bleu.toFixed(3)}  ${r.r1.toFixed(3).padStart(7)}  ${r.rl.toFixed(3).padStart(7)}  ${(r.fact ? "pass" : "fail").padStart(10)}  ${human}`);
  return r;
});
console.log("");
// best achievable accuracy if you pick the single best pass threshold for each metric
for (const key of ["bleu", "r1", "rl"]) {
  let best = 0, bestT = 0;
  for (const t of rows.map((r) => r[key])) {
    const acc = rows.filter((r) => (r[key] >= t) === (r.human === "correct")).length;
    if (acc > best) { best = acc; bestT = t; }
  }
  const label = { bleu: "BLEU", r1: "ROUGE-1", rl: "ROUGE-L" }[key];
  console.log(`${label.padEnd(8)} best threshold ${bestT.toFixed(3)} -> ${best}/${rows.length} verdicts right`);
}
console.log(`fact check                      -> ${rows.filter((r) => r.fact === (r.human === "correct")).length}/${rows.length} verdicts right`);

// Overlap scores depend on which reference someone happened to write.
// Same BLEU / ROUGE-L as the previous example; multi-reference takes the best match.
const tok = (s) => s.toLowerCase().match(/[a-z0-9]+(?:-[a-z0-9]+)*/g) || [];
function ngrams(t, n) {
  const m = new Map();
  for (let i = 0; i + n <= t.length; i++) { const k = t.slice(i, i + n).join(" "); m.set(k, (m.get(k) || 0) + 1); }
  return m;
}
function bleu(cand, refs) {
  const c = tok(cand), rs = refs.map(tok);
  let logSum = 0;
  for (let n = 1; n <= 4; n++) {
    const cg = ngrams(c, n);
    let match = 0, total = 0;
    for (const [g, cnt] of cg) {
      total += cnt;
      match += Math.min(cnt, Math.max(...rs.map((r) => ngrams(r, n).get(g) || 0)));
    }
    const p = n === 1 ? match / total : (match + 1) / (total + 1); // add-one smoothing for n >= 2
    if (p === 0) return 0;
    logSum += Math.log(p) / 4;
  }
  const r = rs.map((x) => x.length).sort((a, b) => Math.abs(a - c.length) - Math.abs(b - c.length) || a - b)[0];
  const bp = c.length > r ? 1 : Math.exp(1 - r / c.length);
  return bp * Math.exp(logSum);
}
function f1(match, cLen, rLen) { if (!match) return 0; const p = match / cLen, r = match / rLen; return (2 * p * r) / (p + r); }
function rouge1(cand, ref) {
  const c = ngrams(tok(cand), 1), r = ngrams(tok(ref), 1);
  let m = 0;
  for (const [g, cnt] of c) m += Math.min(cnt, r.get(g) || 0);
  return f1(m, tok(cand).length, tok(ref).length);
}
function rougeL(cand, ref) {
  const a = tok(cand), b = tok(ref);
  const dp = Array.from({ length: a.length + 1 }, () => new Array(b.length + 1).fill(0));
  for (let i = 1; i <= a.length; i++) for (let j = 1; j <= b.length; j++)
    dp[i][j] = a[i - 1] === b[j - 1] ? dp[i - 1][j - 1] + 1 : Math.max(dp[i - 1][j], dp[i][j - 1]);
  return f1(dp[a.length][b.length], a.length, b.length);
}
function rougeLMulti(cand, refs) { return Math.max(...refs.map((r) => rougeL(cand, r))); }

const REF_A = "Refunds take 5-7 business days.";
const REF_B = "You will get the money back within five to seven working days.";
const answers = [
  ["original wording", "Refunds take 5-7 business days.", "correct"],
  ["paraphrase", "Expect the money back within five to seven working days.", "correct"],
  ["wrong unit", "Refunds take 5-7 business months.", "wrong"],
];
const sets = [["ref A only", [REF_A]], ["ref B only", [REF_B]], ["refs A + B", [REF_A, REF_B]]];
console.log(`ref A: "${REF_A}"`);
console.log(`ref B: "${REF_B}"`);
console.log("");
console.log(("answer             human    " + sets.map(([n]) => n.padEnd(20)).join("")).trimEnd());
console.log(("                            " + sets.map(() => "BLEU   ROUGE-L".padEnd(20)).join("")).trimEnd());
for (const [name, text, human] of answers) {
  const cells = sets.map(([, refs]) => `${bleu(text, refs).toFixed(3)}  ${rougeLMulti(text, refs).toFixed(3)}`.padEnd(20));
  console.log(`${name.padEnd(17)}  ${human.padEnd(7)}  ${cells.join("")}`.trimEnd());
}
console.log("");
const rank = (refs) => answers.map(([n, t]) => [n, bleu(t, refs)]).sort((a, b) => b[1] - a[1]).map((x) => x[0]).join(" > ");
for (const [n, refs] of sets) console.log(`BLEU ranking with ${n}: ${rank(refs)}`);