What is LLM-as-judge and what are its known biases?

// Position bias in pairwise judging, and the swap-and-agree fix.
// The stub judge prefers the better answer, but gives the FIRST slot a hidden
// bonus of 1.5 quality points (a simulated position bias).
const pairs = [
  // true quality (0-10) of model X answer and model Y answer for 12 prompts
  [7, 8], [6, 6], [9, 5], [5, 7], [8, 8], [4, 6], [7, 6], [6, 8], [8, 9], [5, 5], [9, 7], [6, 7],
];
const BIAS = 1.5;
function judge(first, second) {
  // returns "first" or "second"
  return first + BIAS >= second ? "first" : "second";
}
function verdict(qx, qy, xFirst) {
  const r = xFirst ? judge(qx, qy) : judge(qy, qx);
  return (r === "first") === xFirst ? "X" : "Y";
}
const truth = pairs.map(([x, y]) => (x > y ? "X" : y > x ? "Y" : "tie"));
const xFirst = pairs.map(([x, y]) => verdict(x, y, true));
const yFirst = pairs.map(([x, y]) => verdict(x, y, false));
const swapped = xFirst.map((v, i) => (v === yFirst[i] ? v : "tie"));

const winRate = (vs) => {
  let s = 0;
  for (const v of vs) s += v === "X" ? 1 : v === "tie" ? 0.5 : 0;
  return ((100 * s) / vs.length).toFixed(1) + "%";
};
console.log("pair  quality X/Y  truth  X shown first  Y shown first  swap-and-agree");
pairs.forEach(([x, y], i) => {
  const flip = xFirst[i] !== yFirst[i] ? "  <- flipped by order" : "";
  console.log(`${String(i + 1).padStart(4)}  ${String(x).padStart(5)}/${y}      ${truth[i].padEnd(5)}  ${xFirst[i].padEnd(13)}  ${yFirst[i].padEnd(13)}  ${swapped[i]}${flip}`);
});
const flips = xFirst.filter((v, i) => v !== yFirst[i]).length;
console.log("");
console.log(`position consistency: ${pairs.length - flips}/${pairs.length} verdicts survive swapping the order`);
console.log(`X win rate, truth:            ${winRate(truth)}`);
console.log(`X win rate, X always first:   ${winRate(xFirst)}`);
console.log(`X win rate, Y always first:   ${winRate(yFirst)}`);
console.log(`X win rate, swap-and-agree:   ${winRate(swapped)}`);
const agree = (vs) => vs.filter((v, i) => v === truth[i]).length;
console.log(`verdicts matching truth: X first ${agree(xFirst)}/12, Y first ${agree(yFirst)}/12, swap-and-agree ${agree(swapped)}/12`);

// Verbosity bias: a stub judge whose score is quality + 0.01 per word (a simulated bias),
// compared with human preference (quality only), then a length-controlled re-scoring.
// Each row: [quality of concise model C, words of C, quality of verbose model V, words of V]
const rows = [
  [7, 80, 6, 90], [6, 70, 6, 80], [7, 85, 8, 95], [8, 90, 7, 100],
  [7, 80, 6, 140], [6, 75, 6, 135], [7, 70, 8, 130], [9, 100, 8, 160],
  [8, 60, 7, 210], [7, 85, 7, 235], [6, 80, 7, 230], [8, 90, 7, 240],
  [7, 80, 6, 300], [6, 70, 6, 290], [7, 75, 8, 295], [8, 85, 7, 305],
];
const PER_WORD = 0.01;
const judgeScore = (q, w) => q + PER_WORD * w;
const outcome = (diff) => (diff > 1e-9 ? 1 : diff < -1e-9 ? 0 : 0.5); // 1 = V wins
const mean = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
const pct = (x) => (100 * x).toFixed(1) + "%";

const human = rows.map(([qc, , qv]) => outcome(qv - qc));
const judgeDiff = rows.map(([qc, wc, qv, wv]) => judgeScore(qv, wv) - judgeScore(qc, wc));
const judged = judgeDiff.map(outcome);
const agree = (xs) => xs.filter((x, i) => x === human[i]).length;

console.log(`V average length ${mean(rows.map((r) => r[3])).toFixed(0)} words, C average ${mean(rows.map((r) => r[1])).toFixed(0)} words`);
console.log(`V win rate, human preference:   ${pct(mean(human))}`);
console.log(`V win rate, raw judge:          ${pct(mean(judged))}   (matches human on ${agree(judged)}/16)`);

// length control: fit judge score difference against length difference (hundreds of words),
// then remove the length term and re-read each verdict
const dl = rows.map(([, wc, , wv]) => (wv - wc) / 100);
const mx = mean(dl), my = mean(judgeDiff);
let sxy = 0, sxx = 0;
for (let i = 0; i < dl.length; i++) { sxy += (dl[i] - mx) * (judgeDiff[i] - my); sxx += (dl[i] - mx) ** 2; }
const slope = sxy / sxx;
const controlled = judgeDiff.map((j, i) => outcome(j - slope * dl[i]));
console.log(`fitted length effect: ${slope.toFixed(2)} judge points per extra hundred words`);
console.log(`V win rate, length-controlled:  ${pct(mean(controlled))}   (matches human on ${agree(controlled)}/16)`);
console.log("");

// padding probe: same answer, plus 120 words of filler that add nothing
console.log(`padding probe: same content, 80 -> 200 words, judge score ${judgeScore(7, 80).toFixed(1)} -> ${judgeScore(7, 200).toFixed(1)}`);
// a rubric that names concision as a criterion is modelled as a smaller per-word bonus
const strict = (q, w) => q + 0.002 * w;
const strictWins = rows.map(([qc, wc, qv, wv]) => outcome(strict(qv, wv) - strict(qc, wc)));
console.log(`judge with "do not reward length" rubric (0.002 per word): V win rate ${pct(mean(strictWins))}   (matches human on ${agree(strictWins)}/16)`);