Beyond Ragas: how do you eval LLMs with LLM-as-judge, DeepEval, and human eval?

// Calibrating an LLM judge against human labels, then routing to human review.
// The "judge" is a deterministic stub: it scores the features a rubric prompt
// would look at. No model is called.

// Each case: correct = the required facts are present, cites = cites the policy,
// long = over 80 words, toneOk = polite. humanA / humanB = two annotators (1 pass, 0 fail).
const CASES = [
  // id, correct, cites, long, toneOk, humanA, humanB
  ["c01", 1, 1, 0, 1, 1, 1], ["c02", 1, 0, 0, 1, 1, 1], ["c03", 0, 1, 1, 1, 0, 0],
  ["c04", 0, 0, 1, 1, 0, 0], ["c05", 1, 1, 1, 1, 1, 1], ["c06", 0, 1, 0, 1, 0, 0],
  ["c07", 1, 1, 0, 0, 0, 0], ["c08", 0, 0, 0, 1, 0, 0], ["c09", 1, 0, 1, 1, 1, 0],
  ["c10", 0, 1, 1, 1, 0, 1], ["c11", 1, 1, 0, 1, 1, 1], ["c12", 0, 0, 1, 1, 0, 0],
  ["c13", 1, 0, 0, 0, 0, 1], ["c14", 1, 1, 1, 1, 1, 1], ["c15", 0, 1, 1, 1, 0, 0],
  ["c16", 1, 0, 0, 1, 1, 1], ["c17", 0, 0, 0, 1, 0, 0], ["c18", 1, 1, 0, 1, 1, 1],
  ["c19", 0, 1, 1, 0, 0, 0], ["c20", 1, 0, 1, 1, 1, 1],
].map(([id, correct, cites, long, toneOk, a, b]) => ({ id, correct, cites, long, toneOk, a, b }));

// Adjudicated label: where annotators disagree, a third reviewer decides (here: correct AND polite).
const gold = (c) => (c.a === c.b ? c.a : c.correct && c.toneOk ? 1 : 0);

// Judge v1: 1-5 score, rewards length (a known judge bias), pass if score >= 3.
function judgeV1(c) { const s = (c.correct ? 3 : 1) + (c.long ? 1 : 0) + (c.cites ? 1 : 0); return { score: s, pass: s >= 3 ? 1 : 0 }; }
// Judge v2: rubric says "length is not a quality signal; facts first", pass if score >= 4.
function judgeV2(c) { const s = c.correct ? (c.cites ? 5 : 4) : (c.cites ? 2 : 1); return { score: s, pass: s >= 4 ? 1 : 0 }; }

function kappa(x, y) {
  const n = x.length;
  let agree = 0, x1 = 0, y1 = 0;
  for (let i = 0; i < n; i++) { if (x[i] === y[i]) agree++; x1 += x[i]; y1 += y[i]; }
  const po = agree / n;
  const pe = (x1 / n) * (y1 / n) + (1 - x1 / n) * (1 - y1 / n);
  return { po, k: (po - pe) / (1 - pe) };
}
const f3 = (v) => v.toFixed(3);

const A = CASES.map((c) => c.a), B = CASES.map((c) => c.b), G = CASES.map(gold);
const hh = kappa(A, B);
console.log(`human A vs human B: agreement=${f3(hh.po)} kappa=${f3(hh.k)} (${CASES.length} cases)`);
for (const [name, H] of [["human A", A], ["human B", B]]) { const r = kappa(H, G); console.log(`${name} vs adjudicated: agreement=${f3(r.po)} kappa=${f3(r.k)}`); }

for (const [name, judge] of [["judge v1 (length bonus)", judgeV1], ["judge v2 (facts rubric)", judgeV2]]) {
  const J = CASES.map((c) => judge(c).pass);
  const r = kappa(J, G);
  let tp = 0, fp = 0, fn = 0, tn = 0;
  for (let i = 0; i < G.length; i++) {
    if (J[i] && G[i]) tp++; else if (J[i] && !G[i]) fp++; else if (!J[i] && G[i]) fn++; else tn++;
  }
  const wrong = CASES.filter((c, i) => J[i] !== G[i]).map((c) => c.id).join(",");
  console.log(`${name}: agreement=${f3(r.po)} kappa=${f3(r.k)} | judge-pass/human-fail=${fp} judge-fail/human-pass=${fn} | disagree on ${wrong}`);
}

// Routing: send borderline scores and a random-looking 10% audit (every 10th id) to humans.
const ROUTE = CASES.filter((c, i) => judgeV2(c).score === 4 || i % 10 === 9).map((c) => c.id);
console.log(`routed to human review: ${ROUTE.join(",")} (${ROUTE.length}/${CASES.length})`);

// Cost of the layered setup at volume. All prices are assumptions for the arithmetic.
const DAILY = 20000, JUDGE_COST = 0.002, HUMAN_COST = 0.5;
const share = ROUTE.length / CASES.length;
const judgeDay = DAILY * JUDGE_COST, humanDay = DAILY * share * HUMAN_COST, allHuman = DAILY * HUMAN_COST;
console.log(`per day at ${DAILY} outputs: judge $${judgeDay.toFixed(2)} + humans $${humanDay.toFixed(2)} = $${(judgeDay + humanDay).toFixed(2)} vs all-human $${allHuman.toFixed(2)}`);