How do you catch regressions when changing a prompt or upgrading a model?

// Regression gate for a prompt or model change. Each golden case is graded 3
// times per version (outputs are stochastic); a seeded generator stands in for
// the model + grader, so the run is reproducible. Steps: A/A noise floor,
// baseline vs candidate overall and per slice, flipped cases, exact sign test, gate.
let state = 42;
const rand = () => { state = (1664525 * state + 1013904223) % 4294967296; return state / 4294967296; };

const SLICES = ["billing", "shipping", "multilingual"];
// True per-slice pass probability for each version (unknown to the harness).
const P = { base: { billing: 0.88, shipping: 0.86, multilingual: 0.86 }, cand: { billing: 0.97, shipping: 0.96, multilingual: 0.7 } };
const CASES = [];
for (const s of SLICES) for (let i = 0; i < 25; i++) CASES.push({ id: `${s.slice(0, 2)}${String(i + 1).padStart(2, "0")}`, slice: s, critical: i === 0 });
const RUNS = 3;

function evaluate(version) {
  // Returns, per case, how many of RUNS attempts passed.
  return CASES.map((c) => { let k = 0; for (let r = 0; r < RUNS; r++) if (rand() < P[version][c.slice]) k++; return k; });
}
const rate = (res, idx) => idx.reduce((s, i) => s + res[i], 0) / (idx.length * RUNS);
const passed = (k) => k * 2 > RUNS; // majority of runs
const C = (n, k) => { let r = 1; for (let i = 0; i < k; i++) r = (r * (n - i)) / (i + 1); return r; };
function signTest(reg, imp) { // exact two-sided McNemar / sign test on flipped cases
  const n = reg + imp; if (n === 0) return 1;
  let tail = 0; for (let i = 0; i <= Math.min(reg, imp); i++) tail += C(n, i) * Math.pow(0.5, n);
  return Math.min(1, 2 * tail);
}
function flips(a, b, idx) {
  let reg = 0, imp = 0; const ids = [];
  for (const i of idx) { if (b[i] < a[i]) { reg++; ids.push(CASES[i].id); } else if (b[i] > a[i]) imp++; }
  return { reg, imp, ids };
}
const pct = (x) => (x * 100).toFixed(1);
const sgn = (x) => (x >= 0 ? "+" : "") + pct(x);
const ALL = CASES.map((_, i) => i);

const base = evaluate("base"), rerun = evaluate("base"), cand = evaluate("cand");
const aa = flips(base, rerun, ALL);
console.log(`A/A noise floor (baseline vs baseline rerun): ${pct(rate(base, ALL))}% vs ${pct(rate(rerun, ALL))}%, delta ${sgn(rate(rerun, ALL) - rate(base, ALL))} pts, flips ${aa.reg} down / ${aa.imp} up, p=${signTest(aa.reg, aa.imp).toFixed(3)}`);
const d = rate(cand, ALL) - rate(base, ALL), all = flips(base, cand, ALL);
console.log(`overall: baseline ${pct(rate(base, ALL))}% -> candidate ${pct(rate(cand, ALL))}% (${sgn(d)} pts), flips ${all.reg} down / ${all.imp} up, p=${signTest(all.reg, all.imp).toFixed(3)}`);

const failures = [];
if (d < -0.02) failures.push("overall drop > 2 pts");
for (const s of SLICES) {
  const idx = ALL.filter((i) => CASES[i].slice === s);
  const ds = rate(cand, idx) - rate(base, idx), fl = flips(base, cand, idx), p = signTest(fl.reg, fl.imp);
  console.log(`  ${s.padEnd(13)} ${pct(rate(base, idx)).padStart(5)}% -> ${pct(rate(cand, idx)).padStart(5)}%  ${sgn(ds).padStart(6)} pts  flips ${fl.reg} down / ${fl.imp} up  p=${p.toFixed(3)}${fl.ids.length ? "  regressed: " + fl.ids.join(",") : ""}`);
  if (ds < -0.05 && p < 0.05) failures.push(`${s} slice dropped ${pct(-ds)} pts (p=${p.toFixed(3)})`);
}
for (const i of ALL) if (CASES[i].critical && passed(base[i]) && !passed(cand[i])) failures.push(`critical case ${CASES[i].id} regressed`);
console.log(failures.length ? `GATE: BLOCK -> ${failures.join("; ")}` : "GATE: PASS");