How do you version, test and safely roll out prompt changes?

// Prompt registry -> offline gate -> sticky canary -> promote or roll back.
// The "model" is a deterministic stub; integers only, so both languages agree.
const crypto = require("crypto");
const sha = (s) => crypto.createHash("sha256").update(s).digest("hex");

// 1. Registry: immutable versions, content-hashed, labels are movable pointers.
const versions = [];
const labels = {};
const labelLog = [];
function register(name, template, model, params) {
  const spec = { model, name, params, template };      // keys already sorted
  const hash = sha(JSON.stringify(spec)).slice(0, 12);
  const same = versions.find((v) => v.name === name && v.hash === hash);
  if (same) return same;                              // same content, same version
  const v = { ...spec, version: versions.filter((x) => x.name === name).length + 1, hash };
  versions.push(v);
  return v;
}
function setLabel(label, v) { labels[label] = v; labelLog.push(`${label} -> v${v.version} (${v.hash})`); }
function render(v, vars) {
  return v.template.replace(/\{(\w+)\}/g, (_, k) => {
    if (!(k in vars)) throw new Error(`missing variable ${k} in v${v.version}`);
    return vars[k];
  });
}

// 2. Stub model: follows only the rules that are written in the prompt.
function fakeModel(prompt) {
  const t = prompt.split("Ticket: ")[1].toLowerCase();
  const err = /crash|error/.test(t);
  if (prompt.includes("Payment problems are billing unless") && /pay/.test(t) && !err) return "billing";
  if (prompt.includes("Payment problems are billing.") && /pay/.test(t)) return "billing";
  if (err) return "bug";
  if (/charge|refund|invoice/.test(t)) return "billing";
  return "other";
}

// 3. Offline gate: golden set, per-case comparison with the production version.
const golden = [
  ["I was charged twice this month", "billing"],
  ["The app crashes when I open settings", "bug"],
  ["How do I change my avatar?", "other"],
  ["Refund my last invoice please", "billing"],
  ["Error 500 when I tap pay", "bug"],
  ["My payment failed and I am not sure it went through", "billing"],
];
const runSuite = (v) => golden.map(([ticket, want]) => fakeModel(render(v, { ticket })) === want);
function gate(candidate, baseline) {
  const c = runSuite(candidate), b = runSuite(baseline);
  const broke = c.map((ok, i) => (b[i] && !ok ? i + 1 : 0)).filter(Boolean);
  const fixed = c.map((ok, i) => (!b[i] && ok ? i + 1 : 0)).filter(Boolean);
  const pass = c.filter(Boolean).length;
  const verdict = broke.length === 0 && pass >= b.filter(Boolean).length ? "PASS" : "FAIL";
  console.log(`gate v${candidate.version} vs v${baseline.version}: ${pass}/6 pass, fixed [${fixed}], broke [${broke}] -> ${verdict}`);
  return verdict === "PASS";
}

const base = "Classify the support ticket as billing, bug or other. Answer with one word.";
const v1 = register("triage", `${base}\nTicket: {ticket}`, "model-2026-01-15", { max_tokens: 5, temperature: 0 });
setLabel("production", v1);
const again = register("triage", `${base}\nTicket: {ticket}`, "model-2026-01-15", { max_tokens: 5, temperature: 0 });
console.log(`re-register identical content -> v${again.version} (${again.hash})`);
const v2 = register("triage", `${base}\nPayment problems are billing.\nTicket: {ticket}`, "model-2026-01-15", { max_tokens: 5, temperature: 0 });
const v3 = register("triage", `${base}\nPayment problems are billing unless the app shows an error.\nTicket: {ticket}`, "model-2026-01-15", { max_tokens: 5, temperature: 0 });
try { render(v3, {}); } catch (e) { console.log(`render check: ${e.message}`); }
gate(v2, v1);
const v3ok = gate(v3, v1);

// 4. Sticky bucketing: a user keeps the same arm as the percentage ramps up.
const bucket = (user, exp) => parseInt(sha(`${exp}:${user}`).slice(0, 8), 16) % 100;
const users = Array.from({ length: 2000 }, (_, i) => `u${String(i + 1).padStart(4, "0")}`);
// Simulated outcome: 1 = user gave a thumbs-down. Fixed per (user, version) by a hash.
const bad = (user, v, perMille) => parseInt(sha(`outcome:${v.hash}:${user}`).slice(0, 8), 16) % 1000 < perMille;
const quality = { [v1.hash]: 40, [v3.hash]: 30 };   // thumbs-down per 1000 requests

function canary(cand, stages, exp) {
  const ctlV = labels.production;
  let prev = new Set();
  for (const pct of stages) {
    const arm = users.filter((u) => bucket(u, exp) < pct);
    const stayed = [...prev].every((u) => arm.includes(u));
    const ctl = users.filter((u) => bucket(u, exp) >= pct);
    const cBad = arm.filter((u) => bad(u, cand, quality[cand.hash])).length;
    const kBad = ctl.filter((u) => bad(u, ctlV, quality[ctlV.hash])).length;
    const p1 = cBad / arm.length, p0 = kBad / ctl.length;
    const p = (cBad + kBad) / (arm.length + ctl.length);
    const z = (p1 - p0) / Math.sqrt(p * (1 - p) * (1 / arm.length + 1 / ctl.length));
    console.log(`  ${String(pct).padStart(3)}%: canary ${arm.length} users ${(100 * p1).toFixed(1)}% bad | control ${ctl.length} users ${(100 * p0).toFixed(1)}% bad | z=${z.toFixed(2)} sticky=${stayed}`);
    if (z > 2) return `ROLL BACK at ${pct}% (canary significantly worse)`;
    prev = new Set(arm);
  }
  return "PROMOTE";
}
if (v3ok) {
  console.log(`canary v${v3.version}:`);
  const r = canary(v3, [5, 25, 50], "triage-v3");
  console.log(`  -> ${r}`);
  if (r === "PROMOTE") setLabel("production", v3);
}
// 5. A later version that passes the offline gate but users dislike.
const v4 = register("triage", v3.template, "model-2026-09-30", { max_tokens: 5, temperature: 0 });
quality[v4.hash] = 75;
gate(v4, v3);
console.log(`canary v${v4.version} (same text, new model snapshot):`);
console.log(`  -> ${canary(v4, [5, 25, 50], "triage-v4")}`);
console.log("label history:");
labelLog.forEach((l) => console.log(`  ${l}`));
const live = labels.production;
console.log(`request log line: prompt=triage@v${live.version} hash=${live.hash} model=${live.model}`);