How do you design human-in-the-loop approval for agent actions?

// Human-in-the-loop gate: classify by risk, then bind each approval to the
// exact arguments, a deadline and a single use. Fake clock in seconds.
const crypto = require("crypto");

function canonical(v) {  // stable JSON: sorted keys, no spaces
  if (Array.isArray(v)) return "[" + v.map(canonical).join(",") + "]";
  if (v && typeof v === "object") return "{" + Object.keys(v).sort().map((k) => JSON.stringify(k) + ":" + canonical(v[k])).join(",") + "}";
  return JSON.stringify(v);
}
const digest = (tool, args) => crypto.createHash("sha256").update(tool + " " + canonical(args)).digest("hex").slice(0, 12);

// risk = irreversibility (0-2) + blast radius (0-2) + leaves the org (0-1)
const RISK = {
  search_docs: [0, 0, 0], update_ticket: [1, 0, 0], post_team_note: [1, 1, 0], bulk_update_tickets: [1, 2, 0],
  send_email: [2, 1, 1], refund_payment: [2, 1, 1], delete_customer: [2, 2, 0],
};
const tierOf = (s) => (s <= 1 ? "auto" : s === 2 ? "notify" : "approve");
console.log("tool                  irrev blast external score tier");
for (const [tool, [i, b, e]] of Object.entries(RISK))
  console.log(tool.padEnd(22) + String(i).padEnd(6) + String(b).padEnd(6) + String(e).padEnd(9) + String(i + b + e).padEnd(6) + tierOf(i + b + e));

const approvals = new Map();
let nextId = 1;
function requestApproval(now, tool, args) {
  const id = "apr-" + nextId++;
  approvals.set(id, { tool, hash: digest(tool, args), expires: now + 300, used: false });
  return id;
}
function decide(id, decision, editedArgs) {  // the human's click, possibly after editing
  const a = approvals.get(id);
  if (decision === "reject") { a.rejected = true; return; }
  if (editedArgs) a.hash = digest(a.tool, editedArgs);
  a.approved = true;
}
function execute(now, id, tool, args) {  // the executor re-checks everything
  const a = approvals.get(id);
  if (!a || !a.approved) return "REFUSED: not approved";
  if (a.used) return "REFUSED: approval " + id + " already used";
  if (now > a.expires) return "REFUSED: approval expired at t=" + a.expires;
  if (a.tool !== tool || a.hash !== digest(tool, args)) return "REFUSED: args changed since approval (" + a.hash + " vs " + digest(tool, args) + ")";
  a.used = true;
  return "EXECUTED " + tool + " to=" + args.to;
}

const email = { to: "dana@client.com", subject: "Your refund", body: "Refund of $40 issued." };
console.log("");
console.log("1. approve, then execute the same action");
let id = requestApproval(0, "send_email", email);
console.log("   preview " + id + ": send_email " + canonical(email) + " hash " + digest("send_email", email));
decide(id, "approve");
console.log("   t=20  " + execute(20, id, "send_email", email));
console.log("   t=25  " + execute(25, id, "send_email", email));

console.log("2. approve, then the agent swaps the recipient before executing");
id = requestApproval(100, "send_email", email);
decide(id, "approve");
console.log("   t=130 " + execute(130, id, "send_email", { ...email, to: "all-customers@client.com" }));

console.log("3. approve, but execution happens after the deadline");
id = requestApproval(200, "send_email", email);
decide(id, "approve");
console.log("   t=560 " + execute(560, id, "send_email", email));

console.log("4. edit-then-approve: the human fixes the amount");
id = requestApproval(600, "send_email", email);
const edited = { ...email, body: "Refund of $45 issued." };
decide(id, "approve", edited);
console.log("   t=640 original: " + execute(640, id, "send_email", email));
console.log("   t=641 edited:   " + execute(641, id, "send_email", edited));

console.log("5. reject");
id = requestApproval(700, "send_email", email);
decide(id, "reject");
console.log("   t=710 " + execute(710, id, "send_email", email));

// Approval fatigue: how many prompts a reviewer sees in one agent session,
// and how attentive they still are when the one bad action arrives.
// ASSUMED attention model (illustrative, not measured): the reviewer catches a
// bad action shown as their k-th prompt of the session with probability
// max(30%, 95% - 2% * (k - 1)).
let seed = 42;
const rand100 = () => { seed = (seed * 48271) % 2147483647; return seed % 100; };

const actions = [];
for (let i = 0; i < 80; i++) {
  const r = rand100();
  if (r < 55) actions.push({ tool: "search_docs" });
  else if (r < 80) actions.push({ tool: "update_ticket" });
  else if (r < 90) actions.push({ tool: "post_team_note" });
  else if (r < 97) actions.push({ tool: "send_email", toRequester: rand100() < 85 });
  else actions.push({ tool: "refund_payment" });
}
// Two harmful actions hidden late in the session.
actions[49] = { tool: "send_email", toRequester: false, bad: "email to the wrong customer" };
actions[71] = { tool: "refund_payment", bad: "refund 10x the amount" };

const counts = {};
for (const a of actions) counts[a.tool] = (counts[a.tool] || 0) + 1;
console.log("session of " + actions.length + " actions: " + Object.keys(counts).sort().map((k) => k + "=" + counts[k]).join(" "));

const policies = {
  "A confirm every write": (a) => a.tool !== "search_docs",
  "B tiered (score >= 3)": (a) => a.tool === "send_email" || a.tool === "refund_payment",
  "C tiered + scoped grant": (a) => (a.tool === "send_email" && !a.toRequester) || a.tool === "refund_payment",
};
console.log("");
console.log("policy                    prompts  bad #1 at prompt  bad #2 at prompt  expected catches of 2");
for (const [name, needsApproval] of Object.entries(policies)) {
  let k = 0, caughtPct = 0;
  const at = [];
  for (const a of actions) {
    if (!needsApproval(a)) continue;
    k++;
    if (a.bad) { const p = Math.max(30, 95 - 2 * (k - 1)); at.push("#" + k + " (" + p + "%)"); caughtPct += p; }
  }
  console.log(name.padEnd(26) + String(k).padEnd(9) + at[0].padEnd(18) + at[1].padEnd(18) + (caughtPct / 100).toFixed(2));
}
console.log("");
console.log("C grant: \"send_email to the ticket requester is pre-approved for this task\"");
console.log("everything not prompted still lands in the audit log and the end-of-run digest");