Guardrails for AI agents: rate limits, budget caps, and HITL?

// A run governor for an agent loop: step cap plus a dollar budget, enforced
// either after each model call or by reserving the worst case before it.
// ASSUMED prices (illustrative, not a vendor list price): $3 per million input
// tokens, $15 per million output tokens. Money is kept in integer micro-dollars.
const PRICE_IN = 3, PRICE_OUT = 15;           // micro-dollars per token
const MAX_TOKENS = 1024;                      // output cap sent with every call
const BUDGET = 500000;                        // $0.50 per run
const STEP_CAP = 25;

// Stub model + tools: the context grows with every tool result, and step 6
// reads a large file (40,000 tokens) into the context. Output is 400 tokens per step.
function stepUsage(context) { return { input: context, output: 400 }; }
function toolResultTokens(step) { return step === 6 ? 40000 : 1500; }
function usd(micro) {  // integer rounding to 4 decimals, identical in every language
  const t = Math.floor((micro + 50) / 100);
  return "$" + Math.floor(t / 10000) + "." + String(t % 10000).padStart(4, "0");
}

function run(policy, verbose) {
  let context = 2000, spent = 0, step = 0, stop = "step cap " + STEP_CAP;
  while (step < STEP_CAP) {
    if (policy === "reserve") {
      const worst = context * PRICE_IN + MAX_TOKENS * PRICE_OUT;  // known input + max output
      if (spent + worst > BUDGET) { stop = "reserve " + usd(worst) + " would exceed budget"; break; }
    }
    const u = stepUsage(context);
    const cost = u.input * PRICE_IN + u.output * PRICE_OUT;
    spent += cost;
    step++;
    if (verbose) console.log("  step " + String(step).padStart(2) + "  input " + String(u.input).padStart(6) + "  cost " + usd(cost) + "  total " + usd(spent));
    if (policy === "after" && spent > BUDGET) { stop = "budget exceeded after the call"; break; }
    context += u.output + toolResultTokens(step);
  }
  return { steps: step, spent, stop };
}

console.log("reserve-before policy, step by step (budget " + usd(BUDGET) + "):");
const r = run("reserve", true);
console.log("");
console.log("policy             steps  spent      over budget  stopped because");
for (const [name, p] of [["none (step cap)", "none"], ["check after call", "after"], ["reserve before", "reserve"]]) {
  const x = run(p, false);
  const over = Math.max(0, x.spent - BUDGET);
  console.log(name.padEnd(19) + String(x.steps).padEnd(7) + usd(x.spent).padEnd(11) + usd(over).padEnd(13) + x.stop);
}
console.log("");
console.log("on stop: status=paused_for_human, keep the trace, return partial work after " + r.steps + " steps");

// Per-tool rate limits (token buckets), a repeat-call breaker and escalation
// to a human, wrapped around a stub agent that gets stuck. Fake clock, seconds.
// Integer token bucket: 60 units = 1 token, so 1 token per minute is 1 unit per second.
class Bucket {
  constructor(capacity, unitsPerSec) { this.max = capacity * 60; this.units = this.max; this.rate = unitsPerSec; this.last = 0; }
  take(now) {
    this.units = Math.min(this.max, this.units + (now - this.last) * this.rate);
    this.last = now;
    if (this.units >= 60) { this.units -= 60; return true; }
    return false;
  }
  left() { const h = Math.floor((this.units * 100) / 60); return Math.floor(h / 100) + "." + String(h % 100).padStart(2, "0"); }
}
const LIMITS = { search: new Bucket(10, 60), send_email: new Bucket(3, 1) };  // search 1/s, email 3 burst then 1/min
const REPEAT_LIMIT = 3;           // same tool + same args this many times in a row -> break
const MAX_DENIALS = 4;            // too many guard denials in one run -> escalate

// Stub model: a fixed sequence of tool calls, one every 5 seconds. It retries a
// failing search with identical arguments and then tries to email everyone.
const plan = [
  ["search", "q=refund policy"], ["search", "q=refund policy"], ["send_email", "to=a@x.com"],
  ["send_email", "to=b@x.com"], ["send_email", "to=c@x.com"], ["send_email", "to=d@x.com"],
  ["send_email", "to=e@x.com"], ["search", "q=order 7"], ["search", "q=order 7"], ["search", "q=order 7"],
  ["send_email", "to=f@x.com"],
];

let lastKey = "", repeats = 0, denials = 0, status = "completed";
const log = [];
for (let i = 0; i < plan.length; i++) {
  const now = i * 5;
  const [tool, args] = plan[i];
  const key = tool + " " + args;
  repeats = key === lastKey ? repeats + 1 : 1;
  lastKey = key;
  let verdict;
  if (repeats >= REPEAT_LIMIT) {
    verdict = "BREAK: " + repeats + " identical calls in a row";
    status = "paused_for_human";
  } else if (!LIMITS[tool].take(now)) {
    denials++;
    verdict = "DENY: rate limit (" + LIMITS[tool].left() + " tokens left)";
    if (denials >= MAX_DENIALS) status = "paused_for_human";
  } else verdict = "ok";
  console.log("t=" + String(now).padStart(2) + "  " + key.padEnd(24) + verdict);
  log.push({ now, key, verdict });
  if (status !== "completed") break;
}
console.log("");
console.log("run status: " + status + " after " + log.length + " of " + plan.length + " planned calls");
const sent = log.filter((l) => l.key.startsWith("send_email") && l.verdict === "ok").length;
console.log("emails actually sent: " + sent + ", denied by rate limit: " + denials);
console.log("escalation message: \"Agent paused: " + log[log.length - 1].verdict + ". Approve continue, edit plan, or stop.\"");