Cost optimization for LLM apps: how do you keep the bill predictable?

// 30 days of spend, with and without budget controls. One tenant ships a
// retry-loop bug on day 12; a prompt change bloats every request on day 20.
// Hypothetical prices ($ per million tokens) and traffic; seeded noise.
const IN = 3.0, OUT = 15.0, SMALL_FACTOR = 0.27; // small model costs 27% as much
const BUDGET = 25000; // monthly budget in $
const tenants = { acme: 20000, globex: 15000, initech: 10000 }; // requests per day
const TENANT_DAILY_CAP = { acme: 700, globex: 550, initech: 400 }; // $ per day

function simulate(controls) {
  let seed = 11; // Park-Miller, same noise in both runs and both languages
  const rand = () => { seed = (seed * 48271) % 2147483647; return seed / 2147483647; };
  let spent = 0, maxDay = 0, degraded = false, bloatFixed = false;
  const events = [], history = [];
  for (let day = 1; day <= 30; day++) {
    const bloat = day >= 20 && !bloatFixed ? 3000 : 0;
    let perReq = ((4000 + bloat) * IN + 300 * OUT) / 1e6;
    if (degraded) perReq *= SMALL_FACTOR;
    let today = 0, requests = 0;
    for (const [name, base] of Object.entries(tenants)) {
      let n = Math.round(base * (0.9 + 0.2 * rand()));
      if (name === "initech" && day >= 12 && day <= 16) n *= 10; // the retry loop
      let c = n * perReq;
      if (controls && c > TENANT_DAILY_CAP[name]) {
        if (day === 12) events.push(`day ${day}: ${name} hit its daily cap, extra calls get 429 (other tenants unaffected)`);
        c = TENANT_DAILY_CAP[name]; n = Math.floor(c / perReq);
      }
      today += c; requests += n;
    }
    spent += today; maxDay = Math.max(maxDay, today);
    const cpr = today / requests;
    if (controls && history.length >= 7) {
      const avg = history.slice(-7).reduce((a, b) => a + b, 0) / 7;
      if (cpr > 1.3 * avg && !bloatFixed) {
        events.push(`day ${day}: cost per request $${cpr.toFixed(4)} vs 7-day avg $${avg.toFixed(4)}: alert, prompt rolled back`);
        bloatFixed = true;
      }
    }
    history.push(cpr);
    const projected = (spent / day) * 30;
    if (controls && day >= 3 && projected > BUDGET && !events.some((e) => e.includes("forecast"))) {
      events.push(`day ${day}: forecast month end $${projected.toFixed(0)} > budget $${BUDGET}: alert sent`);
    }
    if (controls && !degraded && spent > 0.9 * BUDGET) {
      degraded = true;
      events.push(`day ${day}: 90% of budget used, non-premium traffic moves to the small model`);
    }
  }
  return { spent, maxDay, events };
}

for (const controls of [false, true]) {
  const r = simulate(controls);
  console.log(controls ? "with controls:" : "no controls:");
  if (!r.events.length) console.log("  no caps, no alerts: the invoice is the first signal");
  for (const e of r.events) console.log(`  ${e}`);
  console.log(`  month total $${r.spent.toFixed(0)} (budget $${BUDGET}), worst day $${r.maxDay.toFixed(0)}`);
}

// Part 1: a per-request cost bound. Output length has a long tail; a cap
// turns "unknown" into a number you can multiply by traffic.
// Part 2: pay-per-token vs reserved capacity: when is a fixed bill cheaper?
// All prices are HYPOTHETICAL ($ per million tokens, $ per hour).
const IN = 3.0, OUT = 15.0;
let seed = 5;
const rand = () => { seed = (seed * 48271) % 2147483647; return seed / 2147483647; };

// 1000 requests: input 2000-6000 tokens; output usually short, sometimes runaway
const reqs = [];
for (let i = 0; i < 1000; i++) {
  const input = 2000 + Math.floor(rand() * 4000);
  const r = rand();
  const output = r < 0.97 ? 100 + Math.floor(rand() * 400) : 3000 + Math.floor(rand() * 9000);
  reqs.push([input, output]);
}
const cost = (i, o) => (i * IN + o * OUT) / 1e6;
function stats(cap) {
  const cs = reqs.map(([i, o]) => cost(i, Math.min(o, cap))).sort((a, b) => a - b);
  const at = (p) => cs[Math.ceil((p / 100) * cs.length) - 1];
  const mean = cs.reduce((a, b) => a + b, 0) / cs.length;
  return { mean, p50: at(50), p99: at(99), max: cs[cs.length - 1] };
}
console.log("part 1: cost per request over 1000 requests");
for (const cap of [16000, 1000]) {
  const s = stats(cap);
  const bound = cost(6000, cap);
  console.log(`  max_tokens ${String(cap).padStart(5)}: mean $${s.mean.toFixed(4)}  p50 $${s.p50.toFixed(4)}  p99 $${s.p99.toFixed(4)}  max $${s.max.toFixed(4)}  bound $${bound.toFixed(4)}`);
}
console.log("  bound = max input x input price + max_tokens x output price");
console.log(`  1M requests/month at the 1000-token bound can never exceed $${(cost(6000, 1000) * 1e6).toFixed(0)}`);

console.log("part 2: reserved capacity at $40/hour vs pay per token");
const RESERVED_PER_MONTH = 40 * 24 * 30; // $28800, billed whether used or not
const perReq = stats(1000).mean;
for (const millions of [0.5, 1, 2, 4]) {
  const onDemand = millions * 1e6 * perReq;
  const cheaper = onDemand < RESERVED_PER_MONTH ? "pay per token" : "reserved";
  console.log(`  ${String(millions).padStart(3)}M requests/month: per token $${onDemand.toFixed(0).padStart(6)}  reserved $${RESERVED_PER_MONTH}  cheaper: ${cheaper}`);
}
console.log(`  break-even at ${(RESERVED_PER_MONTH / perReq / 1e6).toFixed(2)}M requests/month (if the reserved units can carry that load)`);