When should you use batch APIs instead of real-time LLM calls?

// A batch job end to end with a simulated provider: build JSONL with custom_id,
// get results back out of order with failures, join by id, retry the failures,
// then compare the monthly bill and wall-clock time against real-time calls.

const tickets = ["refund not received", "app crashes on login", "change my plan", "invoice is wrong",
  "password reset loop", "cancel subscription", "dark mode request", "charged twice"];
const LABEL = { refund: "billing", invoice: "billing", charged: "billing", plan: "account", cancel: "account",
  crashes: "bug", password: "bug", dark: "feature" };
const label = (t) => LABEL[Object.keys(LABEL).find((k) => t.includes(k))];

// 1) One JSONL line per request; custom_id is the join key and the dedupe key.
const lines = tickets.map((t, i) => JSON.stringify({
  custom_id: "ticket-" + (101 + i), method: "POST", url: "/v1/chat/completions",
  body: { model: "MODEL", max_tokens: 5, messages: [{ role: "user", content: "Label: " + t }] },
}));
console.log("input.jsonl: " + lines.length + " lines, first line:");
console.log("  " + lines[0]);

// 2) Simulated provider: results in its own order; one bad request, one expired.
const ORDER = [5, 2, 7, 0, 3, 6, 1, 4];
function runBatch(reqLines, failIds) {
  return ORDER.filter((i) => i < reqLines.length).map((i) => {
    const r = JSON.parse(reqLines[i]);
    const t = r.body.messages[0].content.slice(7);
    if (failIds[r.custom_id]) return { custom_id: r.custom_id, result: failIds[r.custom_id] };
    return { custom_id: r.custom_id, result: "succeeded", label: label(t) };
  });
}
const out = runBatch(lines, { "ticket-104": "errored", "ticket-107": "expired" });
console.log("");
console.log("output order: " + out.map((o) => o.custom_id.slice(7)).join(", "));

// Wrong: zip results with inputs by position. Right: join on custom_id.
const truth = tickets.map(label);
const byPosition = out.filter((o, i) => o.result === "succeeded" && o.label === truth[i]).length;
const byId = new Map(out.map((o) => [o.custom_id, o]));
const ok = [...byId.values()].filter((o) => o.result === "succeeded");
console.log("positional join, labels on the right ticket: " + byPosition + " of " + ok.length);
console.log("custom_id join,  labels on the right ticket: " +
  ok.filter((o) => o.label === truth[Number(o.custom_id.slice(7)) - 101]).length + " of " + ok.length);

// 3) Retry only what failed, as a small follow-up batch (same custom_ids).
const failed = [...byId.values()].filter((o) => o.result !== "succeeded");
console.log("failed: " + failed.map((o) => o.custom_id + " (" + o.result + ")").join(", "));
const retryLines = lines.filter((l) => failed.some((f) => l.includes('"' + f.custom_id + '"')));
for (const o of runBatch(retryLines, {})) byId.set(o.custom_id, o);
const done = [...byId.values()].filter((o) => o.result === "succeeded").length;
console.log("after retry sub-batch of " + retryLines.length + ": " + done + " of " + tickets.length + " succeeded");

// 4) Monthly cost and time. Prices are placeholders: put in your model's price.
const REQS = 2000000, IN_TOK = 1200, OUT_TOK = 150;          // per request
const IN_PRICE = 200, OUT_PRICE = 800;                       // cents per million tokens
const BATCH = 0.5;                                           // providers quote 50% off
const PREFIX = 1000, CACHE_READ = 0.1, HIT = 0.6;            // shared prefix, cached-read price, hit rate
const cents = (inTok, outTok) => (inTok * IN_PRICE + outTok * OUT_PRICE) / 1e6;
const realtime = REQS * cents(IN_TOK, OUT_TOK);
const batch = realtime * BATCH;
const cachedIn = REQS * (HIT * (PREFIX * CACHE_READ + (IN_TOK - PREFIX)) + (1 - HIT) * IN_TOK);
const batchCache = BATCH * (cachedIn * IN_PRICE + REQS * OUT_TOK * OUT_PRICE) / 1e6;
const usd = (c) => "$" + (c / 100).toFixed(2).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
console.log("");
console.log("monthly: " + REQS + " requests x (" + IN_TOK + " in + " + OUT_TOK + " out) tokens");
console.log("  real-time:              " + usd(realtime));
console.log("  batch (50% off):        " + usd(batch));
console.log("  batch + prompt cache:   " + usd(batchCache) + "  (" + (100 * HIT) + "% hits on a " + PREFIX + "-token prefix)");
const RPM = 500;
console.log("  real-time at " + RPM + " RPM:    " + (REQS / RPM / 60).toFixed(1) + " hours of sending, sharing limits with live users");
console.log("  batch:                  results within the 24 h window, separate batch quota");