What is prompt caching and how does it reduce cost and latency?

// Toy prefix cache: the prompt is split into fixed-size blocks, and each block is keyed
// by a hash of its own tokens plus the hash of everything before it (a hash chain).
// A request reuses cached blocks from the start until the first block that differs.
const crypto = require("crypto");

const BLOCK = 4;                       // tokens per cache block (real engines use 16+)
const PRICE_IN = 3.0;                  // $ per million input tokens (illustrative)
const WRITE_X = 1.25, READ_X = 0.1;    // write surcharge and read discount (illustrative)
const MS_PER_UNCACHED = 0.5, MS_PER_CACHED = 0.02; // toy prefill cost per token

const cache = new Map();               // block hash -> true (stands for stored keys/values)

function blockHashes(tokens) {
  const out = [];
  let parent = "root";
  for (let i = 0; i + BLOCK <= tokens.length; i += BLOCK) {
    const h = crypto.createHash("sha256").update(parent + "|" + tokens.slice(i, i + BLOCK).join(" ")).digest("hex").slice(0, 8);
    out.push(h);
    parent = h;
  }
  return out;
}

function run(name, text) {
  const tokens = text.split(" ");
  const hashes = blockHashes(tokens);
  let hit = 0;
  while (hit < hashes.length && cache.has(hashes[hit])) hit++;
  const cached = hit * BLOCK;
  const written = (hashes.length - hit) * BLOCK;   // new full blocks get stored
  const tail = tokens.length - cached - written;   // partial last block, never stored
  for (const h of hashes) cache.set(h, true);
  const units = cached * READ_X + written * WRITE_X + tail;
  const ms = cached * MS_PER_CACHED + (written + tail) * MS_PER_UNCACHED;
  console.log(`${name.padEnd(22)} tokens=${String(tokens.length).padStart(2)} cached=${String(cached).padStart(2)} computed=${String(written + tail).padStart(2)} prefill=${ms.toFixed(2).padStart(5)}ms cost_units=${units.toFixed(2)}`);
  return { tokens: tokens.length, units };
}

const SYSTEM = "You are the support bot for Acme . Rules : never share internal prices . Tools : lookup_order refund_order escalate .";
const DOCS = "Policy : refunds within 30 days . Shipping : 5 business days . Warranty : 1 year on hardware .";

console.log("block size", BLOCK, "tokens; prefix =", SYSTEM.split(" ").length + DOCS.split(" ").length, "tokens");
const rows = [];
rows.push(run("A first question", `${SYSTEM} ${DOCS} User : where is order 1042 ?`));
rows.push(run("B same prefix", `${SYSTEM} ${DOCS} User : can I return my headphones ?`));
rows.push(run("C same prefix again", `${SYSTEM} ${DOCS} User : how long is the warranty ?`));
rows.push(run("D timestamp first", `Time : 10:42 ${SYSTEM} ${DOCS} User : where is order 1042 ?`));
rows.push(run("E docs edited", `${SYSTEM} Policy : refunds within 14 days . Shipping : 5 business days . Warranty : 1 year on hardware . User : hi ?`));

const noCache = rows.reduce((s, r) => s + r.tokens, 0);
const withCache = rows.reduce((s, r) => s + r.units, 0);
console.log(`total cost units: no cache ${noCache.toFixed(2)}, with cache ${withCache.toFixed(2)} (${(100 * (1 - withCache / noCache)).toFixed(1)}% lower)`);
console.log(`per 1M requests of this mix at $${PRICE_IN.toFixed(2)} per M input tokens: $${(noCache / rows.length * PRICE_IN).toFixed(2)} vs $${(withCache / rows.length * PRICE_IN).toFixed(2)}`);

// Break-even: one write then n reads, against n+1 uncached sends of the same prefix.
for (const n of [0, 1, 2, 10]) {
  const cachedCost = WRITE_X + n * READ_X, plain = n + 1;
  console.log(`prefix sent ${n + 1}x: cached ${cachedCost.toFixed(2)} vs uncached ${plain.toFixed(2)} prefix-units -> ${cachedCost < plain ? "cache wins" : "cache loses"}`);
}