How would you design AI inference infra with GPU queue and KV-cache?

// LLM inference fleet: capacity math, static vs continuous batching, paged KV admission,
// and prefix-aware routing. GPUs are simulated: one "step" = one decode iteration.
const fmt = (n) => Math.round(n).toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");

console.log("== 1. Capacity (assumed: 70B model in BF16, 80 layers, 8 KV heads, head dim 128) ==");
const weights = 70e9 * 2;
const kvPerToken = 2 * 80 * 8 * 128 * 2; // K and V, layers, kv heads, head dim, 2 bytes
const node = 8 * 80e9, usable = node * 0.9;
const kvBytes = usable - weights;
const kvTokens = Math.floor(kvBytes / kvPerToken);
console.log(`weights ${fmt(weights / 1e9)} GB; KV cache ${fmt(kvPerToken)} bytes per token`);
console.log(`8 x 80 GB node, 90% usable: ${fmt(kvBytes / 1e9)} GB left for KV = ${fmt(kvTokens)} tokens`);
const stepFloorMs = weights / (8 * 3.35e12) * 1000;
console.log(`decode floor: read all weights once per step = ${stepFloorMs.toFixed(2)} ms per step`);
const reqDay = 20e6, inTok = 1500, outTok = 300, peak = 3;
const outPeak = reqDay * outTok / 86400 * peak, inPeak = reqDay * inTok / 86400 * peak;
const perNode = 128 / 0.025; // assumed: batch 128 at 25 ms per step
console.log(`demand at 3x peak: ${fmt(outPeak)} output tok/s, ${fmt(inPeak)} prompt tok/s`);
console.log(`assumed node rate ${fmt(perNode)} output tok/s -> ${Math.ceil(outPeak / perNode)} nodes for decode`);
console.log(`prompt tokens to compute with a 60% prefix-cache hit rate: ${fmt(inPeak * 0.4)} tok/s`);

console.log("\n== 2. Static vs continuous batching (8 requests, 4 slots) ==");
const lens = [5, 40, 8, 12, 30, 6, 20, 10];
function staticBatching() {
  const done = []; let t = 0;
  for (let i = 0; i < lens.length; i += 4) {
    const batch = lens.slice(i, i + 4);
    for (const l of batch) done.push(t + l);
    t += Math.max(...batch); // the batch holds its slots until the longest finishes
  }
  return { done, steps: t };
}
function continuousBatching() {
  const done = new Array(lens.length).fill(0); const slots = []; let next = 0, t = 0;
  while (next < lens.length || slots.length) {
    while (slots.length < 4 && next < lens.length) { slots.push({ i: next, left: lens[next] }); next++; }
    t++;
    for (const s of slots) s.left--;
    for (const s of slots) if (s.left === 0) done[s.i] = t;
    for (let k = slots.length - 1; k >= 0; k--) if (slots[k].left === 0) slots.splice(k, 1);
  }
  return { done, steps: t };
}
const work = lens.reduce((a, b) => a + b, 0);
for (const [name, r] of [["static", staticBatching()], ["continuous", continuousBatching()]]) {
  const avg = r.done.reduce((a, b) => a + b, 0) / r.done.length;
  console.log(`${name.padEnd(10)} finish steps ${r.done.join(",")}`);
  console.log(`${"".padEnd(10)} total ${r.steps} steps, mean latency ${avg.toFixed(2)} steps, slot use ${(100 * work / (r.steps * 4)).toFixed(1)}%`);
}

console.log("\n== 3. KV admission: reserve max context vs paged blocks of 16 tokens ==");
const maxCtx = 8192, avgUsed = 1800;
console.log(`reserve ${maxCtx} tokens each: ${Math.floor(kvTokens / maxCtx)} concurrent sequences`);
console.log(`paged, ${Math.ceil(avgUsed / 16)} blocks for ${avgUsed} tokens: ${Math.floor(kvTokens / (Math.ceil(avgUsed / 16) * 16))} concurrent sequences`);

console.log("\n== 4. Routing 12 requests to 3 replicas (system prompt 2,000 tokens, user part 100) ==");
const tenants = "AABAAACAABAC".split("");
function fnv1a(s) {
  let h = 0x811c9dc5;
  for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 0x01000193) >>> 0; }
  return h;
}
function route(policy) {
  const cached = [new Set(), new Set(), new Set()], load = [0, 0, 0];
  let computed = 0;
  tenants.forEach((tn, i) => {
    let r;
    if (policy === "round-robin") r = i % 3;
    else if (policy === "prefix-hash") r = fnv1a("system-prompt-" + tn) % 3;
    else { // prefix-aware with a load cap: prefer a replica that has the prefix unless it is 2 ahead
      const min = Math.min(...load);
      const hit = [0, 1, 2].find((k) => cached[k].has(tn) && load[k] - min < 2);
      r = hit !== undefined ? hit : load.indexOf(min);
    }
    computed += (cached[r].has(tn) ? 0 : 2000) + 100;
    cached[r].add(tn); load[r]++;
  });
  const hitRate = 1 - computed / (tenants.length * 2100);
  console.log(`${policy.padEnd(18)} prefill tokens ${fmt(computed)} of ${fmt(tenants.length * 2100)} (saved ${(100 * hitRate).toFixed(1)}%), load ${load.join("/")}`);
}
route("round-robin");
route("prefix-hash");
route("prefix+load-aware");