What techniques reduce LLM latency at each stage of a request?

// A latency budget for one RAG + tools request, and what each lever removes.
// Every rate below is an ASSUMPTION for the arithmetic, not a measurement.
const A = {
  network: 80, // client <-> API round trip
  routerCall: 900, // a big model used to classify intent
  smallRouter: 250, // the same job on a small model
  embed: 40, search: 30, rerank: 120,
  tools: [400, 250, 300], // three independent tool calls
  queue: 150, // waiting for a GPU slot at the provider
  prefillTokPerSec: 20000, // prompt processing speed
  promptTokens: 12000, cachedTokens: 10000, // stable prefix: system + tools + docs
  msPerOutputToken: 20, // decode speed (50 tokens/s)
  fastMsPerOutputToken: 8, // a smaller model (125 tokens/s)
  outputTokens: 600, conciseTokens: 250,
};

function run(cfg) {
  const stages = [];
  stages.push(["network", A.network]);
  stages.push(["intent router", cfg.smallRouter ? A.smallRouter : A.routerCall]);
  stages.push(["retrieval", A.embed + A.search + A.rerank]);
  const tools = cfg.parallelTools ? Math.max(...A.tools) : A.tools.reduce((a, b) => a + b, 0);
  stages.push(["tools", tools]);
  stages.push(["queue", A.queue]);
  const uncached = cfg.promptCache ? A.promptTokens - A.cachedTokens : A.promptTokens;
  stages.push(["prefill", Math.round((uncached / A.prefillTokPerSec) * 1000)]);
  const per = cfg.fastModel ? A.fastMsPerOutputToken : A.msPerOutputToken;
  const out = cfg.concise ? A.conciseTokens : A.outputTokens;
  const beforeDecode = stages.reduce((a, [, ms]) => a + ms, 0);
  const total = beforeDecode + out * per;
  // without streaming the user sees nothing until the last token
  const firstVisible = cfg.stream ? beforeDecode + per : total;
  return { stages, total, firstVisible, decode: out * per };
}

const levers = [
  ["baseline (no streaming)", {}],
  ["+ stream the answer", { stream: true }],
  ["+ prompt cache the prefix", { promptCache: true }],
  ["+ run tools in parallel", { parallelTools: true }],
  ["+ small model routes", { smallRouter: true }],
  ["+ concise output format", { concise: true }],
  ["+ faster model for answer", { fastModel: true }],
];
const base = run({});
console.log("where the time goes (baseline):");
for (const [name, ms] of base.stages) console.log(`  ${name.padEnd(14)} ${String(ms).padStart(6)} ms`);
console.log(`  ${"decode".padEnd(14)} ${String(base.decode).padStart(6)} ms  (${A.outputTokens} tokens x ${A.msPerOutputToken} ms)`);
console.log("");
console.log("lever (cumulative)              first visible   total");
let cfg = {};
for (const [name, add] of levers) {
  cfg = { ...cfg, ...add };
  const r = run(cfg);
  console.log(`  ${name.padEnd(30)} ${String(r.firstVisible).padStart(7)} ms  ${String(r.total).padStart(6)} ms`);
}

// why you track p95, not the mean: 20 total latencies (ms) from a fake day
const sample = [1900, 2100, 2000, 2300, 1800, 2200, 2050, 1950, 2400, 2150,
  2250, 1850, 2000, 2100, 9800, 2050, 2200, 1900, 14500, 2000];
const sorted = [...sample].sort((a, b) => a - b);
const pct = (p) => sorted[Math.ceil((p / 100) * sorted.length) - 1]; // nearest-rank
const mean = sample.reduce((a, b) => a + b, 0) / sample.length;
console.log("");
console.log(`20 requests: mean ${mean.toFixed(0)} ms, p50 ${pct(50)} ms, p95 ${pct(95)} ms, max ${sorted[sorted.length - 1]} ms`);

// Speculative decoding (greedy version): a small draft model guesses k tokens,
// the big model checks all k in ONE forward pass, keeps the agreeing prefix
// and adds its own next token. The output is exactly what the big model alone
// would have written; only the number of big-model passes changes.
const target = ("the refund window is thirty days from delivery and store credit is offered " +
  "after that window closes so customers should contact support with the order number").split(" ");

// the big model: greedy next token for a given prefix
const bigModel = (prefix) => target[prefix.length];
// a draft model that agrees with the big one except at some positions
function makeDraft(wrongEvery) {
  return (prefix) => {
    const i = prefix.length;
    return i % wrongEvery === wrongEvery - 1 ? "uh" : target[i];
  };
}

function plainDecode() {
  const out = [];
  let bigPasses = 0;
  while (out.length < target.length) { out.push(bigModel(out)); bigPasses++; }
  return { out, bigPasses, draftPasses: 0 };
}

function speculative(draft, k) {
  const out = [];
  let bigPasses = 0, draftPasses = 0;
  while (out.length < target.length) {
    const guess = [];
    for (let j = 0; j < k && out.length + guess.length < target.length; j++) {
      guess.push(draft([...out, ...guess])); draftPasses++;
    }
    bigPasses++; // one pass scores every guessed position at once
    let accepted = 0;
    while (accepted < guess.length && guess[accepted] === bigModel([...out, ...guess.slice(0, accepted)])) accepted++;
    out.push(...guess.slice(0, accepted));
    if (out.length < target.length) out.push(bigModel(out)); // the big model's own token, free from the same pass
  }
  return { out, bigPasses, draftPasses };
}

const BIG_MS = 30, DRAFT_MS = 3; // assumed cost of one forward pass
const report = (name, r) => {
  const ms = r.bigPasses * BIG_MS + r.draftPasses * DRAFT_MS;
  const same = r.out.join(" ") === target.join(" ") ? "identical" : "DIFFERENT";
  console.log(`  ${name.padEnd(28)} big passes ${String(r.bigPasses).padStart(2)}  draft passes ${String(r.draftPasses).padStart(2)}  est. ${String(ms).padStart(4)} ms  output ${same}`);
};
console.log(`${target.length} tokens to generate; assume ${BIG_MS} ms per big pass, ${DRAFT_MS} ms per draft pass`);
report("plain decoding", plainDecode());
report("draft wrong 1 in 8, k=4", speculative(makeDraft(8), 4));
report("draft wrong 1 in 8, k=8", speculative(makeDraft(8), 8));
report("draft wrong 1 in 3, k=4", speculative(makeDraft(3), 4));
report("draft always wrong, k=4", speculative(makeDraft(1), 4));