What is semantic caching and when is it safe to use?

// A semantic cache with the rails that make it safe: scope keys, TTL,
// a bypass for transactional requests and an exact-value guard.
// embed() is a toy stand-in for an embedding model: it maps words to
// concepts, so it captures what a text is about and blurs exact values.
const LEX = {
  password: ["password", 1], reset: ["reset", 1], forgot: ["reset", 1], steps: ["procedure", 1],
  how: ["procedure", 0.3], can: ["ability", 0.3], cancel: ["cancel", 1], order: ["order", 1],
  refund: ["refund", 1], policy: ["policy", 1], convert: ["convert", 1], usd: ["usd", 1],
  eur: ["eur", 1], store: ["store", 1], open: ["open", 1], when: ["time", 0.3],
};
const STOP = new Set(["do", "i", "my", "to", "is", "your", "the", "a", "an", "what", "does", "on"]);
const words = (q) => q.toLowerCase().replace(/[?.!,]/g, "").split(" ").filter(Boolean);
function embed(q) {
  const v = {};
  for (const w of words(q)) {
    if (STOP.has(w)) continue;
    const [dim, wt] = /^\d+$/.test(w) ? ["NUM", 1] : LEX[w] || ["w:" + w, 1];
    v[dim] = (v[dim] || 0) + wt;
  }
  return v;
}
function cosine(a, b) {
  let dot = 0, na = 0, nb = 0;
  for (const k of Object.keys(a).sort()) { na += a[k] * a[k]; if (k in b) dot += a[k] * b[k]; }
  for (const k of Object.keys(b).sort()) nb += b[k] * b[k];
  return dot / Math.sqrt(na * nb);
}
const numbers = (q) => words(q).filter((w) => /^\d+$/.test(w)).join(",");
const ACTIONS = new Set(["cancel", "delete", "change"]);
const isTransactional = (q) => { const ws = words(q); return ws.some((w) => ACTIONS.has(w)) && (ws.includes("my") || ACTIONS.has(ws[0])); };

const THRESHOLD = 0.9, TTL = 24 * 3600; // seconds
const cache = [];
function put(query, answer, scope, t) { cache.push({ query, vec: embed(query), answer, scope, t }); }
function lookup(query, scope, t, guarded) {
  if (guarded && isTransactional(query)) return { kind: "BYPASS", why: "transactional, always call the model" };
  const v = embed(query);
  let best = null, bestSim = -1, expired = false;
  for (const e of cache) {
    if (guarded && e.scope !== scope) continue;
    if (guarded && t - e.t > TTL) { expired = true; continue; }
    const sim = cosine(v, e.vec);
    if (sim > bestSim) { best = e; bestSim = sim; }
  }
  if (!best || bestSim < THRESHOLD) return { kind: "MISS", why: best ? `best ${bestSim.toFixed(3)} < ${THRESHOLD}` : expired ? "entries in scope are past the 24h TTL" : "no entry in this scope" };
  if (guarded && numbers(query) !== numbers(best.query)) return { kind: "MISS", why: `sim ${bestSim.toFixed(3)} but numbers differ` };
  return { kind: "HIT", why: `sim ${bestSim.toFixed(3)} to "${best.query}"`, answer: best.answer };
}

const S = "acme|prompt-v3|model-a"; // scope: tenant | prompt version | model
put("How do I reset my password?", "Settings > Security > Reset password.", S, 0);
put("Can I cancel an order?", "Orders can be cancelled within 1 hour.", S, 0);
put("Convert 100 USD to EUR", "100 USD is about 92 EUR.", S, 0);
put("What is your refund policy?", "Refunds within 30 days.", S, 0);
put("When does the store open?", "9am every day.", S, 0);

const probes = [
  ["password reset steps", S, 60],
  ["password reset steps", "globex|prompt-v3|model-a", 60],
  ["password reset steps", "acme|prompt-v4|model-a", 60],
  ["cancel my order", S, 60],
  ["Convert 250 USD to EUR", S, 60],
  ["When does the store open on Sunday?", S, 60],
  ["What is your refund policy?", S, 25 * 3600],
];
for (const [q, scope, t] of probes) {
  console.log(`"${q}"  [${scope.split("|").slice(0, 2).join(" ")}${t > TTL ? ", 25h later" : ""}]`);
  for (const guarded of [false, true]) {
    const r = lookup(q, scope, t, guarded);
    const ans = r.answer ? ` -> "${r.answer}"` : "";
    console.log(`  ${guarded ? "guarded" : "naive  "} ${r.kind.padEnd(6)} ${r.why}${ans}`);
  }
}

// Pick the threshold from labelled data, not from a blog post.
// Each pair: a new query, the cached query it is nearest to, and whether the
// cached answer is actually correct for the new query (labelled by a human).
const LEX = {
  password: ["password", 1], reset: ["reset", 1], forgot: ["reset", 1], steps: ["procedure", 1],
  how: ["procedure", 0.3], can: ["ability", 0.3], cancel: ["cancel", 1], order: ["order", 1],
  refund: ["refund", 1], policy: ["policy", 1], convert: ["convert", 1], usd: ["usd", 1],
  eur: ["eur", 1], store: ["store", 1], open: ["open", 1], when: ["time", 0.3],
};
const STOP = new Set(["do", "i", "my", "to", "is", "your", "the", "a", "an", "what", "does", "on", "for"]);
const words = (q) => q.toLowerCase().replace(/[?.!,]/g, "").split(" ").filter(Boolean);
function embed(q) {
  const v = {};
  for (const w of words(q)) {
    if (STOP.has(w)) continue;
    const [dim, wt] = /^\d+$/.test(w) ? ["NUM", 1] : LEX[w] || ["w:" + w, 1];
    v[dim] = (v[dim] || 0) + wt;
  }
  return v;
}
function cosine(a, b) {
  let dot = 0, na = 0, nb = 0;
  for (const k of Object.keys(a).sort()) { na += a[k] * a[k]; if (k in b) dot += a[k] * b[k]; }
  for (const k of Object.keys(b).sort()) nb += b[k] * b[k];
  return dot / Math.sqrt(na * nb);
}
const numbers = (q) => words(q).filter((w) => /^\d+$/.test(w)).join(",");
const ACTIONS = new Set(["cancel", "delete", "change"]);
const isTransactional = (q) => { const ws = words(q); return ws.some((w) => ACTIONS.has(w)) && (ws.includes("my") || ACTIONS.has(ws[0])); };

const pairs = [
  ["password reset steps", "How do I reset my password?", true],
  ["forgot password how to reset", "How do I reset my password?", true],
  ["reset my password", "How do I reset my password?", true],
  ["refund policy", "What is your refund policy?", true],
  ["Convert 100 USD to EUR please", "Convert 100 USD to EUR", true],
  ["can I cancel my order", "Can I cancel an order?", true],
  ["what time does the store open", "When does the store open?", true],
  ["cancel my order", "Can I cancel an order?", false],
  ["Convert 250 USD to EUR", "Convert 100 USD to EUR", false],
  ["When does the store open on Sunday?", "When does the store open?", false],
  ["refund policy for electronics", "What is your refund policy?", false],
  ["delete my account", "How do I reset my password?", false],
];
console.log("similarity per labelled pair:");
for (const [q, c, ok] of pairs) console.log(`  ${cosine(embed(q), embed(c)).toFixed(3)}  ${ok ? "same " : "DIFF "} "${q}"`);
console.log("");
console.log("threshold  mode     served  correct  WRONG  missed-good");
for (const th of [0.8, 0.85, 0.9, 0.95, 0.99]) {
  for (const guarded of [false, true]) {
    let served = 0, correct = 0, wrong = 0, missed = 0;
    for (const [q, c, ok] of pairs) {
      let hit = cosine(embed(q), embed(c)) >= th;
      if (guarded && (isTransactional(q) || numbers(q) !== numbers(c))) hit = false;
      if (hit) { served++; if (ok) correct++; else wrong++; } else if (ok) missed++;
    }
    console.log(`  ${th.toFixed(2)}     ${(guarded ? "guarded" : "naive").padEnd(8)} ${String(served).padStart(4)}  ${String(correct).padStart(6)}  ${String(wrong).padStart(5)}  ${String(missed).padStart(6)}`);
  }
}