How would you design a distributed cache?

// A distributed cache routing layer in the style of Redis Cluster:
// CRC16 -> 16384 hash slots -> nodes, hash tags, resharding, and a hot key.

function crc16(s) { // CRC-16/XMODEM (poly 0x1021, init 0), the variant Redis Cluster uses
  let crc = 0;
  for (let i = 0; i < s.length; i++) {
    crc ^= s.charCodeAt(i) << 8;
    for (let b = 0; b < 8; b++) crc = crc & 0x8000 ? ((crc << 1) ^ 0x1021) & 0xffff : (crc << 1) & 0xffff;
  }
  return crc;
}
function hashPart(key) { // only the text inside the first {...} is hashed, if non-empty
  const open = key.indexOf("{");
  if (open >= 0) {
    const close = key.indexOf("}", open + 1);
    if (close > open + 1) return key.substring(open + 1, close);
  }
  return key;
}
const slotOf = (key) => crc16(hashPart(key)) % 16384;

console.log("Part 1: key -> slot");
console.log("  crc16(123456789) = 0x" + crc16("123456789").toString(16).toUpperCase());
for (const k of ["user:1000", "{user:42}:profile", "{user:42}:cart", "user:42:profile"]) {
  console.log(`  ${k.padEnd(18)} slot ${String(slotOf(k)).padStart(5)}`);
}

// Slot table: owner[slot] = node index. Three nodes start with even ranges.
const names = ["A", "B", "C", "D"];
const owner = new Array(16384);
for (let s = 0; s < 16384; s++) owner[s] = s <= 5460 ? 0 : s <= 10922 ? 1 : 2;
const KEYS = 30000;
const keys = [];
for (let i = 0; i < KEYS; i++) keys.push("key:" + i);
function countPerNode(n) {
  const c = new Array(n).fill(0);
  for (const k of keys) c[owner[slotOf(k)]]++;
  return c.map((x, i) => names[i] + "=" + x).join(" ");
}
console.log("\nPart 2: 30000 keys on 3 nodes:", countPerNode(3));

const before = keys.map((k) => owner[slotOf(k)]);
// Add node D: move the top 1365 slots of A, 1365 of B and 1366 of C (4096 slots in total).
let moved = 0;
for (const [lo, hi] of [[5460 - 1364, 5460], [10922 - 1364, 10922], [16383 - 1365, 16383]]) {
  for (let s = lo; s <= hi; s++) { owner[s] = 3; moved++; }
}
let keysMoved = 0;
keys.forEach((k, i) => { if (owner[slotOf(k)] !== before[i]) keysMoved++; });
let modMoved = 0;
for (const k of keys) if (crc16(k) % 3 !== crc16(k) % 4) modMoved++;
console.log(`  add node D, ${moved} slots move:`, countPerNode(4));
console.log(`  keys that changed node with slots:    ${keysMoved} (${Math.round((keysMoved * 100) / KEYS)}%)`);
console.log(`  keys that changed node with hash % N: ${modMoved} (${Math.round((modMoved * 100) / KEYS)}%)`);

console.log("\nPart 3: a hot key, 100000 reads in 10 s from 50 app servers");
const load = [0, 0, 0, 0];
const hot = "product:42";
let hotReads = 0;
for (let i = 0; i < 100000; i++) {
  const k = i % 10 < 3 ? hot : keys[(i * 7919) % KEYS]; // 30% of reads hit one key
  if (k === hot) hotReads++;
  load[owner[slotOf(k)]]++;
}
const hotNode = owner[slotOf(hot)];
console.log("  without a local cache:", load.map((x, i) => names[i] + "=" + x).join(" "), "(hot key on " + names[hotNode] + ")");
// Each app server keeps the hot key in process memory for 1 s: at most 1 fetch per server per second.
const l1Fetches = 50 * 10;
load[hotNode] = load[hotNode] - hotReads + l1Fetches;
console.log("  with a 1 s local cache:", load.map((x, i) => names[i] + "=" + x).join(" "));