How would you design OpenTelemetry tracing for 200 microservices?

// W3C traceparent propagation, broken propagation, capacity for 200 services, and trace-ID routing to collectors.
let seed = 7;
function rand() { seed = (seed * 48271) % 2147483647; return seed / 2147483647; }
const hex8 = () => Math.floor(rand() * 4294967296).toString(16).padStart(8, "0");
const newTraceId = () => hex8() + hex8() + hex8() + hex8();
const newSpanId = () => hex8() + hex8();
const RE = /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/;

function parse(header) {
  const m = RE.exec(header || "");
  if (!m) return null;
  const [, version, traceId, parentId, flags] = m;
  if (version === "ff" || /^0+$/.test(traceId) || /^0+$/.test(parentId)) return null;
  return { traceId, parentId, sampled: (parseInt(flags, 16) & 1) === 1 };
}
// Start a span: continue the incoming context if valid, else start a new trace (a new root).
function startSpan(service, header) {
  const ctx = parse(header);
  const span = { service, traceId: ctx ? ctx.traceId : newTraceId(), spanId: newSpanId(), parentId: ctx ? ctx.parentId : null };
  span.outgoing = `00-${span.traceId}-${span.spanId}-01`;
  return span;
}
const show = (s) => console.log(`  ${s.service.padEnd(16)} trace=${s.traceId.slice(0, 8)}.. span=${s.spanId} parent=${s.parentId || "(root)"}`);

console.log("== 1. One request across HTTP and a Kafka message ==");
const gw = startSpan("api-gateway", null);
const orders = startSpan("orders", gw.outgoing);
const inv = startSpan("inventory", orders.outgoing);
const kafkaHeaders = { traceparent: orders.outgoing }; // producer injects into message headers
const billing = startSpan("billing-worker", kafkaHeaders.traceparent);
[gw, orders, inv, billing].forEach(show);
console.log(`  header sent by orders: traceparent: ${orders.outgoing}`);
console.log(`  same trace everywhere: ${[orders, inv, billing].every((s) => s.traceId === gw.traceId)}`);

console.log("\n== 2. Header validation (invalid means: start a new trace) ==");
const tid = gw.traceId;
for (const [label, h] of [["valid", `00-${tid}-00f067aa0ba902b7-01`], ["not sampled", `00-${tid}-00f067aa0ba902b7-00`],
  ["all-zero trace id", `00-${"0".repeat(32)}-00f067aa0ba902b7-01`], ["uppercase hex", `00-${tid.toUpperCase()}-00f067aa0ba902b7-01`],
  ["version ff", `ff-${tid}-00f067aa0ba902b7-01`], ["missing", null]]) {
  const c = parse(h);
  console.log(`  ${label.padEnd(18)} -> ${c ? `continue trace, sampled=${c.sampled}` : "rejected, new root span"}`);
}

console.log("\n== 3. A legacy proxy in the middle that drops headers ==");
const a = startSpan("web", null);
const legacy = { service: "legacy-proxy" }; // not instrumented: forwards the call, not the header
const b = startSpan("search", undefined);
const c2 = startSpan("ranking", b.outgoing);
[a, b, c2].forEach(show);
const ids = new Set([a.traceId, b.traceId, c2.traceId]);
console.log(`  one user request became ${ids.size} traces; ${legacy.service} broke the chain`);

console.log("\n== 4. Capacity for 200 services (assumptions in, arithmetic out) ==");
const peakRps = 20000, avgRps = 10000, hops = 8, spansPerHop = 3, bytesPerSpan = 500, perCollector = 30000;
const peakSpans = peakRps * hops * spansPerHop, avgSpans = avgRps * hops * spansPerHop;
console.log(`  peak spans/s = ${peakRps} req/s x ${hops} services x ${spansPerHop} spans = ${peakSpans}`);
console.log(`  peak ingest = ${(peakSpans * bytesPerSpan) / 1e6} MB/s at ${bytesPerSpan} B/span`);
const perDayTB = (avgSpans * 86400 * bytesPerSpan) / 1e12;
console.log(`  spans/day at average load = ${avgSpans * 86400} -> ${perDayTB.toFixed(2)} TB/day unsampled`);
for (const pct of [100, 10, 1]) console.log(`  keep ${String(pct).padStart(3)}%: ${((perDayTB * pct * 7) / 100).toFixed(2)} TB for 7 days of retention`);
const collectors = Math.ceil((peakSpans / perCollector) * 1.3);
console.log(`  gateway collectors = ceil(${peakSpans} / ${perCollector} x 1.3 headroom) = ${collectors}`);

console.log("\n== 5. Routing spans to tail-sampling collectors ==");
function fnv1a(s) { let h = 0x811c9dc5; for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 16777619) >>> 0; }
  h ^= h >>> 16; h = Math.imul(h, 0x85ebca6b) >>> 0; h ^= h >>> 13; h = Math.imul(h, 0xc2b2ae35) >>> 0; h ^= h >>> 16; return h >>> 0; } // FNV-1a + murmur3 finalizer
const traceIds = Array.from({ length: 20000 }, newTraceId);
let rr = 0, rrSeen = 0;
for (const t of traceIds.slice(0, 1000)) { const hit = new Set(); for (let s = 0; s < 24; s++) hit.add(rr++ % 4); rrSeen += hit.size; }
console.log(`  round robin, 4 collectors: an average trace is split over ${(rrSeen / 1000).toFixed(2)} collectors`);
let hashSeen = 0;
for (const t of traceIds.slice(0, 1000)) { const hit = new Set(); for (let s = 0; s < 24; s++) hit.add(fnv1a(t) % 4); hashSeen += hit.size; }
console.log(`  hash(traceID), 4 collectors: an average trace is split over ${(hashSeen / 1000).toFixed(2)} collectors`);
function ring(n) {
  const pts = [];
  for (let k = 0; k < n; k++) for (let v = 0; v < 100; v++) pts.push([fnv1a(`collector-${k}#${v}`), k]);
  return pts.sort((x, y) => x[0] - y[0]);
}
function lookup(pts, h) { let lo = 0, hi = pts.length; while (lo < hi) { const mid = (lo + hi) >> 1; if (pts[mid][0] < h) lo = mid + 1; else hi = mid; } return pts[lo % pts.length][1]; }
const r4 = ring(4), r5 = ring(5);
let modMoved = 0, ringMoved = 0;
for (const t of traceIds) {
  const h = fnv1a(t);
  if (h % 4 !== h % 5) modMoved++;
  if (lookup(r4, h) !== lookup(r5, h)) ringMoved++;
}
console.log(`  scale 4 -> 5 collectors, ${traceIds.length} in-flight traces change owner:`);
console.log(`    hash mod N:        ${modMoved} (${((modMoved * 100) / traceIds.length).toFixed(1)}%)`);
console.log(`    consistent ring:   ${ringMoved} (${((ringMoved * 100) / traceIds.length).toFixed(1)}%)`);