Head vs tail sampling in OpenTelemetry — when does each break?

// Head vs tail sampling on 100,000 simulated traces (6 spans each): what each keeps, and where each breaks.
let seed = 2024;
function rand() { seed = (seed * 48271) % 2147483647; return seed / 2147483647; }
const N = 100000, SPANS = 6, ROOT = 5, SLOW_MS = 2000; // spans 0..4 are the request (4 is the root)
const pct = (a, b) => ((a * 100) / b).toFixed(1) + "%";

const traces = [];
for (let i = 0; i < N; i++) {
  const r = rand(), error = r < 0.003, slow = r >= 0.003 && r < 0.008;
  const dur = slow ? 2000 + Math.floor(rand() * 18000) : 40 + Math.floor(rand() * 160);
  const ends = []; for (let k = 0; k < ROOT; k++) ends.push(Math.floor((dur * (k + 1)) / ROOT)); // span 4 = root, ends at dur
  ends.push(rand() < 0.02 ? dur + 1000 + Math.floor(rand() * 119000) : dur); // span 5 = async consumer, sometimes late
  const errSpan = error ? Math.floor(rand() * SPANS) : -1;
  traces.push({ id: Math.floor(rand() * 4294967296), error, slow, dur, ends, errSpan });
}
const errors = traces.filter((t) => t.error).length, slows = traces.filter((t) => t.slow).length;
console.log(`population: ${N} traces, ${N * SPANS} spans, ${errors} with an error, ${slows} slower than ${SLOW_MS} ms`);

console.log("\n== HEAD sampling 1% (decided at the root from the trace ID, before anything happens) ==");
const headKeep = (t, ratio) => t.id < ratio * 4294967296;
const head = traces.filter((t) => headKeep(t, 0.01));
const hErr = head.filter((t) => t.error).length, hSlow = head.filter((t) => t.slow).length;
console.log(`kept ${head.length} traces; errors kept ${hErr} of ${errors}; slow kept ${hSlow} of ${slows}`);
console.log(`estimated totals (count x 100): requests ${head.length * 100} (true ${N}), errors ${hErr * 100} (true ${errors})`);

console.log("\n== Why every service must follow one decision (10% rate, 6 services) ==");
let indep = { full: 0, partial: 0 }, consistent = { full: 0, partial: 0 };
for (const t of traces) {
  let k = 0; for (let s = 0; s < SPANS; s++) if (rand() < 0.1) k++;
  if (k === SPANS) indep.full++; else if (k > 0) indep.partial++;
  if (headKeep(t, 0.1)) consistent.full++;
}
console.log(`each service flips its own coin: complete ${indep.full}, broken partial traces ${indep.partial}`);
console.log(`parent-based / same trace-ID ratio everywhere: complete ${consistent.full}, partial ${consistent.partial}`);

function tailDecide(t, seen) {
  if (t.errSpan >= 0 && seen[t.errSpan]) return "error";
  if (seen[ROOT - 1] && t.ends[ROOT - 1] >= SLOW_MS) return "latency"; // root span duration
  return headKeep(t, 0.01) ? "probabilistic" : null;
}
console.log("\n== TAIL sampling: keep errors, keep >= 2 s, keep 1% of the rest ==");
const all = Array(SPANS).fill(true);
let tKept = 0, tErr = 0, tSlow = 0;
for (const t of traces) { const d = tailDecide(t, all); if (d) { tKept++; if (t.error) tErr++; if (t.slow) tSlow++; } }
console.log(`with whole traces: kept ${tKept} (${pct(tKept, N)} of traffic); errors kept ${tErr} of ${errors}; slow kept ${tSlow} of ${slows}`);

console.log("\n== TAIL break 1: spans not routed by trace ID (round robin over 4 collectors) ==");
let rr = 0, full = 0, part = 0, spansKept = 0, errTraces = 0;
for (const t of traces) {
  const owner = []; for (let s = 0; s < SPANS; s++) owner.push(rr++ % 4);
  if (!t.error) continue;
  errTraces++;
  let kept = 0;
  for (let c = 0; c < 4; c++) {
    const seen = owner.map((o) => o === c);
    if (!seen.some(Boolean)) continue;
    if (tailDecide(t, seen)) kept += seen.filter(Boolean).length;
  }
  spansKept += kept; if (kept === SPANS) full++; else part++;
}
console.log(`error traces kept whole ${full}, kept as fragments ${part}; ${pct(spansKept, errTraces * SPANS)} of their spans survived`);

console.log("\n== TAIL break 2: decision_wait vs spans that arrive late ==");
console.log("wait     errors caught  slow caught  kept but incomplete  buffer at 480k spans/s x 500 B");
for (const wait of [1000, 5000, 10000, 30000]) {
  let e = 0, s = 0, inc = 0;
  for (const t of traces) {
    const cutoff = t.ends[0] + wait, seen = t.ends.map((x) => x <= cutoff);
    if (tailDecide(t, seen)) { if (t.error) e++; if (t.slow) s++; if (!seen.every(Boolean)) inc++; }
  }
  const gb = (480000 * (wait / 1000) * 500) / 1e9;
  console.log(`${String(wait / 1000).padStart(2)} s ${String(e).padStart(10)}/${errors} ${String(s).padStart(9)}/${slows} ${String(inc).padStart(15)} ${gb.toFixed(2).padStart(17)} GB`);
}