What are RLHF and DPO, and how do they turn a base model into an assistant?

// Preference tuning on one prompt with four candidate responses.
// 1) fit a reward model from human preference pairs (Bradley-Terry),
// 2) RLHF target: maximise reward minus beta*KL to the SFT model,
// 3) DPO: train the policy directly on the same pairs, no reward model.
const responses = ["helpful", "curt", "refusal", "confident-wrong"];
const ref = [0.30, 0.25, 0.15, 0.30]; // the SFT model: wrong answers are as likely as good ones
// Labeller comparisons: [chosen, rejected, how many labellers]
const pairs = [[0, 1, 7], [1, 0, 3], [0, 2, 8], [2, 0, 2], [0, 3, 9], [3, 0, 1], [1, 2, 6], [2, 1, 4], [1, 3, 7], [3, 1, 3], [2, 3, 6], [3, 2, 4]];
const sigmoid = (z) => 1 / (1 + Math.exp(-z));
const softmax = (z) => { const m = Math.max(...z); const e = z.map((v) => Math.exp(v - m)); const s = e.reduce((a, b) => a + b, 0); return e.map((v) => v / s); };
const kl = (p, q) => p.reduce((s, pi, i) => s + pi * Math.log(pi / q[i]), 0);
const fmt = (p) => p.map((v, i) => responses[i] + "=" + v.toFixed(3)).join(" ");
const total = pairs.reduce((s, p) => s + p[2], 0);
console.log("SFT policy        :", fmt(ref));

// 1) Reward model: P(chosen beats rejected) = sigmoid(r_c - r_r); maximise log-likelihood.
let r = [0, 0, 0, 0];
for (let step = 0; step < 4000; step++) {
  const g = [0, 0, 0, 0];
  for (const [c, l, n] of pairs) { const p = sigmoid(r[c] - r[l]); g[c] += n * (1 - p); g[l] -= n * (1 - p); }
  r = r.map((v, i) => v + 1.0 * g[i] / total);
}
const mean = r.reduce((a, b) => a + b, 0) / 4;
r = r.map((v) => v - mean); // rewards are only defined up to a constant
console.log("reward model      :", fmt(r));
let agree = 0;
for (const [c, l, n] of pairs) if (r[c] > r[l]) agree += n;
console.log("reward model agrees with", agree, "of", total, "labeller votes");

// 2) RLHF objective E[r] - beta*KL(pi || ref) has the closed-form optimum pi ~ ref * exp(r / beta).
//    PPO approximates this by sampling; here we compute the target exactly.
for (const beta of [4.0, 1.0, 0.1]) {
  const pi = softmax(ref.map((p, i) => Math.log(p) + r[i] / beta));
  const er = pi.reduce((s, p, i) => s + p * r[i], 0);
  console.log(`RLHF beta=${beta.toFixed(1)}     :`, fmt(pi), `| E[r]=${er.toFixed(3)} KL=${kl(pi, ref).toFixed(3)}`);
}

// 3) DPO: loss = -log sigmoid(beta * ((log pi_c - log ref_c) - (log pi_r - log ref_r))) over the pairs.
function dpo(beta, steps, lr) {
  let theta = ref.map(Math.log); // start from the SFT model
  for (let step = 0; step < steps; step++) {
    const lp = softmax(theta).map(Math.log);
    const g = [0, 0, 0, 0];
    for (const [c, l, n] of pairs) {
      const margin = beta * ((lp[c] - Math.log(ref[c])) - (lp[l] - Math.log(ref[l])));
      const w = n * beta * (1 - sigmoid(margin)); // the softmax normaliser cancels in the difference
      g[c] += w; g[l] -= w;
    }
    theta = theta.map((v, i) => v + lr * g[i] / total);
  }
  return softmax(theta);
}
const beta = 1.0;
const pi = dpo(beta, 4000, 1.0);
console.log(`DPO beta=${beta.toFixed(1)}      :`, fmt(pi), `| KL=${kl(pi, ref).toFixed(3)}`);
const rlhf = softmax(ref.map((p, i) => Math.log(p) + r[i] / beta));
const gap = Math.max(...pi.map((p, i) => Math.abs(p - rlhf[i])));
console.log(`max |DPO - RLHF optimum| at beta=${beta.toFixed(1)}:`, gap < 1e-6 ? "below 1e-6" : gap.toFixed(6));
const implied = pi.map((p, i) => beta * Math.log(p / ref[i]));
const im = implied.reduce((a, b) => a + b, 0) / 4;
console.log("DPO implied reward:", fmt(implied.map((v) => v - im)));
const order = (x) => x.map((v, i) => [v, i]).sort((a, b) => b[0] - a[0]).map((t) => responses[t[1]]).join(" > ");
console.log("ranking by reward model:", order(r));
console.log("ranking by DPO policy  :", order(implied));