What is the difference between fine-tuning, LoRA and prompt engineering?

// Three ways to adapt a "pretrained" model, on a toy linear layer y = W x.
// Prompt engineering changes the input; full fine-tuning updates every weight;
// LoRA freezes W0 and trains a low-rank update B*A.
let seed = 7;
const rand = () => { seed = (seed * 48271) % 2147483647; return seed / 2147483647 * 2 - 1; };
const D_IN = 9, D_OUT = 8, N = 64; // input = 8 features + 1 "instruction" slot
const mat = (r, c, f) => Array.from({ length: r }, () => Array.from({ length: c }, f));
const W0 = mat(D_OUT, D_IN, () => rand() * 0.5);          // the pretrained weights
const X = mat(N, D_IN - 1, () => rand());                  // 64 inputs, no instruction yet
const withFlag = (x, f) => x.concat([f]);
const matvec = (W, x) => W.map((row) => row.reduce((s, w, j) => s + w * x[j], 0));
const add = (A, B) => A.map((row, i) => row.map((v, j) => v + B[i][j]));
const mul = (A, B) => A.map((row) => B[0].map((_, j) => row.reduce((s, v, k) => s + v * B[k][j], 0)));
const T = (A) => A[0].map((_, j) => A.map((row) => row[j]));
const lossOf = (W, inputs, targets) => {
  let s = 0;
  inputs.forEach((x, n) => matvec(W, x).forEach((v, i) => { s += (v - targets[n][i]) ** 2; }));
  return s / (inputs.length * D_OUT);
};
const gradOf = (W, inputs, targets) => { // dLoss/dW
  const G = mat(D_OUT, D_IN, () => 0);
  inputs.forEach((x, n) => matvec(W, x).forEach((v, i) => {
    const e = 2 * (v - targets[n][i]) / (inputs.length * D_OUT);
    for (let j = 0; j < D_IN; j++) G[i][j] += e * x[j];
  }));
  return G;
};
const f4 = (v) => v.toFixed(4);

// Task A: a skill the model already has, switched on by the instruction slot.
const inA = X.map((x) => withFlag(x, 1));
const yA = inA.map((x) => matvec(W0, x));
console.log("Task A (latent skill) loss, plain prompt :", f4(lossOf(W0, X.map((x) => withFlag(x, 0)), yA)));
console.log("Task A (latent skill) loss, with instruction:", f4(lossOf(W0, inA, yA)), "| trainable params: 0");

// Task B: genuinely new behaviour = W0 plus a rank-k change.
function lowRankDelta(k) {
  let d = mat(D_OUT, D_IN, () => 0);
  for (let t = 0; t < k; t++) d = add(d, mul(mat(D_OUT, 1, () => rand() * 0.6), mat(1, D_IN, () => rand() * 0.6)));
  return d;
}
function fullFT(inputs, targets, steps, lr) {
  let W = W0.map((r) => r.slice());
  for (let s = 0; s < steps; s++) { const G = gradOf(W, inputs, targets); W = W.map((row, i) => row.map((w, j) => w - lr * G[i][j])); }
  return lossOf(W, inputs, targets);
}
function lora(inputs, targets, r, steps, lr) {
  let B = mat(D_OUT, r, () => 0), A = mat(r, D_IN, () => rand() * 0.5); // B starts at 0: no change at step 0
  for (let s = 0; s < steps; s++) {
    const G = gradOf(add(W0, mul(B, A)), inputs, targets);
    const gB = mul(G, T(A)), gA = mul(T(B), G);
    B = B.map((row, i) => row.map((v, j) => v - lr * gB[i][j]));
    A = A.map((row, i) => row.map((v, j) => v - lr * gA[i][j]));
  }
  return { loss: lossOf(add(W0, mul(B, A)), inputs, targets), B, A };
}
for (const k of [1, 3]) {
  const Wt = add(W0, lowRankDelta(k));
  const yB = inA.map((x) => matvec(Wt, x));
  console.log(`Task B (new behaviour, true change has rank ${k}):`);
  console.log("  prompt only (0 params)   loss:", f4(lossOf(W0, inA, yB)));
  console.log("  full fine-tune (" + D_OUT * D_IN + " params) loss:", f4(fullFT(inA, yB, 400, 1.5)));
  for (const r of [1, 4]) {
    const res = lora(inA, yB, r, 400, 1.5);
    console.log(`  LoRA r=${r} (${r * (D_IN + D_OUT)} params)     loss:`, f4(res.loss));
    if (k === 3 && r === 4) { // merge check: W0 + B*A gives the same output as the adapter path
      const x = inA[0], merged = matvec(add(W0, mul(res.B, res.A)), x);
      const path = matvec(W0, x).map((v, i) => v + matvec(res.B, matvec(res.A, x))[i]);
      console.log("  merged vs adapter path, max diff:", Math.max(...merged.map((v, i) => Math.abs(v - path[i]))) < 1e-12 ? "below 1e-12" : "MISMATCH");
    }
  }
}

// Scale it up: LoRA on the q,k,v,o projections of an assumed 7B-class shape.
const hidden = 4096, layers = 32, totalParams = 6.7e9;
for (const r of [8, 16, 64]) {
  const p = layers * 4 * r * (hidden + hidden);
  console.log(`r=${r}: LoRA trains ${p.toLocaleString("en-US")} params = ${(100 * p / totalParams).toFixed(2)}% of ${totalParams / 1e9}B`);
}
const gb = (bytes) => (bytes / 1e9).toFixed(1) + " GB";
console.log("full FT, mixed-precision Adam (16 bytes/param):", gb(totalParams * 16));
console.log("LoRA, frozen 16-bit base (2 bytes/param)      :", gb(totalParams * 2), "+ adapters and their optimizer state");
console.log("QLoRA, frozen 4-bit base (0.5 bytes/param)    :", gb(totalParams * 0.5), "+ adapters and their optimizer state");