How do you design good tool definitions for an LLM?

// Lint tool definitions the way a model will read them: names, descriptions, params, overlap.
const SETS = {
  "bad": [
    {"name": "get_user", "description": "Gets user data.", "params": {"q": {"type": "string"}}, "required": ["q"]},
    {"name": "fetch_account", "description": "Fetch account info for a customer.", "params": {"id": {"type": "string"}}, "required": ["id"]},
    {"name": "lookup profile", "description": "Lookup profile.", "params": {"input": {"type": "string"}}, "required": []},
    {"name": "order", "description": "Order operations.", "params": {"action": {"type": "string"}, "data": {"type": "string"}, "opts": {"type": "string"}, "flag": {"type": "boolean"}, "extra": {"type": "string"}}, "required": []},
    {"name": "refund", "description": "Handles refunds and orders.", "params": {"order": {"type": "string"}, "amount": {"type": "string"}}, "required": ["order"]}
  ],
  "good": [
    {"name": "customers_find", "description": "Find a customer by email, name or phone and return their customer id, plan and account status. Use this first when the user mentions a person but gives no customer id. Do not use for order or refund questions.", "params": {"query": {"type": "string", "description": "Email, full name or phone, e.g. ann@example.com"}}, "required": ["query"]},
    {"name": "orders_get_status", "description": "Get the shipping status and line items of one order by its order id. Use when the user asks where an order is or what it contains. Do not use to cancel or refund.", "params": {"order_id": {"type": "string", "pattern": "^A-[0-9]+$", "description": "Order id, e.g. A-17"}}, "required": ["order_id"]},
    {"name": "orders_cancel", "description": "Cancel or stop an order that has not shipped yet. Use only when the user explicitly asks to cancel or stop it. Do not use for refunds of delivered orders.", "params": {"order_id": {"type": "string", "pattern": "^A-[0-9]+$", "description": "Order id, e.g. A-17"}, "reason": {"type": "string", "enum": ["customer_request", "duplicate", "fraud"], "description": "Why the order is cancelled"}}, "required": ["order_id", "reason"]},
    {"name": "refunds_create", "description": "Refund a delivered order, fully or partly, and return the refund id. Use when the user asks for money back on an order that has arrived. Do not use to cancel an order that has not shipped.", "params": {"order_id": {"type": "string", "pattern": "^A-[0-9]+$", "description": "Order id, e.g. A-17"}, "amount_cents": {"type": ["integer", "null"], "description": "Amount in cents; null refunds the full order"}}, "required": ["order_id", "amount_cents"]}
  ]
};
const STOP = new Set(["the", "and", "for", "use", "this", "that", "not", "one", "its", "their", "when", "has", "yet", "only", "with", "get", "gets", "data", "info"]);
const words = (s) => new Set(s.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 2 && !STOP.has(w)).map((w) => (w.length > 4 && w.endsWith("s") ? w.slice(0, -1) : w)));
const jaccard = (a, b) => { const i = [...a].filter((x) => b.has(x)).length; return i / (a.size + b.size - i); };
const sentences = (s) => s.split(/[.!?](?:\s|$)/).filter((x) => x.trim()).length;

function lint(tools) {
  const findings = {};
  for (const t of tools) {
    const f = [];
    if (!/^[a-zA-Z0-9_-]{1,128}$/.test(t.name)) f.push("name breaks the API name rule ^[a-zA-Z0-9_-]{1,128}$");
    else if (!/^[a-z]+(_[a-z]+)+$/.test(t.name)) f.push("name is not service_verb or verb_object");
    const n = sentences(t.description);
    if (n < 3) f.push(`description has ${n} sentence(s); aim for 3-4`);
    if (!/\bdo not use\b|\bnot for\b/i.test(t.description)) f.push("no when-NOT-to-use guidance");
    const optional = Object.keys(t.params).filter((k) => !t.required.includes(k));
    if (optional.length > 2) f.push(`${optional.length} optional params: each is a decision delegated to the model`);
    for (const [k, p] of Object.entries(t.params)) {
      if (!p.description) f.push(`param ${k}: no description`);
      if (p.type === "string" && !p.enum && !p.pattern && !(p.description || "").includes("e.g.")) f.push(`param ${k}: free string with no enum, pattern or example`);
    }
    findings[t.name] = f;
  }
  for (let i = 0; i < tools.length; i++) for (let j = i + 1; j < tools.length; j++) {
    const a = tools[i], b = tools[j];
    const s = jaccard(words(a.name + " " + a.description), words(b.name + " " + b.description));
    if (s >= 0.25) findings[a.name].push(`overlaps with ${b.name} (similarity ${s.toFixed(2)})`);
  }
  return findings;
}

for (const [label, tools] of Object.entries(SETS)) {
  const findings = lint(tools);
  const total = Object.values(findings).reduce((s, f) => s + f.length, 0);
  const tokens = Math.ceil(JSON.stringify(tools).length / 4);
  console.log(`== ${label} toolset: ${tools.length} tools, ~${tokens} tokens of definitions, ${total} findings`);
  for (const [name, f] of Object.entries(findings)) {
    console.log(`  ${name}:${f.length ? "" : " ok"}`);
    for (const x of f) console.log(`    - ${x}`);
  }
}

// A tool-selection eval: labelled requests -> expected tool, scored per toolset.
// The "model" here is a crude bag-of-words router (a stand-in, not an LLM): it can only
// use what the names and descriptions say, which is the point being measured.
const SETS = {
  "bad": [
    {"name": "get_user", "description": "Gets user data.", "params": {"q": {"type": "string"}}, "required": ["q"]},
    {"name": "fetch_account", "description": "Fetch account info for a customer.", "params": {"id": {"type": "string"}}, "required": ["id"]},
    {"name": "lookup profile", "description": "Lookup profile.", "params": {"input": {"type": "string"}}, "required": []},
    {"name": "order", "description": "Order operations.", "params": {"action": {"type": "string"}, "data": {"type": "string"}, "opts": {"type": "string"}, "flag": {"type": "boolean"}, "extra": {"type": "string"}}, "required": []},
    {"name": "refund", "description": "Handles refunds and orders.", "params": {"order": {"type": "string"}, "amount": {"type": "string"}}, "required": ["order"]}
  ],
  "good": [
    {"name": "customers_find", "description": "Find a customer by email, name or phone and return their customer id, plan and account status. Use this first when the user mentions a person but gives no customer id. Do not use for order or refund questions.", "params": {"query": {"type": "string", "description": "Email, full name or phone, e.g. ann@example.com"}}, "required": ["query"]},
    {"name": "orders_get_status", "description": "Get the shipping status and line items of one order by its order id. Use when the user asks where an order is or what it contains. Do not use to cancel or refund.", "params": {"order_id": {"type": "string", "pattern": "^A-[0-9]+$", "description": "Order id, e.g. A-17"}}, "required": ["order_id"]},
    {"name": "orders_cancel", "description": "Cancel or stop an order that has not shipped yet. Use only when the user explicitly asks to cancel or stop it. Do not use for refunds of delivered orders.", "params": {"order_id": {"type": "string", "pattern": "^A-[0-9]+$", "description": "Order id, e.g. A-17"}, "reason": {"type": "string", "enum": ["customer_request", "duplicate", "fraud"], "description": "Why the order is cancelled"}}, "required": ["order_id", "reason"]},
    {"name": "refunds_create", "description": "Refund a delivered order, fully or partly, and return the refund id. Use when the user asks for money back on an order that has arrived. Do not use to cancel an order that has not shipped.", "params": {"order_id": {"type": "string", "pattern": "^A-[0-9]+$", "description": "Order id, e.g. A-17"}, "amount_cents": {"type": ["integer", "null"], "description": "Amount in cents; null refunds the full order"}}, "required": ["order_id", "amount_cents"]}
  ]
};
const CASES = [
  ["Where is my order A-17?", "order", "orders_get_status"],
  ["Please cancel order A-18, it has not shipped", "order", "orders_cancel"],
  ["I want my money back for order A-19, it arrived broken", "refund", "refunds_create"],
  ["Find the customer with email ann@example.com", "get_user", "customers_find"],
  ["What items are in order A-20?", "order", "orders_get_status"],
  ["Refund the delivered order A-21", "refund", "refunds_create"],
  ["What plan and account status does Bob Lee have?", "fetch_account", "customers_find"],
  ["Stop order A-22 before it ships", "order", "orders_cancel"],
];
const STOP = new Set(["the", "and", "for", "use", "this", "that", "not", "one", "its", "their", "when", "has", "yet", "only", "with", "get", "gets", "data", "info", "what", "where", "want", "please", "does", "have", "before"]);
const words = (s) => new Set(s.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 2 && !STOP.has(w)).map((w) => (w.length > 4 && w.endsWith("s") ? w.slice(0, -1) : w)));
// Iteration 2: the eval showed a confusion, so ONE description was reworded.
SETS["good v2"] = SETS.good.map((t) => t.name !== "orders_cancel" ? t : { ...t,
  description: "Cancel or stop an order that has not shipped yet. Use only when the user explicitly asks to cancel or stop it. Do not use for refunds; use refunds_create." });
const catchAll = (t) => Object.entries(t.params).some(([k, p]) => k === "action" && !p.enum);
const overlap = (a, b) => [...a].filter((x) => b.has(x)).length;

function route(request, tools) {
  const req = words(request);
  let best = null, bestScore = -Infinity;
  for (const t of tools) {
    const parts = t.description.split(/(?<=[.!?])\s+/);
    const pos = words([t.name.replace(/_/g, " "), ...parts.filter((p) => !/^do not use/i.test(p))].join(" "));
    const neg = words(parts.filter((p) => /^do not use/i.test(p)).join(" "));
    const score = overlap(req, pos) - 2 * overlap(req, neg);
    if (score > bestScore) { best = t.name; bestScore = score; }
  }
  return [best, bestScore];
}

for (const [label, tools] of Object.entries(SETS)) {
  let correct = 0, vague = 0;
  const confusions = {};
  console.log(`== ${label} toolset`);
  for (const [request, expBad, expGood] of CASES) {
    const expected = label === "bad" ? expBad : expGood;
    const [picked, score] = route(request, tools);
    const ok = picked === expected;
    if (catchAll(tools.find((t) => t.name === picked))) vague++;
    if (ok) correct++;
    else confusions[`${expected} -> ${picked}`] = (confusions[`${expected} -> ${picked}`] || 0) + 1;
    console.log(`  ${ok ? "PASS" : "FAIL"}  ${request.padEnd(56)} picked ${picked} (score ${score})${ok ? "" : `, expected ${expected}`}`);
  }
  console.log(`  tool-selection accuracy: ${correct}/${CASES.length}`);
  for (const [k, n] of Object.entries(confusions)) console.log(`  confusion: ${k} x${n}`);
  if (vague) console.log(`  ${vague} picks went to a catch-all tool whose free-string "action" argument the model must still guess`);
}