Cost Engineering for LLM Apps: Unit Economics, Routing, and Budgets

// USD per million tokens at the time of writing. Keep this table in one place.
export const PRICES = {
  "claude-opus-5-5": { input: 4, cacheWrite: 5, cacheRead: 0.2, output: 20 },
  "claude-sonnet-5-5": { input: 2, cacheWrite: 2.5, cacheRead: 0.2, output: 10 },
  "claude-haiku-4-5": { input: 1, cacheWrite: 1.25, cacheRead: 0.1, output: 5 },
} as const;
export type Model = keyof typeof PRICES;
 
export interface Workload {
  name: string;
  requestsPerMonth: number;
  inputTokens: number; // per request, total
  cacheablePrefix: number; // stable prefix tokens (tools + system + shared docs)
  outputTokens: number;
}
 
export interface Levers {
  caching: boolean;
  cacheHitRate: number; // share of requests that read the prefix from cache
  routing: Partial<Record<string, { share: number; model: Model }>>; // share of a workload sent to a cheaper model
  batched: string[]; // workloads sent through the Batches API (50% off)
  outputReduction: number; // 0.3 means 30% fewer output tokens
}
 
export interface Breakdown {
  input: number;
  cache: number;
  output: number;
  total: number;
}
 
function costOf(w: Workload, model: Model, requests: number, l: Levers): Breakdown {
  const p = PRICES[model];
  const out = w.outputTokens * (1 - l.outputReduction);
  const discount = l.batched.includes(w.name) ? 0.5 : 1;
  let input: number;
  let cache = 0;
  if (l.caching && w.cacheablePrefix > 0) {
    const reads = requests * l.cacheHitRate;
    const writes = requests - reads;
    input = requests * (w.inputTokens - w.cacheablePrefix) * p.input;
    cache = reads * w.cacheablePrefix * p.cacheRead + writes * w.cacheablePrefix * p.cacheWrite;
  } else {
    input = requests * w.inputTokens * p.input;
  }
  const output = requests * out * p.output;
  const scale = discount / 1_000_000;
  return { input: input * scale, cache: cache * scale, output: output * scale, total: (input + cache + output) * scale };
}
 
export function monthlyCost(workloads: Workload[], baseModel: Model, l: Levers): Breakdown {
  const sum: Breakdown = { input: 0, cache: 0, output: 0, total: 0 };
  for (const w of workloads) {
    const route = l.routing[w.name];
    const parts: [Model, number][] = route
      ? [[route.model, w.requestsPerMonth * route.share], [baseModel, w.requestsPerMonth * (1 - route.share)]]
      : [[baseModel, w.requestsPerMonth]];
    for (const [model, requests] of parts) {
      const c = costOf(w, model, requests, l);
      sum.input += c.input;
      sum.cache += c.cache;
      sum.output += c.output;
      sum.total += c.total;
    }
  }
  return sum;
}

import { monthlyCost, type Levers, type Workload } from "./lib/cost-model.ts";
 
const workloads: Workload[] = [
  { name: "chat", requestsPerMonth: 200_000, inputTokens: 9_000, cacheablePrefix: 7_000, outputTokens: 500 },
  { name: "summaries", requestsPerMonth: 50_000, inputTokens: 4_000, cacheablePrefix: 0, outputTokens: 300 },
];
 
const baseline: Levers = { caching: false, cacheHitRate: 0, routing: {}, batched: [], outputReduction: 0 };
const caching: Levers = { ...baseline, caching: true, cacheHitRate: 0.9 };
const routing: Levers = {
  ...caching,
  routing: { chat: { share: 0.6, model: "claude-sonnet-5-5" }, summaries: { share: 1, model: "claude-haiku-4-5" } },
};
const batching: Levers = { ...routing, batched: ["summaries"] };
const shorter: Levers = { ...batching, outputReduction: 0.3 };
 
const steps: [string, Levers][] = [
  ["Baseline", baseline],
  ["+ prompt caching", caching],
  ["+ model routing", routing],
  ["+ batching", batching],
  ["+ shorter outputs", shorter],
];
for (const [label, levers] of steps) {
  const c = monthlyCost(workloads, "claude-opus-5-5", levers);
  console.log(label.padEnd(18), `$${c.total.toFixed(0).padStart(6)}`, `(input $${c.input.toFixed(0)}, cache $${c.cache.toFixed(0)}, output $${c.output.toFixed(0)})`);
}

Baseline           $ 10300 (input $8000, cache $0, output $2300)
+ prompt caching   $  5652 (input $2400, cache $952, output $2300)
+ model routing    $  3537 (input $1320, cache $742, output $1475)
+ batching         $  3400 (input $1220, cache $742, output $1438)
+ shorter outputs  $  2968 (input $1220, cache $742, output $1006)

import Anthropic from "@anthropic-ai/sdk";
import { zodOutputFormat } from "@anthropic-ai/sdk/helpers/zod";
import { z } from "zod";
 
const client = new Anthropic();
const text = (m: Anthropic.Message) => m.content.flatMap((b) => (b.type === "text" ? [b.text] : [])).join("");
 
export type Feature = "tagging" | "support-chat" | "code-review";
 
// 1. Rules: most routing decisions are known per feature.
const BY_FEATURE: Record<Feature, Anthropic.Model | "classify"> = {
  tagging: "claude-haiku-4-5",
  "code-review": "claude-opus-5-5",
  "support-chat": "classify",
};
 
// 2. Classifier: a small, fast model labels difficulty.
const Difficulty = z.object({
  difficulty: z.enum(["simple", "complex"]).describe("complex = multi-step reasoning, troubleshooting, or account-specific judgment"),
});
 
export async function chooseModel(feature: Feature, input: string): Promise<Anthropic.Model> {
  const rule = BY_FEATURE[feature];
  if (rule !== "classify") return rule;
  const res = await client.messages.parse({
    model: "claude-haiku-4-5",
    max_tokens: 256,
    output_config: { format: zodOutputFormat(Difficulty) },
    messages: [{ role: "user", content: `Classify this support message:\n<message>\n${input}\n</message>` }],
  });
  return res.parsed_output?.difficulty === "simple" ? "claude-sonnet-5-5" : "claude-opus-5-5";
}
 
// 3. Cascade: answer with a mid-size model; escalate when it says it is unsure.
const Answer = z.object({
  answer: z.string(),
  confident: z.boolean().describe("false if the question needs information or reasoning you are unsure about"),
});
 
export async function answerWithCascade(system: string, question: string): Promise<{ answer: string; model: string }> {
  const first = await client.messages.parse({
    model: "claude-sonnet-5-5",
    max_tokens: 4096,
    output_config: { effort: "low", format: zodOutputFormat(Answer) },
    system,
    messages: [{ role: "user", content: question }],
  });
  if (first.parsed_output?.confident) return { answer: first.parsed_output.answer, model: "claude-sonnet-5-5" };
 
  const escalated = await client.messages.create({
    model: "claude-opus-5-5",
    max_tokens: 8192,
    system,
    messages: [{ role: "user", content: question }],
  });
  return { answer: text(escalated), model: "claude-opus-5-5" };
}

import { answerWithCascade, chooseModel } from "./lib/router.ts";
 
console.log(await chooseModel("tagging", "Label this ticket"));
console.log(await chooseModel("support-chat", "How do I download an invoice?"));
 
const result = await answerWithCascade(
  "You are the support assistant for Acme Billing.",
  "Our SSO login fails only for users in the EU tenant since yesterday's certificate rotation. What should we check?",
);
console.log(result.model, result.answer.slice(0, 120));

// Daily per-feature spend check, fed from your cost records (for example a nightly query over trace data).
export interface FeatureSpend {
  feature: string;
  todayUSD: number;
  trailing7DayAvgUSD: number;
  dailyBudgetUSD: number;
}
 
export type Alert = { feature: string; level: "warning" | "critical"; message: string };
 
export function checkBudgets(spend: FeatureSpend[]): Alert[] {
  const alerts: Alert[] = [];
  for (const s of spend) {
    if (s.todayUSD >= s.dailyBudgetUSD) {
      alerts.push({ feature: s.feature, level: "critical", message: `${s.feature} spent $${s.todayUSD.toFixed(2)} of a $${s.dailyBudgetUSD} budget` });
    } else if (s.trailing7DayAvgUSD > 0 && s.todayUSD > s.trailing7DayAvgUSD * 1.5) {
      const pct = Math.round((s.todayUSD / s.trailing7DayAvgUSD - 1) * 100);
      alerts.push({ feature: s.feature, level: "warning", message: `${s.feature} is ${pct}% above its 7-day average` });
    }
  }
  return alerts;
}

import { test } from "node:test";
import assert from "node:assert/strict";
import { checkBudgets } from "./lib/feature-budget.ts";
 
test("flags budget overruns and unusual growth", () => {
  const alerts = checkBudgets([
    { feature: "support-chat", todayUSD: 130, trailing7DayAvgUSD: 110, dailyBudgetUSD: 200 }, // normal
    { feature: "summaries", todayUSD: 95, trailing7DayAvgUSD: 40, dailyBudgetUSD: 150 }, // growth
    { feature: "agent", todayUSD: 310, trailing7DayAvgUSD: 250, dailyBudgetUSD: 300 }, // over budget
  ]);
  assert.deepEqual(alerts.map((a) => [a.feature, a.level]), [["summaries", "warning"], ["agent", "critical"]]);
  assert.match(alerts[0]!.message, /138% above/);
});