Prompt Caching and Batches: Cutting LLM API Costs in TypeScript

import Anthropic from "@anthropic-ai/sdk";
import { readFileSync } from "node:fs";
 
const client = new Anthropic();
// A long, stable prompt: product docs, policies, examples. It must not change between requests.
const SYSTEM = readFileSync(new URL("./prompts/handbook.md", import.meta.url), "utf8");
 
const messages: Anthropic.MessageParam[] = [];
 
async function turn(userText: string) {
  messages.push({ role: "user", content: userText });
  const message = await client.messages.create({
    model: "claude-opus-5-5",
    max_tokens: 4096,
    system: SYSTEM,
    cache_control: { type: "ephemeral" }, // cache everything up to the last block
    messages,
  });
  messages.push({ role: "assistant", content: message.content });
 
  const u = message.usage;
  console.log(
    `in: ${u.input_tokens} uncached, ${u.cache_creation_input_tokens ?? 0} written, ${u.cache_read_input_tokens ?? 0} read | out: ${u.output_tokens}`,
  );
}
 
await turn("What is our refund policy for annual plans?");
await turn("And for monthly plans?");
await turn("Summarize both in a table.");

in: 9 uncached, 12804 written, 0 read | out: 412
in: 6 uncached, 431 written, 12804 read | out: 380
in: 5 uncached, 402 written, 13235 read | out: 655

import Anthropic from "@anthropic-ai/sdk";
 
const client = new Anthropic();
const tools: Anthropic.Tool[] = [
  { name: "get_invoice", description: "Fetch an invoice by ID for the current customer.", input_schema: { type: "object", properties: { invoice_id: { type: "string" } }, required: ["invoice_id"] } },
];
 
const message = await client.messages.create({
  model: "claude-opus-5-5",
  max_tokens: 4096,
  tools, // rendered first, so it is part of every cached prefix
  system: [
    // Breakpoint 1: tools and instructions, shared by every user and every conversation.
    { type: "text", text: "You are the billing assistant for Acme…(long, stable instructions)…", cache_control: { type: "ephemeral", ttl: "1h" } },
  ],
  messages: [
    {
      role: "user",
      content: [
        // Breakpoint 2: this user's conversation so far.
        { type: "text", text: "Why did my invoice total change?", cache_control: { type: "ephemeral" } },
      ],
    },
  ],
});
console.log(message.usage);

// USD per million tokens for claude-opus-5-5 at the time of writing.
export const PRICE = { input: 4, cacheWrite: 5, cacheRead: 0.2, output: 20 };
 
export interface ChatShape {
  turns: number;
  prefix: number; // tools + system prompt tokens
  userPerTurn: number;
  outputPerTurn: number;
}
 
export interface Cost {
  uncached: number;
  write: number;
  read: number;
  output: number;
  total: number;
}
 
const usd = (tokens: number, perMillion: number) => (tokens * perMillion) / 1_000_000;
 
export function conversationCost(s: ChatShape, cached: boolean): Cost {
  let uncached = 0, write = 0, read = 0, previous = 0;
  for (let turn = 1; turn <= s.turns; turn++) {
    // Each request: prefix + all earlier exchanges + the new user message.
    const input = s.prefix + (turn - 1) * (s.userPerTurn + s.outputPerTurn) + s.userPerTurn;
    if (!cached) uncached += input;
    else {
      read += previous; // everything cached by the previous turn
      write += input - previous; // the new part
      previous = input;
    }
  }
  const cost = {
    uncached: usd(uncached, PRICE.input),
    write: usd(write, PRICE.cacheWrite),
    read: usd(read, PRICE.cacheRead),
    output: usd(s.turns * s.outputPerTurn, PRICE.output),
  };
  return { ...cost, total: cost.uncached + cost.write + cost.read + cost.output };
}

import { conversationCost } from "./lib/simulate.ts";
 
const shape = { turns: 10, prefix: 12_100, userPerTurn: 700, outputPerTurn: 400 };
for (const cached of [false, true]) {
  const c = conversationCost(shape, cached);
  const parts = Object.entries(c).map(([k, v]) => `${k} $${v.toFixed(3)}`);
  console.log(cached ? "with caching:   " : "without caching:", parts.join(", "));
}

without caching: uncached $0.710, write $0.000, read $0.000, output $0.080, total $0.790
with caching:    uncached $0.000, write $0.114, read $0.031, output $0.080, total $0.224

import Anthropic from "@anthropic-ai/sdk";
 
const client = new Anthropic();
 
const reviews = [
  { id: "r-101", text: "Arrived broken, and support never answered." },
  { id: "r-102", text: "Exactly as described. Fast shipping!" },
  { id: "r-103", text: "Fine, but the color is darker than in the photos." },
];
 
const batch = await client.messages.batches.create({
  requests: reviews.map((r) => ({
    custom_id: r.id, // how you match results; they can come back in any order
    params: {
      model: "claude-haiku-4-5",
      max_tokens: 16,
      system: "Classify the sentiment of the product review. Answer with one word: positive, negative, or mixed.",
      messages: [{ role: "user", content: r.text }],
    },
  })),
});
console.log("submitted", batch.id);
 
// Poll until done. In production, run this from a scheduled job instead of keeping a process alive.
let status = batch;
while (status.processing_status !== "ended") {
  await new Promise((r) => setTimeout(r, 60_000));
  status = await client.messages.batches.retrieve(batch.id);
  console.log(status.processing_status, status.request_counts);
}
 
for await (const item of await client.messages.batches.results(batch.id)) {
  if (item.result.type === "succeeded") {
    const text = item.result.message.content.flatMap((b) => (b.type === "text" ? [b.text] : [])).join("");
    console.log(item.custom_id, text.trim());
  } else {
    console.log(item.custom_id, "failed:", item.result.type); // errored, canceled, or expired: retry these
  }
}