Tutorial: Trace an LLM Feature with OpenTelemetry and Build an SLO Report

feature          requests  errors  p50 ms  p95 ms  $/request   total $
support-chat           21       1     887     993    0.00345    0.0724
ticket-triage           9       0     100     106    0.00004    0.0003
 
first token p50 100 ms, p95 117 ms; cache read ratio 97%
stop reasons {"end_turn":20,"max_tokens":9}
 
slowest request (trace b7d62006):
  support.answer                 |████████████████████████████████████████| 1019 ms
    retrieve kb_search           |█                                       | 35 ms
    anthropic.messages.create    |  ██████████████████████████████████████| 959 ms (first token 141 ms)
 
ALERTS:
  - error rate 3.3% is above 2%
  - 9 calls stopped with max_tokens

mkdir trace-report && cd trace-report
npm init -y
npm pkg set type=module
npm install @anthropic-ai/sdk @opentelemetry/api @opentelemetry/sdk-trace-node @opentelemetry/sdk-trace-base \
  @opentelemetry/resources @opentelemetry/semantic-conventions @opentelemetry/core
npm install -D typescript @types/node
mkdir src test
npm pkg set scripts.simulate="node --env-file=.env src/simulate.ts" scripts.report="node src/report.ts" scripts.test="node --test" scripts.typecheck="tsc"

import { appendFileSync, mkdirSync } from "node:fs";
import path from "node:path";
import { type Attributes, SpanStatusCode } from "@opentelemetry/api";
import { type ExportResult, ExportResultCode, hrTimeToMilliseconds } from "@opentelemetry/core";
import { resourceFromAttributes } from "@opentelemetry/resources";
import { type ReadableSpan, type SpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
 
export interface SpanRecord {
  traceId: string;
  spanId: string;
  parentId: string | null;
  name: string;
  startMs: number; // epoch milliseconds
  durationMs: number;
  status: "ok" | "error";
  error?: string;
  attributes: Attributes;
}
 
// Personal data that must never leave the process, whatever attribute it ends up in.
const PATTERNS: [RegExp, string][] = [
  [/[\w.+-]+@[\w-]+\.[\w.]+/g, "[email]"],
  [/\b(?:\d[ -]?){13,16}\b/g, "[card]"],
  [/\bsk-ant-[\w-]+/g, "[api-key]"],
];
 
export function redact(attributes: Attributes): Attributes {
  return Object.fromEntries(
    Object.entries(attributes).map(([k, v]) => [k, typeof v === "string" ? PATTERNS.reduce((s, [re, by]) => s.replace(re, by), v) : v]),
  );
}
 
/**
 * Writes finished spans as JSON lines, redacted. In production you would export OTLP to a
 * backend instead; a file keeps this tutorial self-contained and easy to inspect.
 */
export class JsonlFileExporter implements SpanExporter {
  readonly file: string;
 
  constructor(file: string) {
    this.file = file;
    mkdirSync(path.dirname(file), { recursive: true });
  }
 
  export(spans: ReadableSpan[], done: (result: ExportResult) => void): void {
    const lines = spans.map((s) => {
      const record: SpanRecord = {
        traceId: s.spanContext().traceId,
        spanId: s.spanContext().spanId,
        parentId: s.parentSpanContext?.spanId ?? null,
        name: s.name,
        startMs: hrTimeToMilliseconds(s.startTime),
        durationMs: hrTimeToMilliseconds(s.duration),
        status: s.status.code === SpanStatusCode.ERROR ? "error" : "ok",
        ...(s.status.message ? { error: redact({ m: s.status.message }).m as string } : {}),
        attributes: redact(s.attributes),
      };
      return JSON.stringify(record);
    });
    appendFileSync(this.file, `${lines.join("\n")}\n`);
    done({ code: ExportResultCode.SUCCESS });
  }
 
  async shutdown(): Promise<void> {}
}
 
/** Register a global tracer provider that writes spans to `file`. Call shutdown() before exiting. */
export function setupTracing(file: string, serviceName = "support-app"): NodeTracerProvider {
  const provider = new NodeTracerProvider({
    resource: resourceFromAttributes({ [ATTR_SERVICE_NAME]: serviceName }),
    // Simple: export each span as it ends. Production uses BatchSpanProcessor, off the request path.
    spanProcessors: [new SimpleSpanProcessor(new JsonlFileExporter(file))],
  });
  provider.register(); // also installs the async context manager, so child spans find their parent
  return provider;
}

import { type Attributes, SpanStatusCode, trace } from "@opentelemetry/api";
 
const tracer = trace.getTracer("support-app");
 
/**
 * Run `fn` inside a span. The Anthropic SDK's own spans for API calls made inside `fn`
 * become children of it, so one trace shows the whole request.
 */
export function withSpan<T>(name: string, attributes: Attributes, fn: () => Promise<T>): Promise<T> {
  return tracer.startActiveSpan(name, { attributes }, async (span) => {
    try {
      return await fn();
    } catch (error) {
      span.recordException(error as Error);
      span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.message : String(error) });
      throw error;
    } finally {
      span.end();
    }
  });
}

import { createHash } from "node:crypto";
import { setTimeout as sleep } from "node:timers/promises";
import type Anthropic from "@anthropic-ai/sdk";
import { withSpan } from "./spans.ts";
 
// A long, stable policy block: cached after the first request, so later requests read it cheaply.
const POLICY = `You are the support assistant for Acme Billing. Answer in at most four sentences.
${Array.from({ length: 40 }, (_, i) => `Policy ${i + 1}: refunds within 30 days of purchase; exports under Billing → Invoices → Export; cards under Billing → Payment methods; plans under Billing → Plan.`).join("\n")}`;
 
const KB = [
  { id: "kb-refunds", text: "Refunds: full refund within 30 days of purchase. After that, support reviews exceptions." },
  { id: "kb-export", text: "Export: Billing → Invoices → Export → CSV or PDF." },
  { id: "kb-cards", text: "Cards: Billing → Payment methods → Edit." },
];
 
/** Hash user IDs before they reach telemetry: correlatable, not identifying. */
const pseudonym = (userId: string) => createHash("sha256").update(userId).digest("hex").slice(0, 12);
 
async function searchKb(question: string) {
  await sleep(20 + Math.random() * 40); // stand-in for a vector search
  const words = question.toLowerCase().split(/\W+/);
  return KB.filter((d) => words.some((w) => w.length > 3 && d.text.toLowerCase().includes(w)));
}
 
/** The support feature: retrieve, maybe look up an invoice, then stream an answer. One trace. */
export function answerQuestion(client: Anthropic, userId: string, question: string): Promise<string> {
  // The root span names the feature; every span below it, including the SDK's, belongs to it.
  const attributes = { "app.feature": "support-chat", "app.user": pseudonym(userId), "app.question_preview": question.slice(0, 80) };
  return withSpan("support.answer", attributes, async () => {
    const docs = await withSpan("retrieve kb_search", { "app.top_k": 3 }, () => searchKb(question));
    const invoice = /INV-\d+/.exec(question)?.[0];
    const facts = invoice
      ? await withSpan("execute_tool get_invoice", { "gen_ai.tool.name": "get_invoice" }, async () => {
          await sleep(40);
          return `${invoice}: $129, paid on 2026-10-01`;
        })
      : "";
    // No wrapper: the SDK records this call as a child span, with usage, cache tokens, and time to first token.
    const message = await client.messages
      .stream({
        model: "claude-opus-5-5",
        max_tokens: 1024,
        system: [{ type: "text", text: POLICY, cache_control: { type: "ephemeral" } }],
        messages: [{ role: "user", content: `<docs>\n${docs.map((d) => d.text).join("\n")}\n${facts}\n</docs>\n\n${question}` }],
      })
      .finalMessage();
    return message.content.flatMap((b) => (b.type === "text" ? [b.text] : [])).join("");
  });
}
 
/** A second feature on a small model. */
export function triageTicket(client: Anthropic, ticket: string): Promise<string> {
  return withSpan("triage.ticket", { "app.feature": "ticket-triage" }, async () => {
    const message = await client.messages.create({
      model: "claude-haiku-5-5",
      max_tokens: 64, // "a one-word answer needs few tokens"... or does it?
      system: "Label this support ticket with one word: billing, bug, account, or other.",
      messages: [{ role: "user", content: ticket }],
    });
    return message.content.flatMap((b) => (b.type === "text" ? [b.text] : [])).join("").trim();
  });
}

import { rm } from "node:fs/promises";
import Anthropic from "@anthropic-ai/sdk";
import { answerQuestion, triageTicket } from "./app.ts";
import { setupTracing } from "./tracing.ts";
 
// Tracers created before this point are proxies: they start recording once the provider is registered.
const FILE = "traces/spans.jsonl";
await rm(FILE, { force: true });
const provider = setupTracing(FILE);
 
const client = new Anthropic({ maxRetries: 1 });
const QUESTIONS = [
  "Can I get a refund? I bought the plan 10 days ago.",
  "How do I export invoices to CSV?",
  "Why was I charged twice for INV-2041? My email is ana@example.com",
  "How do I change my card? The old one is 4242 4242 4242 4242.",
  "Where do I find my invoices?",
];
const TICKETS = ["Card declined at checkout", "The export button does nothing", "I want to change my login email"];
 
const jobs: (() => Promise<unknown>)[] = [];
for (let i = 0; i < 20; i++) jobs.push(() => answerQuestion(client, `user-${i % 7}`, QUESTIONS[i % QUESTIONS.length]!));
for (let i = 0; i < 9; i++) jobs.push(() => triageTicket(client, TICKETS[i % TICKETS.length]!));
// One request whose stream breaks halfway, so the report has a failure to show.
jobs.push(() => answerQuestion(client, "user-9", "Explain my latest invoice in detail MOCK_BREAK"));
 
let failed = 0;
let next = 0;
await Promise.all(
  Array.from({ length: 4 }, async () => {
    while (next < jobs.length) {
      const job = jobs[next++]!;
      await job().catch(() => failed++);
    }
  }),
);
await provider.shutdown(); // flush every span to the file
console.log(`${jobs.length} requests, ${failed} failed; spans in ${FILE}`);

npm run simulate   # 30 requests, 1 failed; spans in traces/spans.jsonl

import type { Attributes } from "@opentelemetry/api";
 
// USD per million tokens. Keep the table in one place, and check the pricing page when models change.
const PRICES: Record<string, { input: number; output: number; cacheRead: number; cacheWrite: number }> = {
  "claude-opus-5-5": { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 5 },
  "claude-sonnet-5-5": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 },
  "claude-haiku-5-5": { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 },
};
// Haiku 5.5 costs more for prompts above 100K tokens.
const HAIKU_LONG = { input: 0.5, output: 2.5, cacheRead: 0.05, cacheWrite: 0.625 };
 
/** The cost of one API call, from the usage attributes on the SDK's span. */
export function callCost(a: Attributes): number {
  const model = String(a["gen_ai.response.model"] ?? a["gen_ai.request.model"] ?? "");
  const n = (key: string) => Number(a[key] ?? 0);
  const input = n("gen_ai.usage.input_tokens"); // includes cache reads and writes
  const read = n("gen_ai.usage.cache_read.input_tokens");
  const write = n("gen_ai.usage.cache_write.input_tokens");
  const p = model === "claude-haiku-5-5" && input > 100_000 ? HAIKU_LONG : PRICES[model];
  if (!p) return 0;
  return ((input - read - write) * p.input + read * p.cacheRead + write * p.cacheWrite + n("gen_ai.usage.output_tokens") * p.output) / 1_000_000;
}

import { readFile } from "node:fs/promises";
import { callCost } from "./prices.ts";
import type { SpanRecord } from "./tracing.ts";
 
export function percentile(values: number[], p: number): number {
  if (values.length === 0) return 0;
  const sorted = [...values].sort((a, b) => a - b);
  return sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)]!;
}
 
const num = (s: SpanRecord, key: string) => Number(s.attributes[key] ?? 0);
const isApiCall = (s: SpanRecord) => s.name.startsWith("anthropic.");
 
/** One trace as an indented waterfall: each bar shows when a span ran within the request. */
export function waterfall(spans: SpanRecord[], width = 40): string[] {
  const root = spans.find((s) => !s.parentId) ?? spans[0]!;
  const total = Math.max(root.durationMs, 1);
  const depth = (s: SpanRecord): number => {
    const parent = spans.find((p) => p.spanId === s.parentId);
    return parent ? depth(parent) + 1 : 0;
  };
  // Parents before children, then by start time.
  const ordered: SpanRecord[] = [];
  const visit = (s: SpanRecord) => {
    ordered.push(s);
    spans.filter((c) => c.parentId === s.spanId).sort((a, b) => a.startMs - b.startMs).forEach(visit);
  };
  visit(root);
  return ordered.map((s) => {
    const from = Math.min(width - 1, Math.round(((s.startMs - root.startMs) / total) * width));
    const len = Math.max(1, Math.min(width - from, Math.round((s.durationMs / total) * width)));
    const bar = `${" ".repeat(from)}${"█".repeat(len)}`.padEnd(width);
    const ttft = s.attributes["gen_ai.response.time_to_first_chunk"];
    const extra = ttft !== undefined ? ` (first token ${Math.round(Number(ttft) * 1000)} ms)` : "";
    return `${`${"  ".repeat(depth(s))}${s.name}`.padEnd(30)} |${bar}| ${Math.round(s.durationMs)} ms${extra}${s.status === "error" ? "  ERROR" : ""}`;
  });
}
 
export function summarize(spans: SpanRecord[]) {
  const calls = spans.filter(isApiCall);
  const roots = spans.filter((s) => !s.parentId);
  const featureOf = new Map(roots.map((r) => [r.traceId, String(r.attributes["app.feature"] ?? "unknown")]));
 
  const features = [...Map.groupBy(roots, (r) => featureOf.get(r.traceId)!)].map(([feature, rs]) => {
    const traceIds = new Set(rs.map((r) => r.traceId));
    const cost = calls.filter((c) => traceIds.has(c.traceId)).reduce((sum, c) => sum + callCost(c.attributes), 0);
    return {
      feature,
      requests: rs.length,
      errors: rs.filter((r) => r.status === "error").length,
      p50: percentile(rs.map((r) => r.durationMs), 50),
      p95: percentile(rs.map((r) => r.durationMs), 95),
      cost,
    };
  });
  const ttft = calls.filter((c) => c.attributes["gen_ai.response.time_to_first_chunk"] !== undefined).map((c) => num(c, "gen_ai.response.time_to_first_chunk") * 1000);
  const input = calls.reduce((s, c) => s + num(c, "gen_ai.usage.input_tokens"), 0);
  const cacheRead = calls.reduce((s, c) => s + num(c, "gen_ai.usage.cache_read.input_tokens"), 0);
  const count = (key: string) =>
    Object.fromEntries([...Map.groupBy(calls.filter((c) => c.attributes[key] !== undefined), (c) => String(c.attributes[key]))].map(([k, v]) => [k, v.length]));
  return {
    features,
    ttftP50: percentile(ttft, 50),
    ttftP95: percentile(ttft, 95),
    cacheReadRatio: input ? cacheRead / input : 0,
    stopReasons: count("anthropic.message.stop_reason"),
    errorTypes: count("error.type"),
    errorRate: roots.filter((r) => r.status === "error").length / Math.max(roots.length, 1),
    // For each failed call: the request ID to give support, and what went wrong.
    failedCalls: calls.filter((c) => c.status === "error").map((c) => `${c.attributes["anthropic.request.id"] ?? "no request id"} ${c.attributes["error.type"]}`),
  };
}
 
/** Alert rules from the observability article, applied to one window of spans. */
export function alerts(s: ReturnType<typeof summarize>): string[] {
  const out: string[] = [];
  if (s.errorRate > 0.02) out.push(`error rate ${(s.errorRate * 100).toFixed(1)}% is above 2%`);
  if (s.cacheReadRatio < 0.5) out.push(`cache read ratio ${(s.cacheReadRatio * 100).toFixed(0)}% is below 50%`);
  for (const reason of ["max_tokens", "refusal"]) if (s.stopReasons[reason]) out.push(`${s.stopReasons[reason]} calls stopped with ${reason}`);
  return out;
}
 
if (import.meta.filename === process.argv[1]) {
  const spans = (await readFile(process.argv[2] ?? "traces/spans.jsonl", "utf8")).trim().split("\n").map((l) => JSON.parse(l) as SpanRecord);
  const s = summarize(spans);
 
  console.log(`${"feature".padEnd(16)}${"requests".padStart(9)}${"errors".padStart(8)}${"p50 ms".padStart(8)}${"p95 ms".padStart(8)}${"$/request".padStart(11)}${"total $".padStart(10)}`);
  for (const f of s.features) {
    console.log(
      `${f.feature.padEnd(16)}${String(f.requests).padStart(9)}${String(f.errors).padStart(8)}${String(Math.round(f.p50)).padStart(8)}${String(Math.round(f.p95)).padStart(8)}` +
        `${(f.cost / f.requests).toFixed(5).padStart(11)}${f.cost.toFixed(4).padStart(10)}`,
    );
  }
  console.log(`\nfirst token p50 ${Math.round(s.ttftP50)} ms, p95 ${Math.round(s.ttftP95)} ms; cache read ratio ${(s.cacheReadRatio * 100).toFixed(0)}%`);
  console.log(`stop reasons ${JSON.stringify(s.stopReasons)}`);
  if (s.failedCalls.length) console.log(`failed calls: ${s.failedCalls.join("; ")}`);
 
  const slowest = spans.filter((x) => !x.parentId && x.name === "support.answer").sort((a, b) => b.durationMs - a.durationMs)[0];
  if (slowest) {
    console.log(`\nslowest request (trace ${slowest.traceId.slice(0, 8)}):`);
    for (const line of waterfall(spans.filter((x) => x.traceId === slowest.traceId))) console.log(`  ${line}`);
  }
  const fired = alerts(s);
  console.log(fired.length ? `\nALERTS:\n${fired.map((a) => `  - ${a}`).join("\n")}` : "\nno alerts");
}

npm run report

feature          requests  errors  p50 ms  p95 ms  $/request   total $
support-chat           21       1     887     993    0.00345    0.0724
ticket-triage           9       0     100     106    0.00004    0.0003
 
first token p50 100 ms, p95 117 ms; cache read ratio 97%
stop reasons {"end_turn":20,"max_tokens":9}
failed calls: req_mock_b775dc05 client_error
 
slowest request (trace b7d62006):
  support.answer                 |████████████████████████████████████████| 1019 ms
    retrieve kb_search           |█                                       | 35 ms
    anthropic.messages.create    |  ██████████████████████████████████████| 959 ms (first token 141 ms)
 
ALERTS:
  - error rate 3.3% is above 2%
  - 9 calls stopped with max_tokens

    const message = await client.messages.create({
      model: "claude-haiku-5-5",
      max_tokens: 1024, // max_tokens includes thinking; too low and the answer never arrives
      output_config: { effort: "low" }, // a one-word label needs little thinking
      system: "Label this support ticket with one word: billing, bug, account, or other.",
      messages: [{ role: "user", content: ticket }],
    });

feature          requests  errors  p50 ms  p95 ms  $/request   total $
support-chat           21       1     867    1058    0.00345    0.0724
ticket-triage           9       0     102     110    0.00003    0.0002
 
stop reasons {"end_turn":29}
ALERTS:
  - error rate 3.3% is above 2%

grep -o '"app.question_preview":"[^"]*"' traces/spans.jsonl | sort -u

"app.question_preview":"How do I change my card? The old one is [card]."
"app.question_preview":"Why was I charged twice for INV-2041? My email is [email]"
...

import assert from "node:assert/strict";
import { mkdtemp, readFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { test } from "node:test";
import Anthropic from "@anthropic-ai/sdk";
import { triageTicket } from "../src/app.ts";
import { callCost } from "../src/prices.ts";
import { percentile, summarize } from "../src/report.ts";
import { redact, type SpanRecord, setupTracing } from "../src/tracing.ts";
 
const file = path.join(await mkdtemp(path.join(tmpdir(), "spans-")), "spans.jsonl");
const provider = setupTracing(file);
 
test("redaction removes emails, card numbers, and API keys from attribute values", () => {
  assert.deepEqual(redact({ q: "I am ana@example.com, card 4242 4242 4242 4242, key sk-ant-api03-abc", n: 3 }), {
    q: "I am [email], card [card], key [api-key]",
    n: 3,
  });
});
 
test("cost comes from the usage attributes, with cache reads at their own price", () => {
  const cost = callCost({
    "gen_ai.request.model": "claude-opus-5-5",
    "gen_ai.usage.input_tokens": 1_000_000, // of which 800K were read from cache
    "gen_ai.usage.cache_read.input_tokens": 800_000,
    "gen_ai.usage.output_tokens": 100_000,
  });
  assert.equal(cost, 0.8 + 0.16 + 2); // 200K uncached at $4, 800K cached at $0.20, 100K output at $20
  assert.equal(callCost({ "gen_ai.request.model": "claude-haiku-5-5", "gen_ai.usage.input_tokens": 200_000 }), 0.1); // long-prompt price
});
 
test("percentiles pick the value at or above the rank", () => {
  assert.equal(percentile([5, 1, 4, 2, 3], 50), 3);
  assert.equal(percentile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 95), 10);
});
 
test("the SDK's span for the API call nests under the feature span and carries usage", async () => {
  const headers: string[] = [];
  const client = new Anthropic({
    apiKey: "test-key",
    maxRetries: 0,
    fetch: async (_url, init) => {
      headers.push(new Headers(init?.headers).get("traceparent") ?? "");
      return Response.json(
        {
          id: "msg_1", type: "message", role: "assistant", model: "claude-haiku-5-5", stop_reason: "end_turn", stop_sequence: null,
          usage: { input_tokens: 40, output_tokens: 3, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 },
          content: [{ type: "text", text: "billing" }],
        },
        { headers: { "request-id": "req_test_1" } },
      );
    },
  });
  assert.equal(await triageTicket(client, "Card declined"), "billing");
  await provider.forceFlush();
 
  const spans = (await readFile(file, "utf8")).trim().split("\n").map((l) => JSON.parse(l) as SpanRecord);
  const root = spans.find((s) => s.name === "triage.ticket")!;
  const call = spans.find((s) => s.name.startsWith("anthropic."))!;
  assert.equal(call.parentId, root.spanId);
  assert.equal(call.attributes["gen_ai.usage.output_tokens"], 3);
  assert.equal(call.attributes["anthropic.request.id"], "req_test_1");
  assert.match(headers[0]!, new RegExp(`^00-${root.traceId}-`)); // the request carried the trace context
 
  const s = summarize(spans);
  assert.deepEqual(s.features.map((f) => [f.feature, f.requests]), [["ticket-triage", 1]]);
  assert.equal(s.stopReasons.end_turn, 1);
});

npm test   # 4 passing, no API calls