Staying Current in AI Engineering Without Drowning in Hype

import Anthropic from "@anthropic-ai/sdk";
 
const client = new Anthropic();
 
// Models your app uses, and models you have already evaluated and decided about.
const IN_USE = ["claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-4-5"];
const REVIEWED = new Set([...IN_USE, "claude-fable-5-1"]);
 
const available: { id: string; name: string; created: string }[] = [];
for await (const model of client.models.list()) {
  available.push({ id: model.id, name: model.display_name, created: model.created_at.slice(0, 10) });
}
 
const ids = new Set(available.map((m) => m.id));
const fresh = available.filter((m) => !REVIEWED.has(m.id));
const missing = IN_USE.filter((id) => !ids.has(id) && !available.some((m) => m.id.startsWith(`${id}-`)));
 
for (const m of fresh) console.log(`NEW      ${m.id} (${m.name}, ${m.created}): run your evals on it`);
for (const id of missing) console.log(`MISSING  ${id}: check the deprecation schedule and plan a migration`);
if (!fresh.length && !missing.length) console.log("Nothing new. Your model list is current.");

export interface CaseResult {
  id: string;
  passRate: number;
  tokens: number;
}
 
export interface Run {
  version: string;
  overall: number;
  results: CaseResult[];
}
 
export interface Comparison {
  overallDelta: number;
  tokenRatio: number;
  regressions: { id: string; before: number; after: number }[];
  improvements: { id: string; before: number; after: number }[];
  verdict: "adopt" | "investigate" | "reject";
}
 
export function compareRuns(baseline: Run, candidate: Run, threshold = 0.34): Comparison {
  const before = new Map(baseline.results.map((r) => [r.id, r.passRate]));
  const regressions: Comparison["regressions"] = [];
  const improvements: Comparison["improvements"] = [];
  for (const r of candidate.results) {
    const b = before.get(r.id);
    if (b === undefined) continue;
    if (b - r.passRate >= threshold) regressions.push({ id: r.id, before: b, after: r.passRate });
    if (r.passRate - b >= threshold) improvements.push({ id: r.id, before: b, after: r.passRate });
  }
  const tokens = (run: Run) => run.results.reduce((s, r) => s + r.tokens, 0);
  const overallDelta = candidate.overall - baseline.overall;
  const verdict = regressions.length === 0 && overallDelta >= 0 ? "adopt" : overallDelta < -0.05 ? "reject" : "investigate";
  return { overallDelta, tokenRatio: tokens(candidate) / Math.max(1, tokens(baseline)), regressions, improvements, verdict };
}

import { test } from "node:test";
import assert from "node:assert/strict";
import { compareRuns, type Run } from "./lib/compare-runs.ts";
 
const run = (version: string, rates: Record<string, number>, tokens = 1000): Run => {
  const results = Object.entries(rates).map(([id, passRate]) => ({ id, passRate, tokens }));
  return { version, overall: results.reduce((s, r) => s + r.passRate, 0) / results.length, results };
};
 
test("adopts a candidate that is better everywhere", () => {
  const c = compareRuns(run("old", { a: 0.33, b: 1 }), run("new", { a: 1, b: 1 }, 800));
  assert.equal(c.verdict, "adopt");
  assert.equal(c.improvements[0]?.id, "a");
  assert.equal(c.tokenRatio, 0.8);
});
 
test("investigates when one case regresses even if the average improves", () => {
  const c = compareRuns(run("old", { a: 1, b: 0.33, d: 0.33 }), run("new", { a: 0.33, b: 1, d: 1 }));
  assert.equal(c.verdict, "investigate");
  assert.deepEqual(c.regressions.map((r) => r.id), ["a"]);
});
 
test("rejects a clearly worse candidate", () => {
  assert.equal(compareRuns(run("old", { a: 1, b: 1 }), run("new", { a: 0.33, b: 0.67 })).verdict, "reject");
});