An AI Engineer Portfolio That Gets Interviews

# Acme Docs Assistant
 
Ask questions about the Acme SDK docs and get cited answers. Live: https://docs-assistant.example.dev
 
![Demo: asking how to paginate results](./docs/demo.gif)
 
## Results
| Metric | Value |
| --- | --- |
| Retrieval recall@5 (48 questions) | 94% |
| Answer pass rate (judge + code checks, 3 trials) | 89% |
| "Not in the docs" correctly detected (12 out-of-scope questions) | 11 / 12 |
| p95 time to first token | 1.1 s |
| Cost per answer | ~$0.006 |
 
## Architecture
(diagram) Ingest job → Postgres + pgvector → hybrid search → rerank → Claude with citations.
 
## Key decisions
- **Hybrid search over vector-only**: error codes and API names were missed by embeddings (recall 81% → 94%).
- **Relevance threshold** before calling the model: out-of-scope questions get an honest answer for free.
- **Incremental ingestion** with content hashes: re-indexing after a docs change costs cents, not dollars.
 
## Run it locally
`cp .env.example .env.local`, then `docker compose up -d && npm install && npm run db:migrate && npm run dev`.
 
## What I'd do next
Multi-version docs, and a feedback loop that opens docs issues for unclear pages.

import { readFileSync, writeFileSync } from "node:fs";
 
interface CaseResult {
  id: string;
  tags: string[];
  passRate: number;
  tokens: number;
}
 
interface EvalRun {
  version: string;
  overall: number;
  results: CaseResult[];
}
 
const pct = (v: number) => `${(v * 100).toFixed(0)}%`;
 
export function resultsTable(run: EvalRun): string {
  const byTag = new Map<string, number[]>();
  for (const r of run.results) for (const tag of r.tags) byTag.set(tag, [...(byTag.get(tag) ?? []), r.passRate]);
 
  const rows = [...byTag]
    .sort(([a], [b]) => a.localeCompare(b))
    .map(([tag, rates]) => `| ${tag} | ${rates.length} | ${pct(rates.reduce((s, v) => s + v, 0) / rates.length)} |`);
 
  return [
    `**Eval results** (${run.version}, ${run.results.length} case${run.results.length === 1 ? "" : "s"}): **${pct(run.overall)}** overall`,
    "",
    "| Area | Cases | Pass rate |",
    "| --- | --- | --- |",
    ...rows,
  ].join("\n");
}
 
// Replace the block between the markers in README.md with a fresh table.
if (import.meta.main) {
  const run = JSON.parse(readFileSync("evals/results/latest.json", "utf8")) as EvalRun;
  const readme = readFileSync("README.md", "utf8");
  const updated = readme.replace(/<!-- results:start -->[\s\S]*<!-- results:end -->/, `<!-- results:start -->\n${resultsTable(run)}\n<!-- results:end -->`);
  writeFileSync("README.md", updated);
  console.log(updated === readme ? "README already up to date" : "README results updated");
}

import { test } from "node:test";
import assert from "node:assert/strict";
import { resultsTable } from "./scripts/readme-results.ts";
 
test("summarizes pass rates per tag", () => {
  const table = resultsTable({
    version: "support-v3",
    overall: 0.85,
    results: [
      { id: "a", tags: ["refunds"], passRate: 1, tokens: 900 },
      { id: "b", tags: ["refunds"], passRate: 0.5, tokens: 900 },
      { id: "c", tags: ["scope"], passRate: 1, tokens: 700 },
    ],
  });
  assert.match(table, /\*\*85%\*\* overall/);
  assert.match(table, /\| refunds \| 2 \| 75% \|/);
  assert.match(table, /\| scope \| 1 \| 100% \|/);
});