Writing About Your Engineering Work: Case Studies and Technical Posts

# Why our docs assistant missed error codes, and how hybrid search fixed it
 
Readers asked about errors like `E_CARD_DECLINED` and got answers about unrelated payment topics.
Retrieval recall on error-code questions was 52%. Adding keyword search beside vector search raised it to 96%,
for under 2 ms of extra latency.
 
## The problem
…who was affected, with one real (anonymized) example question.
 
## Constraints
- p95 time to first token under 1.5 s
- No new infrastructure: Postgres only
- Cost per answer under $0.01
 
## What we tried
| Approach | Recall@5 (error codes) | Recall@5 (all) | Notes |
| --- | --- | --- | --- |
| Vector only (baseline) | 52% | 88% | Embeddings blur exact tokens |
| Bigger embedding model | 58% | 90% | Higher cost, small gain |
| Hybrid search with RRF | 96% | 94% | One extra indexed query |
 
## The decision
Hybrid search with reciprocal rank fusion, because…
 
## What we learned
…including one thing that did not work.

export interface LintIssue {
  rule: string;
  message: string;
}
 
/** Mechanical checks for a technical post written in Markdown. */
export function lintDraft(markdown: string): LintIssue[] {
  const issues: LintIssue[] = [];
  const body = markdown
    .replace(/\r\n/g, "\n") // drafts saved on Windows use CRLF line endings
    .replace(/`{3}[\s\S]*?`{3}/g, ""); // ignore code blocks
  const blocks = body.split(/\n{2,}/).map((b) => b.trim()).filter(Boolean);
  // Prose paragraphs only: headings, lists, tables, and quotes are not sentences.
  const prose = blocks.filter((b) => !/^([#|>-]|\*\s|\d+\.\s)/.test(b));
 
  if (!/\d/.test(prose[0] ?? "")) {
    issues.push({ rule: "lead-with-result", message: "The first paragraph has no number. State the result up front." });
  }
 
  for (const sentence of prose.flatMap((p) => p.replace(/\n/g, " ").split(/(?<=[.!?])\s+/))) {
    const words = sentence.split(/\s+/).filter(Boolean).length;
    if (words > 35) issues.push({ rule: "long-sentence", message: `${words} words: "${sentence.slice(0, 60)}…"` });
  }
 
  const headings = [...body.matchAll(/^#{2,3} (.+)$/gm)].map((m) => m[1]!.toLowerCase());
  for (const required of ["decision", "learn"]) {
    if (!headings.some((h) => h.includes(required))) {
      issues.push({ rule: "structure", message: `No section about "${required}". Readers look for the decision and the lessons.` });
    }
  }
 
  for (const vague of ["very fast", "much better", "significantly", "blazing", "seamless"]) {
    if (body.toLowerCase().includes(vague)) issues.push({ rule: "vague", message: `"${vague}": replace it with a number.` });
  }
  return issues;
}

import { test } from "node:test";
import assert from "node:assert/strict";
import { lintDraft } from "./lib/draft-lint.ts";
 
test("flags a vague draft", () => {
  const rules = lintDraft("# Post\n\nWe made search much better.\n\n## How\nStuff.").map((i) => i.rule);
  assert.ok(rules.includes("lead-with-result"));
  assert.ok(rules.includes("vague"));
  assert.equal(rules.filter((r) => r === "structure").length, 2);
});
 
test("passes a well-structured draft", () => {
  const draft = "# Post\n\nRecall rose from 52% to 96%.\n\n## The decision\nHybrid search.\n\n## What we learned\nMeasure first.";
  assert.deepEqual(lintDraft(draft), []);
  assert.deepEqual(lintDraft(draft.replace(/\n/g, "\r\n")), []); // same draft saved on Windows
});

import Anthropic from "@anthropic-ai/sdk";
import { zodOutputFormat } from "@anthropic-ai/sdk/helpers/zod";
import { readFileSync } from "node:fs";
import { z } from "zod";
import { lintDraft } from "./lib/draft-lint.ts";
 
const draft = readFileSync(process.argv[2] ?? "posts/draft.md", "utf8");
for (const issue of lintDraft(draft)) console.log(`lint  [${issue.rule}] ${issue.message}`);
 
const Review = z.object({
  oneSentenceTakeaway: z.string().describe("What a reader will remember, in one sentence"),
  notes: z.array(
    z.object({
      quote: z.string().describe("The exact passage, at most 15 words"),
      problem: z.enum(["unclear", "unsupported-claim", "missing-context", "jargon", "too-long"]),
      suggestion: z.string().describe("What to change, without rewriting the passage in full"),
    }),
  ),
});
 
const client = new Anthropic();
const message = await client.messages.parse({
  model: "claude-opus-5-5",
  max_tokens: 4096,
  output_config: { effort: "medium", format: zodOutputFormat(Review) },
  system:
    "You are an experienced technical editor. Find the passages a busy engineer would find unclear, unsupported, or too long. Do not rewrite the post and do not change the author's voice. At most 8 notes, most important first.",
  messages: [{ role: "user", content: `<draft>\n${draft}\n</draft>` }],
});
 
const review = message.parsed_output;
if (review) {
  console.log(`\ntakeaway: ${review.oneSentenceTakeaway}\n`);
  for (const n of review.notes) console.log(`[${n.problem}] "${n.quote}"\n  → ${n.suggestion}`);
}