Tutorial: Ask This Wiki, a Local RAG Pipeline with Hybrid Search

node src/ingest.ts ../wiki/content          # chunk and index every .md and .mdx file
node src/eval.ts                            # recall@5 and MRR for keyword, vector, and hybrid search
node src/ask.ts "What is backpressure in Node.js streams?"

mkdir ask-wiki && cd ask-wiki
npm init -y
npm pkg set type=module
npm install @anthropic-ai/sdk
npm install -D typescript @types/node
mkdir src test
npm pkg set scripts.test="node --test" scripts.typecheck="tsc"

export interface Chunk {
  source: string; // file path relative to the docs folder, used for citations
  title: string; // "Page title > Section > Subsection"
  content: string;
}
 
const MAX_CHARS = 1500; // roughly 300 to 400 tokens of English
 
/** Turn MDX into plain Markdown: drop frontmatter, imports, and component tags, keep the prose. */
export function mdxToMarkdown(source: string): { title: string; markdown: string } {
  const frontmatter = /^---\n([\s\S]*?)\n---\n/.exec(source);
  const title = /^title:\s*"?(.*?)"?\s*$/m.exec(frontmatter?.[1] ?? "")?.[1] ?? "";
  const body = source
    .slice(frontmatter?.[0].length ?? 0)
    .replace(/^import .*$/gm, "")
    .replace(/^\s*<\/?[A-Z][\w.]*(\s[^>]*)?\/?>\s*$/gm, "") // lines that are only a component tag
    .replace(/<[A-Z][\w.]*\s[^>]*\/>/g, ""); // inline self-closing components such as <Term />
  return { title, markdown: body };
}
 
/** Split Markdown on headings, then by size, keeping the heading path as each chunk's title. */
export function chunkMarkdown(source: string, pageTitle: string, markdown: string): Chunk[] {
  const chunks: Chunk[] = [];
  const headings: string[] = [];
  let buffer: string[] = [];
 
  const flush = () => {
    const content = buffer.join("\n\n").trim();
    if (content) chunks.push({ source, title: [pageTitle, ...headings.filter(Boolean)].join(" > "), content });
    buffer = [];
  };
 
  for (const block of markdown.split(/\n{2,}/)) {
    const heading = /^(#{2,4})\s+(.+)$/.exec(block.trim());
    if (heading) {
      flush();
      const level = heading[1]!.length - 1; // ## is the top section level in these pages
      headings.length = level;
      headings[level - 1] = heading[2]!.trim();
      continue;
    }
    if (buffer.join("\n\n").length + block.length > MAX_CHARS) flush();
    buffer.push(block.trim());
  }
  flush();
  return chunks;
}

import { createHash } from "node:crypto";
 
const VOYAGE = process.env.VOYAGE_BASE_URL ?? "https://api.voyageai.com/v1";
const BATCH = 128;
 
/** Which embedding model this process uses. An index can only be searched with the model that built it. */
export const EMBEDDING_MODEL = process.env.VOYAGE_API_KEY ? "voyage-4" : "toy-hash-256";
 
/** The hashing-trick embedding from Foundations Tutorial 4: matches shared words only, but needs no key. */
function toyEmbed(text: string): number[] {
  const v = new Array<number>(256).fill(0);
  for (const word of text.toLowerCase().match(/[a-z0-9]+/g) ?? []) {
    if (word.length < 3) continue;
    const h = createHash("sha256").update(word).digest();
    v[h.readUInt16BE(0) % 256]! += h[2]! & 1 ? 1 : -1;
  }
  const length = Math.hypot(...v) || 1;
  return v.map((x) => x / length);
}
 
export async function embed(texts: string[], inputType: "document" | "query"): Promise<number[][]> {
  if (EMBEDDING_MODEL === "toy-hash-256") return texts.map(toyEmbed);
  const vectors: number[][] = [];
  for (let i = 0; i < texts.length; i += BATCH) {
    const res = await fetch(`${VOYAGE}/embeddings`, {
      method: "POST",
      headers: { "content-type": "application/json", authorization: `Bearer ${process.env.VOYAGE_API_KEY}` },
      body: JSON.stringify({ input: texts.slice(i, i + BATCH), model: EMBEDDING_MODEL, input_type: inputType }),
    });
    if (!res.ok) throw new Error(`Voyage embeddings failed: ${res.status} ${await res.text()}`);
    const { data } = (await res.json()) as { data: { embedding: number[]; index: number }[] };
    vectors.push(...data.sort((a, b) => a.index - b.index).map((d) => d.embedding));
  }
  return vectors;
}
 
export const cosine = (a: number[], b: number[]) => a.reduce((sum, x, i) => sum + x * b[i]!, 0); // vectors are unit length

// BM25: the classic keyword ranking. It rewards query words that are rare in the corpus and
// frequent in the chunk, with diminishing returns for repetition and a penalty for long chunks.
const K1 = 1.2;
const B = 0.75;
const STOP = new Set(["the", "and", "for", "with", "that", "this", "how", "what", "why", "when", "does", "can", "you", "your", "are", "is", "do", "of", "to", "in", "a", "an", "it", "on", "or", "be", "i"]);
 
export const terms = (text: string) => (text.toLowerCase().match(/[a-z0-9_]+/g) ?? []).filter((t) => t.length > 1 && !STOP.has(t));
 
type Counts = Record<string, number>;
 
// Words like "constructor" or "__proto__" appear in technical docs. On a plain object, counts["constructor"]
// finds Object.prototype.constructor, and the arithmetic quietly becomes NaN. Look up own keys only.
const get = (counts: Counts, term: string) => (Object.hasOwn(counts, term) ? counts[term]! : 0);
const bag = (): Counts => Object.create(null) as Counts;
 
export interface Bm25Index {
  docFreq: Counts; // in how many chunks each term appears
  lengths: number[]; // term count per chunk
  termCounts: Counts[]; // term -> count, per chunk
}
 
export function buildBm25(texts: string[]): Bm25Index {
  const docFreq = bag();
  const termCounts = texts.map((text) => {
    const counts = bag();
    for (const t of terms(text)) counts[t] = get(counts, t) + 1;
    for (const t of Object.keys(counts)) docFreq[t] = get(docFreq, t) + 1;
    return counts;
  });
  return { docFreq, termCounts, lengths: termCounts.map((c) => Object.values(c).reduce((a, b) => a + b, 0)) };
}
 
/** Score every chunk for `query`; returns chunk indexes, best first, skipping chunks with no matching term. */
export function bm25Search(index: Bm25Index, query: string, limit: number): number[] {
  const n = index.lengths.length;
  const avgLength = index.lengths.reduce((a, b) => a + b, 0) / n;
  const queryTerms = [...new Set(terms(query))].filter((t) => get(index.docFreq, t) > 0);
  const scores = index.termCounts.map((counts, i) =>
    queryTerms.reduce((score, t) => {
      const tf = get(counts, t);
      if (tf === 0) return score;
      const df = get(index.docFreq, t);
      const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5));
      return score + (idf * tf * (K1 + 1)) / (tf + K1 * (1 - B + (B * index.lengths[i]!) / avgLength));
    }, 0),
  );
  return scores
    .map((score, i) => ({ score, i }))
    .filter((s) => s.score > 0)
    .sort((a, b) => b.score - a.score)
    .slice(0, limit)
    .map((s) => s.i);
}

import { readdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { buildBm25 } from "./bm25.ts";
import { type Chunk, chunkMarkdown, mdxToMarkdown } from "./chunk.ts";
import { EMBEDDING_MODEL, embed } from "./embed.ts";
 
const root = process.argv[2];
if (!root) throw new Error("usage: node src/ingest.ts <folder of .md or .mdx files>");
 
// Every Markdown file under the folder, recursively.
const files = (await readdir(root, { recursive: true })).filter((f) => /\.mdx?$/.test(f)).sort();
 
const chunks: Chunk[] = [];
for (const file of files) {
  const { title, markdown } = mdxToMarkdown(await readFile(join(root, file), "utf8"));
  // readdir returns paths relative to `root`; use forward slashes on every OS.
  chunks.push(...chunkMarkdown(file.replace(/\\/g, "/"), title || file, markdown));
}
 
// Embedding the title with the content helps short chunks match.
const texts = chunks.map((c) => `${c.title}\n\n${c.content}`);
const vectors = await embed(texts, "document");
const round = (v: number[]) => v.map((x) => Math.round(x * 1e5) / 1e5); // smaller file, same ranking
 
await writeFile(
  "index.json",
  JSON.stringify({ model: EMBEDDING_MODEL, chunks, vectors: vectors.map(round), bm25: buildBm25(texts) }),
);
console.log(`indexed ${files.length} files into ${chunks.length} chunks with ${EMBEDDING_MODEL}`);

import { readFile } from "node:fs/promises";
import { type Bm25Index, bm25Search } from "./bm25.ts";
import type { Chunk } from "./chunk.ts";
import { cosine, EMBEDDING_MODEL, embed } from "./embed.ts";
 
interface Index {
  model: string;
  chunks: Chunk[];
  vectors: number[][];
  bm25: Bm25Index;
}
 
const CANDIDATES = 20;
let index: Index | undefined;
 
async function load(): Promise<Index> {
  index ??= JSON.parse(await readFile("index.json", "utf8")) as Index;
  if (index.model !== EMBEDDING_MODEL) {
    throw new Error(`index.json was built with ${index.model} but this process embeds with ${EMBEDDING_MODEL}; re-run ingest`);
  }
  return index;
}
 
/** Merge ranked lists: an item near the top of any list scores well; in both lists, better still. */
export function reciprocalRankFusion(lists: number[][], k = 60): number[] {
  const scores = new Map<number, number>();
  for (const list of lists) list.forEach((id, rank) => scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank + 1)));
  return [...scores].sort((a, b) => b[1] - a[1]).map(([id]) => id);
}
 
export type Mode = "hybrid" | "keyword" | "vector";
 
export async function search(question: string, topK = 5, mode: Mode = "hybrid"): Promise<Chunk[]> {
  const { chunks, vectors, bm25 } = await load();
 
  const byKeyword = mode === "vector" ? [] : bm25Search(bm25, question, CANDIDATES);
  let byMeaning: number[] = [];
  if (mode !== "keyword") {
    const [q] = await embed([question], "query");
    byMeaning = vectors
      .map((v, i) => ({ i, score: cosine(v, q!) }))
      .sort((a, b) => b.score - a.score)
      .slice(0, CANDIDATES)
      .map((s) => s.i);
  }
 
  return reciprocalRankFusion([byMeaning, byKeyword]).slice(0, topK).map((i) => chunks[i]!);
}

import { type Mode, search } from "./search.ts";
 
// Questions a reader might ask, and the page(s) that answer them. Grow this list from real questions.
const CASES = [
  { question: "What is backpressure in Node.js streams?", expected: ["foundations/nodejs-deep-dive.mdx"] },
  { question: "Why must Server Actions validate their input?", expected: ["foundations/react-and-nextjs.mdx"] },
  { question: "When should I use WebSockets instead of Server-Sent Events?", expected: ["foundations/apis-and-authentication.mdx"] },
  { question: "How much does a cache read cost compared with normal input tokens?", expected: ["llm-engineering/prompt-caching-and-cost.mdx", "llm-fundamentals/api-keys-billing-and-security.mdx"] },
  { question: "What does assigning to never in a switch default case do?", expected: ["foundations/modern-javascript-and-typescript.mdx"] },
  { question: "How do I return results for parallel tool calls?", expected: ["llm-engineering/tool-use.mdx"] },
  { question: "What is reciprocal rank fusion?", expected: ["llm-engineering/rag.mdx"] },
  { question: "How do I store embeddings in Postgres with pgvector?", expected: ["foundations/databases.mdx", "llm-engineering/rag.mdx", "foundations/postgres-semantic-notes.mdx"] },
];
 
const K = 5;
const verbose = process.argv.includes("--verbose");
 
// Measure each retriever on its own and together: fusion only helps if both inputs are good.
for (const mode of ["keyword", "vector", "hybrid"] satisfies Mode[]) {
  let found = 0;
  let reciprocalRanks = 0;
  for (const { question, expected } of CASES) {
    const hits = await search(question, K, mode);
    const rank = hits.findIndex((h) => expected.includes(h.source)) + 1; // 0 when missing
    if (rank > 0) {
      found++;
      reciprocalRanks += 1 / rank;
    }
    if (verbose) console.log(`  ${rank > 0 ? `✓ ${rank}` : "✗  "} ${question}\n        ${hits.map((h) => h.source).join(", ")}`);
  }
  console.log(`${mode.padEnd(8)} recall@${K}: ${String(Math.round((found / CASES.length) * 100)).padStart(3)}%   MRR: ${(reciprocalRanks / CASES.length).toFixed(2)}`);
}

node src/ingest.ts ../wiki/content
node src/eval.ts

indexed 59 files into 748 chunks with toy-hash-256
keyword  recall@5: 100%   MRR: 0.72
vector   recall@5:  50%   MRR: 0.40
hybrid   recall@5:  75%   MRR: 0.51

import Anthropic from "@anthropic-ai/sdk";
import { search } from "./search.ts";
 
const client = new Anthropic();
const question = process.argv.slice(2).join(" ");
if (!question) throw new Error('usage: node src/ask.ts "your question"');
 
const hits = await search(question, 5);
 
const message = await client.messages.create({
  model: "claude-opus-5-5",
  max_tokens: 4096,
  output_config: { effort: "low" },
  system:
    "You answer questions about an engineering wiki using only the search results provided. " +
    "If they do not contain the answer, say so plainly instead of guessing.",
  messages: [
    {
      role: "user",
      content: [
        // Search results as content blocks: the API returns citations that point back into them.
        ...hits.map((h): Anthropic.SearchResultBlockParam => ({
          type: "search_result",
          source: h.source,
          title: h.title,
          content: [{ type: "text", text: h.content }],
          citations: { enabled: true },
        })),
        { type: "text", text: question },
      ],
    },
  ],
});
 
// Print the answer with numbered references after each cited passage.
const sources: string[] = [];
let output = "";
for (const block of message.content) {
  if (block.type !== "text") continue;
  output += block.text;
  for (const c of block.citations ?? []) {
    if (c.type !== "search_result_location") continue;
    const ref = `${c.source} (${c.title ?? ""})`;
    if (!sources.includes(ref)) sources.push(ref);
    output += ` [${sources.indexOf(ref) + 1}]`;
  }
}
console.log(output.trim());
if (sources.length > 0) console.log(`\n${sources.map((s, i) => `[${i + 1}] ${s}`).join("\n")}`);
else console.log("\n(no citations: treat this answer as unsupported)");

node --env-file=.env src/ask.ts "What is backpressure in Node.js streams?"

Backpressure is what happens when a stream's destination is slower than its source: chunks pile up
in memory unless the producer waits. [1] pipeline() handles it automatically and cleans up every
stage if one fails. [1]
 
[1] foundations/nodejs-deep-dive.mdx (Node.js Deep Dive: Event Loop, Async I/O, Streams, and ESM > Streams: process data as it arrives)

import assert from "node:assert/strict";
import { test } from "node:test";
import { bm25Search, buildBm25 } from "../src/bm25.ts";
import { chunkMarkdown, mdxToMarkdown } from "../src/chunk.ts";
import { reciprocalRankFusion } from "../src/search.ts";
 
test("MDX loses frontmatter and component tags but keeps prose and the title", () => {
  const { title, markdown } = mdxToMarkdown('---\ntitle: "Streams"\norder: 2\n---\n\n<WhyItMatters>\n\nStreams matter.\n\n</WhyItMatters>\n\nSee <Term id="x" /> here.\n');
  assert.equal(title, "Streams");
  assert.doesNotMatch(markdown, /WhyItMatters|order: 2|<Term/);
  assert.match(markdown, /Streams matter\./);
});
 
test("chunks carry their heading path as a title", () => {
  const chunks = chunkMarkdown("a.mdx", "Guide", "Intro text.\n\n## Setup\n\nInstall it.\n\n### On Windows\n\nUse PowerShell.");
  assert.deepEqual(chunks.map((c) => c.title), ["Guide", "Guide > Setup", "Guide > Setup > On Windows"]);
});
 
test("BM25 ranks the chunk with the rare matching word first", () => {
  const index = buildBm25(["the event loop runs callbacks", "backpressure slows a fast producer", "callbacks and promises in the event loop"]);
  assert.deepEqual(bm25Search(index, "what is backpressure?", 3), [1]);
});
 
test("BM25 survives words that exist on Object.prototype, even after a JSON round trip", () => {
  const index = JSON.parse(JSON.stringify(buildBm25(["call the constructor first", "toString and __proto__ explained", "streams and backpressure"])));
  assert.deepEqual(bm25Search(index, "backpressure", 3), [2]);
  assert.deepEqual(bm25Search(index, "constructor", 3), [0]);
});
 
test("reciprocal rank fusion rewards items found by both lists", () => {
  assert.deepEqual(reciprocalRankFusion([[1, 2, 3], [3, 4, 1]]).slice(0, 2), [1, 3]);
});

npm test   # 5 passing