Project: Build a Docs Q&A Assistant with RAG, Citations, and Evals

CREATE EXTENSION IF NOT EXISTS vector;
 
CREATE TABLE chunks (
  id            bigserial PRIMARY KEY,
  source        text NOT NULL,          -- file path, such as guides/billing.md
  url           text NOT NULL,          -- public URL used in citations
  heading       text NOT NULL,          -- "Billing > Refunds"
  content       text NOT NULL,
  content_hash  text NOT NULL,
  embedding     vector(1024) NOT NULL,
  tsv           tsvector GENERATED ALWAYS AS (to_tsvector('english', heading || ' ' || content)) STORED,
  UNIQUE (source, content_hash)
);
CREATE INDEX ON chunks USING hnsw (embedding vector_cosine_ops);
CREATE INDEX ON chunks USING gin (tsv);
 
CREATE TABLE questions (
  id          bigserial PRIMARY KEY,
  question    text NOT NULL,
  answer      text,
  sources     text[] NOT NULL DEFAULT '{}',
  feedback    smallint,                 -- 1 helpful, -1 not helpful
  created_at  timestamptz NOT NULL DEFAULT now()
);

import pg from "pg";
 
export const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL });
export const toVector = (v: number[]) => JSON.stringify(v);

import { createHash } from "node:crypto";
 
export interface Chunk {
  heading: string;
  content: string;
  hash: string;
}
 
const MAX_CHARS = 1500;
 
export function chunkMarkdown(markdown: string): Chunk[] {
  const chunks: Chunk[] = [];
  const headings: string[] = [];
  let buffer: string[] = [];
  const flush = () => {
    const content = buffer.join("\n\n").trim();
    const heading = headings.filter(Boolean).join(" > ");
    if (content) chunks.push({ heading, content, hash: createHash("sha256").update(`${heading}\n${content}`).digest("hex") });
    buffer = [];
  };
  for (const block of markdown.split(/\n{2,}/)) {
    const h = /^(#{1,3})\s+(.+)$/.exec(block.trim());
    if (h) {
      flush();
      headings.length = h[1]!.length;
      headings[h[1]!.length - 1] = h[2]!.trim();
      continue;
    }
    if (buffer.join("\n\n").length + block.length > MAX_CHARS) flush();
    buffer.push(block.trim());
  }
  flush();
  return chunks;
}

import { readdir, readFile } from "node:fs/promises";
import path from "node:path";
import { chunkMarkdown } from "../lib/chunk.ts";
import { pool, toVector } from "../lib/db.ts";
import { embed } from "../lib/voyage.ts";
 
const DOCS_DIR = process.argv[2] ?? "docs";
const BASE_URL = process.env.DOCS_BASE_URL ?? "https://docs.example.com";
 
const files = (await readdir(DOCS_DIR, { recursive: true })).filter((f) => f.endsWith(".md")).map((f) => f.split(path.sep).join("/"));
let added = 0;
let removed = 0;
 
for (const source of files) {
  const chunks = chunkMarkdown(await readFile(path.join(DOCS_DIR, source), "utf8"));
  const url = `${BASE_URL}/${source.replace(/\.md$/, "").replace(/(^|\/)index$/, "")}`;
 
  const { rows } = await pool.query<{ content_hash: string }>("SELECT content_hash FROM chunks WHERE source = $1", [source]);
  const existing = new Set(rows.map((r) => r.content_hash));
  const current = new Set(chunks.map((c) => c.hash));
  const fresh = chunks.filter((c) => !existing.has(c.hash));
  const stale = [...existing].filter((h) => !current.has(h));
 
  const vectors = fresh.length ? await embed(fresh.map((c) => `${c.heading}\n\n${c.content}`), "document") : [];
  const client = await pool.connect();
  try {
    await client.query("BEGIN");
    if (stale.length) await client.query("DELETE FROM chunks WHERE source = $1 AND content_hash = ANY($2)", [source, stale]);
    for (const [i, c] of fresh.entries()) {
      await client.query(
        "INSERT INTO chunks (source, url, heading, content, content_hash, embedding) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING",
        [source, url, c.heading, c.content, c.hash, toVector(vectors[i]!)],
      );
    }
    await client.query("COMMIT");
  } catch (error) {
    await client.query("ROLLBACK");
    throw error;
  } finally {
    client.release();
  }
  added += fresh.length;
  removed += stale.length;
}
 
// Files deleted from the docs folder.
const gone = await pool.query("DELETE FROM chunks WHERE NOT (source = ANY($1))", [files]);
console.log(`files: ${files.length}, chunks added: ${added}, removed: ${removed + (gone.rowCount ?? 0)}`);
await pool.end();

const API = "https://api.voyageai.com/v1";
 
async function post<T>(path: string, body: unknown): Promise<T> {
  const res = await fetch(`${API}${path}`, {
    method: "POST",
    headers: { "content-type": "application/json", authorization: `Bearer ${process.env.VOYAGE_API_KEY}` },
    body: JSON.stringify(body),
  });
  if (!res.ok) throw new Error(`Voyage ${path} failed: ${res.status}`);
  return (await res.json()) as T;
}
 
export async function embed(texts: string[], inputType: "document" | "query"): Promise<number[][]> {
  const out: number[][] = [];
  for (let i = 0; i < texts.length; i += 128) {
    const { data } = await post<{ data: { embedding: number[]; index: number }[] }>("/embeddings", {
      input: texts.slice(i, i + 128),
      model: "voyage-4",
      input_type: inputType,
    });
    out.push(...data.sort((a, b) => a.index - b.index).map((d) => d.embedding));
  }
  return out;
}
 
export async function rerank(query: string, documents: string[], topK: number) {
  const { data } = await post<{ data: { index: number; relevance_score: number }[] }>("/rerank", { query, documents, model: "rerank-3", top_k: topK });
  return data.map((d) => ({ index: d.index, score: d.relevance_score }));
}

import { pool, toVector } from "./db.ts";
import { embed, rerank } from "./voyage.ts";
 
export interface Hit {
  id: number;
  url: string;
  heading: string;
  content: string;
  score: number;
}
 
const MIN_SCORE = Number(process.env.MIN_RERANK_SCORE ?? 0.3); // tune on your eval set
 
function fuse(lists: Omit<Hit, "score">[][], k = 60) {
  const scores = new Map<number, { hit: Omit<Hit, "score">; s: number }>();
  for (const list of lists) list.forEach((hit, rank) => {
    const e = scores.get(hit.id) ?? { hit, s: 0 };
    e.s += 1 / (k + rank + 1);
    scores.set(hit.id, e);
  });
  return [...scores.values()].sort((a, b) => b.s - a.s).map((e) => e.hit);
}
 
export async function search(question: string, topK = 5): Promise<Hit[]> {
  const [vector] = await embed([question], "query");
  const cols = "id, url, heading, content";
  const [semantic, keyword] = await Promise.all([
    pool.query<Omit<Hit, "score">>(`SELECT ${cols} FROM chunks ORDER BY embedding <=> $1 LIMIT 20`, [toVector(vector!)]),
    pool.query<Omit<Hit, "score">>(
      `SELECT ${cols} FROM chunks WHERE tsv @@ websearch_to_tsquery('english', $1)
       ORDER BY ts_rank_cd(tsv, websearch_to_tsquery('english', $1)) DESC LIMIT 20`,
      [question],
    ),
  ]);
  const candidates = fuse([semantic.rows, keyword.rows]).slice(0, 20);
  if (candidates.length === 0) return [];
  const ranked = await rerank(question, candidates.map((c) => `${c.heading}\n\n${c.content}`), topK);
  return ranked.filter((r) => r.score >= MIN_SCORE).map((r) => ({ ...candidates[r.index]!, score: r.score }));
}

import Anthropic from "@anthropic-ai/sdk";
import { z } from "zod";
import { pool } from "@/lib/db";
import { search } from "@/lib/search";
 
const anthropic = new Anthropic();
const Body = z.object({ question: z.string().trim().min(3).max(1000) });
const NOT_FOUND = "I couldn't find this in the documentation. Try rephrasing, or ask in the community forum.";
 
const sse = (event: string, data: unknown) => `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
 
export async function POST(request: Request) {
  const parsed = Body.safeParse(await request.json().catch(() => null));
  if (!parsed.success) return Response.json({ error: "Ask a question of 3 to 1,000 characters." }, { status: 400 });
  const { question } = parsed.data;
 
  const hits = await search(question);
  const encoder = new TextEncoder();
 
  const body = new ReadableStream<Uint8Array>({
    async start(controller) {
      const send = (event: string, data: unknown) => controller.enqueue(encoder.encode(sse(event, data)));
      let answer = "";
      let sources: { url: string; heading: string }[] = [];
      try {
        if (hits.length === 0) {
          answer = NOT_FOUND;
          send("text", { text: answer });
        } else {
          const stream = anthropic.messages.stream(
            {
              model: "claude-opus-5-5",
              max_tokens: 2048,
              system:
                "You answer questions about this product's documentation using only the search results provided. Be concise and practical; include code from the docs when helpful. If the results do not answer the question, say so instead of guessing.",
              messages: [
                {
                  role: "user",
                  content: [
                    ...hits.map((h): Anthropic.SearchResultBlockParam => ({
                      type: "search_result",
                      source: h.url,
                      title: h.heading,
                      content: [{ type: "text", text: h.content }],
                      citations: { enabled: true },
                    })),
                    { type: "text", text: question },
                  ],
                },
              ],
            },
            { signal: request.signal },
          );
          stream.on("text", (text) => {
            answer += text;
            send("text", { text });
          });
          const final = await stream.finalMessage();
          // Sources = documents the model actually cited, in order of first citation.
          const cited = new Map<string, string>();
          for (const block of final.content) {
            if (block.type !== "text") continue;
            for (const c of block.citations ?? []) if (c.type === "search_result_location") cited.set(c.source, c.title ?? c.source);
          }
          sources = [...cited].map(([url, heading]) => ({ url, heading }));
        }
        const { rows } = await pool.query<{ id: string }>(
          "INSERT INTO questions (question, answer, sources) VALUES ($1, $2, $3) RETURNING id",
          [question, answer, sources.map((s) => s.url)],
        );
        send("sources", { questionId: rows[0]!.id, sources });
        send("done", {});
      } catch (error) {
        if (!request.signal.aborted) {
          console.error("ask failed", error);
          send("error", { message: "Something went wrong. Please try again." });
        }
      } finally {
        try {
          controller.close();
        } catch {
          // client disconnected
        }
      }
    },
  });
  return new Response(body, { headers: { "content-type": "text/event-stream", "cache-control": "no-cache, no-transform" } });
}

import { z } from "zod";
import { pool } from "@/lib/db";
 
const Body = z.object({ questionId: z.coerce.number().int().positive(), helpful: z.boolean() });
 
export async function POST(request: Request) {
  const parsed = Body.safeParse(await request.json().catch(() => null));
  if (!parsed.success) return Response.json({ error: "Invalid feedback" }, { status: 400 });
  await pool.query("UPDATE questions SET feedback = $2 WHERE id = $1", [parsed.data.questionId, parsed.data.helpful ? 1 : -1]);
  return new Response(null, { status: 204 });
}

-- Unhelpful answers from the last 7 days, most recent first: review them and turn the real failures into eval cases.
SELECT id, question, answer, sources, created_at
FROM questions
WHERE feedback = -1 AND created_at > now() - interval '7 days'
ORDER BY created_at DESC;