fix: generate proper markdown for API reference pages in copy/llms features

API reference pages use the APIPage component which dynamically renders from
OpenAPI specs. Previously, the copy page feature and llms-full.txt would
extract JSX code instead of actual documentation.

Changes:
- Added getApiDocContent() to generate markdown from OpenAPI spec
- Updated page.tsx and raw route to use API content generator
- Added api-reference to DOCS_CATEGORIES and llms-full.txt
- Removed api-reference/** from exclusion list

Result: Copy page, /raw/ endpoints, and llms-full.txt now include properly
formatted API documentation with endpoints, parameters, and schemas (~12,900
additional tokens, +35%).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

Author: derrekcoleman

app/[[...slug]]/page.tsx

@@ -12,7 +12,7 @@ import { HTMLAttributes } from "react";
 import { Callout, CalloutProps } from "@/components/theme/callout";
 import { Card, CardProps, Cards } from "@/components/theme/card";
 import { DocsBody, DocsDescription, DocsPage, DocsTitle } from "@/components/theme/page";
-import { getRawDocContent } from "@/lib/files";
+import { getApiDocContent, getRawDocContent } from "@/lib/files";
 import { createMetadata } from "@/lib/metadata";
 import { source } from "@/lib/source";
 import { getMDXComponents } from "@/mdx-components";
@@ -73,7 +73,19 @@ export default async function Page(props: { params: Promise<{ slug?: string[] }>
         filePath = path.join(process.cwd(), "docs", page.file.path);
       }
 
-      const docContent = await getRawDocContent(filePath);
+      // Check if this is an API reference page
+      const isApiReferencePage = page.file.path.includes("reference/endpoints/");
+      const specPath = path.join(process.cwd(), "specs", "competitions.json");
+
+      let docContent;
+      if (isApiReferencePage && !isApiReferenceRootPage) {
+        // For API pages, generate content from OpenAPI spec
+        docContent = await getApiDocContent(filePath, specPath);
+      } else {
+        // For regular pages, use the existing method
+        docContent = await getRawDocContent(filePath);
+      }
+
       markdownContent = `# ${docContent.title}\n\n${docContent.description}\n\n${docContent.content}`;
 
       // Generate markdown URL server-side

app/raw/[...slug]/route.ts

@@ -1,7 +1,7 @@
 import { NextResponse } from "next/server";
 import path from "path";
 
-import { getRawDocContent } from "@/lib/files";
+import { getApiDocContent, getRawDocContent } from "@/lib/files";
 
 export async function GET(_: Request, { params }: { params: Promise<{ slug: string[] }> }) {
   try {
@@ -11,7 +11,22 @@ export async function GET(_: Request, { params }: { params: Promise<{ slug: stri
     // 2. They have the `docs` slug prefix removed (removed from the passed github file url)
     const slug = (await params).slug.map((s) => s.replace(".md", ".mdx"));
     const filePath = path.join(process.cwd(), "docs", slug.join("/"));
-    const { content, title, description } = await getRawDocContent(filePath);
+
+    // Check if this is an API reference page
+    const isApiReferencePage = slug.join("/").includes("reference/endpoints/");
+    const isApiReferenceRootPage = slug.length === 2 && slug[0] === "reference" && slug[1] === "endpoints";
+    const specPath = path.join(process.cwd(), "specs", "competitions.json");
+
+    let docContent;
+    if (isApiReferencePage && !isApiReferenceRootPage) {
+      // For API pages, generate content from OpenAPI spec
+      docContent = await getApiDocContent(filePath, specPath);
+    } else {
+      // For regular pages, use the existing method
+      docContent = await getRawDocContent(filePath);
+    }
+
+    const { content, title, description } = docContent;
     const merged = `# ${title}\n\n${description}\n\n${content}`;
     const filename = slug.pop() || "index.md";
 

lib/files.ts

@@ -10,8 +10,8 @@ import remarkStringify from "remark-stringify";
 import { type DocsFile } from "@/lib/ai";
 
 // Note: these map to the folder names in the `docs` directory
-// We **don't** want to include the `reference/endpoints` folder
 export const DOCS_CATEGORIES = {
+  reference: "API Reference Documentation",
   competitions: "Competitions guides and usage",
   quickstart: "Quickstart guides for building agents with Recall",
   root: "Introduction to the Recall Network",
@@ -31,27 +31,52 @@ export function getCategoryDisplayName(filePath: string): string {
 export async function getDocsContent(docsDir: string): Promise<DocsFile[]> {
   const files = await fg(["**/*.mdx"], {
     cwd: docsDir,
-    ignore: ["reference/endpoints/**", "**/_*.mdx"],
+    ignore: ["**/_*.mdx"],
     absolute: true,
     dot: false,
   });
 
+  const specPath = path.join(docsDir, "..", "specs", "competitions.json");
+
   const scanned = await Promise.all(
     files.map(async (file) => {
-      const fileContent = await fs.readFile(file);
-      const { content, data } = matter(fileContent.toString());
       const relativePath = path.relative(docsDir, file);
       const category = getCategoryDisplayName(relativePath);
-      const processed = await processContent(content);
+
+      // Check if this is an API reference page (not the index page)
+      const isApiReferencePage = relativePath.includes("reference/endpoints/") &&
+                                  !relativePath.endsWith("endpoints/index.mdx");
+
+      let title: string;
+      let description: string;
+      let keywords: string;
+      let processed: string;
+
+      if (isApiReferencePage) {
+        // For API pages, use getApiDocContent to generate markdown from OpenAPI spec
+        const apiContent = await getApiDocContent(file, specPath);
+        title = apiContent.title;
+        description = apiContent.description;
+        keywords = "";
+        processed = apiContent.content;
+      } else {
+        // For regular pages, use the existing method
+        const fileContent = await fs.readFile(file);
+        const { content, data } = matter(fileContent.toString());
+        title = data.title || relativePath;
+        description = data.description || "";
+        keywords = data.keywords || "";
+        processed = await processContent(content);
+      }
 
       // Make sure `index` is removed from the filename and strip the suffix—creating the slug
       const filename = relativePath.replace(/\.mdx$/, "").replace(/\/index$/, "");
       return {
         file: filename,
         category,
-        title: data.title || filename,
-        description: data.description || "",
-        keywords: data.keywords || "",
+        title,
+        description,
+        keywords,
         content: processed,
       };
     })
@@ -87,3 +112,108 @@ async function processContent(content: string): Promise<string> {
 
   return String(file);
 }
+
+interface Operation {
+  path: string;
+  method: string;
+}
+
+export async function getApiDocContent(
+  file: string,
+  specPath: string
+): Promise<{
+  title: string;
+  description: string;
+  content: string;
+}> {
+  const fileExists = await fs
+    .access(file)
+    .then(() => true)
+    .catch(() => false);
+  if (!fileExists) {
+    throw new Error("File not found");
+  }
+
+  const fileContent = await fs.readFile(file);
+  const { data, content } = matter(fileContent.toString());
+
+  // Extract operations from the JSX content
+  const operationsMatch = content.match(/operations=\{(\[[\s\S]*?\])\}/);
+  if (!operationsMatch || !operationsMatch[1]) {
+    // Fall back to regular content if no operations found
+    return getRawDocContent(file);
+  }
+
+  let operations: Operation[];
+  try {
+    // Parse the operations array (it's in JSON-like format)
+    const parsed = eval(operationsMatch[1]);
+    if (!Array.isArray(parsed)) {
+      return getRawDocContent(file);
+    }
+    operations = parsed as Operation[];
+  } catch {
+    // Fall back if parsing fails
+    return getRawDocContent(file);
+  }
+
+  // Read the OpenAPI spec
+  const specFile = await fs.readFile(specPath, "utf8");
+  const spec = JSON.parse(specFile);
+
+  // Generate markdown content from the spec
+  let markdown = "";
+
+  operations.forEach(({ path: opPath, method }) => {
+    const op = spec.paths?.[opPath]?.[method];
+    if (!op) return;
+
+    markdown += `## ${method.toUpperCase()} ${opPath}\n\n`;
+
+    if (op.summary) {
+      markdown += `**${op.summary}**\n\n`;
+    }
+
+    if (op.description) {
+      markdown += `${op.description}\n\n`;
+    }
+
+    // Parameters
+    if (op.parameters?.length) {
+      markdown += "**Parameters:**\n\n";
+      op.parameters.forEach((param: { name: string; in: string; description?: string; required?: boolean; schema?: { type: string } }) => {
+        const required = param.required ? " (required)" : "";
+        const type = param.schema?.type ? `: ${param.schema.type}` : "";
+        markdown += `- \`${param.name}\` (${param.in}${type})${required}: ${param.description || ""}\n`;
+      });
+      markdown += "\n";
+    }
+
+    // Request body
+    if (op.requestBody?.content?.["application/json"]?.schema) {
+      markdown += "**Request Body:**\n\n";
+      markdown += `\`\`\`json\n${JSON.stringify(op.requestBody.content["application/json"].schema, null, 2)}\n\`\`\`\n\n`;
+    }
+
+    // Response
+    const responses = op.responses || {};
+    const successResponse = responses["200"] || responses["201"] || responses["204"];
+    if (successResponse) {
+      markdown += "**Success Response:**\n\n";
+      if (successResponse.description) {
+        markdown += `${successResponse.description}\n\n`;
+      }
+      if (successResponse.content?.["application/json"]?.schema) {
+        markdown += `\`\`\`json\n${JSON.stringify(successResponse.content["application/json"].schema, null, 2)}\n\`\`\`\n\n`;
+      }
+    }
+
+    markdown += "---\n\n";
+  });
+
+  return {
+    title: data.title || file,
+    description: data.description || "",
+    content: markdown.trim(),
+  };
+}