Experimental LLM response optimization

.changeset/llm-integration.md

@@ -0,0 +1,37 @@
+---
+'astro': minor
+---
+
+Add experimental LLM optimization feature for page responses.
+
+LLM applications often work better with Markdown content than HTML, as it's more structured and semantically clear. This feature enables dynamic content negotiation based on the `Accept` header, allowing LLM tools to request content in Markdown format.
+
+When `experimental.llm.optimizePageResponse` is enabled in your Astro config, pages will automatically convert HTML responses to Markdown when a client sends an `Accept: text/markdown` header. This is particularly useful for:
+
+- LLM-powered chat applications that need to process page content
+- Search engines and indexing tools that prefer Markdown
+- Server-side rendering scenarios where you want to serve the same page in multiple formats
+
+**Configuration example:**
+
+```javascript
+// astro.config.mjs
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+  output: 'server',
+  experimental: {
+    llm: {
+      optimizePageResponse: true,
+    },
+  },
+});
+```
+
+When enabled, clients can request Markdown by setting the Accept header:
+
+```bash
+curl -H "Accept: text/markdown" https://example.com/page
+```
+
+The response will include `Content-Type: text/markdown` and the HTML content will be automatically converted to clean, well-formatted Markdown.

packages/astro/package.json

@@ -155,6 +155,7 @@
     "tinyexec": "^1.0.1",
     "tinyglobby": "^0.2.14",
     "tsconfck": "^3.1.6",
+    "turndown": "^7.2.0",
     "ultrahtml": "^1.6.0",
     "unifont": "~0.6.0",
     "unist-util-visit": "^5.0.0",
@@ -188,6 +189,7 @@
     "@types/picomatch": "^3.0.2",
     "@types/prompts": "^2.4.9",
     "@types/semver": "^7.7.0",
+    "@types/turndown": "^5.0.5",
     "@types/yargs-parser": "^21.0.3",
     "astro-scripts": "workspace:*",
     "cheerio": "1.1.2",

packages/astro/src/core/app/types.ts

@@ -95,6 +95,9 @@ export type SSRManifest = {
 	buildClientDir: string | URL;
 	buildServerDir: string | URL;
 	csp: SSRManifestCSP | undefined;
+	llm?: {
+		optimizePageResponse?: boolean;
+	};
 };
 
 export type SSRActions = {

packages/astro/src/core/build/plugins/plugin-manifest.ts

@@ -372,5 +372,6 @@ async function buildManifest(
 		key: encodedKey,
 		sessionConfig: settings.config.session,
 		csp,
+		llm: settings.config.experimental.llm,
 	};
 }

packages/astro/src/core/config/schemas/base.ts

@@ -106,6 +106,9 @@ export const ASTRO_CONFIG_DEFAULTS = {
 		staticImportMetaEnv: false,
 		chromeDevtoolsWorkspace: false,
 		failOnPrerenderConflict: false,
+		llm: {
+			optimizePageResponse: false,
+		},
 	},
 } satisfies AstroUserConfig & { server: { open: boolean } };
 
@@ -526,6 +529,15 @@ export const AstroConfigSchema = z.object({
 				.boolean()
 				.optional()
 				.default(ASTRO_CONFIG_DEFAULTS.experimental.failOnPrerenderConflict),
+			llm: z
+				.object({
+					optimizePageResponse: z
+						.boolean()
+						.optional()
+						.default(ASTRO_CONFIG_DEFAULTS.experimental.llm.optimizePageResponse),
+				})
+				.optional()
+				.default(ASTRO_CONFIG_DEFAULTS.experimental.llm),
 		})
 		.strict(
 			`Invalid or outdated experimental feature.\nCheck for incorrect spelling or outdated Astro version.\nSee https://docs.astro.build/en/reference/experimental-flags/ for a list of all current experiments.`,

packages/astro/src/core/render-context.ts

@@ -8,6 +8,7 @@ import {
 	computePreferredLocale,
 	computePreferredLocaleList,
 } from '../i18n/utils.js';
+import { processLLMResponse } from '../llm/optimize-page-response.js';
 import { renderEndpoint } from '../runtime/server/endpoint.js';
 import { renderPage } from '../runtime/server/index.js';
 import type { ComponentInstance } from '../types/astro.js';
@@ -307,6 +308,12 @@ export class RenderContext {
 		// where the adapter might be expecting to read it.
 		// New code should be using `app.render({ addCookieHeader: true })` instead.
 		attachCookiesToResponse(response, this.cookies);
+
+		// Apply LLM optimization if enabled
+		if (pipeline.manifest.llm?.optimizePageResponse) {
+			return await processLLMResponse(response, this.request);
+		}
+
 		return response;
 	}
 

packages/astro/src/llm/index.ts

@@ -0,0 +1 @@
+export { optimizePageResponse, processLLMResponse } from './optimize-page-response.js';

packages/astro/src/llm/optimize-page-response.ts

@@ -0,0 +1,46 @@
+import TurndownService from 'turndown';
+
+const turndownService = new TurndownService({
+	// Use # for headings instead of underlines
+	headingStyle: 'atx',
+	// Use ``` for code blocks instead of indentation
+	codeBlockStyle: 'fenced',
+});
+
+// Remove elements that aren't useful for LLMs
+turndownService.remove(['script', 'style', 'nav', 'footer']);
+
+export async function optimizePageResponse(
+	html: string,
+): Promise<string> {
+	return turndownService.turndown(html);
+}
+
+export async function processLLMResponse(
+	response: Response,
+	request: Request,
+): Promise<Response> {
+	// Check if the request accepts markdown
+	const acceptHeader = request.headers.get('accept');
+	const wantsMarkdown = acceptHeader?.includes('text/markdown');
+
+	// Only convert HTML responses to markdown
+	if (
+		wantsMarkdown &&
+		response.headers.get('content-type')?.includes('text/html')
+	) {
+		const html = await response.clone().text();
+		const markdown = await optimizePageResponse(html);
+
+		const headers = new Headers(response.headers);
+		headers.set('content-type', 'text/markdown; charset=utf-8');
+
+		return new Response(markdown, {
+			status: response.status,
+			statusText: response.statusText,
+			headers,
+		});
+	}
+
+	return response;
+}

packages/astro/src/types/public/config.ts

@@ -2537,6 +2537,40 @@ export interface AstroUserConfig<
 		 * See the [experimental Chrome DevTools workspace feature documentation](https://docs.astro.build/en/reference/experimental-flags/chrome-devtools-workspace/) for more information.
 		 */
 		chromeDevtoolsWorkspace?: boolean;
+
+		/**
+		 *
+		 * @name experimental.llm
+		 * @type {{ optimizePageResponse?: boolean }}
+		 * @default `undefined`
+		 * @description
+		 * Configuration for LLM (Large Language Model) optimization features.
+		 *
+		 * - `optimizePageResponse`: When enabled, converts HTML page responses to Markdown for better LLM consumption.
+		 *
+		 * ```js
+		 * import { defineConfig } from 'astro/config';
+		 *
+		 * export default defineConfig({
+		 *   experimental: {
+		 *     llm: {
+		 *       optimizePageResponse: true,
+		 *     },
+		 *   },
+		 * });
+		 * ```
+		 */
+		llm?: {
+			/**
+			 * @name experimental.llm.optimizePageResponse
+			 * @type {boolean}
+			 * @default `false`
+			 * @description
+			 * When enabled, converts HTML page responses to Markdown format for better consumption by Large Language Models (LLMs).
+			 * This allows LLMs to request pages in Markdown by setting the Accept header to `text/markdown`.
+			 */
+			optimizePageResponse?: boolean;
+		};
 	};
 }
 

packages/astro/src/vite-plugin-astro-server/plugin.ts

@@ -285,5 +285,6 @@ export function createDevelopmentManifest(settings: AstroSettings): SSRManifest
 		},
 		sessionConfig: settings.config.session,
 		csp,
+		llm: settings.config.experimental.llm,
 	};
 }

packages/astro/test/fixtures/llm-optimization/astro.config.mjs

@@ -0,0 +1,10 @@
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+	output: 'server',
+	experimental: {
+		llm: {
+			optimizePageResponse: true,
+		},
+	},
+});

packages/astro/test/fixtures/llm-optimization/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "test-llm-optimization",
+  "version": "0.0.0",
+  "private": true,
+  "dependencies": {
+    "astro": "workspace:*"
+  }
+}

packages/astro/test/fixtures/llm-optimization/src/pages/index.astro

@@ -0,0 +1,19 @@
+---
+---
+
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>LLM Test Page</title>
+	</head>
+	<body>
+		<h1>Welcome to the LLM Test</h1>
+		<p>This is a test page for LLM optimization.</p>
+		<nav>
+			<a href="/">Home</a>
+		</nav>
+		<footer>
+			<p>Footer content</p>
+		</footer>
+	</body>
+</html>