feat(plugin-llms): init @rspress/plugin-llms

packages/core/package.json

@@ -108,7 +108,7 @@
     "rsbuild-plugin-virtual-module": "0.1.1",
     "tailwindcss": "^3.4.17",
     "typescript": "^5.8.2",
-    "vfile": "^5.3.7"
+    "vfile": "^6.0.3"
   },
   "engines": {
     "node": ">=18.0.0"

packages/core/src/index.ts

@@ -1,3 +1,5 @@
-export * from './node';
+// TODO: do not expose remarkPluginNormalizeLink as publicAPI
+export { dev, build, serve, remarkPluginNormalizeLink } from './node';
 export * from '@rspress/shared';
 export { mergeDocConfig } from '@rspress/shared/node-utils';
+export type { RouteService } from './node/route/RouteService';

packages/core/src/node/PluginDriver.ts

@@ -6,13 +6,15 @@ import type {
   UserConfig,
 } from '@rspress/shared';
 import { isDevDebugMode } from '@rspress/shared';
+import type { RouteService } from './route/RouteService';
 
 type RspressPluginHookKeys =
   | 'beforeBuild'
   | 'afterBuild'
   | 'addPages'
   | 'addRuntimeModules'
   | 'routeGenerated'
+  | 'routeServiceGenerated'
   | 'addSSGRoutes'
   | 'extendPageData'
   | 'modifySearchIndexData';
@@ -186,6 +188,14 @@ export class PluginDriver {
     return this._runParallelAsyncHook('routeGenerated', routes, this.#isProd);
   }
 
+  async routeServiceGenerated(routeService: RouteService) {
+    return this._runParallelAsyncHook(
+      'routeServiceGenerated',
+      routeService,
+      this.#isProd,
+    );
+  }
+
   async addRuntimeModules() {
     const result = await this._runParallelAsyncHook(
       'addRuntimeModules',

packages/core/src/node/index.ts

@@ -1,3 +1,5 @@
 export { dev } from './dev';
 export { build } from './build';
 export { serve } from './serve';
+
+export { remarkPluginNormalizeLink } from './mdx/remarkPlugins/normalizeLink';

packages/core/src/node/mdx/remarkPlugins/normalizeLink.ts

@@ -16,14 +16,80 @@ import { visit } from 'unist-util-visit';
 import type { RouteService } from '../../route/RouteService';
 import { getASTNodeImport } from '../../utils';
 
+function normalizeLink(
+  nodeUrl: string,
+  routeService: RouteService | undefined,
+  relativePath: string,
+  cleanUrls: boolean | string,
+): string {
+  if (!nodeUrl) {
+    return '';
+  }
+  if (nodeUrl.startsWith('#')) {
+    return `#${nodeUrl.slice(1)}`;
+  }
+
+  // eslint-disable-next-line prefer-const
+  let { url, hash } = parseUrl(nodeUrl);
+
+  if (isExternalUrl(url)) {
+    return url + (hash ? `#${hash}` : '');
+  }
+
+  const extname = path.extname(url);
+
+  if ((routeService?.extensions ?? DEFAULT_PAGE_EXTENSIONS).includes(extname)) {
+    url = url.replace(new RegExp(`\\${extname}$`), '');
+  }
+
+  if (url.startsWith('.')) {
+    url = path.posix.join(slash(path.dirname(relativePath)), url);
+  } else if (routeService) {
+    const [pathVersion, pathLang] = routeService.getRoutePathParts(
+      slash(relativePath),
+    );
+    const [urlVersion, urlLang, urlPath] = routeService.getRoutePathParts(url);
+
+    url = addLeadingSlash(urlPath);
+
+    if (pathLang && urlLang !== pathLang) {
+      url = `/${pathLang}${url}`;
+    }
+
+    if (pathVersion && urlVersion !== pathVersion) {
+      url = `/${pathVersion}${url}`;
+    }
+  }
+
+  if (typeof cleanUrls === 'boolean') {
+    url = normalizeHref(url, cleanUrls);
+  } else {
+    url = normalizeHref(url, false);
+    url = url.replace(/\.html$/, cleanUrls);
+  }
+
+  if (hash) {
+    url += `#${hash}`;
+  }
+  return url;
+}
+
+const normalizeImageUrl = (imageUrl: string): string => {
+  if (isExternalUrl(imageUrl) || imageUrl.startsWith('/')) {
+    return '';
+  }
+
+  return imageUrl;
+};
+
 /**
  * Remark plugin to normalize a link href
  */
 export const remarkPluginNormalizeLink: Plugin<
   [
     {
       root: string;
-      cleanUrls: boolean;
+      cleanUrls: boolean | string;
       routeService?: RouteService;
     },
   ],
@@ -33,68 +99,11 @@ export const remarkPluginNormalizeLink: Plugin<
   (tree, file) => {
     const images: MdxjsEsm[] = [];
     visit(tree, 'link', node => {
-      if (!node.url) {
-        return;
-      }
-      if (node.url.startsWith('#')) {
-        node.url = `#${node.url.slice(1)}`;
-        return;
-      }
-
-      // eslint-disable-next-line prefer-const
-      let { url, hash } = parseUrl(node.url);
-
-      if (isExternalUrl(url)) {
-        node.url = url + (hash ? `#${hash}` : '');
-        return;
-      }
-
-      const extname = path.extname(url);
-
-      if (
-        (routeService?.extensions ?? DEFAULT_PAGE_EXTENSIONS).includes(extname)
-      ) {
-        url = url.replace(new RegExp(`\\${extname}$`), '');
-      }
-
+      const { url: nodeUrl } = node;
       const relativePath = path.relative(root, file.path);
-
-      if (url.startsWith('.')) {
-        url = path.posix.join(slash(path.dirname(relativePath)), url);
-      } else if (routeService) {
-        const [pathVersion, pathLang] = routeService.getRoutePathParts(
-          slash(relativePath),
-        );
-        const [urlVersion, urlLang, urlPath] =
-          routeService.getRoutePathParts(url);
-
-        url = addLeadingSlash(urlPath);
-
-        if (pathLang && urlLang !== pathLang) {
-          url = `/${pathLang}${url}`;
-        }
-
-        if (pathVersion && urlVersion !== pathVersion) {
-          url = `/${pathVersion}${url}`;
-        }
-      }
-
-      url = normalizeHref(url, cleanUrls);
-
-      if (hash) {
-        url += `#${hash}`;
-      }
-      node.url = url;
+      node.url = normalizeLink(nodeUrl, routeService, relativePath, cleanUrls);
     });
 
-    const normalizeImageUrl = (imageUrl: string): string => {
-      if (isExternalUrl(imageUrl) || imageUrl.startsWith('/')) {
-        return '';
-      }
-
-      return imageUrl;
-    };
-
     const getMdxSrcAttribute = (tempVar: string) => {
       return {
         type: 'mdxJsxAttribute',

packages/core/src/node/route/init.ts

@@ -23,5 +23,6 @@ export async function initRouteService(
     pluginDriver,
   );
   await routeService.init();
+  await pluginDriver.routeServiceGenerated(routeService);
   return routeService;
 }

packages/core/src/node/runtimeModule/siteData/extractPageData.test.ts

@@ -60,6 +60,8 @@ describe('extractPageData', async () => {
       [
         {
           "_filepath": "<ROOT>/packages/core/src/node/route/fixtures/basic/a.mdx",
+          "_flattenContent": "# Page a
+      ",
           "_html": "<h1 id="page-a">Page a<a aria-hidden="true" href="#page-a">#</a></h1>",
           "_relativePath": "a.mdx",
           "content": "#",
@@ -76,6 +78,8 @@ describe('extractPageData', async () => {
         },
         {
           "_filepath": "<ROOT>/packages/core/src/node/route/fixtures/basic/guide/b.mdx",
+          "_flattenContent": "# Page b
+      ",
           "_html": "<h1 id="page-b">Page b<a aria-hidden="true" href="#page-b">#</a></h1>",
           "_relativePath": "guide/b.mdx",
           "content": "#",
@@ -92,6 +96,7 @@ describe('extractPageData', async () => {
         },
         {
           "_filepath": "<ROOT>/packages/core/src/node/route/fixtures/basic/guide/c.tsx",
+          "_flattenContent": "",
           "_html": "",
           "_relativePath": "guide/c.tsx",
           "content": "",
@@ -106,6 +111,8 @@ describe('extractPageData', async () => {
         },
         {
           "_filepath": "<ROOT>/packages/core/src/node/route/fixtures/basic/index.mdx",
+          "_flattenContent": "# homePage
+      ",
           "_html": "<h1 id="homepage">homePage<a aria-hidden="true" href="#homepage">#</a></h1>",
           "_relativePath": "index.mdx",
           "content": "#",

packages/core/src/node/runtimeModule/siteData/extractPageData.ts

@@ -49,6 +49,7 @@ export async function extractPageData(
           id: index,
           title: '',
           content: '',
+          _flattenContent: '',
           _html: '',
           routePath: route.routePath,
           lang: route.lang,
@@ -66,17 +67,14 @@ export async function extractPageData(
           return defaultIndexInfo;
         }
         let content: string = await fs.readFile(route.absolutePath, 'utf8');
-        const { frontmatter, content: strippedFrontMatter } = loadFrontMatter(
-          content,
-          route.absolutePath,
-          root,
-        );
+        const { frontmatter, content: contentWithoutFrontMatter } =
+          loadFrontMatter(content, route.absolutePath, root);
 
         // 1. Replace rules for frontmatter & content
         applyReplaceRulesToNestedObject(frontmatter, replaceRules);
 
         const { flattenContent } = await flattenMdxContent(
-          applyReplaceRules(strippedFrontMatter, replaceRules),
+          applyReplaceRules(contentWithoutFrontMatter, replaceRules),
           route.absolutePath,
           alias,
         );
@@ -180,6 +178,7 @@ export async function extractPageData(
           toc,
           // for search index
           content,
+          _flattenContent: flattenContent,
           _html: html,
           frontmatter: {
             ...frontmatter,

packages/core/src/node/runtimeModule/siteData/index.ts

@@ -134,7 +134,16 @@ export async function siteDataVMPlugin(context: FactoryContext) {
     search: tempSearchObj ?? { mode: 'local' },
     pages: pages.map(page => {
       // omit some fields for runtime size
-      const { content, id, domain, _filepath, _html, ...rest } = page;
+      const {
+        content,
+        id,
+        domain,
+        _filepath,
+        _html,
+        _flattenContent,
+        ...rest
+      } = page;
+      // FIXME: should not have differences from development
       // In production, we cannot expose the complete filepath for security reasons
       return isProduction() ? rest : { ...rest, _filepath };
     }),

packages/core/tests/prismLanguages.test.ts

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import type { PageIndexInfo } from 'src';
 import type { PluginDriver } from 'src/node/PluginDriver';
 import type { RouteService } from 'src/node/route/RouteService';
 import { describe, expect, it } from 'vitest';
@@ -32,7 +33,7 @@ describe('automatic import of prism languages', () => {
       },
     } as RouteService,
     pluginDriver: {
-      extendPageData(pageData) {
+      extendPageData(pageData: PageIndexInfo) {
         pageData.extraHighlightLanguages = ['jsx', 'tsx'];
       },
       modifySearchIndexData() {},

packages/document/docs/en/plugin/official-plugins/_meta.json

@@ -1,5 +1,6 @@
 [
   "overview",
+  "llms",
   "medium-zoom",
   "client-redirects",
   "last-updated",