feat: nextjs and mdx compiler (#6071)

* feat: nextjs and mdx compiler

* chore: just some text

* chore: some comments

* chore: attempts on improving the performance

* chore: code review changes and fixes

* chore: properly adjust sizes when needed

* feat: include simple own optimised syntax highlighter

* chore: more text

* chore: should always have one child at least

* chore: minor doc changes

* chore: minor changes on types

* chore: removal of legacyMain completely

Author: ovflowd

global.d.ts

@@ -36,7 +36,7 @@ const nextConfig = {
   eslint: { dirs: ['.'], ignoreDuringBuilds: true },
   // Next.js WebPack Bundler does not know how to handle `.mjs` files on `node_modules`
   // This is not an issue when using TurboPack as it uses SWC and it is ESM-only
-  // Once we migrate to Next.js 14 we might be able to remove this
+  // Once Next.js uses Turbopack for their build process we can remove this
   webpack: function (config) {
     config.module.rules.push({
       test: /\.m?js$/,
@@ -58,6 +58,8 @@ const nextConfig = {
       '@radix-ui/react-toast',
       'tailwindcss',
     ],
+    // Removes the warning regarding the WebPack Build Worker
+    webpackBuildWorker: false,
   },
 };
 

next.config.mjs

@@ -82,15 +82,6 @@ export const MD_EXTENSION_REGEX = /((\/)?(index))?\.mdx?$/i;
  */
 export const DEFAULT_LOCALE_CODE = defaultLocale.code;
 
-/**
- * This indicates the path to the Legacy JavaScript File that is used
- * on the legacy Website.
- *
- * @deprecated The Legacy Website is due to be removed soon and this file
- * and its usages should be removed
- */
-export const LEGACY_JAVASCRIPT_FILE = `${BASE_PATH}/static/js/legacyMain.js`;
-
 /**
  * This is a list of all static routes or pages from the Website that we do not
  * want to allow to be statically built on our Static Export Build.

next.constants.mjs

@@ -3,13 +3,12 @@
 import { readFileSync } from 'node:fs';
 import { join, normalize, sep } from 'node:path';
 
-import { serialize } from 'next-mdx-remote/serialize';
 import { VFile } from 'vfile';
 
 import { DEFAULT_LOCALE_CODE, MD_EXTENSION_REGEX } from './next.constants.mjs';
 import { getMarkdownFiles } from './next.helpers.mjs';
 import { availableLocales } from './next.locales.mjs';
-import { nextRehypePlugins, nextRemarkPlugins } from './next.mdx.mjs';
+import { compileMDX } from './next.mdx.compiler.mjs';
 
 // allows us to run a glob to get markdown files based on a language folder
 const getPathsByLanguage = async (locale = DEFAULT_LOCALE_CODE, ignored = []) =>
@@ -148,31 +147,15 @@ export const generateStaticProps = async (source = '', filename = '') => {
     // Gets the file extension of the file, to determine which parser and plugins to use
     const fileExtension = filename.endsWith('.mdx') ? 'mdx' : 'md';
 
-    // This act as a MDX "compiler" but, lightweight. It parses the Markdown
-    // string source into a React Component tree, and then it serializes it
-    // it also supports Remark plugins, and MDX components
-    // Note.: We use the filename extension to define the mode of execution
-    const { compiledSource } = await serialize(sourceAsVirtualFile, {
-      parseFrontmatter: true,
-      mdxOptions: {
-        rehypePlugins: nextRehypePlugins(fileExtension),
-        remarkPlugins: nextRemarkPlugins(fileExtension),
-        format: fileExtension,
-      },
-    });
-
-    // After the MDX gets processed with the remarkPlugins, some extra `data` that might come along
-    // the `frontmatter` comes from `serialize` built-in support to `remark-frontmatter`
-    const { headings, matter: rawFrontmatter } = sourceAsVirtualFile.data;
-
-    // This serialises the Frontmatter into a JSON object that is compatible with the
-    // `getStaticProps` supported data type for props. (No prop value can be an object or not a primitive)
-    const frontmatter = JSON.parse(JSON.stringify(rawFrontmatter));
-
-    // this defines the basic props that should be passed back to the `DynamicPage` component
-    // We only want the `compiledSource` as we use `MDXProvider` for custom components along the journey
-    // And then we want the frontmatter and heading information from the VFile `data`
-    staticProps.props = { content: compiledSource, headings, frontmatter };
+    // This compiles our MDX source (VFile) into a final MDX-parsed VFile
+    // that then is passed as a string to the MDXProvider which will run the MDX Code
+    const { content, headings, frontmatter } = await compileMDX(
+      sourceAsVirtualFile,
+      fileExtension
+    );
+
+    // Passes the compiled MDX Source to the MDX Provider and some extra data
+    staticProps.props = { content: String(content), headings, frontmatter };
     staticProps.notFound = false;
   }
 

next.dynamic.mjs

@@ -0,0 +1,52 @@
+'use strict';
+
+import { compile, runSync } from '@mdx-js/mdx';
+import * as jsxRuntime from 'react/jsx-runtime';
+import { matter } from 'vfile-matter';
+
+import { NEXT_REHYPE_PLUGINS, NEXT_REMARK_PLUGINS } from './next.mdx.mjs';
+
+/**
+ * This is our custom simple MDX Compiler that is used to compile Markdown and MDX
+ * this returns a serializable VFile as a string that then gets passed to our MDX Provider
+ *
+ * @param {import('vfile').VFile} source
+ * @param {'md' | 'mdx'} fileExtension
+ * @returns {Promise<{ content: import('vfile').VFile; headings: import('@vcarl/remark-headings').Heading[]; frontmatter: Record<string, any>}>}
+ */
+export async function compileMDX(source, fileExtension) {
+  // Parses the Frontmatter to the VFile and removes from the original source
+  // cleaning the frontmatter to the source that is going to be parsed by the MDX Compiler
+  matter(source, { strip: true });
+
+  // This is a minimal MDX Compiler that is lightweight and only parses the MDX
+  const compiledSource = await compile(source, {
+    rehypePlugins: NEXT_REHYPE_PLUGINS,
+    remarkPlugins: NEXT_REMARK_PLUGINS,
+    format: fileExtension,
+    // This instructs the MDX compiler to generate a minimal JSX-body
+    // to be consumed within MDX's `run` method, instead of a standalone React Application
+    outputFormat: 'function-body',
+    // Ensure compatibility with Server Components
+    providerImportSource: undefined,
+  });
+
+  // Retrieve some parsed data from the VFile metadata
+  // such as frontmatter and Markdown headings
+  const { headings, matter: frontmatter } = source.data;
+
+  return { content: compiledSource, headings, frontmatter };
+}
+
+/**
+ * This evaluates our MDX VFile into actual JSX eval'd code
+ * which is actually used by the MDX Provider
+ *
+ * @param {string} source
+ * @returns {import('mdx/types').MDXContent}
+ */
+export function runMDX(source) {
+  const { default: content } = runSync(source, jsxRuntime);
+
+  return content;
+}

next.mdx.compiler.mjs

@@ -1,89 +1,29 @@
 'use strict';
 
-/// <reference types="remark-parse" />
-/// <reference types="remark-stringify" />
+import remarkHeadings from '@vcarl/remark-headings';
+import rehypeAutolinkHeadings from 'rehype-autolink-headings';
+import rehypeSlug from 'rehype-slug';
+import remarkGfm from 'remark-gfm';
 
-/**
- * @typedef {import('mdast').Root} Root
- * @typedef {import('unified').Processor<Root>} Processor
- */
-
-import * as remarkHeadings from '@vcarl/remark-headings';
-import * as mdastAutoLink from 'mdast-util-gfm-autolink-literal';
-import * as mdastTable from 'mdast-util-gfm-table';
-import * as rehypeAutolinkHeadings from 'rehype-autolink-headings';
-import * as rehypeRaw from 'rehype-raw';
-import * as rehypeShikiji from 'rehype-shikiji';
-import * as rehypeSlug from 'rehype-slug';
-
-import { LANGUAGES, DEFAULT_THEME } from './shiki.config.mjs';
-
-/**
- * This function is used to add individual `mdast` plugins to the unified/mdx
- * processor with the intent of being able to customize plugins
- *
- * @returns {void}
- */
-function nextMdastPlugins() {
-  const self = /** @type {Processor} */ (this);
-  const data = self.data();
-
-  const fromMarkdownExtensions =
-    data.fromMarkdownExtensions || (data.fromMarkdownExtensions = []);
-
-  const toMarkdownExtensions =
-    data.toMarkdownExtensions || (data.toMarkdownExtensions = []);
-
-  // Converts plain URLs on Markdown to HTML Anchor Tags
-  fromMarkdownExtensions.push(mdastAutoLink.gfmAutolinkLiteralFromMarkdown());
-  toMarkdownExtensions.push(mdastAutoLink.gfmAutolinkLiteralToMarkdown());
-
-  // Converts plain Markdown Tables (GFM) to HTML Tables
-  fromMarkdownExtensions.push(mdastTable.gfmTableFromMarkdown);
-  toMarkdownExtensions.push(mdastTable.gfmTableToMarkdown());
-}
+import rehypeShikiji from './next.mdx.shiki.mjs';
 
 /**
  * Provides all our Rehype Plugins that are used within MDX
  *
- * @param {'md' | 'mdx'} fileExtension
- * @returns {import('unified').Plugin[]}
+ * @type {import('unified').Plugin[]}
  */
-export function nextRehypePlugins(fileExtension) {
-  const rehypePlugins = [
-    // Generates `id` attributes for headings (H1, ...)
-    rehypeSlug.default,
-    [
-      // Automatically add anchor links to headings (H1, ...)
-      rehypeAutolinkHeadings.default,
-      {
-        behaviour: 'append',
-        properties: { ariaHidden: true, tabIndex: -1, class: 'anchor' },
-      },
-    ],
-    [
-      // Syntax Highlighter for Code Blocks
-      rehypeShikiji.default,
-      { theme: DEFAULT_THEME, langs: LANGUAGES },
-    ],
-  ];
-
-  if (fileExtension === 'md') {
-    // We add this plugin at the top of the array as it is supposed to parse raw HTML
-    // before any other plugins (such as adding headings, etc)
-    // before any of the other plugins being applied
-    rehypePlugins.unshift(rehypeRaw.default);
-  }
-
-  return rehypePlugins;
-}
+export const NEXT_REHYPE_PLUGINS = [
+  // Generates `id` attributes for headings (H1, ...)
+  rehypeSlug,
+  // Automatically add anchor links to headings (H1, ...)
+  [rehypeAutolinkHeadings, { properties: { tabIndex: -1, class: 'anchor' } }],
+  // Adds our syntax highlighter (Shikiji) to Codeboxes
+  rehypeShikiji,
+];
 
 /**
  * Provides all our Remark Plugins that are used within MDX
  *
- * @param {'md' | 'mdx'} fileExtension
- * @returns {import('unified').Plugin[]}
+ * @type {import('unified').Plugin[]}
  */
-export function nextRemarkPlugins() {
-  return [remarkHeadings.default, nextMdastPlugins];
-}
+export const NEXT_REMARK_PLUGINS = [remarkGfm, remarkHeadings];

next.mdx.mjs

@@ -0,0 +1,84 @@
+'use strict';
+
+import classNames from 'classnames';
+import { toString } from 'hast-util-to-string';
+import { getHighlighterCore } from 'shikiji/core';
+import { getWasmInlined } from 'shikiji/wasm';
+import { visit } from 'unist-util-visit';
+
+import { LANGUAGES, DEFAULT_THEME } from './shiki.config.mjs';
+
+// This creates a memoized minimal Shikiji Syntax Highlighter
+const memoizedShikiji = await getHighlighterCore({
+  themes: [DEFAULT_THEME],
+  langs: LANGUAGES,
+  loadWasm: getWasmInlined,
+});
+
+// This is what Remark will use as prefix within a <pre> className
+// to attribute the current language of the <pre> element
+const languagePrefix = 'language-';
+
+export default function rehypeShikiji() {
+  return async function (tree) {
+    visit(tree, 'element', (node, index, parent) => {
+      // We only want to process <pre>...</pre> elements
+      if (!parent || index == null || node.tagName !== 'pre') {
+        return;
+      }
+
+      // We want the contents of the <pre> element, hence we attempt to get the first child
+      const preElement = node.children[0];
+
+      // If thereÄs nothing inside the <pre> element... What are we doing here?
+      if (!preElement || !preElement.properties) {
+        return;
+      }
+
+      // Ensure that we're not visiting a <code> element but it's inner contents
+      // (keep iterating further down until we reach where we want)
+      if (preElement.type !== 'element' || preElement.tagName !== 'code') {
+        return;
+      }
+
+      // Get the <pre> element class names
+      const preClassNames = preElement.properties.className;
+
+      // The current classnames should be an array and it should have a length
+      if (typeof preClassNames !== 'object' || preClassNames.length === 0) {
+        return;
+      }
+
+      // We want to retrieve the language class name from the class names
+      const codeLanguage = preClassNames.find(
+        c => typeof c === 'string' && c.startsWith(languagePrefix)
+      );
+
+      // If we didn't find any `language-` classname then we shouldn't highlight
+      if (typeof codeLanguage !== 'string') {
+        return;
+      }
+
+      // Retrieve the whole <pre> contents as a parsed DOM string
+      const preElementContents = toString(preElement);
+
+      // Grabs the relevant alias/name of the language
+      const languageId = codeLanguage.slice(languagePrefix.length);
+
+      // Parses the <pre> contents and returns a HAST tree with the highlighted code
+      const { children } = memoizedShikiji.codeToHast(preElementContents, {
+        theme: DEFAULT_THEME,
+        lang: languageId,
+      });
+
+      // Adds the original language back to the <pre> element
+      children[0].properties.class = classNames(
+        children[0].properties.class,
+        codeLanguage
+      );
+
+      // Replaces the <pre> element with the updated one
+      parent.children.splice(index, 1, ...children);
+    });
+  };
+}