-
Notifications
You must be signed in to change notification settings - Fork 6.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: nextjs and mdx compiler (#6071)
* feat: nextjs and mdx compiler * chore: just some text * chore: some comments * chore: attempts on improving the performance * chore: code review changes and fixes * chore: properly adjust sizes when needed * feat: include simple own optimised syntax highlighter * chore: more text * chore: should always have one child at least * chore: minor doc changes * chore: minor changes on types * chore: removal of legacyMain completely
- Loading branch information
Showing
18 changed files
with
5,600 additions
and
3,123 deletions.
There are no files selected for viewing
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| 'use strict'; | ||
|
|
||
| import { compile, runSync } from '@mdx-js/mdx'; | ||
| import * as jsxRuntime from 'react/jsx-runtime'; | ||
| import { matter } from 'vfile-matter'; | ||
|
|
||
| import { NEXT_REHYPE_PLUGINS, NEXT_REMARK_PLUGINS } from './next.mdx.mjs'; | ||
|
|
||
| /** | ||
| * This is our custom simple MDX Compiler that is used to compile Markdown and MDX | ||
| * this returns a serializable VFile as a string that then gets passed to our MDX Provider | ||
| * | ||
| * @param {import('vfile').VFile} source | ||
| * @param {'md' | 'mdx'} fileExtension | ||
| * @returns {Promise<{ content: import('vfile').VFile; headings: import('@vcarl/remark-headings').Heading[]; frontmatter: Record<string, any>}>} | ||
| */ | ||
| export async function compileMDX(source, fileExtension) { | ||
| // Parses the Frontmatter to the VFile and removes from the original source | ||
| // cleaning the frontmatter to the source that is going to be parsed by the MDX Compiler | ||
| matter(source, { strip: true }); | ||
|
|
||
| // This is a minimal MDX Compiler that is lightweight and only parses the MDX | ||
| const compiledSource = await compile(source, { | ||
| rehypePlugins: NEXT_REHYPE_PLUGINS, | ||
| remarkPlugins: NEXT_REMARK_PLUGINS, | ||
| format: fileExtension, | ||
| // This instructs the MDX compiler to generate a minimal JSX-body | ||
| // to be consumed within MDX's `run` method, instead of a standalone React Application | ||
| outputFormat: 'function-body', | ||
| // Ensure compatibility with Server Components | ||
| providerImportSource: undefined, | ||
| }); | ||
|
|
||
| // Retrieve some parsed data from the VFile metadata | ||
| // such as frontmatter and Markdown headings | ||
| const { headings, matter: frontmatter } = source.data; | ||
|
|
||
| return { content: compiledSource, headings, frontmatter }; | ||
| } | ||
|
|
||
| /** | ||
| * This evaluates our MDX VFile into actual JSX eval'd code | ||
| * which is actually used by the MDX Provider | ||
| * | ||
| * @param {string} source | ||
| * @returns {import('mdx/types').MDXContent} | ||
| */ | ||
| export function runMDX(source) { | ||
| const { default: content } = runSync(source, jsxRuntime); | ||
|
|
||
| return content; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,89 +1,29 @@ | ||
| 'use strict'; | ||
|
|
||
| /// <reference types="remark-parse" /> | ||
| /// <reference types="remark-stringify" /> | ||
| import remarkHeadings from '@vcarl/remark-headings'; | ||
| import rehypeAutolinkHeadings from 'rehype-autolink-headings'; | ||
| import rehypeSlug from 'rehype-slug'; | ||
| import remarkGfm from 'remark-gfm'; | ||
|
|
||
| /** | ||
| * @typedef {import('mdast').Root} Root | ||
| * @typedef {import('unified').Processor<Root>} Processor | ||
| */ | ||
|
|
||
| import * as remarkHeadings from '@vcarl/remark-headings'; | ||
| import * as mdastAutoLink from 'mdast-util-gfm-autolink-literal'; | ||
| import * as mdastTable from 'mdast-util-gfm-table'; | ||
| import * as rehypeAutolinkHeadings from 'rehype-autolink-headings'; | ||
| import * as rehypeRaw from 'rehype-raw'; | ||
| import * as rehypeShikiji from 'rehype-shikiji'; | ||
| import * as rehypeSlug from 'rehype-slug'; | ||
|
|
||
| import { LANGUAGES, DEFAULT_THEME } from './shiki.config.mjs'; | ||
|
|
||
| /** | ||
| * This function is used to add individual `mdast` plugins to the unified/mdx | ||
| * processor with the intent of being able to customize plugins | ||
| * | ||
| * @returns {void} | ||
| */ | ||
| function nextMdastPlugins() { | ||
| const self = /** @type {Processor} */ (this); | ||
| const data = self.data(); | ||
|
|
||
| const fromMarkdownExtensions = | ||
| data.fromMarkdownExtensions || (data.fromMarkdownExtensions = []); | ||
|
|
||
| const toMarkdownExtensions = | ||
| data.toMarkdownExtensions || (data.toMarkdownExtensions = []); | ||
|
|
||
| // Converts plain URLs on Markdown to HTML Anchor Tags | ||
| fromMarkdownExtensions.push(mdastAutoLink.gfmAutolinkLiteralFromMarkdown()); | ||
| toMarkdownExtensions.push(mdastAutoLink.gfmAutolinkLiteralToMarkdown()); | ||
|
|
||
| // Converts plain Markdown Tables (GFM) to HTML Tables | ||
| fromMarkdownExtensions.push(mdastTable.gfmTableFromMarkdown); | ||
| toMarkdownExtensions.push(mdastTable.gfmTableToMarkdown()); | ||
| } | ||
| import rehypeShikiji from './next.mdx.shiki.mjs'; | ||
|
|
||
| /** | ||
| * Provides all our Rehype Plugins that are used within MDX | ||
| * | ||
| * @param {'md' | 'mdx'} fileExtension | ||
| * @returns {import('unified').Plugin[]} | ||
| * @type {import('unified').Plugin[]} | ||
| */ | ||
| export function nextRehypePlugins(fileExtension) { | ||
| const rehypePlugins = [ | ||
| // Generates `id` attributes for headings (H1, ...) | ||
| rehypeSlug.default, | ||
| [ | ||
| // Automatically add anchor links to headings (H1, ...) | ||
| rehypeAutolinkHeadings.default, | ||
| { | ||
| behaviour: 'append', | ||
| properties: { ariaHidden: true, tabIndex: -1, class: 'anchor' }, | ||
| }, | ||
| ], | ||
| [ | ||
| // Syntax Highlighter for Code Blocks | ||
| rehypeShikiji.default, | ||
| { theme: DEFAULT_THEME, langs: LANGUAGES }, | ||
| ], | ||
| ]; | ||
|
|
||
| if (fileExtension === 'md') { | ||
| // We add this plugin at the top of the array as it is supposed to parse raw HTML | ||
| // before any other plugins (such as adding headings, etc) | ||
| // before any of the other plugins being applied | ||
| rehypePlugins.unshift(rehypeRaw.default); | ||
| } | ||
|
|
||
| return rehypePlugins; | ||
| } | ||
| export const NEXT_REHYPE_PLUGINS = [ | ||
| // Generates `id` attributes for headings (H1, ...) | ||
| rehypeSlug, | ||
| // Automatically add anchor links to headings (H1, ...) | ||
| [rehypeAutolinkHeadings, { properties: { tabIndex: -1, class: 'anchor' } }], | ||
| // Adds our syntax highlighter (Shikiji) to Codeboxes | ||
| rehypeShikiji, | ||
| ]; | ||
|
|
||
| /** | ||
| * Provides all our Remark Plugins that are used within MDX | ||
| * | ||
| * @param {'md' | 'mdx'} fileExtension | ||
| * @returns {import('unified').Plugin[]} | ||
| * @type {import('unified').Plugin[]} | ||
| */ | ||
| export function nextRemarkPlugins() { | ||
| return [remarkHeadings.default, nextMdastPlugins]; | ||
| } | ||
| export const NEXT_REMARK_PLUGINS = [remarkGfm, remarkHeadings]; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| 'use strict'; | ||
|
|
||
| import classNames from 'classnames'; | ||
| import { toString } from 'hast-util-to-string'; | ||
| import { getHighlighterCore } from 'shikiji/core'; | ||
| import { getWasmInlined } from 'shikiji/wasm'; | ||
| import { visit } from 'unist-util-visit'; | ||
|
|
||
| import { LANGUAGES, DEFAULT_THEME } from './shiki.config.mjs'; | ||
|
|
||
| // This creates a memoized minimal Shikiji Syntax Highlighter | ||
| const memoizedShikiji = await getHighlighterCore({ | ||
| themes: [DEFAULT_THEME], | ||
| langs: LANGUAGES, | ||
| loadWasm: getWasmInlined, | ||
| }); | ||
|
|
||
| // This is what Remark will use as prefix within a <pre> className | ||
| // to attribute the current language of the <pre> element | ||
| const languagePrefix = 'language-'; | ||
|
|
||
| export default function rehypeShikiji() { | ||
| return async function (tree) { | ||
| visit(tree, 'element', (node, index, parent) => { | ||
| // We only want to process <pre>...</pre> elements | ||
| if (!parent || index == null || node.tagName !== 'pre') { | ||
| return; | ||
| } | ||
|
|
||
| // We want the contents of the <pre> element, hence we attempt to get the first child | ||
| const preElement = node.children[0]; | ||
|
|
||
| // If thereÄs nothing inside the <pre> element... What are we doing here? | ||
| if (!preElement || !preElement.properties) { | ||
| return; | ||
| } | ||
|
|
||
| // Ensure that we're not visiting a <code> element but it's inner contents | ||
| // (keep iterating further down until we reach where we want) | ||
| if (preElement.type !== 'element' || preElement.tagName !== 'code') { | ||
| return; | ||
| } | ||
|
|
||
| // Get the <pre> element class names | ||
| const preClassNames = preElement.properties.className; | ||
|
|
||
| // The current classnames should be an array and it should have a length | ||
| if (typeof preClassNames !== 'object' || preClassNames.length === 0) { | ||
| return; | ||
| } | ||
|
|
||
| // We want to retrieve the language class name from the class names | ||
| const codeLanguage = preClassNames.find( | ||
| c => typeof c === 'string' && c.startsWith(languagePrefix) | ||
| ); | ||
|
|
||
| // If we didn't find any `language-` classname then we shouldn't highlight | ||
| if (typeof codeLanguage !== 'string') { | ||
| return; | ||
| } | ||
|
|
||
| // Retrieve the whole <pre> contents as a parsed DOM string | ||
| const preElementContents = toString(preElement); | ||
|
|
||
| // Grabs the relevant alias/name of the language | ||
| const languageId = codeLanguage.slice(languagePrefix.length); | ||
|
|
||
| // Parses the <pre> contents and returns a HAST tree with the highlighted code | ||
| const { children } = memoizedShikiji.codeToHast(preElementContents, { | ||
| theme: DEFAULT_THEME, | ||
| lang: languageId, | ||
| }); | ||
|
|
||
| // Adds the original language back to the <pre> element | ||
| children[0].properties.class = classNames( | ||
| children[0].properties.class, | ||
| codeLanguage | ||
| ); | ||
|
|
||
| // Replaces the <pre> element with the updated one | ||
| parent.children.splice(index, 1, ...children); | ||
| }); | ||
| }; | ||
| } |
Oops, something went wrong.