Add build time math rendering

While very useful on its own (and combined with the passthrough render hooks), this also serves as a proof of concept of using WASI (WebAssembly System Interface) modules in Hugo.

This will be marked _experimental_ in the documentation. Not because it will be removed or changed in a dramatic way, but we need to think a little more how to best set up/configure similar services, define where these WASM files gets stored, maybe we can allow user provided WASM files plugins via Hugo Modules mounts etc.

See these issues for more context:

* gohugoio#12736
* gohugoio#12737

See gohugoio#11927

Author: bep

cache/filecache/filecache_config.go

@@ -46,6 +46,7 @@ const (
 	CacheKeyAssets      = "assets"
 	CacheKeyModules     = "modules"
 	CacheKeyGetResource = "getresource"
+	CacheKeyMisc        = "misc"
 )
 
 type Configs map[string]FileCacheConfig
@@ -70,10 +71,14 @@ var defaultCacheConfigs = Configs{
 		MaxAge: -1,
 		Dir:    resourcesGenDir,
 	},
-	CacheKeyGetResource: FileCacheConfig{
+	CacheKeyGetResource: {
 		MaxAge: -1, // Never expire
 		Dir:    cacheDirProject,
 	},
+	CacheKeyMisc: {
+		MaxAge: -1,
+		Dir:    cacheDirProject,
+	},
 }
 
 type FileCacheConfig struct {
@@ -120,6 +125,11 @@ func (f Caches) AssetsCache() *Cache {
 	return f[CacheKeyAssets]
 }
 
+// MiscCache gets the file cache for miscellaneous stuff.
+func (f Caches) MiscCache() *Cache {
+	return f[CacheKeyMisc]
+}
+
 // GetResourceCache gets the file cache for remote resources.
 func (f Caches) GetResourceCache() *Cache {
 	return f[CacheKeyGetResource]

cache/filecache/filecache_config_test.go

@@ -59,7 +59,7 @@ dir = "/path/to/c4"
 	c.Assert(err, qt.IsNil)
 	fs := afero.NewMemMapFs()
 	decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
-	c.Assert(len(decoded), qt.Equals, 6)
+	c.Assert(len(decoded), qt.Equals, 7)
 
 	c2 := decoded["getcsv"]
 	c.Assert(c2.MaxAge.String(), qt.Equals, "11h0m0s")
@@ -106,7 +106,7 @@ dir = "/path/to/c4"
 	c.Assert(err, qt.IsNil)
 	fs := afero.NewMemMapFs()
 	decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
-	c.Assert(len(decoded), qt.Equals, 6)
+	c.Assert(len(decoded), qt.Equals, 7)
 
 	for _, v := range decoded {
 		c.Assert(v.MaxAge, qt.Equals, time.Duration(0))
@@ -129,7 +129,7 @@ func TestDecodeConfigDefault(t *testing.T) {
 
 	fs := afero.NewMemMapFs()
 	decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
-	c.Assert(len(decoded), qt.Equals, 6)
+	c.Assert(len(decoded), qt.Equals, 7)
 
 	imgConfig := decoded[filecache.CacheKeyImages]
 	jsonConfig := decoded[filecache.CacheKeyGetJSON]

common/hugio/writers.go

@@ -81,3 +81,33 @@ func ToReadCloser(r io.Reader) io.ReadCloser {
 		io.NopCloser(nil),
 	}
 }
+
+type ReadWriteCloser interface {
+	io.Reader
+	io.Writer
+	io.Closer
+}
+
+// PipeReadWriteCloser is a convenience type to create a pipe with a ReadCloser and a WriteCloser.
+type PipeReadWriteCloser struct {
+	*io.PipeReader
+	*io.PipeWriter
+}
+
+// NewPipeReadWriteCloser creates a new PipeReadWriteCloser.
+func NewPipeReadWriteCloser() PipeReadWriteCloser {
+	pr, pw := io.Pipe()
+	return PipeReadWriteCloser{pr, pw}
+}
+
+func (c PipeReadWriteCloser) Close() (err error) {
+	if err = c.PipeReader.Close(); err != nil {
+		return
+	}
+	err = c.PipeWriter.Close()
+	return
+}
+
+func (c PipeReadWriteCloser) WriteString(s string) (int, error) {
+	return c.PipeWriter.Write([]byte(s))
+}

deps/deps.go

@@ -23,6 +23,7 @@ import (
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/identity"
+	"github.com/gohugoio/hugo/internal/warpc"
 	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/resources/page"
 	"github.com/gohugoio/hugo/resources/postpub"
@@ -93,6 +94,10 @@ type Deps struct {
 	// This is common/global for all sites.
 	BuildState *BuildState
 
+	// Holds RPC dispatchers for Katex etc.
+	// TODO(bep) rethink this re. a plugin setup, but this will have to do for now.
+	WasmDispatchers *warpc.Dispatchers
+
 	*globalErrHandler
 }
 
@@ -343,6 +348,9 @@ func (d *Deps) Close() error {
 	if d.MemCache != nil {
 		d.MemCache.Stop()
 	}
+	if d.WasmDispatchers != nil {
+		d.WasmDispatchers.Close()
+	}
 	return d.BuildClosers.Close()
 }
 

go.mod

@@ -69,6 +69,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/tdewolff/minify/v2 v2.20.37
 	github.com/tdewolff/parse/v2 v2.7.15
+	github.com/tetratelabs/wazero v1.7.4-0.20240805170331-2b12e189eeec
 	github.com/yuin/goldmark v1.7.4
 	github.com/yuin/goldmark-emoji v1.0.3
 	go.uber.org/automaxprocs v1.5.3

go.sum

@@ -233,8 +233,6 @@ github.com/gohugoio/httpcache v0.7.0 h1:ukPnn04Rgvx48JIinZvZetBfHaWE7I01JR2Q2RrQ
 github.com/gohugoio/httpcache v0.7.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
 github.com/gohugoio/hugo-goldmark-extensions/extras v0.2.0 h1:MNdY6hYCTQEekY0oAfsxWZU1CDt6iH+tMLgyMJQh/sg=
 github.com/gohugoio/hugo-goldmark-extensions/extras v0.2.0/go.mod h1:oBdBVuiZ0fv9xd8xflUgt53QxW5jOCb1S+xntcN4SKo=
-github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0 h1:PCtO5l++psZf48yen2LxQ3JiOXxaRC6v0594NeHvGZg=
-github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0/go.mod h1:g9CCh+Ci2IMbPUrVJuXbBTrA+rIIx5+hDQ4EXYaQDoM=
 github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.3.0 h1:7PY5PIJ2mck7v6R52yCFvvYHvsPMEbulgRviw3I9lP4=
 github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.3.0/go.mod h1:r8g5S7bHfdj0+9ShBog864ufCsVODKQZNjYYY8OnJpM=
 github.com/gohugoio/locales v0.14.0 h1:Q0gpsZwfv7ATHMbcTNepFd59H7GoykzWJIxi113XGDc=
@@ -461,6 +459,8 @@ github.com/tdewolff/parse/v2 v2.7.15/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W
 github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
 github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
 github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
+github.com/tetratelabs/wazero v1.7.4-0.20240805170331-2b12e189eeec h1:KeQseLFSWb9qjW4PSWxciTBk1hbG7KsVx3rs1hIQnbQ=
+github.com/tetratelabs/wazero v1.7.4-0.20240805170331-2b12e189eeec/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
 github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

hugolib/site_new.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"html/template"
 	"os"
+	"path/filepath"
 	"sort"
 	"time"
 
@@ -34,6 +35,7 @@ import (
 	"github.com/gohugoio/hugo/hugolib/doctree"
 	"github.com/gohugoio/hugo/hugolib/pagesfromdata"
 	"github.com/gohugoio/hugo/identity"
+	"github.com/gohugoio/hugo/internal/warpc"
 	"github.com/gohugoio/hugo/langs"
 	"github.com/gohugoio/hugo/langs/i18n"
 	"github.com/gohugoio/hugo/lazy"
@@ -157,6 +159,15 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 		MemCache:            memCache,
 		TemplateProvider:    tplimpl.DefaultTemplateProvider,
 		TranslationProvider: i18n.NewTranslationProvider(),
+		WasmDispatchers: warpc.AllDispatchers(
+			warpc.Options{
+				CompilationCacheDir: filepath.Join(conf.Dirs().CacheDir, "_warpc"),
+
+				// Katex is relatively slow.
+				PoolSize: 8,
+				Infof:    logger.InfoCommand("wasm").Logf,
+			},
+		),
 	}
 
 	if err := firstSiteDeps.Init(); err != nil {

internal/warpc/build.sh

@@ -0,0 +1,5 @@
+# TODO1 clean up when done.
+go generate ./gen
+javy compile js/greet.bundle.js -d -o wasm/greet.wasm
+javy compile js/renderkatex.bundle.js -d -o wasm/renderkatex.wasm
+touch warpc_test.go

internal/warpc/gen/main.go

@@ -0,0 +1,55 @@
+//go:generate go run main.go
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/evanw/esbuild/pkg/api"
+)
+
+var scripts = []string{
+	"greet.js",
+	"renderkatex.js",
+}
+
+func main() {
+	for _, script := range scripts {
+		filename := filepath.Join("../js", script)
+		err := buildJSBundle(filename)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func buildJSBundle(filename string) error {
+	minify := true
+	result := api.Build(
+		api.BuildOptions{
+			EntryPoints:       []string{filename},
+			Bundle:            true,
+			MinifyWhitespace:  minify,
+			MinifyIdentifiers: minify,
+			MinifySyntax:      minify,
+			Target:            api.ES2020,
+			Outfile:           strings.Replace(filename, ".js", ".bundle.js", 1),
+			SourceRoot:        "../js",
+		})
+
+	if len(result.Errors) > 0 {
+		return fmt.Errorf("build failed: %v", result.Errors)
+	}
+	if len(result.OutputFiles) != 1 {
+		return fmt.Errorf("expected 1 output file, got %d", len(result.OutputFiles))
+	}
+
+	of := result.OutputFiles[0]
+	if err := os.WriteFile(filepath.FromSlash(of.Path), of.Contents, 0o644); err != nil {
+		return fmt.Errorf("write file failed: %v", err)
+	}
+	return nil
+}

internal/warpc/js/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+package-lock.json

internal/warpc/js/common.js

@@ -0,0 +1,56 @@
+// Read JSONL from stdin.
+export function readInput(handle) {
+	const buffSize = 1024;
+	let currentLine = [];
+	const buffer = new Uint8Array(buffSize);
+
+	// Read all the available bytes
+	while (true) {
+		// Stdin file descriptor
+		const fd = 0;
+		let bytesRead = 0;
+		try {
+			bytesRead = Javy.IO.readSync(fd, buffer);
+		} catch (e) {
+			// IO.readSync fails with os error 29 when stdin closes.
+			if (e.message.includes('os error 29')) {
+				break;
+			}
+			throw new Error('Error reading from stdin');
+		}
+
+		if (bytesRead < 0) {
+			throw new Error('Error reading from stdin');
+			break;
+		}
+
+		if (bytesRead === 0) {
+			break;
+		}
+
+		currentLine = [...currentLine, ...buffer.subarray(0, bytesRead)];
+
+		// Split array into chunks by newline.
+		let i = 0;
+		for (let j = 0; i < currentLine.length; i++) {
+			if (currentLine[i] === 10) {
+				const chunk = currentLine.splice(j, i + 1);
+				const arr = new Uint8Array(chunk);
+				const json = JSON.parse(new TextDecoder().decode(arr));
+				handle(json);
+				j = i + 1;
+			}
+		}
+		// Remove processed data.
+		currentLine = currentLine.slice(i);
+	}
+}
+
+// Write JSONL to stdout
+export function writeOutput(output) {
+	const encodedOutput = new TextEncoder().encode(JSON.stringify(output) + '\n');
+	const buffer = new Uint8Array(encodedOutput);
+	// Stdout file descriptor
+	const fd = 1;
+	Javy.IO.writeSync(fd, buffer);
+}

internal/warpc/js/greet.bundle.js

@@ -0,0 +1,9 @@
+import { readInput, writeOutput } from './common';
+
+const greet = function (input) {
+	writeOutput({ header: input.header, data: { greeting: 'Hello ' + input.data.name + '!' } });
+};
+
+console.log('Greet module loaded');
+
+readInput(greet);

internal/warpc/js/greet.js

@@ -0,0 +1,14 @@
+{
+  "name": "js",
+  "version": "1.0.0",
+  "main": "greet.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "devDependencies": {
+    "katex": "^0.16.11"
+  }
+}