test(wasm): Basic CST walking in Go

Author: pdubroy

packages/wasm/Makefile

@@ -15,5 +15,9 @@ print-runtime:
 go-test-es5: test/go/testmain
 	cd test/go && ./testmain -wasm ../data/_es5.wasm -file ../data/_html5shiv-3.7.3.js
 
+.PHONY: go-test-es5-lite
+go-test-es5-lite: test/go/testmain
+	cd test/go && ./testmain -wasm ../data/_es5.wasm -input "var x = 3; function foo() {}"
+
 test/go/testmain: test/go/testmain.go test/go/matcher.go
 	cd test/go && go mod tidy && go build -o testmain

packages/wasm/test/go/cst.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// Code for walking a CST by hand, by accessing the raw memory.
+// Ultimately, we will want a higher-level API for this, but for now,
+// this is useful for testing/debugging.
+
+const (
+	BYTES_PER_CST_REC = 12
+	SIZEOF_UINT32     = 4
+
+	// Node type constants
+	NODE_TYPE_NONTERMINAL = 0
+	NODE_TYPE_TERMINAL    = -1
+	NODE_TYPE_ITER        = -2
+)
+
+type CstNode struct {
+	Count     uint32   // Number of child nodes
+	MatchLen  uint32   // Length of the matched text
+	Type      int32    // Nonterminal: 0, terminal: -1, iter: -2
+	ChildRefs []uint32 // Addresses of child nodes
+}
+
+type CstWalker struct {
+	matcher *WasmMatcher
+}
+
+func NewCstWalker(matcher *WasmMatcher) *CstWalker {
+	return &CstWalker{
+		matcher: matcher,
+	}
+}
+
+// Read a CST node from memory at the given address
+func (p *CstWalker) GetRawCstNode(addr uint32) (*CstNode, error) {
+	memory := p.matcher.GetModule().Memory()
+	if memory == nil {
+		return nil, fmt.Errorf("WebAssembly module has no memory")
+	}
+
+	// Read the node data
+	data, ok := memory.Read(addr, BYTES_PER_CST_REC)
+	if !ok {
+		return nil, fmt.Errorf("failed to read CST node data at address %d", addr)
+	}
+
+	node := &CstNode{
+		Count:    readUint32(data, 0),
+		MatchLen: readUint32(data, 4),
+		Type:     readInt32(data, 8),
+	}
+
+	// Read child references if any
+	if node.Count > 0 {
+		childData, ok := memory.Read(addr+12, SIZEOF_UINT32*node.Count)
+		if !ok {
+			return nil, fmt.Errorf("failed to read CST child references at address %d", addr+12)
+		}
+
+		node.ChildRefs = make([]uint32, node.Count)
+		for i := uint32(0); i < node.Count; i++ {
+			node.ChildRefs[i] = readUint32(childData, i*SIZEOF_UINT32)
+		}
+	}
+
+	return node, nil
+}
+
+func readUint32(data []byte, offset uint32) uint32 {
+	return *(*uint32)(unsafe.Pointer(&data[offset]))
+}
+
+func readInt32(data []byte, offset uint32) int32 {
+	return *(*int32)(unsafe.Pointer(&data[offset]))
+}

packages/wasm/test/go/matcher.go

@@ -19,21 +19,26 @@ const (
 
 // WasmMatcher is a Go implementation of the JavaScript WasmMatcher class for Ohm
 type WasmMatcher struct {
-	runtime wazero.Runtime
-	module api.Module
-	input string
-	pos int
-	ctx context.Context
-	ruleIds map[string]int
+	runtime          wazero.Runtime
+	module           api.Module
+	input            string
+	pos              int
+	ctx              context.Context
+	ruleIds          map[string]int
 	defaultStartRule string
 }
 
+// GetModule returns the WebAssembly module
+func (m *WasmMatcher) GetModule() api.Module {
+	return m.module
+}
+
 func NewWasmMatcher(ctx context.Context) *WasmMatcher {
 	return &WasmMatcher{
-		runtime:     wazero.NewRuntime(ctx),
-		ctx:         ctx,
-		ruleIds:     make(map[string]int),
-		pos:         0,
+		runtime: wazero.NewRuntime(ctx),
+		ctx:     ctx,
+		ruleIds: make(map[string]int),
+		pos:     0,
 	}
 }
 

packages/wasm/test/go/testmain.go

@@ -6,14 +6,17 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 func main() {
+	fmt.Println("Ohm WebAssembly Matcher - Go Implementation")
 	// Parse command line arguments
 	wasmFile := flag.String("wasm", "test/data/_add.wasm", "Path to WebAssembly file")
 	inputText := flag.String("input", "", "Input text to match against the grammar")
 	inputFile := flag.String("file", "", "Path to file containing input text to match")
 	startRule := flag.String("rule", "", "Start rule for the grammar (defaults to grammar's start rule)")
+	verbose := flag.Bool("verbose", false, "Display verbose information about CST nodes")
 	flag.Parse()
 
 	// Create a context
@@ -76,8 +79,81 @@ func main() {
 			fmt.Printf("Error getting CST root: %v\n", err)
 		} else {
 			fmt.Printf("CST root node ID: %d\n", cstRoot)
+
+			// Create a CST walker with the matcher
+			cstWalker := NewCstWalker(matcher)
+
+			// Read the CST node
+			node, err := cstWalker.GetRawCstNode(cstRoot)
+			if err != nil {
+				fmt.Printf("Error reading CST node: %v\n", err)
+			} else {
+				fmt.Printf("CST Node - Count: %d, MatchLen: %d, Type: %d\n",
+					node.Count, node.MatchLen, node.Type)
+
+				if len(node.ChildRefs) > 0 {
+					fmt.Printf("Child references: %v\n", node.ChildRefs)
+				}
+
+				// Unparse the CST to get the original text
+				fmt.Println("\nUnparsing the CST to reconstruct the input:")
+				unparsedText := unparse(cstWalker, cstRoot, matcher.GetInput())
+				fmt.Printf("Unparsed text: %q\n", unparsedText)
+				fmt.Printf("Original input: %q\n", matcher.GetInput())
+				fmt.Printf("Match: %v\n", unparsedText == matcher.GetInput())
+			}
+
+			// Display more verbose information if requested
+			if *verbose {
+				fmt.Println("\nCST node details have been displayed above.")
+				fmt.Println("Node Types:")
+				fmt.Printf(" - Terminal nodes (type %d): Leaf nodes that consume input\n", NODE_TYPE_TERMINAL)
+				fmt.Printf(" - Iteration nodes (type %d): Used for repetition operations\n", NODE_TYPE_ITER)
+				fmt.Printf(" - Non-terminal nodes (type %d): Internal nodes with children\n", NODE_TYPE_NONTERMINAL)
+				fmt.Println("\nThe unparse function reconstructs the original input by collecting text from all terminal nodes in order.")
+			}
 		}
 	} else {
 		fmt.Println("Match failed")
 	}
 }
+
+// unparse walks the CST starting from the given node and reconstructs the original text
+// It returns the reconstructed text from the terminal nodes
+func unparse(walker *CstWalker, nodeAddr uint32, input string) string {
+	var result strings.Builder
+	pos := uint32(0)
+	unparseNode(walker, nodeAddr, &pos, input, &result)
+	return result.String()
+}
+
+// unparseNode is a helper function that recursively processes nodes and builds the result
+func unparseNode(walker *CstWalker, nodeAddr uint32, pos *uint32, input string, result *strings.Builder) {
+	// Read the current node
+	node, err := walker.GetRawCstNode(nodeAddr)
+	if err != nil {
+		fmt.Printf("Error reading CST node: %v\n", err)
+		return
+	}
+
+	// Handle terminal nodes - append the consumed text to the result
+	if node.Type == NODE_TYPE_TERMINAL {
+		if *pos < uint32(len(input)) && node.MatchLen > 0 {
+			end := *pos + node.MatchLen
+			if end > uint32(len(input)) {
+				end = uint32(len(input))
+			}
+			matchedText := input[*pos:end]
+			result.WriteString(matchedText)
+
+			// Update position only after processing terminal nodes
+			*pos += node.MatchLen
+		}
+		return
+	}
+
+	// For all other node types (nonterminal, iteration, etc.), process children recursively
+	for _, childAddr := range node.ChildRefs {
+		unparseNode(walker, childAddr, pos, input, result)
+	}
+}