feat: support versioned lt file format (#733)

* support versioned lt file format

This updates the LT file format to a versioned format which supports
embedded metadata.  The original legacy format is still supported.

* support metadata for trace file generation

This adds support for reading and writing embedded metadata (and other
versioning information) to the trace file.

Author: DavePearce

pkg/binfile/binfile.go

@@ -40,9 +40,9 @@ type BinaryFile struct {
 
 // NewBinaryFile constructs a new binary file with the default header for the
 // currently supported version.
-func NewBinaryFile(headerdata []byte, attributes []Attribute, schema *hir.Schema) *BinaryFile {
+func NewBinaryFile(metadata []byte, attributes []Attribute, schema *hir.Schema) *BinaryFile {
 	return &BinaryFile{
-		Header{ZKBINARY, BINFILE_MAJOR_VERSION, BINFILE_MINOR_VERSION, headerdata},
+		Header{ZKBINARY, BINFILE_MAJOR_VERSION, BINFILE_MINOR_VERSION, metadata},
 		attributes,
 		*schema,
 	}

pkg/cmd/check.go

@@ -102,8 +102,8 @@ var checkCmd = &cobra.Command{
 			traces = ReadBatchedTraceFile(args[0])
 		} else {
 			// unbatched (i.e. normal) mode
-			columns := ReadTraceFile(args[0])
-			traces = [][]trace.RawColumn{columns}
+			tracefile := ReadTraceFile(args[0])
+			traces = [][]trace.RawColumn{tracefile.Columns}
 		}
 		//
 		stats.Log("Reading trace file")

pkg/cmd/debug.go

@@ -67,7 +67,7 @@ var debugCmd = &cobra.Command{
 		binfile := ReadConstraintFiles(stdlib, debug, legacy, args)
 		// Print meta-data (if requested)
 		if metadata {
-			printMetadata(&binfile.Header)
+			printBinaryFileMetadata(&binfile.Header)
 		}
 		// Print stats (if requested)
 		if stats {
@@ -166,7 +166,7 @@ func printStats(hirSchema *hir.Schema, hir bool, mir bool, air bool, optConfig m
 	tbl.Print()
 }
 
-func printMetadata(header *binfile.Header) {
+func printBinaryFileMetadata(header *binfile.Header) {
 	fmt.Printf("Format: %d.%d\n", header.MajorVersion, header.MinorVersion)
 	// Attempt to parse metadata
 	metadata, err := header.GetMetaData()

pkg/cmd/generate.go

@@ -78,14 +78,19 @@ func generateJavaIntegration(filename string, pkgname string, srcmap *corset.Sou
 	}
 	// Strip suffix to determine classname
 	classname := strings.TrimSuffix(basename, ".java")
+	metadata, err := binfile.Header.GetMetaData()
+	// Error check
+	if err != nil {
+		return "", err
+	}
 	// begin generation
-	generateJavaHeader(pkgname, &builder)
-	generateJavaModule(classname, srcmap.Root, &binfile.Schema, indentBuilder{0, &builder})
+	generateJavaHeader(pkgname, metadata, &builder)
+	generateJavaModule(classname, srcmap.Root, metadata, &binfile.Schema, indentBuilder{0, &builder})
 	//
 	return builder.String(), nil
 }
 
-func generateJavaHeader(pkgname string, builder *strings.Builder) {
+func generateJavaHeader(pkgname string, metadata map[string]string, builder *strings.Builder) {
 	builder.WriteString(license)
 	// Write package line
 	if pkgname != "" {
@@ -94,9 +99,29 @@ func generateJavaHeader(pkgname string, builder *strings.Builder) {
 	//
 	builder.WriteString(javaImports)
 	builder.WriteString(javaWarning)
+	//
+	if len(metadata) > 0 {
+		// Write embedded metadata for the record.
+		builder.WriteString(" * <p>Embedded Metata</p>\n")
+		builder.WriteString(" * <ul>\n")
+		//
+		for k, v := range metadata {
+			builder.WriteString(" * <li>")
+			builder.WriteString(k)
+			builder.WriteString(": ")
+			builder.WriteString(v)
+			builder.WriteString("</li>\n")
+		}
+		//
+		builder.WriteString(" * </ul>\n")
+	}
+	//
+	builder.WriteString(" */\n")
 }
 
-func generateJavaModule(className string, mod corset.SourceModule, schema *hir.Schema, builder indentBuilder) {
+func generateJavaModule(className string, mod corset.SourceModule, metadata map[string]string, schema *hir.Schema,
+	builder indentBuilder) {
+	//
 	var nFields uint
 	// Attempt to find module
 	mid, ok := schema.Modules().Find(func(m sc.Module) bool { return m.Name == mod.Name })
@@ -108,6 +133,11 @@ func generateJavaModule(className string, mod corset.SourceModule, schema *hir.S
 	generateJavaClassHeader(mod.Name == "", className, builder)
 	generateJavaModuleConstants(mod.Constants, builder.Indent())
 	generateJavaModuleSubmoduleFields(mod.Submodules, builder.Indent())
+	//
+	if mod.Name == "" {
+		generateJavaModuleMetadata(metadata, builder.Indent())
+	}
+	//
 	generateJavaModuleHeaders(mid, mod, schema, builder.Indent())
 	//
 	if nFields = generateJavaModuleRegisterFields(mid, schema, builder.Indent()); nFields > 0 {
@@ -125,7 +155,7 @@ func generateJavaModule(className string, mod corset.SourceModule, schema *hir.S
 	// Generate any submodules
 	for _, submod := range mod.Submodules {
 		if !submod.Virtual {
-			generateJavaModule(toPascalCase(submod.Name), submod, schema, builder.Indent())
+			generateJavaModule(toPascalCase(submod.Name), submod, metadata, schema, builder.Indent())
 		} else {
 			generateJavaModuleColumnSetters(className, submod, schema, builder.Indent())
 		}
@@ -323,6 +353,23 @@ func generateJavaModuleSubmoduleFields(submodules []corset.SourceModule, builder
 	}
 }
 
+func generateJavaModuleMetadata(metadata map[string]string, builder indentBuilder) {
+	// Write field declaration
+	builder.WriteIndentedString("public static Map<String,String> metadata() {\n")
+	// Initialise map using Java static initialiser
+	if len(metadata) > 0 {
+		i1Builder := builder.Indent()
+		i1Builder.WriteIndentedString("Map<String,String> metadata = new HashMap<>();\n")
+
+		for k, v := range metadata {
+			i1Builder.WriteIndentedString("metadata.put(\"", k, "\",\"", v, "\");\n")
+		}
+		//
+		i1Builder.WriteIndentedString("return metadata;\n")
+		builder.WriteIndentedString("}\n\n")
+	}
+}
+
 func generateJavaModuleConstructor(classname string, mid uint, mod corset.SourceModule,
 	schema *hir.Schema, builder indentBuilder) {
 	//
@@ -686,24 +733,28 @@ const javaWarning string = `
  * WARNING: This code is generated automatically.
  *
  * <p>Any modifications to this code may be overwritten and could lead to unexpected behavior.
- * Please DO NOT ATTEMPT TO MODIFY this code directly.
- */
+ * Please DO NOT ATTEMPT TO MODIFY this code directly</p>.
+ *
 `
 
 const javaImports string = `
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.math.BigInteger;
+import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
 import java.util.ArrayList;
 import java.util.BitSet;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import net.consensys.linea.zktracer.types.UnsignedByte;
 import org.apache.tuweni.bytes.Bytes;
 `
 
+// nolint
 const javaTraceOf string = `
    /**
     * Construct a new trace which will be written to a given file.
@@ -713,21 +764,46 @@ const javaTraceOf string = `
     *
     * @throws IOException If an I/O error occurs.
     */
-   public static Trace of(RandomAccessFile file, List<ColumnHeader> rawHeaders) throws IOException {
-      // First align headers according to register indices.
-      ColumnHeader[] headers = alignHeaders(rawHeaders);
-      // Second determine file size
-      long headerSize = determineHeaderSize(headers);
-      long dataSize = determineHeaderSize(headers);
+   public static Trace of(RandomAccessFile file, List<ColumnHeader> rawHeaders, byte[] metadata) throws IOException {
+      // Construct trace file header bytes
+      byte[] header = constructTraceFileHeader(metadata);
+      // Align headers according to register indices.
+      ColumnHeader[] columnHeaders = alignHeaders(rawHeaders);
+      // Determine file size
+      long headerSize = determineColumnHeadersSize(columnHeaders) + header.length;
+      long dataSize = determineColumnDataSize(columnHeaders);
       file.setLength(headerSize + dataSize);
-      // Write header
-      writeHeader(file,headers,headerSize);
+      // Write headers
+      writeHeaders(file,header,columnHeaders,headerSize);
       // Initialise buffers
-      MappedByteBuffer[] buffers = initialiseByteBuffers(file,headers,headerSize);
+      MappedByteBuffer[] buffers = initialiseByteBuffers(file,columnHeaders,headerSize);
       // Done
       return new Trace(buffers);
    }
 
+  /**
+   * Construct trace file header containing the given metadata bytes.
+   *
+   * @param metadata Metadata bytes to be embedded in the trace file.
+   *
+   * @return bytes making up the header.
+   */
+   private static byte[] constructTraceFileHeader(byte[] metadata) {
+     ByteBuffer buffer = ByteBuffer.allocate(16 + metadata.length);
+	 // File identifier
+     buffer.put(new byte[]{'z','k','t','r','a','c','e','r'});
+     // Major version
+     buffer.putShort((short) 1);
+     // Minor version
+     buffer.putShort((short) 0);
+     // Metadata length
+     buffer.putInt(metadata.length);
+     // Metadata
+     buffer.put(metadata);
+     // Done
+     return buffer.array();
+   }
+
   /**
    * Align headers ensures that the order in which columns are seen matches the order found in the trace schema.
    *
@@ -750,7 +826,7 @@ const javaTraceOf string = `
     * @param headers Set of headers for the columns being written.
     * @return Number of bytes requires for the trace file header.
     */
-   private static long determineHeaderSize(ColumnHeader[] headers) {
+   private static long determineColumnHeadersSize(ColumnHeader[] headers) {
       long nBytes = 4; // column count
 
       for (ColumnHeader header : headers) {
@@ -769,7 +845,7 @@ const javaTraceOf string = `
     * @param headers Set of headers for the columns being written.
     * @return Number of bytes required for storing all column data, excluding the header.
     */
-   private static long determineDataSize(ColumnHeader[] headers) {
+   private static long determineColumnDataSize(ColumnHeader[] headers) {
       long nBytes = 0;
 
       for (ColumnHeader header : headers) {
@@ -781,20 +857,24 @@ const javaTraceOf string = `
 
    /**
     * Write header information for the trace file.
+    *
     * @param file Trace file being written.
+    * @param header Trace file header
     * @param headers Column headers.
     * @param size Overall size of the header.
     */
-   private static void writeHeader(RandomAccessFile file, ColumnHeader[] headers, long size) throws IOException {
-      final var header = file.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, size);
+   private static void writeHeaders(RandomAccessFile file, byte[] header, ColumnHeader[] headers, long size) throws IOException {
+      final var buffer = file.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, size);
+      // Write trace file header
+      buffer.put(header);
       // Write column count as uint32
-      header.putInt(headers.length);
+      buffer.putInt(headers.length);
       // Write column headers one-by-one
       for(ColumnHeader h : headers) {
-         header.putShort((short) h.name.length());
-         header.put(h.name.getBytes());
-         header.put((byte) h.bytesPerElement);
-         header.putInt((int) h.length);
+         buffer.putShort((short) h.name.length());
+         buffer.put(h.name.getBytes());
+         buffer.put((byte) h.bytesPerElement);
+         buffer.putInt((int) h.length);
       }
    }
 

pkg/cmd/inspect.go

@@ -50,13 +50,13 @@ var inspectCmd = &cobra.Command{
 		//
 		stats.Log("Reading constraints file")
 		// Parse trace file
-		columns := ReadTraceFile(args[0])
+		tracefile := ReadTraceFile(args[0])
 		//
 		stats.Log("Reading trace file")
 		//
 		builder := sc.NewTraceBuilder(&binf.Schema).Expand(true).Defensive(defensive).Parallel(true)
 		//
-		trace, errors := builder.Build(columns)
+		trace, errors := builder.Build(tracefile.Columns)
 		//
 		if len(errors) == 0 {
 			// Run the inspector.

pkg/cmd/trace.go

@@ -23,6 +23,7 @@ import (
 	"github.com/consensys/go-corset/pkg/mir"
 	sc "github.com/consensys/go-corset/pkg/schema"
 	"github.com/consensys/go-corset/pkg/trace"
+	"github.com/consensys/go-corset/pkg/trace/lt"
 	"github.com/consensys/go-corset/pkg/util"
 	"github.com/consensys/go-corset/pkg/util/collection/hash"
 	"github.com/consensys/go-corset/pkg/util/collection/set"
@@ -71,14 +72,19 @@ var traceCmd = &cobra.Command{
 		mir := GetFlag(cmd, "mir")
 		hir := GetFlag(cmd, "hir")
 		batched := GetFlag(cmd, "batched")
+		metadata := GetFlag(cmd, "metadata")
 		// Parse trace file(s)
 		if batched {
 			// batched mode
 			traces = ReadBatchedTraceFile(args[0])
 		} else {
 			// unbatched (i.e. normal) mode
-			columns := ReadTraceFile(args[0])
-			traces = [][]trace.RawColumn{columns}
+			tracefile := ReadTraceFile(args[0])
+			traces = [][]trace.RawColumn{tracefile.Columns}
+			// Print meta-data (if requested)
+			if metadata {
+				printTraceFileMetadata(&tracefile.Header)
+			}
 		}
 		//
 		if expand && !air && !mir && !hir {
@@ -144,6 +150,7 @@ func init() {
 	traceCmd.Flags().Bool("air", false, "expand to AIR level")
 	traceCmd.Flags().Bool("batched", false,
 		"specify trace file is batched (i.e. contains multiple traces, one for each line)")
+	traceCmd.Flags().Bool("metadata", false, "Print embedded metadata")
 }
 
 const air_LEVEL = 0
@@ -254,6 +261,23 @@ func sliceColumns(cols []trace.RawColumn, start uint, end uint) {
 	}
 }
 
+func printTraceFileMetadata(header *lt.Header) {
+	fmt.Printf("Format: %d.%d\n", header.MajorVersion, header.MinorVersion)
+	// Attempt to parse metadata
+	metadata, err := header.GetMetaData()
+	//
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	} else if metadata != nil {
+		fmt.Println("Metadata:")
+		//
+		for k, v := range metadata {
+			fmt.Printf("\t%s: %s\n", k, v)
+		}
+	}
+}
+
 func printTrace(start uint, max_width uint, cols []trace.RawColumn) {
 	n := uint(len(cols))
 	height := maxHeightColumns(cols)

pkg/cmd/tracediff.go

@@ -26,15 +26,15 @@ var traceDiffCmd = &cobra.Command{
 		filename1 := args[0]
 		filename2 := args[1]
 		// Read trace files
-		columns1 := ReadTraceFile(filename1)
-		columns2 := ReadTraceFile(filename2)
+		tracefile1 := ReadTraceFile(filename1)
+		tracefile2 := ReadTraceFile(filename2)
 		// Sanity check
-		if len(columns1) != len(columns2) {
-			fmt.Printf("differing number of columns (%d v %d)", len(columns1), len(columns2))
+		if len(tracefile1.Columns) != len(tracefile2.Columns) {
+			fmt.Printf("differing number of columns (%d v %d)", len(tracefile1.Columns), len(tracefile2.Columns))
 			os.Exit(2)
 		}
 		//
-		errors := parallelDiff(columns1, columns2)
+		errors := parallelDiff(tracefile1.Columns, tracefile2.Columns)
 		// report any differences
 		for _, err := range errors {
 			fmt.Println(err)