feat: support trace diff with differing columns (#882)

This enables the "trace diff" command to work even when the columns
differ between two traces.  In such, case the columns different to each
trace are reported, and the diff proceeds on the set of common columns.

Author: DavePearce

pkg/cmd/tracediff.go

@@ -3,10 +3,12 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"slices"
 	"sort"
 
 	"github.com/consensys/gnark-crypto/ecc/bls12-377/fr"
 	"github.com/consensys/go-corset/pkg/trace"
+	"github.com/consensys/go-corset/pkg/util/collection/set"
 	"github.com/consensys/go-corset/pkg/util/field"
 	"github.com/spf13/cobra"
 )
@@ -28,10 +30,20 @@ var traceDiffCmd = &cobra.Command{
 		// Read trace files
 		tracefile1 := ReadTraceFile(filename1)
 		tracefile2 := ReadTraceFile(filename2)
+		// Extract column names
+		trace1cols := extractColumnNames(tracefile1.Columns)
+		trace2cols := extractColumnNames(tracefile2.Columns)
 		// Sanity check
-		if len(tracefile1.Columns) != len(tracefile2.Columns) {
-			fmt.Printf("differing number of columns (%d v %d)", len(tracefile1.Columns), len(tracefile2.Columns))
-			os.Exit(2)
+		if !slices.Equal(trace1cols, trace2cols) {
+			var common set.SortedSet[string]
+			//
+			common.InsertSorted(&trace1cols)
+			common.Intersect(&trace2cols)
+			//
+			reportExtraColumns(filename1, trace1cols, common)
+			reportExtraColumns(filename2, trace2cols, common)
+			tracefile1.Columns = filterCommonColumns(tracefile1.Columns, common)
+			tracefile2.Columns = filterCommonColumns(tracefile2.Columns, common)
 		}
 		//
 		errors := parallelDiff(tracefile1.Columns, tracefile2.Columns)
@@ -42,6 +54,36 @@ var traceDiffCmd = &cobra.Command{
 	},
 }
 
+func extractColumnNames(columns []trace.RawColumn) set.SortedSet[string] {
+	var names set.SortedSet[string]
+	//
+	for _, c := range columns {
+		names.Insert(c.QualifiedName())
+	}
+	//
+	return names
+}
+
+func reportExtraColumns(name string, columns []string, common set.SortedSet[string]) {
+	for _, c := range columns {
+		if !common.Contains(c) {
+			fmt.Printf("column %s only in trace %s\n", c, name)
+		}
+	}
+}
+
+func filterCommonColumns(columns []trace.RawColumn, common set.SortedSet[string]) []trace.RawColumn {
+	var ncolumns []trace.RawColumn
+	//
+	for _, c := range columns {
+		if common.Contains(c.QualifiedName()) {
+			ncolumns = append(ncolumns, c)
+		}
+	}
+	//
+	return ncolumns
+}
+
 func parallelDiff(columns1 []trace.RawColumn, columns2 []trace.RawColumn) []error {
 	errors := make([]error, 0)
 	ncols := len(columns1)

pkg/util/collection/set/sorted_set.go

@@ -79,7 +79,27 @@ func (p *SortedSet[T]) InsertSorted(q *SortedSet[T]) {
 	// Allocate space
 	ndata := make([]T, len(left)+len(right)-n)
 	// Merge
-	mergeSorted(ndata, left, right)
+	unionSorted(ndata, left, right)
+	// Finally copy over new data
+	*p = ndata
+}
+
+// Intersect removes all elements from this set which are not in the given set.
+func (p *SortedSet[T]) Intersect(q *SortedSet[T]) {
+	left := *p
+	right := *q
+	// Check containment
+	n := countDuplicates(left, right)
+	// Check for total inclusion
+	if n == len(left) {
+		// Left set completely included within right, so actually there is
+		// nothing to do.
+		return
+	}
+	// Allocate space
+	ndata := make([]T, n)
+	// Intersect
+	intersectSorted(ndata, left, right)
 	// Finally copy over new data
 	*p = ndata
 }
@@ -157,9 +177,9 @@ func countDuplicates[T cmp.Ordered](left []T, right []T) int {
 	return n
 }
 
-// Merge two sets of sorted arrays (left and right) into a target array.  This
+// Union two sets of sorted arrays (left and right) into a target array.  This
 // assumes the target array is big enough.
-func mergeSorted[T cmp.Ordered](target []T, left []T, right []T) {
+func unionSorted[T cmp.Ordered](target []T, left []T, right []T) {
 	i := 0
 	j := 0
 	k := 0
@@ -184,3 +204,24 @@ func mergeSorted[T cmp.Ordered](target []T, left []T, right []T) {
 		copy(target[k:], right[j:])
 	}
 }
+
+// Intersect two sets of sorted arrays (left and right) into a target array.
+// This assumes the target array is big enough.
+func intersectSorted[T cmp.Ordered](target []T, left []T, right []T) {
+	i := 0
+	j := 0
+	k := 0
+	// Merge overlap of both sets
+	for i < len(left) && j < len(right) {
+		if left[i] < right[j] {
+			i++
+		} else if left[i] > right[j] {
+			j++
+		} else {
+			target[k] = left[i]
+			i++
+			j++
+			k++
+		}
+	}
+}