feat: cherry pick defcall into v11 (#1331)

This adds support for parsing defcall declarations, along with an
appropriate AST node.  The syntax supports source selectors, and
updates encoding / decoding as necessary.

This adds support for the resolution, processing and typing stages for
the DefCall construct.  An initial implementation of translation is also
included, but this does not yet actually translate the call.

This updates the trace propagation algorithm to recognise function
call's.  These can then trigger calls into assembly functions.  One
problem is that these are evaluated in the base field, meaning they
can overflow (in principle).  To address this, a check is used to protect
against unexpected overflow during trace propagation of function calls.

Author: DavePearce

pkg/asm/propagate.go

@@ -19,7 +19,10 @@ import (
 
 	"github.com/consensys/go-corset/pkg/asm/io"
 	"github.com/consensys/go-corset/pkg/ir"
+	"github.com/consensys/go-corset/pkg/ir/mir"
 	"github.com/consensys/go-corset/pkg/schema"
+	sc "github.com/consensys/go-corset/pkg/schema"
+	"github.com/consensys/go-corset/pkg/trace"
 	"github.com/consensys/go-corset/pkg/trace/lt"
 	"github.com/consensys/go-corset/pkg/util/collection/array"
 	"github.com/consensys/go-corset/pkg/util/field"
@@ -78,7 +81,7 @@ func Propagate[F field.Element[F], T io.Instruction[T]](p MixedProgram[F, T], tr
 	// Construct suitable executior for the given program
 	var (
 		errors []error
-		n      = len(p.program.Functions())
+		n      = uint(len(p.program.Functions()))
 		//
 		executor = io.NewExecutor(p.program)
 		// Clone heap in trace file, since will mutate this.
@@ -91,7 +94,7 @@ func Propagate[F field.Element[F], T io.Instruction[T]](p MixedProgram[F, T], tr
 		return lt.TraceFile{}, errors
 	}
 	// Write seed instances
-	errors = writeInstances(p.program, trace.Modules[:n], executor)
+	errors = writeInstances(p, n, trace.Modules, executor)
 	// Read out generated instances
 	modules := readInstances(&heap, p.program, executor)
 	// Append external modules (which are unaffected by propagation).
@@ -103,15 +106,24 @@ func Propagate[F field.Element[F], T io.Instruction[T]](p MixedProgram[F, T], tr
 // WriteInstances writes all of the instances defined in the given trace columns
 // into the executor which, in turn, forces it to execute the relevant
 // functions, and functions they call, etc.
-func writeInstances[T io.Instruction[T]](p io.Program[T], trace []lt.Module[word.BigEndian],
-	executor *io.Executor[T]) []error {
+func writeInstances[F field.Element[F], T io.Instruction[T]](p MixedProgram[F, T], n uint,
+	trace []lt.Module[word.BigEndian], executor *io.Executor[T]) []error {
 	//
 	var errors []error
 	//
-	for i, m := range trace {
-		errs := writeFunctionInstances(uint(i), p, m, executor)
+	for i, m := range trace[:n] {
+		errs := writeFunctionInstances(uint(i), p.program, m, executor)
 		errors = append(errors, errs...)
 	}
+	// Write all from non-assembly modules
+	for i, m := range trace[n:] {
+		var extern = p.externs[i]
+		// Write instances from any external calls
+		for _, call := range extractExternalCalls(extern) {
+			errs := writeExternCall(call, p.program, m, executor)
+			errors = append(errors, errs...)
+		}
+	}
 	//
 	return errors
 }
@@ -142,6 +154,68 @@ func writeFunctionInstances[T io.Instruction[T]](fid uint, p io.Program[T], mod
 	return errors
 }
 
+// Extract any external function calls found within the given module, returning
+// them as an array.
+func extractExternalCalls[F field.Element[F], M sc.Module[F]](extern M) []mir.FunctionCall[F] {
+	var calls []mir.FunctionCall[F]
+	//
+	for iter := extern.Constraints(); iter.HasNext(); {
+		c := iter.Next()
+		// This should always hold
+		if hc, ok := c.(mir.Constraint[F]); ok {
+			// Check whether its a call or not
+			if call, ok := hc.Unwrap().(mir.FunctionCall[F]); ok {
+				// Yes, so record it
+				calls = append(calls, call)
+			}
+		}
+	}
+	//
+	return calls
+}
+
+// Write any function instances arising from the given call.
+func writeExternCall[F field.Element[F], T io.Instruction[T]](call mir.FunctionCall[F], p io.Program[T], mod RawModule,
+	executor *io.Executor[T]) []error {
+	//
+	var (
+		trMod   = &ltModuleAdaptor[F]{mod}
+		height  = mod.Height()
+		fn      = p.Function(call.Callee)
+		inputs  = make([]big.Int, fn.NumInputs())
+		outputs = make([]big.Int, fn.NumOutputs())
+		errors  []error
+	)
+	//
+	if call.Selector.HasValue() {
+		var selector = call.Selector.Unwrap()
+		// Invoke each user-defined instance in turn
+		for i := range height {
+			// execute if selector enabled
+			if enabled, _, err := selector.TestAt(int(i), trMod, nil); enabled {
+				// Extract external columns
+				extractExternColumns(int(i), call, trMod, inputs, outputs)
+				// Execute function call to produce outputs
+				errs := executeAndCheck(call.Callee, fn.Name(), inputs, outputs, executor)
+				errors = append(errors, errs...)
+			} else if err != nil {
+				errors = append(errors, err)
+			}
+		}
+	} else {
+		// Invoke each user-defined instance in turn
+		for i := range height {
+			// Extract external columns
+			extractExternColumns(int(i), call, trMod, inputs, outputs)
+			// Execute function call to produce outputs
+			errs := executeAndCheck(call.Callee, fn.Name(), inputs, outputs, executor)
+			errors = append(errors, errs...)
+		}
+	}
+	//
+	return errors
+}
+
 func executeAndCheck[T io.Instruction[T]](fid uint, name string, inputs, outputs []big.Int,
 	executor *io.Executor[T]) []error {
 	var (
@@ -195,6 +269,34 @@ func extractFunctionColumns(row uint, mod RawModule, inputs, outputs []big.Int)
 	}
 }
 
+func extractExternColumns[F field.Element[F]](row int, call mir.FunctionCall[F], mod trace.Module[F],
+	inputs, outputs []big.Int) []error {
+	//
+	// Extract function arguments
+	errs1 := extractExternTerms(row, call.Arguments, mod, inputs)
+	// Extract function returns
+	errs2 := extractExternTerms(row, call.Returns, mod, outputs)
+	//
+	return append(errs1, errs2...)
+}
+
+func extractExternTerms[F field.Element[F]](row int, terms []mir.Term[F], mod trace.Module[F], values []big.Int,
+) []error {
+	var errors []error
+	//
+	for i, arg := range terms {
+		var (
+			ith      big.Int
+			val, err = arg.EvalAt(row, mod, nil)
+		)
+		ith.SetBytes(val.Bytes())
+		values[i] = ith
+		//
+		errors = append(errors, err)
+	}
+	//
+	return errors
+}
 func extractFunctionPadding(registers []schema.Register, inputs, outputs []big.Int) {
 	var numInputs = len(inputs)
 	//
@@ -279,3 +381,55 @@ func toArgumentString(args []big.Int) string {
 	//
 	return builder.String()
 }
+
+// The purpose of the lt adaptor is to make an lt.TraceFile look like a Trace.
+// In general, this is not safe.  However, we use this once we already know that
+// the trace has been aligned.  Also, it is only used in a specific context.
+type ltModuleAdaptor[F field.Element[F]] struct {
+	module lt.Module[word.BigEndian]
+}
+
+func (p *ltModuleAdaptor[F]) Name() string {
+	return p.module.Name
+}
+
+func (p *ltModuleAdaptor[F]) Width() uint {
+	return uint(len(p.module.Columns))
+}
+
+func (p *ltModuleAdaptor[F]) Height() uint {
+	return p.module.Height()
+}
+
+func (p *ltModuleAdaptor[F]) Column(cid uint) trace.Column[F] {
+	return &ltColumnAdaptor[F]{p.module.Columns[cid]}
+}
+
+func (p *ltModuleAdaptor[F]) ColumnOf(col string) trace.Column[F] {
+	panic("unsupported operation")
+}
+
+type ltColumnAdaptor[F field.Element[F]] struct {
+	column lt.Column[word.BigEndian]
+}
+
+func (p *ltColumnAdaptor[F]) Name() string {
+	return p.column.Name
+}
+
+func (p *ltColumnAdaptor[F]) Get(row int) F {
+	var (
+		v = p.column.Data.Get(uint(row))
+		w F
+	)
+	// Convert
+	return w.SetBytes(v.Bytes())
+}
+
+func (p *ltColumnAdaptor[F]) Data() array.Array[F] {
+	panic("unsupported operation")
+}
+
+func (p *ltColumnAdaptor[F]) Padding() F {
+	panic("unsupported operation")
+}

pkg/corset/ast/declaration.go

@@ -147,6 +147,97 @@ func (p *DefAlias) Lisp() sexp.SExp {
 	return sexp.NewSymbol(p.Name)
 }
 
+// ============================================================================
+// defcall
+// ============================================================================
+
+// DefCall captures a function call between a lisp module and an assembly
+// function.  A key feature of this is that it triggers trace propagation.
+type DefCall struct {
+	// Returns for the call
+	Returns []Expr
+	// Function being called
+	Function string
+	// Arguments for the call
+	Arguments []Expr
+	// Optional source selector
+	Selector util.Option[Expr]
+	// determines whether or not this has been finalised.
+	finalised bool
+}
+
+// NewDefCall creates a new (unfinalised) function call.
+func NewDefCall(returns []Expr, fun string, args []Expr, selector util.Option[Expr]) *DefCall {
+	//
+	return &DefCall{returns, fun, args, selector, false}
+}
+
+// Definitions returns the set of symbols defined by this declaration.  Observe
+// that these may not yet have been finalised.
+func (p *DefCall) Definitions() iter.Iterator[SymbolDefinition] {
+	return iter.NewArrayIterator[SymbolDefinition](nil)
+}
+
+// Dependencies needed to signal declaration.
+func (p *DefCall) Dependencies() iter.Iterator[Symbol] {
+	var deps []Symbol
+	//
+	deps = append(deps, DependenciesOfExpressions(p.Arguments)...)
+	deps = append(deps, DependenciesOfExpressions(p.Returns)...)
+	// Include selector dependencies (if applicable)
+	if p.Selector.HasValue() {
+		deps = append(deps, p.Selector.Unwrap().Dependencies()...)
+	}
+	// Combine deps
+	return iter.NewArrayIterator(deps)
+}
+
+// Defines checks whether this declaration defines the given symbol.  The symbol
+// in question needs to have been resolved already for this to make sense.
+func (p *DefCall) Defines(symbol Symbol) bool {
+	return false
+}
+
+// IsFinalised checks whether this declaration has already been finalised.  If
+// so, then we don't need to finalise it again.
+func (p *DefCall) IsFinalised() bool {
+	return p.finalised
+}
+
+// Finalise this declaration, which means that all source and target expressions
+// have been resolved.
+func (p *DefCall) Finalise() {
+	p.finalised = true
+}
+
+// Lisp converts this node into its lisp representation.  This is primarily used
+// for debugging purposes.
+func (p *DefCall) Lisp() sexp.SExp {
+	returns := make([]sexp.SExp, len(p.Returns))
+	args := make([]sexp.SExp, len(p.Arguments))
+	// Returns
+	for i, t := range p.Returns {
+		returns[i] = t.Lisp()
+	}
+	// Arguments
+	for i, t := range p.Arguments {
+		args[i] = t.Lisp()
+	}
+	//
+	list := sexp.NewList([]sexp.SExp{
+		sexp.NewSymbol("defcall"),
+		sexp.NewList(returns),
+		sexp.NewSymbol(p.Function),
+		sexp.NewList(args),
+	})
+	// Include selector (if applicable)
+	if p.Selector.HasValue() {
+		list.Append(p.Selector.Unwrap().Lisp())
+	}
+	//
+	return list
+}
+
 // ============================================================================
 // defcolumns
 // ============================================================================

pkg/corset/compiler/parser.go

@@ -287,6 +287,10 @@ func (p *Parser) parseDeclaration(module file.Path, s *sexp.List) (ast.Declarati
 	//
 	if s.MatchSymbols(1, "defalias") {
 		decl, errors = p.parseDefAlias(s.Elements)
+	} else if s.Len() == 4 && s.MatchSymbols(1, "defcall") {
+		decl, errors = p.parseDefCall(false, s.Elements)
+	} else if s.Len() == 5 && s.MatchSymbols(1, "defcall") {
+		decl, errors = p.parseDefCall(true, s.Elements)
 	} else if s.MatchSymbols(1, "defcolumns") {
 		decl, errors = p.parseDefColumns(module, s)
 	} else if s.Len() == 3 && s.MatchSymbols(1, "defcomputed") {
@@ -875,6 +879,37 @@ func (p *Parser) parseDefInterleavedSourceArray(source *sexp.Array) (ast.TypedSy
 	return nil, errors
 }
 
+func (p *Parser) parseDefCall(hasSelector bool, elements []sexp.SExp) (ast.Declaration, []SyntaxError) {
+	var (
+		errors             []SyntaxError
+		returns, retErrors = p.parseDefLookupSources("return", elements[1])
+		args, argErrors    = p.parseDefLookupSources("argument", elements[3])
+		selector           = util.None[ast.Expr]()
+	)
+	// Sanity check function name
+	if !isIdentifier(elements[2]) {
+		return nil, p.translator.SyntaxErrors(elements[2], "malformed function name")
+	}
+	// Extract function name
+	fun := elements[2].AsSymbol().Value
+	// Combine any and all errors
+	errors = append(errors, argErrors...)
+	errors = append(errors, retErrors...)
+	// Parse selector (if applicable)
+	if hasSelector {
+		sel, errs := p.translator.Translate(elements[4])
+		selector = util.Some(sel)
+		//
+		errors = append(errors, errs...)
+	}
+	// Error check
+	if len(errors) != 0 {
+		return nil, errors
+	}
+	//
+	return ast.NewDefCall(returns, fun, args, selector), nil
+}
+
 // Parse a lookup declaration
 func (p *Parser) parseDefLookup(module file.Path, elements []sexp.SExp) (ast.Declaration, []SyntaxError) {
 	// Extract items