feat: enable recursive definition for a defcomputedcolumn (#1088)

* add tests / updated parser

This adds some tests for recursive computed columns, and updates the
parser to support the annotations ":fwd" and ":bwd".

* Recursive computation now operational

* detect recursive definitions

This adds support for detecting and report errors for recursive
definitions.

* mark tests as ignored (for now)

These tests cannot be resolved without further work, and have been
marked as ignored (alongw with issue number).

* fix ModuleScope.IsVisible()

This was not sufficiently flexible to account for all lookup scenarios.
Instead, have copied the essential structure of Bind() to get this to
work correctly.

Author: DavePearce

pkg/corset/ast/binding.go

@@ -24,6 +24,8 @@ import (
 type Binding interface {
 	// Determine whether this binding is finalised or not.
 	IsFinalised() bool
+	// Determine whether this binding can be defined recursively or not.
+	IsRecursive() bool
 }
 
 // FunctionBinding is a special kind of binding which captures the essence of
@@ -99,6 +101,21 @@ func (p *FunctionSignature) Apply(args []Expr, srcmap *source.Maps[Node]) Expr {
 // ColumnBinding
 // ============================================================================
 
+const (
+	// NOT_COMPUTED signals a column is not a computed column.
+	NOT_COMPUTED = 0
+	// COMPUTED signals a column is a (non-recursive) computed column.
+	COMPUTED = 1
+	// COMPUTED_FWD signals a column is a (forward recursive) computed column.
+	// This means its value is computed starting from the first row (hence it
+	// cannot use a forward shift in its declaration).
+	COMPUTED_FWD = 2
+	// COMPUTED_BWD signals a column is a (backward recursive) computed column.
+	// This means its value is computed starting from the first row (hence it
+	// cannot use a backward shift in its declaration).
+	COMPUTED_BWD = 3
+)
+
 // ColumnBinding represents something bound to a given column.
 type ColumnBinding struct {
 	// Context determines the real (i.e. non-virtual) enclosing module of this
@@ -115,8 +132,8 @@ type ColumnBinding struct {
 	MustProve bool
 	// Column's length Multiplier
 	Multiplier uint
-	// Determines whether this is a Computed column, or not.
-	Computed bool
+	// Determines the kind of this column.
+	Kind uint8
 	// Display modifier
 	Display string
 }
@@ -126,11 +143,21 @@ func (p *ColumnBinding) AbsolutePath() *util.Path {
 	return &p.Path
 }
 
+// IsComputed checks whether this binding is for a computed column (or not).
+func (p *ColumnBinding) IsComputed() bool {
+	return p.Kind != NOT_COMPUTED
+}
+
 // IsFinalised checks whether this binding has been finalised yet or not.
 func (p *ColumnBinding) IsFinalised() bool {
 	return p.Multiplier != 0
 }
 
+// IsRecursive implementation for Binding interface.
+func (p *ColumnBinding) IsRecursive() bool {
+	return p.Kind == COMPUTED_FWD || p.Kind == COMPUTED_BWD
+}
+
 // Finalise this binding by providing the necessary missing information.
 func (p *ColumnBinding) Finalise(multiplier uint, datatype Type) {
 	p.Multiplier = multiplier
@@ -175,6 +202,12 @@ func (p *ConstantBinding) IsFinalised() bool {
 	return p.finalised
 }
 
+// IsRecursive implementation for Binding interface.
+func (p *ConstantBinding) IsRecursive() bool {
+	// Constants can never be defined recursively
+	return false
+}
+
 // Finalise this binding.
 func (p *ConstantBinding) Finalise() {
 	p.finalised = true
@@ -211,6 +244,11 @@ func (p *LocalVariableBinding) IsFinalised() bool {
 	return p.Index != math.MaxUint
 }
 
+// IsRecursive implementation for Binding interface.
+func (p *LocalVariableBinding) IsRecursive() bool {
+	return false
+}
+
 // Finalise this local variable binding by allocating it an identifier.
 func (p *LocalVariableBinding) Finalise(index uint) {
 	p.Index = index
@@ -258,6 +296,12 @@ func (p *DefunBinding) IsFinalised() bool {
 	return p.finalised
 }
 
+// IsRecursive implementation for Binding interface.
+func (p *DefunBinding) IsRecursive() bool {
+	// Functions can never be defined recursively (for now, at least).
+	return false
+}
+
 // Signature returns the corresponding function signature for this user-defined
 // function.
 func (p *DefunBinding) Signature() *FunctionSignature {
@@ -294,6 +338,12 @@ func (p *PerspectiveBinding) IsFinalised() bool {
 	return p.resolved
 }
 
+// IsRecursive implementation for Binding interface.
+func (p *PerspectiveBinding) IsRecursive() bool {
+	// Recursive perspectives don't make sense!
+	return false
+}
+
 // Finalise this binding, which indicates the selector expression has been
 // finalised.
 func (p *PerspectiveBinding) Finalise() {

pkg/corset/ast/declaration.go

@@ -214,17 +214,15 @@ var _ SymbolDefinition = &DefColumn{}
 // NewDefColumn constructs a new (non-computed) column declaration.  Such a
 // column is automatically finalised, since all information is provided at the
 // point of creation.
-func NewDefColumn(context util.Path, name util.Path, datatype Type, mustProve bool, multiplier uint,
-	computed bool, display string) *DefColumn {
-	binding := ColumnBinding{context, name, datatype, mustProve, multiplier, computed, display}
+func NewDefColumn(binding ColumnBinding) *DefColumn {
 	return &DefColumn{binding}
 }
 
 // NewDefComputedColumn constructs a new column declaration for a computed
 // column.  Such a column cannot be finalised yet, since its type and multiplier
 // remains to be determined, etc.
 func NewDefComputedColumn(context util.Path, name util.Path) *DefColumn {
-	binding := ColumnBinding{context, name, nil, false, 0, true, "hex"}
+	binding := ColumnBinding{context, name, nil, false, 0, COMPUTED, "hex"}
 	return &DefColumn{binding}
 }
 
@@ -240,6 +238,12 @@ func (e *DefColumn) Binding() Binding {
 	return &e.binding
 }
 
+// InnerBinding returns the allocated binding for this symbol (which may or may
+// not be finalised).
+func (e *DefColumn) InnerBinding() ColumnBinding {
+	return e.binding
+}
+
 // Name returns the (unqualified) name of this symbol.  For example, "X" for
 // a column X defined in a module m1.
 func (e *DefColumn) Name() string {

pkg/corset/compiler/environment.go

@@ -170,7 +170,7 @@ func (p *GlobalEnvironment) initColumnsAndRegisters(modules []*ModuleScope) {
 	// Apply aliases
 	for _, m := range modules {
 		for id, binding_id := range m.ids {
-			if binding, ok := m.bindings[binding_id].(*ast.ColumnBinding); ok && !id.IsFunction() {
+			if binding, ok := m.bindings[binding_id].binding.(*ast.ColumnBinding); ok && !id.IsFunction() {
 				orig := binding.Path.String()
 				alias := m.path.Extend(id.name).String()
 				p.columnMap[alias] = p.columnMap[orig]

pkg/corset/compiler/externs.go

@@ -42,14 +42,20 @@ type ExternSymbolDefinition struct {
 // NewExternSymbolDefinition creates a new external symbol definition for a
 // given register in a given module.
 func NewExternSymbolDefinition(path util.Path, reg schema.Register) *ExternSymbolDefinition {
+	var kind uint8 = ast.NOT_COMPUTED
+	//
+	if reg.IsComputed() {
+		kind = ast.COMPUTED
+	}
+	//
 	return &ExternSymbolDefinition{
 		binding: ast.ColumnBinding{
 			ColumnContext: path,
 			Path:          *path.Extend(reg.Name),
 			DataType:      ast.NewUintType(reg.Width),
 			MustProve:     true,
 			Multiplier:    1,
-			Computed:      reg.IsComputed(),
+			Kind:          kind,
 			Display:       "hex", // default
 		},
 	}

pkg/corset/compiler/intrinsics.go

@@ -68,6 +68,11 @@ func (p *IntrinsicDefinition) IsFinalised() bool {
 	return true
 }
 
+// IsRecursive implementation for Binding interface.
+func (p *IntrinsicDefinition) IsRecursive() bool {
+	return false
+}
+
 // Binding returns the binding associated with this intrinsic.
 func (p *IntrinsicDefinition) Binding() ast.Binding {
 	return p

pkg/corset/compiler/natives.go

@@ -64,6 +64,11 @@ func (p *NativeDefinition) IsNative() bool {
 	return true
 }
 
+// IsRecursive implementation for Binding interface.
+func (p *NativeDefinition) IsRecursive() bool {
+	return false
+}
+
 // Arity indicates whether or not this is a function and, if so, what arity
 // (i.e. how many arguments) the function has.
 func (p *NativeDefinition) Arity() util.Option[uint] {

pkg/corset/compiler/parser.go

@@ -388,13 +388,20 @@ func (p *Parser) parseColumnDeclaration(context util.Path, path util.Path, compu
 	e sexp.SExp) (*ast.DefColumn, *SyntaxError) {
 	//
 	var (
-		error      *SyntaxError
-		name       util.Path
-		multiplier uint = 1
-		datatype   ast.Type
-		mustProve  bool
-		display    string
+		error *SyntaxError
+		// Initial binding with defaults
+		binding = ast.ColumnBinding{
+			ColumnContext: context,
+			Kind:          ast.NOT_COMPUTED,
+			Multiplier:    1,
+			MustProve:     false,
+			Display:       "hex",
+		}
 	)
+	// Update computed status
+	if computed {
+		binding.Kind = ast.COMPUTED
+	}
 	// Check whether extended declaration or not.
 	if l := e.AsList(); l != nil {
 		// Check at least the name provided.
@@ -404,43 +411,40 @@ func (p *Parser) parseColumnDeclaration(context util.Path, path util.Path, compu
 			return nil, p.translator.SyntaxError(l.Elements[0], "invalid column name")
 		}
 		// Column name is always first
-		name = *path.Extend(l.Elements[0].String(false))
+		binding.Path = *path.Extend(l.Elements[0].String(false))
 		//	Parse type (if applicable)
-		if datatype, mustProve, display, error = p.parseColumnDeclarationAttributes(e, l.Elements[1:]); error != nil {
+		if binding, error = p.parseColumnDeclarationAttributes(e, binding, l.Elements[1:]); error != nil {
 			return nil, error
 		}
 	} else if computed {
 		// Only computed columns can be given without attributes.
-		name = *path.Extend(e.String(false))
+		binding.Path = *path.Extend(e.String(false))
 	} else {
 		return nil, p.translator.SyntaxError(e, "column is untyped")
 	}
 	// Final sanity checks
-	if computed && datatype == nil {
+	if computed && binding.DataType == nil {
 		// computed columns initially have multiplier 0 in order to signal that
 		// this needs to be subsequently determined from context.
-		multiplier = 0
-		datatype = ast.INT_TYPE
-	} else if !datatype.HasUnderlying() {
+		binding.Multiplier = 0
+		binding.DataType = ast.INT_TYPE
+	} else if !binding.DataType.HasUnderlying() {
 		return nil, p.translator.SyntaxError(e, "invalid column type")
 	}
 	//
-	def := ast.NewDefColumn(context, name, datatype, mustProve, multiplier, computed, display)
+	def := ast.NewDefColumn(binding)
 	// Update source mapping
 	p.mapSourceNode(e, def)
 	//
 	return def, nil
 }
 
-func (p *Parser) parseColumnDeclarationAttributes(node sexp.SExp, attrs []sexp.SExp) (ast.Type, bool, string,
-	*SyntaxError) {
+func (p *Parser) parseColumnDeclarationAttributes(node sexp.SExp, binding ast.ColumnBinding,
+	attrs []sexp.SExp) (ast.ColumnBinding, *SyntaxError) {
 	//
 	var (
-		dataType  ast.Type
-		mustProve bool = false
 		array_min uint
 		array_max uint
-		display   string = "hex"
 		err       *SyntaxError
 	)
 
@@ -449,49 +453,67 @@ func (p *Parser) parseColumnDeclarationAttributes(node sexp.SExp, attrs []sexp.S
 		symbol := ith.AsSymbol()
 		// Sanity check
 		if symbol == nil {
-			return nil, false, "", p.translator.SyntaxError(ith, "unknown column attribute")
+			return binding, p.translator.SyntaxError(ith, "unknown column attribute")
 		}
 		//
 		switch symbol.Value {
 		case ":display":
 			// skip these for now, as they are only relevant to the inspector.
 			if i+1 == len(attrs) {
-				return nil, false, "", p.translator.SyntaxError(ith, "incomplete display definition")
+				return binding, p.translator.SyntaxError(ith, "incomplete display definition")
 			} else if attrs[i+1].AsSymbol() == nil {
-				return nil, false, "", p.translator.SyntaxError(ith, "malformed display definition")
+				return binding, p.translator.SyntaxError(ith, "malformed display definition")
 			}
 			//
-			display = attrs[i+1].AsSymbol().String(false)
+			binding.Display = attrs[i+1].AsSymbol().String(false)
 			// Check what display attribute we have
-			switch display {
+			switch binding.Display {
 			case ":dec", ":hex", ":bytes", ":opcode":
-				display = display[1:]
+				binding.Display = binding.Display[1:]
 				// all good
 				i = i + 1
 			default:
 				// not good
-				return nil, false, "", p.translator.SyntaxError(ith, "unknown display definition")
+				return binding, p.translator.SyntaxError(ith, "unknown display definition")
 			}
 		case ":array":
 			if array_min, array_max, err = p.parseArrayDimension(attrs[i+1]); err != nil {
-				return nil, false, "", err
+				return binding, err
 			}
 			// skip dimension
 			i++
+		case ":fwd":
+			switch binding.Kind {
+			case ast.NOT_COMPUTED:
+				return binding, p.translator.SyntaxError(ith, "input columns cannot be recursive")
+			case ast.COMPUTED_BWD:
+				return binding, p.translator.SyntaxError(ith, "conflicting direction of recursion")
+			default:
+				binding.Kind = ast.COMPUTED_FWD
+			}
+		case ":bwd":
+			switch binding.Kind {
+			case ast.NOT_COMPUTED:
+				return binding, p.translator.SyntaxError(ith, "input columns cannot be recursive")
+			case ast.COMPUTED_FWD:
+				return binding, p.translator.SyntaxError(ith, "conflicting direction of recursion")
+			default:
+				binding.Kind = ast.COMPUTED_BWD
+			}
 		default:
-			if dataType, mustProve, err = p.parseType(ith); err != nil {
-				return nil, false, "", err
+			if binding.DataType, binding.MustProve, err = p.parseType(ith); err != nil {
+				return binding, err
 			}
 		}
 	}
 	// Done
-	if dataType == nil {
-		return nil, false, "", p.translator.SyntaxError(node, "column is untyped")
+	if binding.DataType == nil {
+		return binding, p.translator.SyntaxError(node, "column is untyped")
 	} else if array_max != 0 {
-		return ast.NewArrayType(dataType, array_min, array_max), mustProve, display, nil
+		binding.DataType = ast.NewArrayType(binding.DataType, array_min, array_max)
 	}
-	//
-	return dataType, mustProve, display, nil
+	// Success!
+	return binding, nil
 }
 
 func (p *Parser) parseArrayDimension(s sexp.SExp) (uint, uint, *SyntaxError) {