feat(ast): implement and require builtin statements

BREAKING CHANGE: The option string is now a keyword.
In addition to adding and implementing the builtin keyword, option was
changed to be a keyword.

ast/ast.go

@@ -82,6 +82,7 @@ func (*BadStatement) node()        {}
 func (*ExpressionStatement) node() {}
 func (*ReturnStatement) node()     {}
 func (*OptionStatement) node()     {}
+func (*BuiltinStatement) node()    {}
 func (*VariableAssignment) node()  {}
 func (*MemberAssignment) node()    {}
 
@@ -295,6 +296,7 @@ func (*MemberAssignment) stmt()    {}
 func (*ExpressionStatement) stmt() {}
 func (*ReturnStatement) stmt()     {}
 func (*OptionStatement) stmt()     {}
+func (*BuiltinStatement) stmt()    {}
 
 type Assignment interface {
 	Statement
@@ -395,6 +397,31 @@ func (s *OptionStatement) Copy() Node {
 	return ns
 }
 
+// BuiltinStatement declares a builtin identifier and its type
+type BuiltinStatement struct {
+	BaseNode
+	ID *Identifier `json:"id"`
+	// TODO(nathanielc): Add type expression here
+	// Type TypeExpression
+}
+
+// Type is the abstract type
+func (*BuiltinStatement) Type() string { return "BuiltinStatement" }
+
+// Copy returns a deep copy of an BuiltinStatement Node
+func (s *BuiltinStatement) Copy() Node {
+	if s == nil {
+		return s
+	}
+	ns := new(BuiltinStatement)
+	*ns = *s
+	ns.BaseNode = s.BaseNode.Copy()
+
+	ns.ID = s.ID.Copy().(*Identifier)
+
+	return ns
+}
+
 // VariableAssignment represents the declaration of a variable
 type VariableAssignment struct {
 	BaseNode

ast/asttest/cmpopts.go

@@ -14,6 +14,7 @@ var IgnoreBaseNodeOptions = []cmp.Option{
 	cmpopts.IgnoreFields(ast.BinaryExpression{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.Block{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.BooleanLiteral{}, "BaseNode"),
+	cmpopts.IgnoreFields(ast.BuiltinStatement{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.CallExpression{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.ConditionalExpression{}, "BaseNode"),
 	cmpopts.IgnoreFields(ast.DateTimeLiteral{}, "BaseNode"),

ast/json.go

@@ -213,6 +213,17 @@ func (s *OptionStatement) UnmarshalJSON(data []byte) error {
 	s.Assignment = a
 	return nil
 }
+func (s *BuiltinStatement) MarshalJSON() ([]byte, error) {
+	type Alias BuiltinStatement
+	raw := struct {
+		Type string `json:"type"`
+		*Alias
+	}{
+		Type:  s.Type(),
+		Alias: (*Alias)(s),
+	}
+	return json.Marshal(raw)
+}
 func (d *VariableAssignment) MarshalJSON() ([]byte, error) {
 	type Alias VariableAssignment
 	raw := struct {
@@ -971,6 +982,8 @@ func unmarshalNode(msg json.RawMessage) (Node, error) {
 		node = new(Block)
 	case "OptionStatement":
 		node = new(OptionStatement)
+	case "BuiltinStatement":
+		node = new(BuiltinStatement)
 	case "ExpressionStatement":
 		node = new(ExpressionStatement)
 	case "ReturnStatement":

ast/json_test.go

@@ -115,6 +115,13 @@ func TestJSONMarshal(t *testing.T) {
 			},
 			want: `{"type":"OptionStatement","assignment":{"type":"VariableAssignment","id":{"type":"Identifier","name":"task"},"init":{"type":"ObjectExpression","properties":[{"type":"Property","key":{"type":"Identifier","name":"name"},"value":{"type":"StringLiteral","value":"foo"}},{"type":"Property","key":{"type":"Identifier","name":"every"},"value":{"type":"DurationLiteral","values":[{"magnitude":1,"unit":"h"}]}}]}}}`,
 		},
+		{
+			name: "builtin statement",
+			node: &ast.BuiltinStatement{
+				ID: &ast.Identifier{Name: "task"},
+			},
+			want: `{"type":"BuiltinStatement","id":{"type":"Identifier","name":"task"}}`,
+		},
 		{
 			name: "qualified option statement",
 			node: &ast.OptionStatement{

ast/walk.go

@@ -118,6 +118,14 @@ func walk(v Visitor, node Node) {
 		if w != nil && n.Assignment != nil {
 			walk(w, n.Assignment)
 		}
+	case *BuiltinStatement:
+		if n == nil {
+			return
+		}
+		w := v.Visit(n)
+		if w != nil {
+			walk(w, n.ID)
+		}
 	case *ExpressionStatement:
 		if n == nil {
 			return

compile.go

@@ -288,24 +288,28 @@ func registerPackageValue(pkgpath, name string, value values.Value, replace bool
 // c is a function reference of type CreateOperationSpec
 // sig is a function signature type that specifies the names and types of each argument for the function.
 func FunctionValue(name string, c CreateOperationSpec, sig semantic.FunctionPolySignature) values.Value {
-	return &function{
-		t:             semantic.NewFunctionPolyType(sig),
-		name:          name,
-		createOpSpec:  c,
-		hasSideEffect: false,
-	}
+	return functionValue(name, c, sig, false)
 }
 
 // FunctionValueWithSideEffect creates a values.Value from the operation spec and signature.
 // Name is the name of the function as it would be called.
 // c is a function reference of type CreateOperationSpec
 // sig is a function signature type that specifies the names and types of each argument for the function.
 func FunctionValueWithSideEffect(name string, c CreateOperationSpec, sig semantic.FunctionPolySignature) values.Value {
+	return functionValue(name, c, sig, true)
+}
+
+func functionValue(name string, c CreateOperationSpec, sig semantic.FunctionPolySignature, sideEffects bool) values.Value {
+	if c == nil {
+		c = func(args Arguments, a *Administration) (OperationSpec, error) {
+			return nil, fmt.Errorf("function %q is not implemented", name)
+		}
+	}
 	return &function{
 		t:             semantic.NewFunctionPolyType(sig),
 		name:          name,
 		createOpSpec:  c,
-		hasSideEffect: true,
+		hasSideEffect: sideEffects,
 	}
 }
 
@@ -350,6 +354,11 @@ func evalBuiltInPackages() error {
 			return errors.Wrapf(err, "package does not exist %q", astPkg.Path)
 		}
 
+		// Validate packages before evaluating them
+		if err := validatePackageBuiltins(pkg, astPkg); err != nil {
+			return errors.Wrapf(err, "package has invalid builtins %q", astPkg.Path)
+		}
+
 		itrp := interpreter.NewInterpreter()
 		if _, err := itrp.Eval(semPkg, preludeScope.Nest(pkg), stdlib); err != nil {
 			return errors.Wrapf(err, "failed to evaluate builtin package %q", astPkg.Path)
@@ -358,6 +367,37 @@ func evalBuiltInPackages() error {
 	return nil
 }
 
+// validatePackageBuiltins ensures that all package builtins have both an AST builtin statement and a registered value.
+func validatePackageBuiltins(pkg *interpreter.Package, astPkg *ast.Package) error {
+	builtinStmts := make(map[string]*ast.BuiltinStatement)
+	ast.Walk(ast.CreateVisitor(func(n ast.Node) {
+		if bs, ok := n.(*ast.BuiltinStatement); ok {
+			builtinStmts[bs.ID.Name] = bs
+		}
+	}), astPkg)
+
+	missing := make([]string, 0, len(builtinStmts))
+	extra := make([]string, 0, len(builtinStmts))
+
+	for n := range builtinStmts {
+		if _, ok := pkg.Get(n); !ok {
+			missing = append(missing, n)
+			continue
+		}
+		// TODO(nathanielc): Ensure that the value's type matches the type expression
+	}
+	pkg.Range(func(k string, v values.Value) {
+		if _, ok := builtinStmts[k]; !ok {
+			extra = append(extra, k)
+			return
+		}
+	})
+	if len(missing) > 0 || len(extra) > 0 {
+		return fmt.Errorf("missing builtin values %v, extra builtin values %v", missing, extra)
+	}
+	return nil
+}
+
 var TableObjectType = semantic.NewObjectPolyType(
 	//TODO: When values.Value support polytyped values, we can add the commented fields back in
 	map[string]semantic.PolyType{

compile_internal_test.go

@@ -0,0 +1,97 @@
+package flux
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/influxdata/flux/ast"
+	"github.com/influxdata/flux/interpreter"
+	"github.com/influxdata/flux/values"
+)
+
+func TestValidatePackageBuiltins(t *testing.T) {
+	testCases := []struct {
+		name   string
+		pkg    *interpreter.Package
+		astPkg *ast.Package
+		err    error
+	}{
+		{
+			name: "no errors",
+			pkg: interpreter.NewPackageWithValues("test", values.NewObjectWithValues(map[string]values.Value{
+				"foo": values.NewInt(0),
+			})),
+			astPkg: &ast.Package{
+				Files: []*ast.File{{
+					Body: []ast.Statement{
+						&ast.BuiltinStatement{
+							ID: &ast.Identifier{Name: "foo"},
+						},
+					},
+				}},
+			},
+		},
+		{
+			name: "extra values",
+			pkg: interpreter.NewPackageWithValues("test", values.NewObjectWithValues(map[string]values.Value{
+				"foo": values.NewInt(0),
+			})),
+			astPkg: &ast.Package{},
+			err:    errors.New("missing builtin values [], extra builtin values [foo]"),
+		},
+		{
+			name: "missing values",
+			pkg:  interpreter.NewPackageWithValues("test", values.NewObject()),
+			astPkg: &ast.Package{
+				Files: []*ast.File{{
+					Body: []ast.Statement{
+						&ast.BuiltinStatement{
+							ID: &ast.Identifier{Name: "foo"},
+						},
+					},
+				}},
+			},
+			err: errors.New("missing builtin values [foo], extra builtin values []"),
+		},
+		{
+			name: "missing and values",
+			pkg: interpreter.NewPackageWithValues("test", values.NewObjectWithValues(map[string]values.Value{
+				"foo": values.NewInt(0),
+				"bar": values.NewInt(0),
+			})),
+			astPkg: &ast.Package{
+				Files: []*ast.File{{
+					Body: []ast.Statement{
+						&ast.BuiltinStatement{
+							ID: &ast.Identifier{Name: "foo"},
+						},
+						&ast.BuiltinStatement{
+							ID: &ast.Identifier{Name: "baz"},
+						},
+					},
+				}},
+			},
+			err: errors.New("missing builtin values [baz], extra builtin values [bar]"),
+		},
+	}
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			err := validatePackageBuiltins(tc.pkg, tc.astPkg)
+			switch {
+			case err == nil && tc.err == nil:
+				// Test passes
+			case err == nil && tc.err != nil:
+				t.Errorf("expected error %v", tc.err)
+			case err != nil && tc.err == nil:
+				t.Errorf("unexpected error %v", err)
+			case err != nil && tc.err != nil:
+				if err.Error() != tc.err.Error() {
+					t.Errorf("differing error messages -want/+got:\n%s", cmp.Diff(tc.err.Error(), err.Error()))
+				}
+				// else test passes
+			}
+		})
+	}
+}

docs/SPEC.md

@@ -784,7 +784,7 @@ The _main_ package is special for a few reasons:
 
 A statement controls execution.
 
-    Statement = OptionStatement
+    Statement = OptionAssignment
               | BuiltinStatement
               | VariableAssignment
               | ReturnStatement
@@ -948,7 +948,7 @@ These preassigned values are defined in the source files for the various built-i
 ### System built-ins
 
 When a built-in value is not expressible in Flux its value may be defined by the hosting environment.
-All such values must have a corresponding "builtin" statement to declare the existence and type of the built-in value.
+All such values must have a corresponding builtin statement to declare the existence and type of the built-in value.
 
     BuiltinStatement = "builtin" identifer ":" TypeExpression
 

execute/transformation.go

@@ -43,9 +43,18 @@ type CreateNewPlannerTransformation func(id DatasetID, mode AccumulationMode, sp
 
 var procedureToTransformation = make(map[plan.ProcedureKind]CreateNewPlannerTransformation)
 
+// RegisterTransformation adds a new registration mapping of procedure kind to transformation.
 func RegisterTransformation(k plan.ProcedureKind, c CreateNewPlannerTransformation) {
 	if procedureToTransformation[k] != nil {
 		panic(fmt.Errorf("duplicate registration for transformation with procedure kind %v", k))
 	}
 	procedureToTransformation[k] = c
 }
+
+// ReplaceTransformation changes an existing transformation registration.
+func ReplaceTransformation(k plan.ProcedureKind, c CreateNewPlannerTransformation) {
+	if procedureToTransformation[k] == nil {
+		panic(fmt.Errorf("missing registration for transformation with procedure kind %v", k))
+	}
+	procedureToTransformation[k] = c
+}

internal/parser/grammar.md

@@ -11,17 +11,16 @@ The parser directly implements the following grammar.
     ImportList                 = { ImportDeclaration } .
     ImportDeclaration          = "import" [identifier] string_lit
     StatementList              = { Statement } .
-    Statement                  = OptionStatement
+    Statement                  = OptionAssignment
+                               | BuiltinStatement
                                | IdentStatement
                                | ReturnStatement
                                | ExpressionStatement .
     IdentStatement             = identifer ( AssignStatement | ExpressionSuffix ) .
-    OptionStatement            = "option" OptionStatementSuffix .
-    OptionStatementSuffix      = OptionAssignment | AssignStatement | ExpressionSuffix .
-    OptionAssignment           = identifier OptionAssignmentSuffix .
-    OptionAssignmentSuffix     = Assignment
-                               | "." identifier Assignment .
-    VariableAssignment         = identifer AssignStatement .
+    OptionAssignment           = "option" identifier OptionAssignmentSuffix .
+    OptionAssignmentSuffix     = AssignStatement
+                               | "." identifier AssignStatement .
+    BuiltinStatement           = "builtin" identifier .
     AssignStatement            = "=" Expression .
     ReturnStatement            = "return" Expression .
     ExpressionStatement        = Expression .