Add build-tool cli with a crd validation command

Makefile

@@ -10,21 +10,25 @@ update_boilerplate:
 linux_compile:
 	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts/flytepropeller ./cmd/controller/main.go
 	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts/kubectl-flyte ./cmd/kubectl-flyte/main.go
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts/build-tool ./cmd/build-tool/main.go
 
 .PHONY: compile
 compile:
 	mkdir -p ./bin
 	go build -o bin/flytepropeller ./cmd/controller/main.go
 	go build -o bin/kubectl-flyte ./cmd/kubectl-flyte/main.go && cp bin/kubectl-flyte ${GOPATH}/bin
+	go build -o bin/build-tool ./cmd/build-tool/main.go && cp bin/build-tool ${GOPATH}/bin
 
 cross_compile:
 	@glide install
 	@mkdir -p ./bin/cross
 	GOOS=linux GOARCH=amd64 go build -o bin/cross/flytepropeller ./cmd/controller/main.go
 	GOOS=linux GOARCH=amd64 go build -o bin/cross/kubectl-flyte ./cmd/kubectl-flyte/main.go
+	GOOS=linux GOARCH=amd64 go build -o bin/cross/build-tool ./cmd/build-tool/main.go
 
 op_code_generate:
 	@RESOURCE_NAME=flyteworkflow OPERATOR_PKG=github.com/lyft/flytepropeller ./hack/update-codegen.sh
+	@openapi-gen -i github.com/lyft/flytepropeller/pkg/apis/flyteworkflow/v1alpha1 -p github.com/lyft/flytepropeller/pkg/apis/flyteworkflow/v1alpha1
 
 benchmark:
 	mkdir -p ./bin/benchmark
@@ -41,6 +45,7 @@ clean:
 # Generate golden files. Add test packages that generate golden files here.
 golden:
 	go test ./cmd/kubectl-flyte/cmd -update
+	go test ./cmd/build-tool/cmd -update
 	go test ./pkg/compiler/test -update
 
 .PHONY: test_unit_codecov

cmd/build-tool/cmd/crd/flyteworkflow.go

@@ -0,0 +1,43 @@
+package crd
+
+import (
+	"github.com/lyft/flytepropeller/pkg/apis/flyteworkflow/v1alpha1"
+	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"log"
+
+	"github.com/kubeflow/crd-validation/pkg/crd/exporter"
+	"github.com/kubeflow/crd-validation/pkg/utils"
+)
+
+const (
+	// CRDName is the name for FlyteWorkflow.
+	CRDNameFlyteWorkflow = "github.com/lyft/flytepropeller/pkg/apis/flyteworkflow/v1alpha1.FlyteWorkflow"
+
+	generatedFileFlyteWorkflow = "flyteworkflow-crd-v1alpha1.yaml"
+)
+
+// FlyteWorkflowGenerator is the type for FlyteWorkflow CRD generator.
+type FlyteWorkflowGenerator struct {
+	*exporter.Exporter
+}
+
+// Creates a new CRD generator which outputs to a file.
+func NewFlyteWorkflowGenerator(outputDir string) *FlyteWorkflowGenerator {
+	return &FlyteWorkflowGenerator{
+		Exporter: exporter.NewFileExporter(outputDir, generatedFileFlyteWorkflow),
+	}
+}
+
+// Creates a new CRD generator which outputs to stdout.
+func NewFlyteWorkflowGeneratorStdout() *FlyteWorkflowGenerator {
+	return &FlyteWorkflowGenerator{
+		Exporter: exporter.NewStdoutExporter(),
+	}
+}
+
+// Generate generates the crd.
+func (t FlyteWorkflowGenerator) Generate(original *apiextensions.CustomResourceDefinition) *apiextensions.CustomResourceDefinition {
+	log.Println("Generating validation")
+	original.Spec.Validation = utils.GetCustomResourceValidation(CRDNameFlyteWorkflow, v1alpha1.GetOpenAPIDefinitions)
+	return original
+}

cmd/build-tool/cmd/crd_validation.go

@@ -0,0 +1,95 @@
+package cmd
+
+import (
+	"github.com/spf13/viper"
+	"log"
+
+	compilerErrors "github.com/lyft/flytepropeller/pkg/compiler/errors"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/kubeflow/crd-validation/pkg/config"
+	"github.com/lyft/flytepropeller/cmd/build-tool/cmd/crd"
+)
+
+const (
+	configKey      = "config-file"
+	baseCrdKey      = "base-crd"
+)
+
+
+const crdValidationCmdName = "crd-validation"
+
+type CrdValidationOpts struct {
+	*RootOptions
+	configFile  string
+	baseCrdFile string
+	dryRun      bool
+}
+
+func NewCrdValidationCommand(opts *RootOptions) *cobra.Command {
+
+	crdValidationOpts := &CrdValidationOpts{
+		RootOptions: opts,
+	}
+
+	crdValidationCmd := &cobra.Command{
+		Use:     crdValidationCmdName,
+		Aliases: []string{"validate"},
+		Short:   "Augment a CRD YAML file with validation section based on a base CRD file",
+		Long:    ``,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := requiredFlags(cmd, baseCrdKey); err != nil {
+				return err
+			}
+
+			compilerErrors.SetIncludeSource()
+
+			return crdValidationOpts.generateValidation()
+		},
+	}
+
+	crdValidationCmd.Flags().StringVarP(&crdValidationOpts.configFile, configKey, "c", "", "Path of the config file for the execution of CRD validation")
+	crdValidationCmd.Flags().StringVarP(&crdValidationOpts.baseCrdFile, baseCrdKey, "b", "", "Path to base CRD file.")
+	crdValidationCmd.Flags().BoolVarP(&crdValidationOpts.dryRun, "dry-run", "d", false, "Compiles and transforms, but does not create a workflow. OutputsRef ts to STDOUT.")
+
+	return crdValidationCmd
+}
+
+func (c *CrdValidationOpts) initConfig() error {
+	if c.configFile != "" { // enable ability to specify config file via flag
+		viper.SetConfigFile(c.configFile)
+		log.Println("Using config file:", viper.ConfigFileUsed())
+	}
+
+	viper.SetConfigType("yaml")  // Set config type to yaml
+
+	// If a config file is found, read it in.
+	if err := viper.ReadInConfig(); err != nil {
+		return errors.Wrapf(err, "Failed to read config file.")
+	} else {
+		log.Println("Using config file:", viper.ConfigFileUsed())
+	}
+	return nil
+}
+
+
+func (c *CrdValidationOpts) generateValidation() error {
+
+	err := c.initConfig()
+	var generator *crd.FlyteWorkflowGenerator
+	if err != nil {
+		log.Println("Output will be written to Stdout")
+		generator = crd.NewFlyteWorkflowGeneratorStdout()
+	} else {
+		crdValidationConfig := config.GetCrdValidationConfig()
+		generator = crd.NewFlyteWorkflowGenerator(crdValidationConfig.OutputDir)
+	}
+
+	original := config.NewCustomResourceDefinition(c.baseCrdFile)
+	final := generator.Generate(original)
+	generator.Export(final)
+
+	return nil
+}

cmd/build-tool/cmd/root.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/lyft/flytestdlib/logger"
+	"github.com/lyft/flytestdlib/version"
+	"github.com/spf13/pflag"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	err := flag.CommandLine.Parse([]string{})
+	if err != nil {
+		logger.Error(context.TODO(), "Error in initializing: %v", err)
+		os.Exit(-1)
+	}
+}
+
+type RootOptions struct {
+	configFile string
+}
+
+func (r *RootOptions) executeRootCmd() error {
+	ctx := context.TODO()
+	logger.Infof(ctx, "Go Version: %s", runtime.Version())
+	logger.Infof(ctx, "Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)
+	version.LogBuildInformation("build-tool")
+	return fmt.Errorf("use one of the sub-commands")
+}
+
+// NewCommand returns a new instance of an argo command
+func NewBuildToolCommand() *cobra.Command {
+	rootOpts := &RootOptions{}
+	command := &cobra.Command{
+		Use:   "build-tool",
+		Short: "build-tool are utility commands that help validating crds, etc.",
+		Long: `Flyte is a serverless workflow processing platform built for native execution on K8s.
+      It is extensible and flexible to allow adding new operators and comes with many operators built in`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return rootOpts.executeRootCmd()
+		},
+	}
+
+	command.AddCommand(NewCrdValidationCommand(rootOpts))
+	return command
+}
+
+

cmd/build-tool/cmd/utils.go

@@ -0,0 +1,18 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+func requiredFlags(cmd *cobra.Command, flags ...string) error {
+	for _, flag := range flags {
+		f := cmd.Flag(flag)
+		if f == nil {
+			return fmt.Errorf("unable to find Key [%v]", flag)
+		}
+	}
+
+	return nil
+}

cmd/build-tool/main.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/lyft/flytepropeller/cmd/build-tool/cmd"
+)
+
+func main() {
+	rootCmd := cmd.NewBuildToolCommand()
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

pkg/apis/flyteworkflow/v1alpha1/branch.go

@@ -71,6 +71,7 @@ func (in *IfBlock) GetThenNode() *NodeID {
 
 type BranchNodeSpec struct {
 	If       IfBlock    `json:"if"`
+	// +listType=atomic
 	ElseIf   []*IfBlock `json:"elseIf,omitempty"`
 	Else     *NodeID    `json:"else,omitempty"`
 	ElseFail *Error     `json:"elseFail,omitempty"`

pkg/apis/flyteworkflow/v1alpha1/doc.go

@@ -1,4 +1,5 @@
 // +k8s:deepcopy-gen=package
+// +k8s:openapi-gen=true
 
 // Package v1alpha1 is the v1alpha1 version of the API.
 // +groupName=flyteworkflow.flyte.net

pkg/apis/flyteworkflow/v1alpha1/nodes.go

@@ -99,9 +99,11 @@ type NodeSpec struct {
 	BranchNode    *BranchNodeSpec               `json:"branch,omitempty"`
 	TaskRef       *TaskID                       `json:"task,omitempty"`
 	WorkflowNode  *WorkflowNodeSpec             `json:"workflow,omitempty"`
+	// +listType=atomic
 	InputBindings []*Binding                    `json:"inputBindings,omitempty"`
 	Config        *typesv1.ConfigMap            `json:"config,omitempty"`
 	RetryStrategy *RetryStrategy                `json:"retry,omitempty"`
+	// +listType=atomic
 	OutputAliases []Alias                       `json:"outputAlias,omitempty"`
 
 	// SecurityContext holds pod-level security attributes and common container settings.
@@ -115,6 +117,7 @@ type NodeSpec struct {
 	// +optional
 	// +patchMergeKey=name
 	// +patchStrategy=merge
+	// +listType=atomic
 	ImagePullSecrets []typesv1.LocalObjectReference `json:"imagePullSecrets,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,15,rep,name=imagePullSecrets"`
 	// Specifies the hostname of the Pod
 	// If not specified, the pod's hostname will be set to a system-defined value.
@@ -133,6 +136,7 @@ type NodeSpec struct {
 	SchedulerName string `json:"schedulerName,omitempty" protobuf:"bytes,19,opt,name=schedulerName"`
 	// If specified, the pod's tolerations.
 	// +optional
+	// +listType=atomic
 	Tolerations []typesv1.Toleration `json:"tolerations,omitempty" protobuf:"bytes,22,opt,name=tolerations"`
 	// Node execution timeout
 	ExecutionDeadline *v1.Duration `json:"executionDeadline,omitempty"`