Update files based on Operator redesign

.github/workflows/e2e_tests.yaml

@@ -107,12 +107,6 @@ jobs:
           echo "Printing CodeFlare operator logs"
           kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
 
-      - name: Print MCAD controller logs
-        if: always() && steps.deploy.outcome == 'success'
-        run: |
-          echo "Printing MCAD controller logs"
-          kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log
-
       - name: Print KubeRay operator logs
         if: always() && steps.deploy.outcome == 'success'
         run: |

Dockerfile

@@ -9,8 +9,7 @@ RUN go mod download
 
 # Copy the go source
 COPY main.go main.go
-COPY api/ api/
-COPY controllers/ controllers/
+COPY pkg/ pkg/
 
 # Build
 USER root
@@ -19,7 +18,6 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
 WORKDIR /
 COPY --from=builder /workspace/manager .
-COPY config/internal config/internal
 
 USER 65532:65532
 ENTRYPOINT ["/manager"]

Makefile

@@ -12,25 +12,23 @@ VERSION ?= v0.0.0-dev
 BUNDLE_VERSION ?= $(VERSION:v%=%)
 
 # INSTASCALE_VERSION defines the default version of the InstaScale controller
-INSTASCALE_VERSION ?= v0.0.6
+INSTASCALE_VERSION ?= v0.0.9
+INSTASCALE_REPO ?= github.com/project-codeflare/instascale
 
 # MCAD_VERSION defines the default version of the MCAD controller
-MCAD_VERSION ?= v1.33.0
-# MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources
-MCAD_REF ?= release-${MCAD_VERSION}
+MCAD_VERSION ?= 725a614debe3d34d1547c1659ef5ad49f8f6c5df
 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher
 # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version)
-# The image is still published using the MCAD_REF format (i.e release-vX.Y.Z)
 MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION}
 
 # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing)
-KUBERAY_VERSION ?= v0.5.0
+KUBERAY_VERSION ?= v0.6.0
 
 # RAY_VERSION defines the default version of Ray (used for testing)
 RAY_VERSION ?= 2.5.0
 
 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK
-CODEFLARE_SDK_VERSION ?= 0.6.1
+CODEFLARE_SDK_VERSION ?= 0.8.0
 
 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against
 # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to
@@ -66,12 +64,6 @@ IMAGE_ORG_BASE ?= quay.io/project-codeflare
 # codeflare.dev/codeflare-operator-bundle:$VERSION and codeflare.dev/codeflare-operator-catalog:$VERSION.
 IMAGE_TAG_BASE ?= $(IMAGE_ORG_BASE)/codeflare-operator
 
-# MCAD_IMAGE defines the default container image for the MCAD controller
-MCAD_IMAGE ?= $(IMAGE_ORG_BASE)/mcad-controller:$(MCAD_REF)
-
-# INSTASCALE_IMAGE defines the default container image for the InstaScale controller
-INSTASCALE_IMAGE ?= $(IMAGE_ORG_BASE)/instascale-controller:$(INSTASCALE_VERSION)
-
 # RAY_IMAGE defines the default container image for Ray (used for testing)
 RAY_IMAGE ?= rayproject/ray:$(RAY_VERSION)
 
@@ -129,25 +121,10 @@ help: ## Display this help.
 
 ##@ Development
 
-DEFAULTS_FILE := controllers/defaults.go
 DEFAULTS_TEST_FILE := tests/support/defaults.go
 
 .PHONY: defaults
 defaults:
-	$(info Regenerating $(DEFAULTS_FILE))
-	@echo "package controllers" > $(DEFAULTS_FILE)
-	@echo "" >> $(DEFAULTS_FILE)
-	@echo "// ***********************" >> $(DEFAULTS_FILE)
-	@echo "//  DO NOT EDIT THIS FILE"  >> $(DEFAULTS_FILE)
-	@echo "// ***********************" >> $(DEFAULTS_FILE)
-	@echo "" >> $(DEFAULTS_FILE)
-	@echo "const (" >> $(DEFAULTS_FILE)
-	@echo "  MCADImage = \"$(MCAD_IMAGE)\"" >> $(DEFAULTS_FILE)
-	@echo "  InstaScaleImage = \"$(INSTASCALE_IMAGE)\"" >> $(DEFAULTS_FILE)
-	@echo "" >> $(DEFAULTS_FILE)
-	@echo ")" >> $(DEFAULTS_FILE)
-	@echo "" >> $(DEFAULTS_FILE)
-
 	$(info Regenerating $(DEFAULTS_TEST_FILE))
 	@echo "package support" > $(DEFAULTS_TEST_FILE)
 	@echo "" >> $(DEFAULTS_TEST_FILE)
@@ -163,51 +140,11 @@ defaults:
 	@echo ")" >> $(DEFAULTS_TEST_FILE)
 	@echo "" >> $(DEFAULTS_TEST_FILE)
 
-	gofmt -w $(DEFAULTS_FILE) $(DEFAULTS_TEST_FILE)
+	gofmt -w $(DEFAULTS_TEST_FILE)
 
 .PHONY: manifests
-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-
-.PHONY: generate
-generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
-	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
-
-.PHONY: generate-client ## Generate client packages and organize the goimports
-generate-client: generate-client-files imports
-
-.PHONY: generate-client-files
-generate-client-files: code-generator
-	rm -rf client
-	$(APPLYCONFIGURATION_GEN) \
-		--input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \
-		--go-header-file="hack/boilerplate.go.txt" \
-		--output-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \
-		--output-base="." \
-		--trim-path-prefix "github.com/project-codeflare/codeflare-operator"
-	$(CLIENT_GEN) \
-		--input="codeflare/v1alpha1" \
-		--input-base="github.com/project-codeflare/codeflare-operator/api" \
-		--apply-configuration-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \
-		--go-header-file="hack/boilerplate.go.txt" \
-		--clientset-name "versioned"  \
-		--output-package="github.com/project-codeflare/codeflare-operator/client/clientset" \
-		--output-base="." \
-		--trim-path-prefix "github.com/project-codeflare/codeflare-operator"
-	$(LISTER_GEN) \
-		--input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \
-		--go-header-file="hack/boilerplate.go.txt" \
-		--output-base="." \
-		--output-package="github.com/project-codeflare/codeflare-operator/client/listers" \
-		--trim-path-prefix "github.com/project-codeflare/codeflare-operator"
-	$(INFORMER_GEN) \
-		--input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \
-		--versioned-clientset-package="github.com/project-codeflare/codeflare-operator/client/clientset/versioned" \
-		--listers-package="github.com/project-codeflare/codeflare-operator/client/listers" \
-		--go-header-file="hack/boilerplate.go.txt" \
-		--output-base="." \
-		--output-package="github.com/project-codeflare/codeflare-operator/client/informer" \
-		--trim-path-prefix "github.com/project-codeflare/codeflare-operator"
+manifests: controller-gen ## Generate RBAC objects.
+	$(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..."
 
 .PHONY: fmt
 fmt: ## Run go fmt against code.
@@ -223,14 +160,16 @@ vet: ## Run go vet against code.
 .PHONY: modules
 modules: ## Update Go dependencies.
 	go get $(MCAD_REPO)@$(MCAD_VERSION)
+	go get $(INSTASCALE_REPO)@$(INSTASCALE_VERSION)
 	go get github.com/ray-project/kuberay/ray-operator
+	go mod tidy
 
 .PHONY: build
-build: modules defaults generate fmt vet ## Build manager binary.
+build: modules defaults fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
 .PHONY: run
-run: modules defaults manifests generate fmt vet ## Run a controller from your host.
+run: modules defaults manifests fmt vet ## Run a controller from your host.
 	go run ./main.go
 
 .PHONY: image-build
@@ -249,24 +188,28 @@ endif
 
 .PHONY: install
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD)
+	$(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml
 	$(KUSTOMIZE) build config/crd | kubectl apply -f -
 	git restore config/*
 
 .PHONY: uninstall
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	$(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml
 	$(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+	git restore config/*
 
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD)
+	$(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 	git restore config/*
 
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	$(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml
 	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+	git restore config/*
 
 ##@ Build Dependencies
 
@@ -277,15 +220,12 @@ $(LOCALBIN):
 
 ## Tool Binaries
 KUSTOMIZE ?= $(LOCALBIN)/kustomize
-APPLYCONFIGURATION_GEN ?= $(LOCALBIN)/applyconfiguration-gen
-CLIENT_GEN ?= $(LOCALBIN)/client-gen
-LISTER_GEN ?= $(LOCALBIN)/lister-gen
-INFORMER_GEN ?= $(LOCALBIN)/informer-gen
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports
 OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk
 GH_CLI ?= $(LOCALBIN)/gh
+SED ?= /usr/bin/sed
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v4.5.4
@@ -311,29 +251,6 @@ $(GH_CLI): $(LOCALBIN)
 	rm -rf $(GH_CLI_DL_FILENAME)
 	rm $(GH_CLI_DL_FILENAME).tar.gz
 
-.PHONY: code-generator
-code-generator: $(APPLYCONFIGURATION_GEN) $(CLIENT_GEN) $(LISTER_GEN) $(INFORMER_GEN)
-
-.PHONY: applyconfiguration-gen
-applyconfiguration-gen: $(APPLYCONFIGURATION_GEN)
-$(APPLYCONFIGURATION_GEN): $(LOCALBIN)
-	test -s $(LOCALBIN)/applyconfiguration-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/applyconfiguration-gen@$(CODEGEN_VERSION)
-
-.PHONY: client-gen
-client-gen: $(CLIENT_GEN)
-$(CLIENT_GEN): $(LOCALBIN)
-	test -s $(LOCALBIN)/client-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/client-gen@$(CODEGEN_VERSION)
-
-.PHONY: lister-gen
-lister-gen: $(LISTER_GEN)
-$(LISTER_GEN): $(LOCALBIN)
-	test -s $(LOCALBIN)/lister-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/lister-gen@$(CODEGEN_VERSION)
-
-.PHONY: informer-gen
-informer-gen: $(INFORMER_GEN)
-$(INFORMER_GEN): $(LOCALBIN)
-	test -s $(LOCALBIN)/informer-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/informer-gen@$(CODEGEN_VERSION)
-
 .PHONY: controller-gen
 controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
 $(CONTROLLER_GEN): $(LOCALBIN)
@@ -363,7 +280,7 @@ validate-bundle: install-operator-sdk
 .PHONY: bundle
 bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files.
 	$(OPERATOR_SDK) generate kustomize manifests -q
-	$(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD)
+	$(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml
 	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
 	cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion
 	cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion
@@ -430,7 +347,7 @@ catalog-build-from-index: opm ## Build a catalog image.
 	mkdir catalog
 	$(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml
 	$(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml
-	sed -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n  replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml
+	$(SED) -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n  replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml
 	$(OPM) validate catalog
 	$(OPM) generate dockerfile catalog
 	podman build . -f catalog.Dockerfile -t $(CATALOG_IMG)
@@ -441,7 +358,7 @@ catalog-push: ## Push a catalog image.
 	podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT)
 
 .PHONY: test-unit
-test-unit: defaults manifests generate fmt vet envtest ## Run unit tests.
+test-unit: defaults manifests fmt vet envtest ## Run unit tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(go list ./... | grep -v /test/) -coverprofile cover.out
 
 .PHONY: test-e2e
@@ -463,3 +380,7 @@ imports: openshift-goimports ## Organize imports in go files using openshift-goi
 .PHONY: verify-imports
 verify-imports: openshift-goimports ## Run import verifications.
 	./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS)
+
+.PHONY: scorecard-bundle
+scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image.
+	$(OPERATOR_SDK) scorecard bundle

tests/e2e/mnist_raycluster_sdk_test.go

@@ -169,14 +169,21 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 							Name: "test",
 							// FIXME: switch to base Python image once the dependency on OpenShift CLI is removed
 							// See https://github.com/project-codeflare/codeflare-sdk/pull/146
-							Image:   "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e",
+							Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e",
+							Env: []corev1.EnvVar{
+								corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"},
+							},
 							Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_raycluster_sdk.py" + " " + namespace.Name},
 							// Command: []string{"/bin/sh", "-c", "pip install /test/codeflare_sdk-0.0.0.dev0-py3-none-any.whl && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name},
 							VolumeMounts: []corev1.VolumeMount{
 								{
 									Name:      "test",
 									MountPath: "/test",
 								},
+								{
+									Name:      "workdir",
+									MountPath: "/workdir",
+								},
 							},
 						},
 					},
@@ -191,6 +198,12 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 								},
 							},
 						},
+						{
+							Name: "workdir",
+							VolumeSource: corev1.VolumeSource{
+								EmptyDir: &corev1.EmptyDirVolumeSource{},
+							},
+						},
 					},
 					RestartPolicy:      corev1.RestartPolicyNever,
 					ServiceAccountName: serviceAccount.Name,
@@ -202,9 +215,6 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 	test.Expect(err).NotTo(HaveOccurred())
 	test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)
 
-	// Retrieving the job logs once it has completed or timed out
-	defer WriteJobLogs(test, job.Namespace, job.Name)
-
 	test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
 	test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
 		Or(

tests/e2e/setup.sh

@@ -18,21 +18,9 @@ set -euo pipefail
 : "${KUBERAY_VERSION}"
 
 echo Deploying KubeRay "${KUBERAY_VERSION}"
-kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=90s"
+kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s"
 
-kubectl create ns codeflare-system --dry-run=client -o yaml | kubectl apply -f -
-
-echo Deploying MCAD controller
-cat <<EOF | kubectl apply -n codeflare-system -f -
-apiVersion: codeflare.codeflare.dev/v1alpha1
-kind: MCAD
-metadata:
-  name: mcad
-spec:
-  controllerResources: {}
-EOF
-
-cat <<EOF | kubectl apply -n codeflare-system -f -
+cat <<EOF | kubectl apply -f -
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -54,20 +42,17 @@ rules:
       - delete
 EOF
 
-cat <<EOF | kubectl apply -n codeflare-system -f -
+cat <<EOF | kubectl apply -f -
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: mcad-controller-rayclusters
 subjects:
   - kind: ServiceAccount
-    name: mcad-controller-mcad
-    namespace: codeflare-system
+    name: codeflare-operator-controller-manager
+    namespace: openshift-operators
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: mcad-controller-rayclusters
 EOF
-
-echo "Waiting for the MCAD deployment to become ready..."
-kubectl wait --timeout=120s --for=condition=Available=true deployment -n codeflare-system mcad-controller-mcad