From 73660f5e510c7a3df35ef2e5fe80bb646e6ed5eb Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 10 Oct 2023 09:27:49 +0100 Subject: [PATCH] Update files based on Operator redesign --- .github/workflows/e2e_tests.yaml | 6 -- Dockerfile | 4 +- Makefile | 131 +++++-------------------- tests/e2e/mnist_raycluster_sdk_test.go | 18 +++- tests/e2e/setup.sh | 25 +---- 5 files changed, 46 insertions(+), 138 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 1264b9ab9..e6ebb22c7 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -107,12 +107,6 @@ jobs: echo "Printing CodeFlare operator logs" kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log - - name: Print MCAD controller logs - if: always() && steps.deploy.outcome == 'success' - run: | - echo "Printing MCAD controller logs" - kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log - - name: Print KubeRay operator logs if: always() && steps.deploy.outcome == 'success' run: | diff --git a/Dockerfile b/Dockerfile index 302b31e1c..2ea7fead4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,7 @@ RUN go mod download # Copy the go source COPY main.go main.go -COPY api/ api/ -COPY controllers/ controllers/ +COPY pkg/ pkg/ # Build USER root @@ -19,7 +18,6 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7 WORKDIR / COPY --from=builder /workspace/manager . -COPY config/internal config/internal USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 955c2de08..c0d41155d 100644 --- a/Makefile +++ b/Makefile @@ -12,25 +12,23 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.6 +INSTASCALE_VERSION ?= v0.0.9 +INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.33.0 -# MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources -MCAD_REF ?= release-${MCAD_VERSION} +MCAD_VERSION ?= 725a614debe3d34d1547c1659ef5ad49f8f6c5df MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) -# The image is still published using the MCAD_REF format (i.e release-vX.Y.Z) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v0.5.0 +KUBERAY_VERSION ?= v0.6.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.6.1 +CODEFLARE_SDK_VERSION ?= 0.8.0 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to @@ -66,12 +64,6 @@ IMAGE_ORG_BASE ?= quay.io/project-codeflare # codeflare.dev/codeflare-operator-bundle:$VERSION and codeflare.dev/codeflare-operator-catalog:$VERSION. IMAGE_TAG_BASE ?= $(IMAGE_ORG_BASE)/codeflare-operator -# MCAD_IMAGE defines the default container image for the MCAD controller -MCAD_IMAGE ?= $(IMAGE_ORG_BASE)/mcad-controller:$(MCAD_REF) - -# INSTASCALE_IMAGE defines the default container image for the InstaScale controller -INSTASCALE_IMAGE ?= $(IMAGE_ORG_BASE)/instascale-controller:$(INSTASCALE_VERSION) - # RAY_IMAGE defines the default container image for Ray (used for testing) RAY_IMAGE ?= rayproject/ray:$(RAY_VERSION) @@ -129,25 +121,10 @@ help: ## Display this help. ##@ Development -DEFAULTS_FILE := controllers/defaults.go DEFAULTS_TEST_FILE := tests/support/defaults.go .PHONY: defaults defaults: - $(info Regenerating $(DEFAULTS_FILE)) - @echo "package controllers" > $(DEFAULTS_FILE) - @echo "" >> $(DEFAULTS_FILE) - @echo "// ***********************" >> $(DEFAULTS_FILE) - @echo "// DO NOT EDIT THIS FILE" >> $(DEFAULTS_FILE) - @echo "// ***********************" >> $(DEFAULTS_FILE) - @echo "" >> $(DEFAULTS_FILE) - @echo "const (" >> $(DEFAULTS_FILE) - @echo " MCADImage = \"$(MCAD_IMAGE)\"" >> $(DEFAULTS_FILE) - @echo " InstaScaleImage = \"$(INSTASCALE_IMAGE)\"" >> $(DEFAULTS_FILE) - @echo "" >> $(DEFAULTS_FILE) - @echo ")" >> $(DEFAULTS_FILE) - @echo "" >> $(DEFAULTS_FILE) - $(info Regenerating $(DEFAULTS_TEST_FILE)) @echo "package support" > $(DEFAULTS_TEST_FILE) @echo "" >> $(DEFAULTS_TEST_FILE) @@ -163,51 +140,11 @@ defaults: @echo ")" >> $(DEFAULTS_TEST_FILE) @echo "" >> $(DEFAULTS_TEST_FILE) - gofmt -w $(DEFAULTS_FILE) $(DEFAULTS_TEST_FILE) + gofmt -w $(DEFAULTS_TEST_FILE) .PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: generate-client ## Generate client packages and organize the goimports -generate-client: generate-client-files imports - -.PHONY: generate-client-files -generate-client-files: code-generator - rm -rf client - $(APPLYCONFIGURATION_GEN) \ - --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ - --go-header-file="hack/boilerplate.go.txt" \ - --output-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \ - --output-base="." \ - --trim-path-prefix "github.com/project-codeflare/codeflare-operator" - $(CLIENT_GEN) \ - --input="codeflare/v1alpha1" \ - --input-base="github.com/project-codeflare/codeflare-operator/api" \ - --apply-configuration-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \ - --go-header-file="hack/boilerplate.go.txt" \ - --clientset-name "versioned" \ - --output-package="github.com/project-codeflare/codeflare-operator/client/clientset" \ - --output-base="." \ - --trim-path-prefix "github.com/project-codeflare/codeflare-operator" - $(LISTER_GEN) \ - --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ - --go-header-file="hack/boilerplate.go.txt" \ - --output-base="." \ - --output-package="github.com/project-codeflare/codeflare-operator/client/listers" \ - --trim-path-prefix "github.com/project-codeflare/codeflare-operator" - $(INFORMER_GEN) \ - --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ - --versioned-clientset-package="github.com/project-codeflare/codeflare-operator/client/clientset/versioned" \ - --listers-package="github.com/project-codeflare/codeflare-operator/client/listers" \ - --go-header-file="hack/boilerplate.go.txt" \ - --output-base="." \ - --output-package="github.com/project-codeflare/codeflare-operator/client/informer" \ - --trim-path-prefix "github.com/project-codeflare/codeflare-operator" +manifests: controller-gen ## Generate RBAC objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." .PHONY: fmt fmt: ## Run go fmt against code. @@ -223,14 +160,16 @@ vet: ## Run go vet against code. .PHONY: modules modules: ## Update Go dependencies. go get $(MCAD_REPO)@$(MCAD_VERSION) + go get $(INSTASCALE_REPO)@$(INSTASCALE_VERSION) go get github.com/ray-project/kuberay/ray-operator + go mod tidy .PHONY: build -build: modules defaults generate fmt vet ## Build manager binary. +build: modules defaults fmt vet ## Build manager binary. go build -o bin/manager main.go .PHONY: run -run: modules defaults manifests generate fmt vet ## Run a controller from your host. +run: modules defaults manifests fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build @@ -249,24 +188,28 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl apply -f - git restore config/* .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + git restore config/* .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - git restore config/* .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + git restore config/* ##@ Build Dependencies @@ -277,15 +220,12 @@ $(LOCALBIN): ## Tool Binaries KUSTOMIZE ?= $(LOCALBIN)/kustomize -APPLYCONFIGURATION_GEN ?= $(LOCALBIN)/applyconfiguration-gen -CLIENT_GEN ?= $(LOCALBIN)/client-gen -LISTER_GEN ?= $(LOCALBIN)/lister-gen -INFORMER_GEN ?= $(LOCALBIN)/informer-gen CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk GH_CLI ?= $(LOCALBIN)/gh +SED ?= /usr/bin/sed ## Tool Versions KUSTOMIZE_VERSION ?= v4.5.4 @@ -311,29 +251,6 @@ $(GH_CLI): $(LOCALBIN) rm -rf $(GH_CLI_DL_FILENAME) rm $(GH_CLI_DL_FILENAME).tar.gz -.PHONY: code-generator -code-generator: $(APPLYCONFIGURATION_GEN) $(CLIENT_GEN) $(LISTER_GEN) $(INFORMER_GEN) - -.PHONY: applyconfiguration-gen -applyconfiguration-gen: $(APPLYCONFIGURATION_GEN) -$(APPLYCONFIGURATION_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/applyconfiguration-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/applyconfiguration-gen@$(CODEGEN_VERSION) - -.PHONY: client-gen -client-gen: $(CLIENT_GEN) -$(CLIENT_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/client-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/client-gen@$(CODEGEN_VERSION) - -.PHONY: lister-gen -lister-gen: $(LISTER_GEN) -$(LISTER_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/lister-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/lister-gen@$(CODEGEN_VERSION) - -.PHONY: informer-gen -informer-gen: $(INFORMER_GEN) -$(INFORMER_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/informer-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/informer-gen@$(CODEGEN_VERSION) - .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. $(CONTROLLER_GEN): $(LOCALBIN) @@ -363,7 +280,7 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion @@ -430,7 +347,7 @@ catalog-build-from-index: opm ## Build a catalog image. mkdir catalog $(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml $(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml - sed -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml + $(SED) -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml $(OPM) validate catalog $(OPM) generate dockerfile catalog podman build . -f catalog.Dockerfile -t $(CATALOG_IMG) @@ -441,7 +358,7 @@ catalog-push: ## Push a catalog image. podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT) .PHONY: test-unit -test-unit: defaults manifests generate fmt vet envtest ## Run unit tests. +test-unit: defaults manifests fmt vet envtest ## Run unit tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(go list ./... | grep -v /test/) -coverprofile cover.out .PHONY: test-e2e @@ -463,3 +380,7 @@ imports: openshift-goimports ## Organize imports in go files using openshift-goi .PHONY: verify-imports verify-imports: openshift-goimports ## Run import verifications. ./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS) + +.PHONY: scorecard-bundle +scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. + $(OPERATOR_SDK) scorecard bundle \ No newline at end of file diff --git a/tests/e2e/mnist_raycluster_sdk_test.go b/tests/e2e/mnist_raycluster_sdk_test.go index 976f54130..86cfc5342 100644 --- a/tests/e2e/mnist_raycluster_sdk_test.go +++ b/tests/e2e/mnist_raycluster_sdk_test.go @@ -169,7 +169,10 @@ func TestMNISTRayClusterSDK(t *testing.T) { Name: "test", // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Env: []corev1.EnvVar{ + corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, + }, Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_raycluster_sdk.py" + " " + namespace.Name}, // Command: []string{"/bin/sh", "-c", "pip install /test/codeflare_sdk-0.0.0.dev0-py3-none-any.whl && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ @@ -177,6 +180,10 @@ func TestMNISTRayClusterSDK(t *testing.T) { Name: "test", MountPath: "/test", }, + { + Name: "workdir", + MountPath: "/workdir", + }, }, }, }, @@ -191,6 +198,12 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, }, }, + { + Name: "workdir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, }, RestartPolicy: corev1.RestartPolicyNever, ServiceAccountName: serviceAccount.Name, @@ -202,9 +215,6 @@ func TestMNISTRayClusterSDK(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - // Retrieving the job logs once it has completed or timed out - defer WriteJobLogs(test, job.Namespace, job.Name) - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( diff --git a/tests/e2e/setup.sh b/tests/e2e/setup.sh index dbc973dd2..d5c99cbf8 100755 --- a/tests/e2e/setup.sh +++ b/tests/e2e/setup.sh @@ -18,21 +18,9 @@ set -euo pipefail : "${KUBERAY_VERSION}" echo Deploying KubeRay "${KUBERAY_VERSION}" -kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=90s" +kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s" -kubectl create ns codeflare-system --dry-run=client -o yaml | kubectl apply -f - - -echo Deploying MCAD controller -cat <