From 1d798c512744399c702414e41fb19992115e3d59 Mon Sep 17 00:00:00 2001 From: yetone Date: Wed, 4 Jan 2023 14:13:36 +0000 Subject: [PATCH] feat: add e2e tests --- .github/workflows/e2e.yaml | 216 +++++++++++++++++++ Makefile | 4 + go.mod | 1 + go.sum | 1 + scripts/quick-install-yatai-image-builder.sh | 86 +++++--- tests/e2e/e2e_suite_test.go | 16 ++ tests/e2e/e2e_test.go | 118 ++++++++++ tests/e2e/example.yaml | 8 + tests/e2e/installation_test.sh | 14 ++ tests/gh-actions/install_helm.sh | 5 + tests/gh-actions/install_kind.sh | 10 + tests/gh-actions/kind-cluster-1-24.yaml | 25 +++ tests/utils/utils.go | 122 +++++++++++ 13 files changed, 600 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/e2e.yaml create mode 100644 tests/e2e/e2e_suite_test.go create mode 100644 tests/e2e/e2e_test.go create mode 100644 tests/e2e/example.yaml create mode 100755 tests/e2e/installation_test.sh create mode 100755 tests/gh-actions/install_helm.sh create mode 100755 tests/gh-actions/install_kind.sh create mode 100644 tests/gh-actions/kind-cluster-1-24.yaml create mode 100644 tests/utils/utils.go diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 0000000..91b59f4 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,216 @@ +name: e2e +on: + issue_comment: + types: [created] + +env: + E2E_CHECK_NAME: e2e tests + +jobs: + triage: + runs-on: ubuntu-latest + name: Comment evaluate + outputs: + run-e2e: ${{ startsWith(github.event.comment.body,'/run-e2e') && steps.checkUserMember.outputs.isTeamMember == 'true' }} + pr_num: ${{ steps.parser.outputs.pr_num }} + image_tag: "pr-${{ steps.parser.outputs.pr_num }}-${{ steps.parser.outputs.commit_sha }}" + commit_sha: ${{ steps.parser.outputs.commit_sha }} + version_buildflags: ${{ steps.parser.outputs.version_buildflags }} + image_build_hash: ${{ steps.parser.outputs.image_build_hash }} + + steps: + - uses: actions/checkout@v3 + + - uses: tspascoal/get-user-teams-membership@v2 + id: checkUserMember + with: + username: ${{ github.actor }} + team: 'dev' + GITHUB_TOKEN: ${{ secrets.GH_CHECKING_USER_AUTH }} + + - name: Update comment with the execution url + if: ${{ startsWith(github.event.comment.body,'/run-e2e') && steps.checkUserMember.outputs.isTeamMember == 'true' }} + uses: peter-evans/create-or-update-comment@v2 + with: + comment-id: ${{ github.event.comment.id }} + body: | + **Update:** You can check the progress [here](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}) + reactions: rocket + + - name: Parse git info + if: ${{ startsWith(github.event.comment.body,'/run-e2e') && steps.checkUserMember.outputs.isTeamMember == 'true' }} + id: parser + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Get PR number + PR_URL="${{ github.event.issue.pull_request.url }}" + PR_NUM=${PR_URL##*/} + echo "Checking out from PR #$PR_NUM based on URL: $PR_URL" + echo "::set-output name=pr_num::$PR_NUM" + # Get commit SHA + git config --global --add safe.directory "$GITHUB_WORKSPACE" + gh pr checkout $PR_NUM + SHA=$(git log -n 1 --pretty=format:"%H") + echo "::set-output name=commit_sha::$SHA" + GIT_COMMIT=$(git describe --match=NeVeRmAtCh --tags --always --dirty | cut -c 1-7) + BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + VERSION=$(git describe --tags `git rev-list --tags --max-count=1` | sed 's/v\(\)/\1/') + PKG=github.com/bentoml/yatai-image-builder + VERSION_BUILDFLAGS="-X '${PKG}/version.GitCommit=${GIT_COMMIT}' -X '${PKG}/version.Version=${VERSION}' -X '${PKG}/version.BuildDate=${BUILD_DATE}'" + echo "::set-output name=version_buildflags::$VERSION_BUILDFLAGS" + echo "::set-output name=image_build_hash::${{ hashFiles('Dockerfile', 'main.go', './apis/**', './controllers/**', './utils/**', './version/**', './yatai-client/**', '**/go.sum', '**go.mod') }}" + + build-test-images: + needs: triage + if: needs.triage.outputs.run-e2e == 'true' + runs-on: ubuntu-latest + steps: + - name: Set status in-progress + uses: LouisBrunner/checks-action@v1.5.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + sha: ${{ needs.triage.outputs.commit_sha }} + name: ${{ env.E2E_CHECK_NAME }} + status: in_progress + details_url: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + + - uses: actions/checkout@v3 + + - name: Register workspace path + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Checkout Pull Request + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + id: checkout + run: | + gh pr checkout ${{ needs.triage.outputs.pr_num }} + + - name: Set up Docker Buildx + id: buildx + # Use the action from the master, as we've seen some inconsistencies with @v1 + # Issue: https://github.com/docker/build-push-action/issues/286 + uses: docker/setup-buildx-action@master + with: + install: true + + - name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + # Key is named differently to avoid collision + key: ${{ runner.os }}-multi-buildx-${{ needs.triage.outputs.image_build_hash }} + restore-keys: | + ${{ runner.os }}-multi-buildx + + - name: Build test image + uses: docker/build-push-action@v2 + with: + build-args: 'VERSION_BUILDFLAGS=${{ needs.triage.outputs.version_buildflags }}' + context: . + push: true + tags: quay.io/bentoml/test-yatai-image-builder:${{ needs.triage.outputs.image_tag }} + cache-from: type=local,src=/tmp/.buildx-cache + # Note the mode=max here + # More: https://github.com/moby/buildkit#--export-cache-options + # And: https://github.com/docker/buildx#--cache-tonametypetypekeyvalue + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + run-test: + needs: [triage, build-test-images] + if: needs.triage.outputs.run-e2e == 'true' + runs-on: ubuntu-latest + steps: + - name: Set status in-progress + uses: LouisBrunner/checks-action@v1.5.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + sha: ${{ needs.triage.outputs.commit_sha }} + name: ${{ env.E2E_CHECK_NAME }} + status: in_progress + details_url: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + + - name: Checkout + uses: actions/checkout@v3 + + - name: Register workspace path + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Checkout Pull Request + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + id: checkout + run: | + gh pr checkout ${{ needs.triage.outputs.pr_num }} + + - name: Install KinD + run: ./tests/gh-actions/install_kind.sh + + - name: Install Helm + run: ./tests/gh-actions/install_helm.sh + + - name: Create KinD Cluster + run: kind create cluster --config tests/gh-actions/kind-cluster-1-24.yaml + + - uses: oNaiPs/secrets-to-env-action@v1 + with: + secrets: ${{ toJSON(secrets) }} + + - name: Run e2e test + continue-on-error: true + id: test + env: + YATAI_DEPLOYMENT_IMG_REPO: test-yatai-image-builder + YATAI_DEPLOYMENT_IMG_TAG: ${{ needs.triage.outputs.image_tag }} + run: | + ./tests/e2e/installation_test.sh + make test-e2e + + - name: Set status success + uses: LouisBrunner/checks-action@v1.5.0 + if: steps.test.outcome == 'success' + with: + token: ${{ secrets.GITHUB_TOKEN }} + sha: ${{ needs.triage.outputs.commit_sha }} + name: ${{ env.E2E_CHECK_NAME }} + conclusion: success + details_url: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + + - name: React to comment with success + uses: dkershner6/reaction-action@v1 + if: steps.test.outcome == 'success' + with: + token: ${{ secrets.GITHUB_TOKEN }} + commentId: ${{ github.event.comment.id }} + reaction: "hooray" + + - name: React to comment with failure + uses: dkershner6/reaction-action@v1 + if: steps.test.outcome != 'success' + with: + token: ${{ secrets.GITHUB_TOKEN }} + commentId: ${{ github.event.comment.id }} + reaction: "confused" + + - name: Set status failure + uses: LouisBrunner/checks-action@v1.5.0 + if: steps.test.outcome != 'success' + with: + token: ${{ secrets.GITHUB_TOKEN }} + sha: ${{ needs.triage.outputs.commit_sha }} + name: ${{ env.E2E_CHECK_NAME }} + conclusion: failure + details_url: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} diff --git a/Makefile b/Makefile index 0da5c63..dc15ed7 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,10 @@ vet: ## Run go vet against code. test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out +.PHONY: test-e2e # You will need to have a Kind cluster up in running to run this target +test-e2e: + go test ./tests/e2e/ -v -ginkgo.v -timeout 20m + ##@ Build .PHONY: build diff --git a/go.mod b/go.mod index d047930..46b5cc5 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/huandu/xstrings v1.3.2 github.com/iancoleman/strcase v0.2.0 github.com/onsi/ginkgo v1.16.5 + github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.19.0 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 diff --git a/go.sum b/go.sum index c66fd75..8ebae97 100644 --- a/go.sum +++ b/go.sum @@ -317,6 +317,7 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= diff --git a/scripts/quick-install-yatai-image-builder.sh b/scripts/quick-install-yatai-image-builder.sh index 73bbbb4..28f2b6c 100755 --- a/scripts/quick-install-yatai-image-builder.sh +++ b/scripts/quick-install-yatai-image-builder.sh @@ -194,13 +194,9 @@ DOCKER_REGISTRY_PASSWORD='' DOCKER_REGISTRY_SECURE=false DOCKER_REGISTRY_BENTO_REPOSITORY_NAME=yatai-bentos -helm_repo_name=bentoml -helm_repo_url=https://bentoml.github.io/helm-charts - -# check if DEVEL_HELM_REPO is true -if [ "${DEVEL_HELM_REPO}" = "true" ]; then - helm_repo_name=bentoml-devel - helm_repo_url=https://bentoml.github.io/helm-charts-devel +YATAI_ENDPOINT=${YATAI_ENDPOINT:-http://yatai.yatai-system.svc.cluster.local} +if [ "${YATAI_ENDPOINT}" = "empty" ]; then + YATAI_ENDPOINT="" fi UPGRADE_CRDS=${UPGRADE_CRDS:-false} @@ -216,27 +212,65 @@ if [ "${UPGRADE_CRDS}" = "true" ]; then echo "✅ Bento CRD are established" fi -helm repo remove ${helm_repo_name} 2> /dev/null || true -helm repo add ${helm_repo_name} ${helm_repo_url} -helm repo update ${helm_repo_name} +USE_LOCAL_HELM_CHART=${USE_LOCAL_HELM_CHART:-false} -# if $VERSION is not set, use the latest version -if [ -z "$VERSION" ]; then - VERSION=$(helm search repo ${helm_repo_name} --devel="$DEVEL" -l | grep "${helm_repo_name}/yatai-image-builder " | awk '{print $2}' | head -n 1) -fi +if [ "${USE_LOCAL_HELM_CHART}" = "true" ]; then + YATAI_IMAGE_BUILDER_IMG_REGISTRY=${YATAI_IMAGE_BUILDER_IMG_REGISTRY:-quay.io/bentoml} + YATAI_IMAGE_BUILDER_IMG_REPO=${YATAI_IMAGE_BUILDER_IMG_REPO:-yatai-image-builder} + YATAI_IMAGE_BUILDER_IMG_TAG=${YATAI_IMAGE_BUILDER_IMG_TAG:-0.0.1} + + echo "🤖 installing yatai-image-builder from local helm chart..." + helm upgrade --install yatai-image-builder ./helm/yatai-image-builder -n ${namespace} \ + --set registry=${YATAI_IMAGE_BUILDER_IMG_REGISTRY} \ + --set image.repository=${YATAI_IMAGE_BUILDER_IMG_REPO} \ + --set image.tag=${YATAI_IMAGE_BUILDER_IMG_TAG} \ + --set yatai.endpoint=${YATAI_ENDPOINT} \ + --set dockerRegistry.server=${DOCKER_REGISTRY_SERVER} \ + --set dockerRegistry.inClusterServer=${DOCKER_REGISTRY_IN_CLUSTER_SERVER} \ + --set dockerRegistry.username=${DOCKER_REGISTRY_USERNAME} \ + --set dockerRegistry.password=${DOCKER_REGISTRY_PASSWORD} \ + --set dockerRegistry.secure=${DOCKER_REGISTRY_SECURE} \ + --set dockerRegistry.bentoRepositoryName=${DOCKER_REGISTRY_BENTO_REPOSITORY_NAME} \ + --set aws.accessKeyID=${AWS_ACCESS_KEY_ID} \ + --set aws.secretAccessKeyExistingSecretName=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_NAME} \ + --set aws.secretAccessKeyExistingSecretKey=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY} \ + --skip-crds=${UPGRADE_CRDS} +else + helm_repo_name=bentoml + helm_repo_url=https://bentoml.github.io/helm-charts -echo "🤖 installing yatai-image-builder ${VERSION} from helm repo ${helm_repo_name}..." -helm upgrade --install yatai-image-builder ${helm_repo_name}/yatai-image-builder -n ${namespace} \ - --set dockerRegistry.server=${DOCKER_REGISTRY_SERVER} \ - --set dockerRegistry.inClusterServer=${DOCKER_REGISTRY_IN_CLUSTER_SERVER} \ - --set dockerRegistry.username=${DOCKER_REGISTRY_USERNAME} \ - --set dockerRegistry.password=${DOCKER_REGISTRY_PASSWORD} \ - --set dockerRegistry.secure=${DOCKER_REGISTRY_SECURE} \ - --set dockerRegistry.bentoRepositoryName=${DOCKER_REGISTRY_BENTO_REPOSITORY_NAME} \ - --set yatai.endpoint=${YATAI_ENDPOINT} \ - --skip-crds=${UPGRADE_CRDS} \ - --version=${VERSION} \ - --devel=${DEVEL} + # check if DEVEL_HELM_REPO is true + if [ "${DEVEL_HELM_REPO}" = "true" ]; then + helm_repo_name=bentoml-devel + helm_repo_url=https://bentoml.github.io/helm-charts-devel + fi + + helm repo remove ${helm_repo_name} 2> /dev/null || true + helm repo add ${helm_repo_name} ${helm_repo_url} + helm repo update ${helm_repo_name} + + # if $VERSION is not set, use the latest version + if [ -z "$VERSION" ]; then + VERSION=$(helm search repo ${helm_repo_name} --devel="$DEVEL" -l | grep "${helm_repo_name}/yatai-image-builder " | awk '{print $2}' | head -n 1) + fi + + echo "🤖 installing yatai-image-builder ${VERSION} from helm repo ${helm_repo_name}..." + helm upgrade --install yatai-image-builder ${helm_repo_name}/yatai-image-builder -n ${namespace} \ + --set yatai.endpoint=${YATAI_ENDPOINT} \ + --set dockerRegistry.server=${DOCKER_REGISTRY_SERVER} \ + --set dockerRegistry.inClusterServer=${DOCKER_REGISTRY_IN_CLUSTER_SERVER} \ + --set dockerRegistry.username=${DOCKER_REGISTRY_USERNAME} \ + --set dockerRegistry.password=${DOCKER_REGISTRY_PASSWORD} \ + --set dockerRegistry.secure=${DOCKER_REGISTRY_SECURE} \ + --set dockerRegistry.bentoRepositoryName=${DOCKER_REGISTRY_BENTO_REPOSITORY_NAME} \ + --set yatai.endpoint=${YATAI_ENDPOINT} \ + --set aws.accessKeyID=${AWS_ACCESS_KEY_ID} \ + --set aws.secretAccessKeyExistingSecretName=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_NAME} \ + --set aws.secretAccessKeyExistingSecretKey=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY} \ + --skip-crds=${UPGRADE_CRDS} \ + --version=${VERSION} \ + --devel=${DEVEL} +fi kubectl -n ${namespace} rollout restart deploy/yatai-image-builder diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go new file mode 100644 index 0000000..ceb0c18 --- /dev/null +++ b/tests/e2e/e2e_suite_test.go @@ -0,0 +1,16 @@ +package e2e + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Run e2e tests using the Ginkgo runner. +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + fmt.Fprintf(GinkgoWriter, "Starting yatai-image-builder e2e suite\n") + RunSpecs(t, "yatai-image-builder e2e suite") +} diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go new file mode 100644 index 0000000..393b9d1 --- /dev/null +++ b/tests/e2e/e2e_test.go @@ -0,0 +1,118 @@ +//nolint:wrapcheck,gosec +package e2e + +import ( + "context" + "fmt" + "os/exec" + "time" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client/config" + + commonconsts "github.com/bentoml/yatai-common/consts" + resourcesclient "github.com/bentoml/yatai-image-builder/generated/resources/clientset/versioned/typed/resources/v1alpha1" + + //nolint:golint + //nolint:revive + . "github.com/onsi/ginkgo/v2" + + //nolint:golint + //nolint:revive + . "github.com/onsi/gomega" + + "github.com/bentoml/yatai-image-builder/tests/utils" +) + +var _ = Describe("yatai-image-builder", Ordered, func() { + AfterAll(func() { + By("Showing image builder pod events") + cmd := exec.Command("kubectl", "-n", "yatai", "describe", "pod", "-l", fmt.Sprintf("%s=true", commonconsts.KubeLabelIsBentoImageBuilder)) + logs, _ := utils.Run(cmd) + fmt.Println(string(logs)) + By("Showing image builder pod logs") + cmd = exec.Command("kubectl", "-n", "yatai", "logs", "--tail", "200", "-l", fmt.Sprintf("%s=true", commonconsts.KubeLabelIsBentoImageBuilder)) + logs, _ = utils.Run(cmd) + fmt.Println(string(logs)) + By("Showing yatai-image-builder events") + cmd = exec.Command("kubectl", "-n", "yatai-image-builder", "describe", "pod", "-l", "app.kubernetes.io/name=yatai-image-builder") + logs, _ = utils.Run(cmd) + fmt.Println(string(logs)) + By("Showing yatai-image-builder logs") + cmd = exec.Command("kubectl", "-n", "yatai-image-builder", "logs", "--tail", "200", "-l", "app.kubernetes.io/name=yatai-image-builder") + logs, _ = utils.Run(cmd) + fmt.Println(string(logs)) + By("Cleaning up BentoRequest resources") + cmd = exec.Command("kubectl", "delete", "-f", "tests/e2e/example.yaml") + _, _ = utils.Run(cmd) + }) + + Context("BentoRequest Operator", func() { + It("Should run successfully", func() { + By("Creating a BentoRequest CR") + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "apply", "-f", "tests/e2e/example.yaml") + _, err := utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + + By("Sleeping for 5 seconds") + time.Sleep(5 * time.Second) + + restConf := config.GetConfigOrDie() + cliset, err := kubernetes.NewForConfig(restConf) + Expect(err).To(BeNil(), "failed to create kubernetes clientset") + + By("Checking the generated image builder pod") + EventuallyWithOffset(1, func() error { + pods, err := cliset.CoreV1().Pods("yatai").List(context.Background(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=true", commonconsts.KubeLabelIsBentoImageBuilder), + }) + if err != nil { + return err + } + if len(pods.Items) != 1 { + return fmt.Errorf("expected 1 pod, got %d", len(pods.Items)) + } + pod := pods.Items[0] + if pod.Status.Phase == corev1.PodFailed { + return fmt.Errorf("pod failed: %s", pod.Status.Message) + } + if pod.Status.Phase != corev1.PodSucceeded { + return fmt.Errorf("pod not finished yet") + } + return nil + }, 10*time.Minute, time.Second).Should(Succeed()) + + bentorequestcli, err := resourcesclient.NewForConfig(restConf) + Expect(err).To(BeNil(), "failed to create bentorequest clientset") + + By("Sleeping for 5 seconds") + time.Sleep(5 * time.Second) + + By("Checking the generated Bento CR") + EventuallyWithOffset(1, func() error { + ctx := context.Background() + + logrus.Infof("Getting BentoRequest CR %s", "test-bento") + bentoRequest, err := bentorequestcli.BentoRequests("yatai").Get(ctx, "test-bento", metav1.GetOptions{}) + if err != nil { + return err + } + + logrus.Infof("Getting Bento CR %s", "test-bento") + bento, err := bentorequestcli.Bentoes("yatai").Get(ctx, "test-bento", metav1.GetOptions{}) + if err != nil { + return err + } + + Expect(bentoRequest.Spec.BentoTag).To(Equal(bento.Spec.Tag), "BentoRequest and Bento tag should match") + Expect(bento.Spec.Image).To(Not(BeEmpty()), "Bento CR image should not be empty") + return nil + }).Should(Succeed()) + }) + }) +}) diff --git a/tests/e2e/example.yaml b/tests/e2e/example.yaml new file mode 100644 index 0000000..fed705f --- /dev/null +++ b/tests/e2e/example.yaml @@ -0,0 +1,8 @@ +apiVersion: resources.yatai.ai/v1alpha1 +kind: BentoRequest +metadata: + name: test-bento + namespace: yatai +spec: + bentoTag: iris_classifier:r4zint4b567i4usu + downloadUrl: s3://yetone/bentos/test-bento.bento diff --git a/tests/e2e/installation_test.sh b/tests/e2e/installation_test.sh new file mode 100755 index 0000000..ef2c002 --- /dev/null +++ b/tests/e2e/installation_test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -xe + +kubectl create ns yatai-system +kubectl create ns yatai-image-builder + +echo "🚀 Creating AWS Secret Access Key..." +kubectl create secret generic aws-secret-access-key --from-literal=AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} --namespace yatai-image-builder +echo "🚀 Installing yatai-image-builder..." +YATAI_ENDPOINT='empty' USE_LOCAL_HELM_CHART=true UPGRADE_CRDS=false AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_NAME=aws-secret-access-key AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY=AWS_SECRET_ACCESS_KEY bash ./scripts/quick-install-yatai-image-builder.sh +echo "yatai-image-builder helm release values:" +helm get values yatai-image-builder -n yatai-image-builder + diff --git a/tests/gh-actions/install_helm.sh b/tests/gh-actions/install_helm.sh new file mode 100755 index 0000000..be6731b --- /dev/null +++ b/tests/gh-actions/install_helm.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e +echo "Installing helm ..." + +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash diff --git a/tests/gh-actions/install_kind.sh b/tests/gh-actions/install_kind.sh new file mode 100755 index 0000000..4de9be7 --- /dev/null +++ b/tests/gh-actions/install_kind.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e +echo "Fetching KinD executable ..." +sudo swapoff -a +sudo rm -f /swapfile +sudo mkdir -p /tmp/etcd +sudo mount -t tmpfs tmpfs /tmp/etcd +curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.17.0/kind-linux-amd64 +chmod +x ./kind +sudo mv kind /usr/local/bin \ No newline at end of file diff --git a/tests/gh-actions/kind-cluster-1-24.yaml b/tests/gh-actions/kind-cluster-1-24.yaml new file mode 100644 index 0000000..eca9d4a --- /dev/null +++ b/tests/gh-actions/kind-cluster-1-24.yaml @@ -0,0 +1,25 @@ +# This testing option is available for testing projects that don't yet support k8s 1.25 +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +# Configure registry for KinD. +containerdConfigPatches: +- |- + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."$REGISTRY_NAME:$REGISTRY_PORT"] + endpoint = ["http://$REGISTRY_NAME:$REGISTRY_PORT"] +# This is needed in order to support projected volumes with service account tokens. +# See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600268272383600 +kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta2 + kind: ClusterConfiguration + metadata: + name: config + apiServer: + extraArgs: + "service-account-issuer": "kubernetes.default.svc" + "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key" +nodes: +- role: control-plane + image: kindest/node:v1.24.7@sha256:577c630ce8e509131eab1aea12c022190978dd2f745aac5eb1fe65c0807eb315 +- role: worker + image: kindest/node:v1.24.7@sha256:577c630ce8e509131eab1aea12c022190978dd2f745aac5eb1fe65c0807eb315 diff --git a/tests/utils/utils.go b/tests/utils/utils.go new file mode 100644 index 0000000..57040c3 --- /dev/null +++ b/tests/utils/utils.go @@ -0,0 +1,122 @@ +//nolint:wrapcheck +package utils + +import ( + "errors" + "fmt" + "os" + "os/exec" + "strings" + "syscall" + + . "github.com/onsi/ginkgo/v2" //nolint:golint,revive,stylecheck +) + +// Run executes the provided command within this context +func Run(cmd *exec.Cmd) ([]byte, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + fmt.Fprintf(GinkgoWriter, "running dir: %s\n", cmd.Dir) + + // To allow make commands be executed from the project directory which is subdir on SDK repo + // TODO:(user) You might not need the following code + if err := os.Chdir(cmd.Dir); err != nil { + fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + } + + cmd.Env = append(os.Environ(), "GO111MODULE=on") + command := strings.Join(cmd.Args, " ") + fmt.Fprintf(GinkgoWriter, "running: %s\n", command) + output, err := cmd.CombinedOutput() + if err != nil { + return output, fmt.Errorf("%s failed with error: (%w) %s", command, err, string(output)) + } + + return output, nil +} + +func RunAsDaemon(cmd *exec.Cmd) (*os.Process, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + fmt.Fprintf(GinkgoWriter, "running dir: %s\n", cmd.Dir) + + // To allow make commands be executed from the project directory which is subdir on SDK repo + // TODO:(user) You might not need the following code + if err := os.Chdir(cmd.Dir); err != nil { + fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + } + + cmd.Env = append(os.Environ(), "GO111MODULE=on") + cmd.Stdin = nil + cmd.Stdout = nil + cmd.Stderr = nil + cmd.ExtraFiles = nil + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + command := strings.Join(cmd.Args, " ") + fmt.Fprintf(GinkgoWriter, "running: %s\n", command) + err := cmd.Start() + if err != nil { + return nil, fmt.Errorf("%s failed with error: (%w)", command, err) + } + + return cmd.Process, nil +} + +// LoadImageToKindCluster loads a local docker image to the kind cluster +func LoadImageToKindClusterWithName(name string) error { + cluster := "kind" + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + kindOptions := []string{"load", "docker-image", name, "--name", cluster} + cmd := exec.Command("kind", kindOptions...) + _, err := Run(cmd) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + elements := strings.Split(output, "\n") + for _, element := range elements { + if element != "" { + res = append(res, element) + } + } + + return res +} + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, err + } + wd = strings.ReplaceAll(wd, "/tests/e2e", "") + return wd, nil +} + +// ReplaceInFile replaces all instances of old with new in the file at path. +func ReplaceInFile(path, old, new string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + // false positive + // nolint:gosec + b, err := os.ReadFile(path) + if err != nil { + return err + } + if !strings.Contains(string(b), old) { + return errors.New("unable to find the content to be replaced") + } + s := strings.ReplaceAll(string(b), old, new) + err = os.WriteFile(path, []byte(s), info.Mode()) + if err != nil { + return err + } + return nil +}