Skip to content

Commit

Permalink
[Fix][kubectl-plugin] Create separate namespaces for each kubectl plu…
Browse files Browse the repository at this point in the history
…gin e2e test

Closes: #2729
Signed-off-by: Chi-Sheng Liu <[email protected]>
  • Loading branch information
MortalHappiness committed Jan 14, 2025
1 parent b15f5af commit 00c9e96
Show file tree
Hide file tree
Showing 6 changed files with 125 additions and 43 deletions.
6 changes: 1 addition & 5 deletions .github/workflows/kubectl-plugin-e2e-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ on:
- '**.adoc'
- '**.md'
- 'LICENSE'
workflow_dispatch:

jobs:
build:
Expand Down Expand Up @@ -76,11 +77,6 @@ jobs:
make deploy -e IMG="${IMG}"
kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator
- name: Deploy RayCluster
run: |
kubectl apply -f ./ray-operator/config/samples/ray-cluster.sample.yaml
kubectl wait --timeout=300s --for 'jsonpath={.status.state}=ready' raycluster/raycluster-kuberay
- name: Run e2e tests
run: |
export KUBERAY_TEST_TIMEOUT_SHORT=1m
Expand Down
19 changes: 15 additions & 4 deletions kubectl-plugin/test/e2e/kubectl_ray_cluster_get_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,20 @@ import (
"k8s.io/cli-runtime/pkg/printers"
)

var _ = Describe("Calling ray plugin `get` command", Ordered, func() {
var _ = Describe("Calling ray plugin `get` command", func() {
var namespace string

BeforeEach(func() {
namespace = createTestNamespace()
deployTestRayCluster(namespace)
DeferCleanup(func() {
deleteTestNamespace(namespace)
namespace = ""
})
})

It("succeed in getting ray cluster information", func() {
cmd := exec.Command("kubectl", "ray", "get", "cluster", "--namespace", "default")
cmd := exec.Command("kubectl", "ray", "get", "cluster", "--namespace", namespace)
output, err := cmd.CombinedOutput()

expectedOutputTablePrinter := printers.NewTablePrinter(printers.PrintOptions{})
Expand All @@ -34,7 +45,7 @@ var _ = Describe("Calling ray plugin `get` command", Ordered, func() {
expectedTestResultTable.Rows = append(expectedTestResultTable.Rows, v1.TableRow{
Cells: []interface{}{
"raycluster-kuberay",
"default",
namespace,
"1",
"1",
"2",
Expand All @@ -53,7 +64,7 @@ var _ = Describe("Calling ray plugin `get` command", Ordered, func() {
})

It("should not succeed", func() {
cmd := exec.Command("kubectl", "ray", "get", "cluster", "fakeclustername", "anotherfakeclustername")
cmd := exec.Command("kubectl", "ray", "get", "cluster", "--namespace", namespace, "fakeclustername", "anotherfakeclustername")
output, err := cmd.CombinedOutput()

Expect(err).To(HaveOccurred())
Expand Down
39 changes: 20 additions & 19 deletions kubectl-plugin/test/e2e/kubectl_ray_job_submit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,20 @@ const (
runtimeEnvSampleFileName = "runtime-env-sample.yaml"
)

var _ = Describe("Calling ray plugin `job submit` command on Ray Job", Ordered, func() {
var _ = Describe("Calling ray plugin `job submit` command on Ray Job", func() {
var namespace string

BeforeEach(func() {
namespace = createTestNamespace()
deployTestRayCluster(namespace)
DeferCleanup(func() {
deleteTestNamespace(namespace)
namespace = ""
})
})

It("succeed in submitting RayJob", func() {
cmd := exec.Command("kubectl", "ray", "job", "submit", "-f", rayJobFilePath, "--working-dir", kubectlRayJobWorkingDir, "--", "python", entrypointSampleFileName)
cmd := exec.Command("kubectl", "ray", "job", "submit", "--namespace", namespace, "-f", rayJobFilePath, "--working-dir", kubectlRayJobWorkingDir, "--", "python", entrypointSampleFileName)
output, err := cmd.CombinedOutput()

Expect(err).NotTo(HaveOccurred())
Expand All @@ -33,35 +44,30 @@ var _ = Describe("Calling ray plugin `job submit` command on Ray Job", Ordered,

// Use kubectl to check status of the rayjob
// Retrieve Job ID
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
output, err = cmd.CombinedOutput()
Expect(err).ToNot(HaveOccurred())

Expect(cmdOutputJobID).To(Equal(string(output)))

// Retrieve Job Status
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
output, err = cmd.CombinedOutput()
Expect(err).ToNot(HaveOccurred())

Expect(string(output)).To(Equal("SUCCEEDED"))

// Retrieve Job Deployment Status
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
output, err = cmd.CombinedOutput()
Expect(err).ToNot(HaveOccurred())

Expect(string(output)).To(Equal("Complete"))

// Cleanup
cmd = exec.Command("kubectl", "delete", "rayjob", "rayjob-sample")
_, err = cmd.CombinedOutput()
Expect(err).ToNot(HaveOccurred())
})

It("succeed in submitting RayJob with runtime environment set with working dir", func() {
runtimeEnvFilePath := path.Join(kubectlRayJobWorkingDir, runtimeEnvSampleFileName)
cmd := exec.Command("kubectl", "ray", "job", "submit", "-f", rayJobNoEnvFilePath, "--runtime-env", runtimeEnvFilePath, "--", "python", entrypointSampleFileName)
cmd := exec.Command("kubectl", "ray", "job", "submit", "--namespace", namespace, "-f", rayJobNoEnvFilePath, "--runtime-env", runtimeEnvFilePath, "--", "python", entrypointSampleFileName)
output, err := cmd.CombinedOutput()

Expect(err).NotTo(HaveOccurred())
Expand All @@ -74,29 +80,24 @@ var _ = Describe("Calling ray plugin `job submit` command on Ray Job", Ordered,

// Use kubectl to check status of the rayjob
// Retrieve Job ID
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
output, err = cmd.CombinedOutput()
Expect(err).ToNot(HaveOccurred())

Expect(cmdOutputJobID).To(Equal(string(output)))

// Retrieve Job Status
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
output, err = cmd.CombinedOutput()
Expect(err).ToNot(HaveOccurred())

Expect(string(output)).To(Equal("SUCCEEDED"))

// Retrieve Job Deployment Status
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
output, err = cmd.CombinedOutput()
Expect(err).ToNot(HaveOccurred())

Expect(string(output)).To(Equal("Complete"))

// Cleanup
cmd = exec.Command("kubectl", "delete", "rayjob", "rayjob-sample")
_, err = cmd.CombinedOutput()
Expect(err).ToNot(HaveOccurred())
})
})
25 changes: 18 additions & 7 deletions kubectl-plugin/test/e2e/kubectl_ray_log_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,23 @@ var requiredFileSet = map[string]string{
"raylet.out": "Ray Event initialized for RAYLET",
}

var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, func() {
var _ = Describe("Calling ray plugin `log` command on Ray Cluster", func() {
var namespace string

BeforeEach(func() {
namespace = createTestNamespace()
deployTestRayCluster(namespace)
DeferCleanup(func() {
deleteTestNamespace(namespace)
namespace = ""
})
})

It("succeed in retrieving all ray cluster logs", func() {
expectedDirPath := "./raycluster-kuberay"
expectedOutputStringFormat := `No output directory specified, creating dir under current directory using resource name\.\nCommand set to retrieve both head and worker node logs\.\nDownloading log for Ray Node raycluster-kuberay-head-\w+\nDownloading log for Ray Node raycluster-kuberay-workergroup-worker-\w+`

cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--node-type", "all")
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--node-type", "all")
output, err := cmd.CombinedOutput()

Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -75,7 +86,7 @@ var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, fun
expectedDirPath := "./raycluster-kuberay"
expectedOutputStringFormat := `No output directory specified, creating dir under current directory using resource name\.\nCommand set to retrieve only head node logs\.\nDownloading log for Ray Node raycluster-kuberay-head-\w+`

cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--node-type", "head")
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--node-type", "head")
output, err := cmd.CombinedOutput()
Expect(err).NotTo(HaveOccurred())
Expect(strings.TrimSpace(string(output))).Should(MatchRegexp(expectedOutputStringFormat))
Expand Down Expand Up @@ -128,7 +139,7 @@ var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, fun
expectedDirPath := "./raycluster-kuberay"
expectedOutputStringFormat := `No output directory specified, creating dir under current directory using resource name\.\nCommand set to retrieve only worker node logs\.\nDownloading log for Ray Node raycluster-kuberay-workergroup-worker-\w+`

cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--node-type", "worker")
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--node-type", "worker")
output, err := cmd.CombinedOutput()

Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -185,7 +196,7 @@ var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, fun
err := os.MkdirAll(expectedDirPath, 0o755)
Expect(err).NotTo(HaveOccurred())

cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--node-type", "all", "--out-dir", expectedDirPath)
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--node-type", "all", "--out-dir", expectedDirPath)
output, err := cmd.CombinedOutput()

Expect(err).NotTo(HaveOccurred())
Expand All @@ -202,15 +213,15 @@ var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, fun
})

It("should not succeed with non-existent cluster", func() {
cmd := exec.Command("kubectl", "ray", "log", "fakeclustername")
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "fakeclustername")
output, err := cmd.CombinedOutput()

Expect(err).To(HaveOccurred())
Expect(strings.TrimSpace(string(output))).To(ContainSubstring("No ray nodes found for resource fakecluster"))
})

It("should not succeed with non-existent directory set", func() {
cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--out-dir", "./fake-directory")
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--out-dir", "./fake-directory")
output, err := cmd.CombinedOutput()

Expect(err).To(HaveOccurred())
Expand Down
27 changes: 19 additions & 8 deletions kubectl-plugin/test/e2e/kubectl_ray_session_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,20 @@ import (
. "github.com/onsi/gomega"
)

var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
var _ = Describe("Calling ray plugin `session` command", func() {
var namespace string

BeforeEach(func() {
namespace = createTestNamespace()
deployTestRayCluster(namespace)
DeferCleanup(func() {
deleteTestNamespace(namespace)
namespace = ""
})
})

It("succeed in forwarding RayCluster and should be able to cancel", func() {
cmd := exec.Command("kubectl", "ray", "session", "raycluster-kuberay")
cmd := exec.Command("kubectl", "ray", "session", "--namespace", namespace, "raycluster-kuberay")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()

Expand Down Expand Up @@ -53,7 +64,7 @@ var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
})

It("should reconnect after pod connection is lost", func() {
sessionCmd := exec.Command("kubectl", "ray", "session", "raycluster-kuberay")
sessionCmd := exec.Command("kubectl", "ray", "session", "--namespace", namespace, "raycluster-kuberay")

err := sessionCmd.Start()
Expect(err).NotTo(HaveOccurred())
Expand All @@ -65,20 +76,20 @@ var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
}, 3*time.Second, 500*time.Millisecond).ShouldNot(HaveOccurred())

// Get the current head pod name
cmd := exec.Command("kubectl", "get", "raycluster/raycluster-kuberay", "-o", "jsonpath={.status.head.podName}")
cmd := exec.Command("kubectl", "get", "--namespace", namespace, "raycluster/raycluster-kuberay", "-o", "jsonpath={.status.head.podName}")
output, err := cmd.CombinedOutput()
Expect(err).NotTo(HaveOccurred())
oldPodName := string(output)
var newPodName string

// Delete the pod
cmd = exec.Command("kubectl", "delete", "pod", oldPodName)
cmd = exec.Command("kubectl", "delete", "--namespace", namespace, "pod", oldPodName)
err = cmd.Run()
Expect(err).NotTo(HaveOccurred())

// Wait for the new pod to be created
Eventually(func() error {
cmd := exec.Command("kubectl", "get", "raycluster/raycluster-kuberay", "-o", "jsonpath={.status.head.podName}")
cmd := exec.Command("kubectl", "get", "--namespace", namespace, "raycluster/raycluster-kuberay", "-o", "jsonpath={.status.head.podName}")
output, err := cmd.CombinedOutput()
newPodName = string(output)
if err != nil {
Expand All @@ -91,7 +102,7 @@ var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
}, 60*time.Second, 1*time.Second).ShouldNot(HaveOccurred())

// Wait for the new pod to be ready
cmd = exec.Command("kubectl", "wait", "pod", newPodName, "--for=condition=Ready", "--timeout=60s")
cmd = exec.Command("kubectl", "wait", "--namespace", namespace, "pod", newPodName, "--for=condition=Ready", "--timeout=60s")
err = cmd.Run()
Expect(err).NotTo(HaveOccurred())

Expand All @@ -107,7 +118,7 @@ var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
})

It("should not succeed", func() {
cmd := exec.Command("kubectl", "ray", "session", "fakeclustername")
cmd := exec.Command("kubectl", "ray", "session", "--namespace", namespace, "fakeclustername")
output, err := cmd.CombinedOutput()

Expect(err).To(HaveOccurred())
Expand Down
52 changes: 52 additions & 0 deletions kubectl-plugin/test/e2e/support.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package e2e

import (
"math/rand"
"os/exec"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789"

func randStringBytes(n int) string {
// Reference: https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-go/22892986
b := make([]byte, n)
for i := range b {
b[i] = letterBytes[rand.Intn(len(letterBytes))] //nolint:gosec // Don't need cryptographically secure random number
}
return string(b)
}

func createTestNamespace() string {
GinkgoHelper()
suffix := randStringBytes(5)
ns := "test-ns-" + suffix
cmd := exec.Command("kubectl", "create", "namespace", ns)
err := cmd.Run()
Expect(err).NotTo(HaveOccurred())
nsWithPrefix := "namespace/" + ns
cmd = exec.Command("kubectl", "wait", "--timeout=20s", "--for", "jsonpath={.status.phase}=Active", nsWithPrefix)
err = cmd.Run()
Expect(err).NotTo(HaveOccurred())
return ns
}

func deleteTestNamespace(ns string) {
GinkgoHelper()
cmd := exec.Command("kubectl", "delete", "namespace", ns)
err := cmd.Run()
Expect(err).NotTo(HaveOccurred())
}

func deployTestRayCluster(ns string) {
GinkgoHelper()
// Print current working directory
cmd := exec.Command("kubectl", "apply", "-f", "../../../ray-operator/config/samples/ray-cluster.sample.yaml", "-n", ns)
err := cmd.Run()
Expect(err).NotTo(HaveOccurred())
cmd = exec.Command("kubectl", "wait", "--timeout=300s", "--for", "jsonpath={.status.state}=ready", "raycluster/raycluster-kuberay", "-n", ns)
err = cmd.Run()
Expect(err).NotTo(HaveOccurred())
}

0 comments on commit 00c9e96

Please sign in to comment.