Skip to content

Commit 00c9e96

Browse files
MortalHappinessMortalHappiness
committed
[Fix][kubectl-plugin] Create separate namespaces for each kubectl plugin e2e test
Closes: #2729 Signed-off-by: Chi-Sheng Liu <[email protected]>
1 parent b15f5af commit 00c9e96

File tree

6 files changed

+125
-43
lines changed

6 files changed

+125
-43
lines changed

‎.github/workflows/kubectl-plugin-e2e-tests.yaml

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ on:
2121
- '**.adoc'
2222
- '**.md'
2323
- 'LICENSE'
24+
workflow_dispatch:
2425

2526
jobs:
2627
build:
@@ -76,11 +77,6 @@ jobs:
7677
make deploy -e IMG="${IMG}"
7778
kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator
7879
79-
- name: Deploy RayCluster
80-
run: |
81-
kubectl apply -f ./ray-operator/config/samples/ray-cluster.sample.yaml
82-
kubectl wait --timeout=300s --for 'jsonpath={.status.state}=ready' raycluster/raycluster-kuberay
83-
8480
- name: Run e2e tests
8581
run: |
8682
export KUBERAY_TEST_TIMEOUT_SHORT=1m

‎kubectl-plugin/test/e2e/kubectl_ray_cluster_get_test.go

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,20 @@ import (
1111
"k8s.io/cli-runtime/pkg/printers"
1212
)
1313

14-
var _ = Describe("Calling ray plugin `get` command", Ordered, func() {
14+
var _ = Describe("Calling ray plugin `get` command", func() {
15+
var namespace string
16+
17+
BeforeEach(func() {
18+
namespace = createTestNamespace()
19+
deployTestRayCluster(namespace)
20+
DeferCleanup(func() {
21+
deleteTestNamespace(namespace)
22+
namespace = ""
23+
})
24+
})
25+
1526
It("succeed in getting ray cluster information", func() {
16-
cmd := exec.Command("kubectl", "ray", "get", "cluster", "--namespace", "default")
27+
cmd := exec.Command("kubectl", "ray", "get", "cluster", "--namespace", namespace)
1728
output, err := cmd.CombinedOutput()
1829

1930
expectedOutputTablePrinter := printers.NewTablePrinter(printers.PrintOptions{})
@@ -34,7 +45,7 @@ var _ = Describe("Calling ray plugin `get` command", Ordered, func() {
3445
expectedTestResultTable.Rows = append(expectedTestResultTable.Rows, v1.TableRow{
3546
Cells: []interface{}{
3647
"raycluster-kuberay",
37-
"default",
48+
namespace,
3849
"1",
3950
"1",
4051
"2",
@@ -53,7 +64,7 @@ var _ = Describe("Calling ray plugin `get` command", Ordered, func() {
5364
})
5465

5566
It("should not succeed", func() {
56-
cmd := exec.Command("kubectl", "ray", "get", "cluster", "fakeclustername", "anotherfakeclustername")
67+
cmd := exec.Command("kubectl", "ray", "get", "cluster", "--namespace", namespace, "fakeclustername", "anotherfakeclustername")
5768
output, err := cmd.CombinedOutput()
5869

5970
Expect(err).To(HaveOccurred())

‎kubectl-plugin/test/e2e/kubectl_ray_job_submit_test.go

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,20 @@ const (
1818
runtimeEnvSampleFileName = "runtime-env-sample.yaml"
1919
)
2020

21-
var _ = Describe("Calling ray plugin `job submit` command on Ray Job", Ordered, func() {
21+
var _ = Describe("Calling ray plugin `job submit` command on Ray Job", func() {
22+
var namespace string
23+
24+
BeforeEach(func() {
25+
namespace = createTestNamespace()
26+
deployTestRayCluster(namespace)
27+
DeferCleanup(func() {
28+
deleteTestNamespace(namespace)
29+
namespace = ""
30+
})
31+
})
32+
2233
It("succeed in submitting RayJob", func() {
23-
cmd := exec.Command("kubectl", "ray", "job", "submit", "-f", rayJobFilePath, "--working-dir", kubectlRayJobWorkingDir, "--", "python", entrypointSampleFileName)
34+
cmd := exec.Command("kubectl", "ray", "job", "submit", "--namespace", namespace, "-f", rayJobFilePath, "--working-dir", kubectlRayJobWorkingDir, "--", "python", entrypointSampleFileName)
2435
output, err := cmd.CombinedOutput()
2536

2637
Expect(err).NotTo(HaveOccurred())
@@ -33,35 +44,30 @@ var _ = Describe("Calling ray plugin `job submit` command on Ray Job", Ordered,
3344

3445
// Use kubectl to check status of the rayjob
3546
// Retrieve Job ID
36-
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
47+
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
3748
output, err = cmd.CombinedOutput()
3849
Expect(err).ToNot(HaveOccurred())
3950

4051
Expect(cmdOutputJobID).To(Equal(string(output)))
4152

4253
// Retrieve Job Status
43-
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
54+
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
4455
output, err = cmd.CombinedOutput()
4556
Expect(err).ToNot(HaveOccurred())
4657

4758
Expect(string(output)).To(Equal("SUCCEEDED"))
4859

4960
// Retrieve Job Deployment Status
50-
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
61+
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
5162
output, err = cmd.CombinedOutput()
5263
Expect(err).ToNot(HaveOccurred())
5364

5465
Expect(string(output)).To(Equal("Complete"))
55-
56-
// Cleanup
57-
cmd = exec.Command("kubectl", "delete", "rayjob", "rayjob-sample")
58-
_, err = cmd.CombinedOutput()
59-
Expect(err).ToNot(HaveOccurred())
6066
})
6167

6268
It("succeed in submitting RayJob with runtime environment set with working dir", func() {
6369
runtimeEnvFilePath := path.Join(kubectlRayJobWorkingDir, runtimeEnvSampleFileName)
64-
cmd := exec.Command("kubectl", "ray", "job", "submit", "-f", rayJobNoEnvFilePath, "--runtime-env", runtimeEnvFilePath, "--", "python", entrypointSampleFileName)
70+
cmd := exec.Command("kubectl", "ray", "job", "submit", "--namespace", namespace, "-f", rayJobNoEnvFilePath, "--runtime-env", runtimeEnvFilePath, "--", "python", entrypointSampleFileName)
6571
output, err := cmd.CombinedOutput()
6672

6773
Expect(err).NotTo(HaveOccurred())
@@ -74,29 +80,24 @@ var _ = Describe("Calling ray plugin `job submit` command on Ray Job", Ordered,
7480

7581
// Use kubectl to check status of the rayjob
7682
// Retrieve Job ID
77-
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
83+
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
7884
output, err = cmd.CombinedOutput()
7985
Expect(err).ToNot(HaveOccurred())
8086

8187
Expect(cmdOutputJobID).To(Equal(string(output)))
8288

8389
// Retrieve Job Status
84-
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
90+
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
8591
output, err = cmd.CombinedOutput()
8692
Expect(err).ToNot(HaveOccurred())
8793

8894
Expect(string(output)).To(Equal("SUCCEEDED"))
8995

9096
// Retrieve Job Deployment Status
91-
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
97+
cmd = exec.Command("kubectl", "get", "--namespace", namespace, "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
9298
output, err = cmd.CombinedOutput()
9399
Expect(err).ToNot(HaveOccurred())
94100

95101
Expect(string(output)).To(Equal("Complete"))
96-
97-
// Cleanup
98-
cmd = exec.Command("kubectl", "delete", "rayjob", "rayjob-sample")
99-
_, err = cmd.CombinedOutput()
100-
Expect(err).ToNot(HaveOccurred())
101102
})
102103
})

‎kubectl-plugin/test/e2e/kubectl_ray_log_test.go

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,23 @@ var requiredFileSet = map[string]string{
1515
"raylet.out": "Ray Event initialized for RAYLET",
1616
}
1717

18-
var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, func() {
18+
var _ = Describe("Calling ray plugin `log` command on Ray Cluster", func() {
19+
var namespace string
20+
21+
BeforeEach(func() {
22+
namespace = createTestNamespace()
23+
deployTestRayCluster(namespace)
24+
DeferCleanup(func() {
25+
deleteTestNamespace(namespace)
26+
namespace = ""
27+
})
28+
})
29+
1930
It("succeed in retrieving all ray cluster logs", func() {
2031
expectedDirPath := "./raycluster-kuberay"
2132
expectedOutputStringFormat := `No output directory specified, creating dir under current directory using resource name\.\nCommand set to retrieve both head and worker node logs\.\nDownloading log for Ray Node raycluster-kuberay-head-\w+\nDownloading log for Ray Node raycluster-kuberay-workergroup-worker-\w+`
2233

23-
cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--node-type", "all")
34+
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--node-type", "all")
2435
output, err := cmd.CombinedOutput()
2536

2637
Expect(err).NotTo(HaveOccurred())
@@ -75,7 +86,7 @@ var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, fun
7586
expectedDirPath := "./raycluster-kuberay"
7687
expectedOutputStringFormat := `No output directory specified, creating dir under current directory using resource name\.\nCommand set to retrieve only head node logs\.\nDownloading log for Ray Node raycluster-kuberay-head-\w+`
7788

78-
cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--node-type", "head")
89+
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--node-type", "head")
7990
output, err := cmd.CombinedOutput()
8091
Expect(err).NotTo(HaveOccurred())
8192
Expect(strings.TrimSpace(string(output))).Should(MatchRegexp(expectedOutputStringFormat))
@@ -128,7 +139,7 @@ var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, fun
128139
expectedDirPath := "./raycluster-kuberay"
129140
expectedOutputStringFormat := `No output directory specified, creating dir under current directory using resource name\.\nCommand set to retrieve only worker node logs\.\nDownloading log for Ray Node raycluster-kuberay-workergroup-worker-\w+`
130141

131-
cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--node-type", "worker")
142+
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--node-type", "worker")
132143
output, err := cmd.CombinedOutput()
133144

134145
Expect(err).NotTo(HaveOccurred())
@@ -185,7 +196,7 @@ var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, fun
185196
err := os.MkdirAll(expectedDirPath, 0o755)
186197
Expect(err).NotTo(HaveOccurred())
187198

188-
cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--node-type", "all", "--out-dir", expectedDirPath)
199+
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--node-type", "all", "--out-dir", expectedDirPath)
189200
output, err := cmd.CombinedOutput()
190201

191202
Expect(err).NotTo(HaveOccurred())
@@ -202,15 +213,15 @@ var _ = Describe("Calling ray plugin `log` command on Ray Cluster", Ordered, fun
202213
})
203214

204215
It("should not succeed with non-existent cluster", func() {
205-
cmd := exec.Command("kubectl", "ray", "log", "fakeclustername")
216+
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "fakeclustername")
206217
output, err := cmd.CombinedOutput()
207218

208219
Expect(err).To(HaveOccurred())
209220
Expect(strings.TrimSpace(string(output))).To(ContainSubstring("No ray nodes found for resource fakecluster"))
210221
})
211222

212223
It("should not succeed with non-existent directory set", func() {
213-
cmd := exec.Command("kubectl", "ray", "log", "raycluster-kuberay", "--out-dir", "./fake-directory")
224+
cmd := exec.Command("kubectl", "ray", "log", "--namespace", namespace, "raycluster-kuberay", "--out-dir", "./fake-directory")
214225
output, err := cmd.CombinedOutput()
215226

216227
Expect(err).To(HaveOccurred())

‎kubectl-plugin/test/e2e/kubectl_ray_session_test.go

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,20 @@ import (
1111
. "github.com/onsi/gomega"
1212
)
1313

14-
var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
14+
var _ = Describe("Calling ray plugin `session` command", func() {
15+
var namespace string
16+
17+
BeforeEach(func() {
18+
namespace = createTestNamespace()
19+
deployTestRayCluster(namespace)
20+
DeferCleanup(func() {
21+
deleteTestNamespace(namespace)
22+
namespace = ""
23+
})
24+
})
25+
1526
It("succeed in forwarding RayCluster and should be able to cancel", func() {
16-
cmd := exec.Command("kubectl", "ray", "session", "raycluster-kuberay")
27+
cmd := exec.Command("kubectl", "ray", "session", "--namespace", namespace, "raycluster-kuberay")
1728
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
1829
defer cancel()
1930

@@ -53,7 +64,7 @@ var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
5364
})
5465

5566
It("should reconnect after pod connection is lost", func() {
56-
sessionCmd := exec.Command("kubectl", "ray", "session", "raycluster-kuberay")
67+
sessionCmd := exec.Command("kubectl", "ray", "session", "--namespace", namespace, "raycluster-kuberay")
5768

5869
err := sessionCmd.Start()
5970
Expect(err).NotTo(HaveOccurred())
@@ -65,20 +76,20 @@ var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
6576
}, 3*time.Second, 500*time.Millisecond).ShouldNot(HaveOccurred())
6677

6778
// Get the current head pod name
68-
cmd := exec.Command("kubectl", "get", "raycluster/raycluster-kuberay", "-o", "jsonpath={.status.head.podName}")
79+
cmd := exec.Command("kubectl", "get", "--namespace", namespace, "raycluster/raycluster-kuberay", "-o", "jsonpath={.status.head.podName}")
6980
output, err := cmd.CombinedOutput()
7081
Expect(err).NotTo(HaveOccurred())
7182
oldPodName := string(output)
7283
var newPodName string
7384

7485
// Delete the pod
75-
cmd = exec.Command("kubectl", "delete", "pod", oldPodName)
86+
cmd = exec.Command("kubectl", "delete", "--namespace", namespace, "pod", oldPodName)
7687
err = cmd.Run()
7788
Expect(err).NotTo(HaveOccurred())
7889

7990
// Wait for the new pod to be created
8091
Eventually(func() error {
81-
cmd := exec.Command("kubectl", "get", "raycluster/raycluster-kuberay", "-o", "jsonpath={.status.head.podName}")
92+
cmd := exec.Command("kubectl", "get", "--namespace", namespace, "raycluster/raycluster-kuberay", "-o", "jsonpath={.status.head.podName}")
8293
output, err := cmd.CombinedOutput()
8394
newPodName = string(output)
8495
if err != nil {
@@ -91,7 +102,7 @@ var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
91102
}, 60*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
92103

93104
// Wait for the new pod to be ready
94-
cmd = exec.Command("kubectl", "wait", "pod", newPodName, "--for=condition=Ready", "--timeout=60s")
105+
cmd = exec.Command("kubectl", "wait", "--namespace", namespace, "pod", newPodName, "--for=condition=Ready", "--timeout=60s")
95106
err = cmd.Run()
96107
Expect(err).NotTo(HaveOccurred())
97108

@@ -107,7 +118,7 @@ var _ = Describe("Calling ray plugin `session` command", Ordered, func() {
107118
})
108119

109120
It("should not succeed", func() {
110-
cmd := exec.Command("kubectl", "ray", "session", "fakeclustername")
121+
cmd := exec.Command("kubectl", "ray", "session", "--namespace", namespace, "fakeclustername")
111122
output, err := cmd.CombinedOutput()
112123

113124
Expect(err).To(HaveOccurred())

‎kubectl-plugin/test/e2e/support.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package e2e
2+
3+
import (
4+
"math/rand"
5+
"os/exec"
6+
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
)
10+
11+
const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789"
12+
13+
func randStringBytes(n int) string {
14+
// Reference: https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-go/22892986
15+
b := make([]byte, n)
16+
for i := range b {
17+
b[i] = letterBytes[rand.Intn(len(letterBytes))] //nolint:gosec // Don't need cryptographically secure random number
18+
}
19+
return string(b)
20+
}
21+
22+
func createTestNamespace() string {
23+
GinkgoHelper()
24+
suffix := randStringBytes(5)
25+
ns := "test-ns-" + suffix
26+
cmd := exec.Command("kubectl", "create", "namespace", ns)
27+
err := cmd.Run()
28+
Expect(err).NotTo(HaveOccurred())
29+
nsWithPrefix := "namespace/" + ns
30+
cmd = exec.Command("kubectl", "wait", "--timeout=20s", "--for", "jsonpath={.status.phase}=Active", nsWithPrefix)
31+
err = cmd.Run()
32+
Expect(err).NotTo(HaveOccurred())
33+
return ns
34+
}
35+
36+
func deleteTestNamespace(ns string) {
37+
GinkgoHelper()
38+
cmd := exec.Command("kubectl", "delete", "namespace", ns)
39+
err := cmd.Run()
40+
Expect(err).NotTo(HaveOccurred())
41+
}
42+
43+
func deployTestRayCluster(ns string) {
44+
GinkgoHelper()
45+
// Print current working directory
46+
cmd := exec.Command("kubectl", "apply", "-f", "../../../ray-operator/config/samples/ray-cluster.sample.yaml", "-n", ns)
47+
err := cmd.Run()
48+
Expect(err).NotTo(HaveOccurred())
49+
cmd = exec.Command("kubectl", "wait", "--timeout=300s", "--for", "jsonpath={.status.state}=ready", "raycluster/raycluster-kuberay", "-n", ns)
50+
err = cmd.Run()
51+
Expect(err).NotTo(HaveOccurred())
52+
}

0 commit comments

Comments
 (0)