e2e: add debugshell to e2e tests

.github/workflows/e2e.yml

@@ -18,6 +18,9 @@ on:
       self-hosted:
         description: "Self Hosted"
         type: boolean
+      debug-shell:
+        description: "Debug shell"
+        type: boolean
     secrets:
       GITHUB_TOKEN_IN:
         required: true
@@ -36,7 +39,7 @@ env:
 
 jobs:
   test:
-    name: "${{ inputs.test-name }}"
+    name: "${{ inputs.test-name }}${{ inputs.debug-shell && ' (with debug shell)' || '' }}"
     runs-on: ${{ inputs.runner }}
     permissions:
       contents: read
@@ -46,6 +49,7 @@ jobs:
       NODE_INSTALLER_TARGET_CONF_TYPE: ${{ inputs.platform == 'Metal-QEMU-SNP' && 'none' || 'k3s' }}
       TEST_NAME: ${{ inputs.test-name }}
       CONTRAST_GHCR_READ: ${{ secrets.CONTRAST_GHCR_READ }}
+      DEBUG_SHELL: ${{ inputs.debug-shell }}
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
@@ -70,7 +74,7 @@ jobs:
           EOF
       - name: Build and push container images
         run: |
-          just coordinator initializer port-forwarder openssl service-mesh-proxy memdump node-installer "${PLATFORM}"
+          just coordinator initializer port-forwarder openssl service-mesh-proxy memdump debugshell node-installer "${PLATFORM}"
       - name: Get credentials for CI cluster
         if: (!inputs.self-hosted)
         run: |
@@ -89,7 +93,8 @@ jobs:
             --platform "${PLATFORM}" \
             --node-installer-target-conf-type "${NODE_INSTALLER_TARGET_CONF_TYPE}" \
             --namespace-suffix="-ci" \
-            --sync-ticket-file workspace/just.sync-ticket
+            --sync-ticket-file workspace/just.sync-ticket \
+            --insecure-enable-debug-shell-access="${DEBUG_SHELL}"
       - name: Download logs
         if: always()
         run: |
@@ -98,7 +103,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: e2e_pod_logs-${{ inputs.platform }}-${{ inputs.test-name }}
+          name: e2e_pod_logs-${{ inputs.platform }}-${{ inputs.test-name }}${{ inputs.debug-shell && '-debug-shell' || '' }}
           path: workspace/logs/export-no-stream/logs
       - name: Notify teams channel of failure
         if: failure() && github.event_name == 'schedule' && github.run_attempt == 1

.github/workflows/e2e_manual.yml

@@ -39,6 +39,11 @@ on:
         required: false
         type: boolean
         default: false
+      debug-shell:
+        description: "Debug shell"
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   determine-platform-params:
@@ -82,6 +87,7 @@ jobs:
       platform: ${{ inputs.platform }}
       runner: ${{ needs.determine-platform-params.outputs.runner }}
       self-hosted: ${{ fromJSON(needs.determine-platform-params.outputs.self-hosted) }}
+      debug-shell: ${{ inputs.debug-shell }}
     secrets:
       GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN }}
       CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}

.github/workflows/e2e_nightly.yml

@@ -21,6 +21,7 @@ jobs:
   test_matrix:
     strategy:
       matrix:
+        debug-shell: [true, false]
         platform:
           - name: Metal-QEMU-SNP
             runner: SNP
@@ -63,6 +64,7 @@ jobs:
       platform: ${{ matrix.platform.name }}
       runner: ${{ matrix.platform.runner }}
       self-hosted: ${{ matrix.platform.self-hosted }}
+      debug-shell: ${{ matrix.debug-shell }}
     secrets:
       GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN || secrets.GITHUB_TOKEN }}
       CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}

.github/workflows/e2e_on_pull_request.yml

@@ -45,6 +45,7 @@ jobs:
       platform: ${{ matrix.platform }}
       runner: ${{ matrix.runner }}
       self-hosted: ${{ matrix.self-hosted }}
+      debug-shell: true
     secrets:
       GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN }}
       CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}

e2e/internal/contrasttest/contrasttest.go

@@ -44,6 +44,7 @@ type testFlags struct {
 	NamespaceSuffix             string
 	NodeInstallerTargetConfType string
 	SyncTicketFile              string
+	InsecureEnableDebugShell    bool
 }
 
 // RegisterFlags registers the flags that are shared between all tests.
@@ -54,6 +55,7 @@ func RegisterFlags() {
 	flag.StringVar(&Flags.PlatformStr, "platform", "", "Deployment platform")
 	flag.StringVar(&Flags.NodeInstallerTargetConfType, "node-installer-target-conf-type", "", "Type of node installer target configuration to generate (k3s,...)")
 	flag.StringVar(&Flags.SyncTicketFile, "sync-ticket-file", "", "file that contains the sync ticket")
+	flag.BoolVar(&Flags.InsecureEnableDebugShell, "insecure-enable-debug-shell-access", false, "enable the debug shell service")
 }
 
 // ContrastTest is the Contrast test helper struct.
@@ -168,6 +170,7 @@ func (ct *ContrastTest) RunGenerate(ctx context.Context) error {
 		ct.commonArgs(),
 		"--image-replacements", ct.ImageReplacementsFile,
 		"--reference-values", ct.Platform.String(),
+		fmt.Sprintf("--insecure-enable-debug-shell-access=%t", Flags.InsecureEnableDebugShell),
 		ct.WorkDir,
 	)
 

e2e/internal/kubeclient/kubeclient.go

@@ -21,6 +21,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -165,33 +166,19 @@ func (c *Kubeclient) Exec(ctx context.Context, namespace, pod string, argv []str
 	stdout string, stderr string, err error,
 ) {
 	c.log.Debug("executing command in pod", "namespace", namespace, "pod", pod, "argv", argv)
-	buf := &bytes.Buffer{}
-	errBuf := &bytes.Buffer{}
-	request := c.Client.CoreV1().RESTClient().
-		Post().
-		Namespace(namespace).
-		Resource("pods").
-		Name(pod).
-		SubResource("exec").
-		VersionedParams(&corev1.PodExecOptions{
-			Command: argv,
-			Stdin:   false,
-			Stdout:  true,
-			Stderr:  true,
-			TTY:     false,
-		}, scheme.ParameterCodec)
-	exec, err := remotecommand.NewSPDYExecutor(c.config, http.MethodPost, request.URL())
+	podList, err := c.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("metadata.name", pod).String(),
+	})
 	if err != nil {
-		return "", "", fmt.Errorf("creating executor: %w", err)
+		return "", "", fmt.Errorf("listing pods: %w", err)
 	}
-
-	err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
-		Stdout: buf,
-		Stderr: errBuf,
-		Tty:    false,
-	})
-
-	return buf.String(), errBuf.String(), err
+	if len(podList.Items) == 0 {
+		return "", "", fmt.Errorf("pod not found: %s/%s", namespace, pod)
+	}
+	if len(podList.Items[0].Spec.Containers) == 0 {
+		return "", "", fmt.Errorf("pod %s/%s has no containers", namespace, pod)
+	}
+	return c.ExecContainer(ctx, namespace, pod, podList.Items[0].Spec.Containers[0].Name, argv)
 }
 
 // ExecContainer executes a process in the container of a pod and returns the stdout and stderr.

justfile

@@ -58,7 +58,8 @@ e2e target=default_deploy_target platform=default_platform: soft-clean coordinat
             --platform {{ platform }} \
             --node-installer-target-conf-type ${node_installer_target_conf_type} \
             --namespace-suffix=${namespace_suffix-} \
-            --sync-ticket-file ./{{ workspace_dir }}/just.sync-ticket
+            --sync-ticket-file ./{{ workspace_dir }}/just.sync-ticket \
+            --insecure-enable-debug-shell-access=${debug:-false}
 
 # Generate policies, apply Kubernetes manifests.
 deploy target=default_deploy_target cli=default_cli platform=default_platform: (runtime target platform) (apply "runtime" platform) (populate target platform) (generate cli platform) (apply target platform)