From cab18de168210573dc9d0be7a0805e71694675ec Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Mon, 24 Jun 2024 13:24:29 +0200
Subject: [PATCH 01/68] fix e2e tests and bugs

---
 Dockerfile                                    |  2 +-
 bootstrap/config/manager/manager.yaml         |  3 +-
 .../bootstrap_v1alpha1_ck8sconfig.yaml        |  2 +-
 ...bootstrap_v1alpha3_ck8sconfigtemplate.yaml |  2 +-
 .../config/default/kustomization.yaml         |  2 +-
 controlplane/config/manager/manager.yaml      |  2 +-
 ...ontrolplane_v1alpha3_ck8scontrolplane.yaml |  2 +-
 docs/main.md                                  |  4 +-
 pkg/ck8s/manifests/k8sd-proxy-template.yaml   |  3 +-
 .../cluster-template-kcp-remediation.yaml     | 14 ++-----
 .../cluster-template-md-remediation.yaml      | 12 ++----
 .../cluster-template.yaml                     | 37 +++++++++++--------
 12 files changed, 41 insertions(+), 44 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6113fcf7..790690cb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,7 +29,7 @@ FROM ${builder_image} as builder
 WORKDIR /workspace
 
 # Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
-ARG goproxy=off
+ARG goproxy=https://proxy.golang.org
 # Run this with docker build --build-arg package=./controlplane/kubeadm or --build-arg package=./bootstrap/kubeadm
 ENV GOPROXY=$goproxy
 
diff --git a/bootstrap/config/manager/manager.yaml b/bootstrap/config/manager/manager.yaml
index 36980726..978e5485 100644
--- a/bootstrap/config/manager/manager.yaml
+++ b/bootstrap/config/manager/manager.yaml
@@ -27,6 +27,7 @@ spec:
         - /manager
         args:
         - --enable-leader-election
-        image: controller:latest
+        # TODO: how to only set this for the e2e tests
+        image: controller:dev
         name: manager
       terminationGracePeriodSeconds: 10
diff --git a/bootstrap/config/samples/bootstrap_v1alpha1_ck8sconfig.yaml b/bootstrap/config/samples/bootstrap_v1alpha1_ck8sconfig.yaml
index 6d3fc62c..f1afdde5 100644
--- a/bootstrap/config/samples/bootstrap_v1alpha1_ck8sconfig.yaml
+++ b/bootstrap/config/samples/bootstrap_v1alpha1_ck8sconfig.yaml
@@ -1,4 +1,4 @@
-apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfig
 metadata:
   name: ck8sconfig-sample
diff --git a/bootstrap/config/samples/bootstrap_v1alpha3_ck8sconfigtemplate.yaml b/bootstrap/config/samples/bootstrap_v1alpha3_ck8sconfigtemplate.yaml
index f336faf1..eb3029ad 100644
--- a/bootstrap/config/samples/bootstrap_v1alpha3_ck8sconfigtemplate.yaml
+++ b/bootstrap/config/samples/bootstrap_v1alpha3_ck8sconfigtemplate.yaml
@@ -1,4 +1,4 @@
-apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate
 metadata:
   name: ck8sconfigtemplate-sample
diff --git a/controlplane/config/default/kustomization.yaml b/controlplane/config/default/kustomization.yaml
index f09a39aa..89fccd20 100644
--- a/controlplane/config/default/kustomization.yaml
+++ b/controlplane/config/default/kustomization.yaml
@@ -28,7 +28,7 @@ patchesStrategicMerge:
   # Protect the /metrics endpoint by putting it behind auth.
   # If you want your controller-manager to expose the /metrics
   # endpoint w/o any authn/z, please comment the following line.
-- manager_auth_proxy_patch.yaml
+#- manager_auth_proxy_patch.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
diff --git a/controlplane/config/manager/manager.yaml b/controlplane/config/manager/manager.yaml
index b6c85a52..efd9de16 100644
--- a/controlplane/config/manager/manager.yaml
+++ b/controlplane/config/manager/manager.yaml
@@ -27,7 +27,7 @@ spec:
         - /manager
         args:
         - --enable-leader-election
-        image: controller:latest
+        image: controller:dev
         name: manager
         resources:
           limits:
diff --git a/controlplane/config/samples/controlplane_v1alpha3_ck8scontrolplane.yaml b/controlplane/config/samples/controlplane_v1alpha3_ck8scontrolplane.yaml
index b11f9088..5a0f24d2 100644
--- a/controlplane/config/samples/controlplane_v1alpha3_ck8scontrolplane.yaml
+++ b/controlplane/config/samples/controlplane_v1alpha3_ck8scontrolplane.yaml
@@ -1,4 +1,4 @@
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ck8scontrolplane-sample
diff --git a/docs/main.md b/docs/main.md
index 2f41b926..10ceed1a 100644
--- a/docs/main.md
+++ b/docs/main.md
@@ -14,7 +14,7 @@ The default behaviour is to `snap install k8s` using the matching track (e.g. in
 You can override this behaviour by changing the default installation script by setting the following fields on the config template:
 
 ```yaml
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ${CLUSTER_NAME}-control-plane
@@ -47,7 +47,7 @@ For airgap deployments, or environment you can specify `airGapped: true` to prev
 Any extra yaml files placed in `/capi/manifests` will be applied once on the cluster after bootstrapping. Files are applied in alphabetical order, so you can use this in case of dependencies. Example:
 
 ```yaml
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ${CLUSTER_NAME}-control-plane
diff --git a/pkg/ck8s/manifests/k8sd-proxy-template.yaml b/pkg/ck8s/manifests/k8sd-proxy-template.yaml
index 2832a1b7..ad99d7b7 100644
--- a/pkg/ck8s/manifests/k8sd-proxy-template.yaml
+++ b/pkg/ck8s/manifests/k8sd-proxy-template.yaml
@@ -31,7 +31,8 @@ spec:
         effect: NoSchedule
       containers:
       - name: k8sd-proxy
-        image: ghcr.io/canonical/cluster-api-k8s/socat:1.8.0.0
+        # TODO: change back to ghcr image once this is public
+        image: alpine/socat:1.8.0.0
         env:
         # TODO: Make this more robust by possibly finding/parsing the right IP.
         # This works as a start but might not be sufficient as the kubelet IP might not match microcluster IP.
diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml
index a8677d02..99497a09 100644
--- a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml
+++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml
@@ -13,7 +13,7 @@ spec:
       - 10.46.0.0/16
     serviceDomain: cluster.local
   controlPlaneRef:
-    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta2
     kind: CK8sControlPlane
     name: ${CLUSTER_NAME}-control-plane
   infrastructureRef:
@@ -27,7 +27,7 @@ metadata:
   name: ${CLUSTER_NAME}
 spec: {}
 ---
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ${CLUSTER_NAME}-control-plane
@@ -69,12 +69,6 @@ spec:
       permissions: "0777"
     preK3sCommands:
     - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}"
-    serverConfig:
-      tlsSan:
-        - localhost
-        - 127.0.0.1
-        - 0.0.0.0
-        - host.docker.internal
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: DockerMachineTemplate
@@ -111,7 +105,7 @@ spec:
       clusterName: ${CLUSTER_NAME}
       bootstrap:
         configRef:
-          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
           kind: CK8sConfigTemplate
           name: ${CLUSTER_NAME}-md-0
       infrastructureRef:
@@ -129,7 +123,7 @@ spec:
     spec:
       customImage: kindest/node:${KIND_IMAGE_VERSION}
 ---
-apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate
 metadata:
   name: ${CLUSTER_NAME}-md-0
diff --git a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml
index 3bf354eb..0a7c2245 100644
--- a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml
+++ b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml
@@ -16,7 +16,7 @@ spec:
       - 10.46.0.0/16
     serviceDomain: cluster.local
   controlPlaneRef:
-    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta2
     kind: CK8sControlPlane
     name: ${CLUSTER_NAME}-control-plane
   infrastructureRef:
@@ -30,7 +30,7 @@ metadata:
   name: ${CLUSTER_NAME}
 spec: {}
 ---
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ${CLUSTER_NAME}-control-plane
@@ -42,10 +42,6 @@ spec:
     name: ${CLUSTER_NAME}-control-plane
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
-  spec:
-    serverConfig:
-      tlsSan:
-        - 0.0.0.0
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: DockerMachineTemplate
@@ -83,7 +79,7 @@ spec:
       clusterName: ${CLUSTER_NAME}
       bootstrap:
         configRef:
-          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
           kind: CK8sConfigTemplate
           name: ${CLUSTER_NAME}-md-0
       infrastructureRef:
@@ -101,7 +97,7 @@ spec:
     spec:
       customImage: kindest/node:${KIND_IMAGE_VERSION}
 ---
-apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate
 metadata:
   name: ${CLUSTER_NAME}-md-0
diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml
index 062164cb..27ea144f 100644
--- a/test/e2e/data/infrastructure-docker/cluster-template.yaml
+++ b/test/e2e/data/infrastructure-docker/cluster-template.yaml
@@ -10,13 +10,13 @@ spec:
   clusterNetwork:
     pods:
       cidrBlocks:
-      - 10.45.0.0/16
+      - 10.1.0.0/16
     services:
       cidrBlocks:
-      - 10.46.0.0/16
+      - 10.152.0.0/16
     serviceDomain: cluster.local
   controlPlaneRef:
-    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta2
     kind: CK8sControlPlane
     name: ${CLUSTER_NAME}-control-plane
   infrastructureRef:
@@ -30,22 +30,24 @@ metadata:
   name: ${CLUSTER_NAME}
 spec: {}
 ---
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ${CLUSTER_NAME}-control-plane
   namespace: ${NAMESPACE}
 spec:
-  infrastructureTemplate:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-    kind: DockerMachineTemplate
-    name: ${CLUSTER_NAME}-control-plane
+  machineTemplate:
+    infrastructureTemplate:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: DockerMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane
+  spec:
+    airGapped: true
+    controlPlane:
+      extraKubeAPIServerArgs:
+        --anonymous-auth: "true"
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
-  spec:
-    serverConfig:
-      tlsSan:
-        - 0.0.0.0
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: DockerMachineTemplate
@@ -55,7 +57,8 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      # TODO: make this customable
+      customImage: k8s-snap:dev
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -82,7 +85,7 @@ spec:
       clusterName: ${CLUSTER_NAME}
       bootstrap:
         configRef:
-          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
           kind: CK8sConfigTemplate
           name: ${CLUSTER_NAME}-md-0
       infrastructureRef:
@@ -98,9 +101,10 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      # TODO: make this customable
+      customImage: k8s-snap:dev
 ---
-apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate
 metadata:
   name: ${CLUSTER_NAME}-md-0
@@ -108,3 +112,4 @@ metadata:
 spec:
   template:
     spec:
+      airGapped: true

From ea8a78ff2f7cacd1ab6089d993cd2fdde9828842 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 26 Jun 2024 14:10:49 +0200
Subject: [PATCH 02/68] fixup linter issues and missing flags

---
 .../cluster-template-kcp-remediation.yaml                    | 4 ++++
 .../cluster-template-md-remediation.yaml                     | 5 +++++
 2 files changed, 9 insertions(+)

diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml
index 99497a09..fee3ad57 100644
--- a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml
+++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml
@@ -40,6 +40,10 @@ spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
   spec:
+    airGapped: true
+    controlPlane:
+      extraKubeAPIServerArgs:
+        --anonymous-auth: "true"
     files:
     - path: /wait-signal.sh
       content: |
diff --git a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml
index 0a7c2245..e2895a31 100644
--- a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml
+++ b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml
@@ -40,6 +40,11 @@ spec:
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
     kind: DockerMachineTemplate
     name: ${CLUSTER_NAME}-control-plane
+  spec:
+    airGapped: true
+    controlPlane:
+      extraKubeAPIServerArgs:
+        --anonymous-auth: "true"
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
 ---

From 9e05e15beadf40367d1a16f41eec3cf578c2ec1a Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 27 Jun 2024 19:39:49 +0200
Subject: [PATCH 03/68] fix remaining tests

---
 .github/workflows/e2e.yaml                    | 33 +++++++
 pkg/cloudinit/controlplane_init_test.go       |  7 ++
 .../cluster-template-kcp-remediation.yaml     | 20 ++--
 .../cluster-template-md-remediation.yaml      | 14 +--
 test/e2e/helpers.go                           | 91 +++++++++++++++++++
 test/e2e/md_remediation_test.go               |  3 +-
 6 files changed, 152 insertions(+), 16 deletions(-)
 create mode 100644 .github/workflows/e2e.yaml

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
new file mode 100644
index 00000000..83368651
--- /dev/null
+++ b/.github/workflows/e2e.yaml
@@ -0,0 +1,33 @@
+name: E2E Tests
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  e2e-tests:
+    name: Build provider images
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Install requirements
+        run: |
+          sudo apt install make
+          sudo snap install go --classic
+          sudo snap install docker
+      - name: Build provider images
+        run: sudo make docker-build-e2e
+      - name: Build images
+        run: |
+          cd templates/docker
+          sudo docker build . -t k8s-snap:dev
+      - name: Run e2e tests
+        run: sudo make test-e2e
diff --git a/pkg/cloudinit/controlplane_init_test.go b/pkg/cloudinit/controlplane_init_test.go
index 5ad86d68..cf33eccb 100644
--- a/pkg/cloudinit/controlplane_init_test.go
+++ b/pkg/cloudinit/controlplane_init_test.go
@@ -46,6 +46,13 @@ func TestNewInitControlPlane(t *testing.T) {
 
 	// TODO: add tests for expected files and commands
 	g.Expect(err).To(BeNil())
+	g.Expect(config.WriteFiles).To(ContainElement(cloudinit.File{
+		Path:        "/tmp/file",
+		Content:     "test file",
+		Permissions: "0400",
+		Owner:       "root:root",
+	}))
+	g.Expect(config.BootCommands).To(ContainElement("bootcmd"))
 	g.Expect(config.RunCommands).To(Equal([]string{
 		"set -x",
 		"prerun1",
diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml
index fee3ad57..8c8cb43b 100644
--- a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml
+++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml
@@ -33,12 +33,13 @@ metadata:
   name: ${CLUSTER_NAME}-control-plane
   namespace: ${NAMESPACE}
 spec:
-  infrastructureTemplate:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-    kind: DockerMachineTemplate
-    name: ${CLUSTER_NAME}-control-plane
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
+  machineTemplate:
+    infrastructureTemplate:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: DockerMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane
   spec:
     airGapped: true
     controlPlane:
@@ -66,12 +67,13 @@ spec:
           echo "signal $signal"
 
           if [ "$signal" == "pass" ]; then
-             curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test
-             exit 0
+            curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test
+            exit 0
           fi
         done
       permissions: "0777"
-    preK3sCommands:
+      owner: root:root
+    preRunCommands:
     - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}"
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
@@ -82,7 +84,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -125,7 +127,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate
diff --git a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml
index e2895a31..7b2bfe64 100644
--- a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml
+++ b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml
@@ -36,10 +36,11 @@ metadata:
   name: ${CLUSTER_NAME}-control-plane
   namespace: ${NAMESPACE}
 spec:
-  infrastructureTemplate:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-    kind: DockerMachineTemplate
-    name: ${CLUSTER_NAME}-control-plane
+  machineTemplate:
+    infrastructureTemplate:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: DockerMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane
   spec:
     airGapped: true
     controlPlane:
@@ -56,7 +57,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -100,7 +101,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate
@@ -110,6 +111,7 @@ metadata:
 spec:
   template:
     spec:
+      airGapped: true
 ---
 # MachineHealthCheck object with
 # - a selector that targets all the machines with label e2e.remediation.label=""
diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go
index 24e8dec1..8c423f5f 100644
--- a/test/e2e/helpers.go
+++ b/test/e2e/helpers.go
@@ -22,15 +22,19 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/noderefutil"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -587,6 +591,40 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont
 	}, input.WaitForMachinesToBeUpgraded...)
 }
 
+type WaitForNodesReadyInput struct {
+	Lister            framework.Lister
+	KubernetesVersion string
+	Count             int
+	WaitForNodesReady []interface{}
+}
+
+// WaitForNodesReady waits until there are exactly the given count nodes and they have the correct Kubernetes minor version
+// and are ready.
+func WaitForNodesReady(ctx context.Context, input WaitForNodesReadyInput) {
+	Eventually(func() (bool, error) {
+		nodeList := &corev1.NodeList{}
+		if err := input.Lister.List(ctx, nodeList); err != nil {
+			return false, err
+		}
+		nodeReadyCount := 0
+		for _, node := range nodeList.Items {
+			n := node
+			match, err := CompareVersions(node.Status.NodeInfo.KubeletVersion, input.KubernetesVersion, "minor")
+			if err != nil {
+				return false, fmt.Errorf("failed to compare versions: %w", err)
+			}
+			if !match {
+				return false, nil
+			}
+			if !noderefutil.IsNodeReady(&n) {
+				return false, nil
+			}
+			nodeReadyCount++
+		}
+		return input.Count == nodeReadyCount, nil
+	}, input.WaitForNodesReady...).Should(BeTrue())
+}
+
 // byClusterOptions returns a set of ListOptions that allows to identify all the objects belonging to a Cluster.
 func byClusterOptions(name, namespace string) []client.ListOption {
 	return []client.ListOption{
@@ -596,3 +634,56 @@ func byClusterOptions(name, namespace string) []client.ListOption {
 		},
 	}
 }
+
+func parseVersion(version string) ([]int, error) {
+	// Remove the leading "v" if it exists
+	if strings.HasPrefix(version, "v") {
+		version = version[1:]
+	}
+
+	parts := strings.Split(version, ".")
+	if len(parts) != 3 {
+		return nil, fmt.Errorf("invalid version format")
+	}
+
+	intParts := make([]int, len(parts))
+	for i, part := range parts {
+		num, err := strconv.Atoi(part)
+		if err != nil {
+			return nil, fmt.Errorf("invalid version part: %w", err)
+		}
+		intParts[i] = num
+	}
+
+	return intParts, nil
+}
+
+func compareVersionParts(v1, v2 []int, level string) bool {
+	switch strings.ToLower(level) {
+	case "major":
+		return v1[0] == v2[0]
+	case "minor":
+		return v1[0] == v2[0] && v1[1] == v2[1]
+	case "patch":
+		return v1[0] == v2[0] && v1[1] == v2[1] && v1[2] == v2[2]
+	default:
+		return v1[0] == v2[0] && v1[1] == v2[1] && v1[2] == v2[2]
+	}
+}
+
+// CompareVersions compares two versions and returns true if they are equal at the specified level.
+// The level can be "major", "minor", "patch" or an empty string to compare the full version.
+// Returns an error if the versions are not in the correct format.
+// Example: CompareVersions("1.19.0", "1.19.1", "patch") returns false
+func CompareVersions(version1, version2, level string) (bool, error) {
+	v1, err := parseVersion(version1)
+	if err != nil {
+		return false, fmt.Errorf("failed to parse version1: %w", err)
+	}
+
+	v2, err := parseVersion(version2)
+	if err != nil {
+		return false, fmt.Errorf("failed to parse version2: %w", err)
+	}
+	return compareVersionParts(v1, v2, level), nil
+}
diff --git a/test/e2e/md_remediation_test.go b/test/e2e/md_remediation_test.go
index 9b2c9d4f..4f707ba2 100644
--- a/test/e2e/md_remediation_test.go
+++ b/test/e2e/md_remediation_test.go
@@ -108,7 +108,8 @@ var _ = Describe("When testing MachineDeployment remediation", func() {
 			By("Waiting until nodes are ready")
 			workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, result.Cluster.Name)
 			workloadClient := workloadProxy.GetClient()
-			framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{
+
+			WaitForNodesReady(ctx, WaitForNodesReadyInput{
 				Lister:            workloadClient,
 				KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
 				Count:             int(result.ExpectedTotalNodes()),

From 38c72a996cf1dacbc633aa8c4b84b60c2e7cadbf Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Fri, 28 Jun 2024 08:07:55 +0200
Subject: [PATCH 04/68] fixup docker build

---
 Dockerfile                       | 10 +---------
 test/e2e/cluster_upgrade_test.go |  3 +++
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 790690cb..a1ab82c3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -37,11 +37,6 @@ ENV GOPROXY=$goproxy
 COPY go.mod go.mod
 COPY go.sum go.sum
 
-# Cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN --mount=type=cache,target=/go/pkg/mod \
-    go mod download
-
 # Copy the sources
 COPY ./ ./
 
@@ -50,10 +45,7 @@ ARG package=.
 ARG ARCH
 ARG ldflags
 
-# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
-RUN --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/go/pkg/mod \
-    CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
+Run CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
     go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \
     -o manager ${package}
 
diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go
index e5b290d2..4d8070da 100644
--- a/test/e2e/cluster_upgrade_test.go
+++ b/test/e2e/cluster_upgrade_test.go
@@ -25,6 +25,9 @@ import (
 )
 
 var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() {
+	// TODO(bschimke): Remove once we find a way to run e2e tests with other infrastructure providers that support snap.
+	Skip("Skipping the upgrade tests as snap does not work on CAPD.")
+
 	Context("Upgrading a cluster with 1 control plane", func() {
 		ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput {
 			return ClusterUpgradeSpecInput{

From 493631deed7ce730e3f2193f10b97b6fd9b87fdf Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Fri, 28 Jun 2024 08:43:21 +0200
Subject: [PATCH 05/68] ci fixup

---
 Dockerfile | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a1ab82c3..6a315477 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -31,12 +31,17 @@ WORKDIR /workspace
 # Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
 ARG goproxy=https://proxy.golang.org
 # Run this with docker build --build-arg package=./controlplane/kubeadm or --build-arg package=./bootstrap/kubeadm
-ENV GOPROXY=$goproxy
+ENV GOPROXY=direct
 
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
 
+# Cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
 # Copy the sources
 COPY ./ ./
 
@@ -45,7 +50,10 @@ ARG package=.
 ARG ARCH
 ARG ldflags
 
-Run CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
+# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
     go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \
     -o manager ${package}
 

From a45894e43ed85c05c2540799415f282c283eb2af Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Fri, 28 Jun 2024 12:45:55 +0200
Subject: [PATCH 06/68] ci fixup

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6a315477..6113fcf7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,9 +29,9 @@ FROM ${builder_image} as builder
 WORKDIR /workspace
 
 # Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
-ARG goproxy=https://proxy.golang.org
+ARG goproxy=off
 # Run this with docker build --build-arg package=./controlplane/kubeadm or --build-arg package=./bootstrap/kubeadm
-ENV GOPROXY=direct
+ENV GOPROXY=$goproxy
 
 # Copy the Go Modules manifests
 COPY go.mod go.mod

From 8c21eca27d7189c2094db0f5cbc920e26adf90ca Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Fri, 28 Jun 2024 13:02:11 +0200
Subject: [PATCH 07/68] cleanup temlate todos

---
 pkg/ck8s/manifests/k8sd-proxy-template.yaml               | 3 +--
 pkg/cloudinit/controlplane_init_test.go                   | 8 --------
 test/e2e/data/infrastructure-docker/cluster-template.yaml | 1 -
 3 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/pkg/ck8s/manifests/k8sd-proxy-template.yaml b/pkg/ck8s/manifests/k8sd-proxy-template.yaml
index ad99d7b7..2832a1b7 100644
--- a/pkg/ck8s/manifests/k8sd-proxy-template.yaml
+++ b/pkg/ck8s/manifests/k8sd-proxy-template.yaml
@@ -31,8 +31,7 @@ spec:
         effect: NoSchedule
       containers:
       - name: k8sd-proxy
-        # TODO: change back to ghcr image once this is public
-        image: alpine/socat:1.8.0.0
+        image: ghcr.io/canonical/cluster-api-k8s/socat:1.8.0.0
         env:
         # TODO: Make this more robust by possibly finding/parsing the right IP.
         # This works as a start but might not be sufficient as the kubelet IP might not match microcluster IP.
diff --git a/pkg/cloudinit/controlplane_init_test.go b/pkg/cloudinit/controlplane_init_test.go
index cf33eccb..79935853 100644
--- a/pkg/cloudinit/controlplane_init_test.go
+++ b/pkg/cloudinit/controlplane_init_test.go
@@ -46,20 +46,12 @@ func TestNewInitControlPlane(t *testing.T) {
 
 	// TODO: add tests for expected files and commands
 	g.Expect(err).To(BeNil())
-	g.Expect(config.WriteFiles).To(ContainElement(cloudinit.File{
-		Path:        "/tmp/file",
-		Content:     "test file",
-		Permissions: "0400",
-		Owner:       "root:root",
-	}))
-	g.Expect(config.BootCommands).To(ContainElement("bootcmd"))
 	g.Expect(config.RunCommands).To(Equal([]string{
 		"set -x",
 		"prerun1",
 		"prerun2",
 		"/capi/scripts/install.sh",
 		"/capi/scripts/bootstrap.sh",
-		"/capi/scripts/load-images.sh",
 		"/capi/scripts/wait-apiserver-ready.sh",
 		"/capi/scripts/deploy-manifests.sh",
 		"/capi/scripts/configure-token.sh",
diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml
index 27ea144f..b33ac49d 100644
--- a/test/e2e/data/infrastructure-docker/cluster-template.yaml
+++ b/test/e2e/data/infrastructure-docker/cluster-template.yaml
@@ -101,7 +101,6 @@ metadata:
 spec:
   template:
     spec:
-      # TODO: make this customable
       customImage: k8s-snap:dev
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta2

From c3a7b27587bc4c00e82f7c51a7490135809db965 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Mon, 1 Jul 2024 14:45:05 +0200
Subject: [PATCH 08/68] another ci try

---
 .github/workflows/e2e.yaml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 83368651..00322b42 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -12,17 +12,13 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@v2
-        with:
-          egress-policy: audit
       - name: Check out repo
         uses: actions/checkout@v4
       - name: Install requirements
         run: |
           sudo apt install make
           sudo snap install go --classic
-          sudo snap install docker
+          sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Build images

From 3338b2a13172bf27092008bee3cff10d25b2b52f Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Mon, 1 Jul 2024 15:20:13 +0200
Subject: [PATCH 09/68] put skip into beforeall

---
 test/e2e/cluster_upgrade_test.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go
index 4d8070da..a1ddd556 100644
--- a/test/e2e/cluster_upgrade_test.go
+++ b/test/e2e/cluster_upgrade_test.go
@@ -25,8 +25,10 @@ import (
 )
 
 var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() {
-	// TODO(bschimke): Remove once we find a way to run e2e tests with other infrastructure providers that support snap.
-	Skip("Skipping the upgrade tests as snap does not work on CAPD.")
+	BeforeAll(func() {
+		// TODO(bschimke): Remove once we find a way to run e2e tests with other infrastructure providers that support snap.
+		Skip("Skipping the upgrade tests as snap does not work on CAPD.")
+	})
 
 	Context("Upgrading a cluster with 1 control plane", func() {
 		ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput {

From 1feb5909e94d49c7baf178e2d03e5934d2b2f558 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Mon, 1 Jul 2024 15:44:51 +0200
Subject: [PATCH 10/68] fix tests

---
 test/e2e/cluster_upgrade_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go
index a1ddd556..a058b65c 100644
--- a/test/e2e/cluster_upgrade_test.go
+++ b/test/e2e/cluster_upgrade_test.go
@@ -25,7 +25,7 @@ import (
 )
 
 var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() {
-	BeforeAll(func() {
+	BeforeEach(func() {
 		// TODO(bschimke): Remove once we find a way to run e2e tests with other infrastructure providers that support snap.
 		Skip("Skipping the upgrade tests as snap does not work on CAPD.")
 	})

From 98ad5e8fd27f6f23eb16f4dcb05511d75fbcd116 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Tue, 2 Jul 2024 09:02:53 +0200
Subject: [PATCH 11/68] try to get some disk space back

---
 .github/workflows/e2e.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 00322b42..6145739d 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -25,5 +25,11 @@ jobs:
         run: |
           cd templates/docker
           sudo docker build . -t k8s-snap:dev
+      - name: Free disk space
+        run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Run e2e tests
         run: sudo make test-e2e

From 306be858ca6287f69a41d3c923c6612792807826 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Tue, 2 Jul 2024 09:05:48 +0200
Subject: [PATCH 12/68] another ci attempt

---
 .github/workflows/e2e.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 6145739d..f754c7f5 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -27,9 +27,9 @@ jobs:
           sudo docker build . -t k8s-snap:dev
       - name: Free disk space
         run: |
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf /opt/ghc
-        sudo rm -rf "/usr/local/share/boost"
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Run e2e tests
         run: sudo make test-e2e

From 601d9fd2822c4f5a2a08be39d5c3dd949b4aa824 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 09:22:44 +0200
Subject: [PATCH 13/68] replace image tags for e2e

---
 .github/workflows/e2e.yaml               | 2 +-
 bootstrap/config/manager/manager.yaml    | 3 +--
 controlplane/config/manager/manager.yaml | 2 +-
 pkg/cloudinit/controlplane_init_test.go  | 1 +
 test/e2e/config/ck8s-docker.yaml         | 6 ++++++
 5 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index f754c7f5..65626ba2 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -8,7 +8,7 @@ permissions:
 
 jobs:
   e2e-tests:
-    name: Build provider images
+    name: Run e2e tests
     runs-on: ubuntu-22.04
 
     steps:
diff --git a/bootstrap/config/manager/manager.yaml b/bootstrap/config/manager/manager.yaml
index 978e5485..36980726 100644
--- a/bootstrap/config/manager/manager.yaml
+++ b/bootstrap/config/manager/manager.yaml
@@ -27,7 +27,6 @@ spec:
         - /manager
         args:
         - --enable-leader-election
-        # TODO: how to only set this for the e2e tests
-        image: controller:dev
+        image: controller:latest
         name: manager
       terminationGracePeriodSeconds: 10
diff --git a/controlplane/config/manager/manager.yaml b/controlplane/config/manager/manager.yaml
index efd9de16..b6c85a52 100644
--- a/controlplane/config/manager/manager.yaml
+++ b/controlplane/config/manager/manager.yaml
@@ -27,7 +27,7 @@ spec:
         - /manager
         args:
         - --enable-leader-election
-        image: controller:dev
+        image: controller:latest
         name: manager
         resources:
           limits:
diff --git a/pkg/cloudinit/controlplane_init_test.go b/pkg/cloudinit/controlplane_init_test.go
index 79935853..5ad86d68 100644
--- a/pkg/cloudinit/controlplane_init_test.go
+++ b/pkg/cloudinit/controlplane_init_test.go
@@ -52,6 +52,7 @@ func TestNewInitControlPlane(t *testing.T) {
 		"prerun2",
 		"/capi/scripts/install.sh",
 		"/capi/scripts/bootstrap.sh",
+		"/capi/scripts/load-images.sh",
 		"/capi/scripts/wait-apiserver-ready.sh",
 		"/capi/scripts/deploy-manifests.sh",
 		"/capi/scripts/configure-token.sh",
diff --git a/test/e2e/config/ck8s-docker.yaml b/test/e2e/config/ck8s-docker.yaml
index 011e6bdb..a622c8c1 100644
--- a/test/e2e/config/ck8s-docker.yaml
+++ b/test/e2e/config/ck8s-docker.yaml
@@ -64,6 +64,9 @@ providers:
       # is modified
       - name: v0.1.99 # next; use manifest from source files
         value: "../../../bootstrap/config/default"
+        replacements:
+          - old: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:latest"
+            new: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev"
     files:
       - sourcePath: "../../../metadata.yaml"
         targetName: "metadata.yaml"
@@ -72,6 +75,9 @@ providers:
     versions:
       - name: v0.1.99 # next; use manifest from source files
         value: "../../../controlplane/config/default"
+        replacements:
+          - old: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:latest"
+            new: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev"
     files:
       - sourcePath: "../../../metadata.yaml"
         targetName: "metadata.yaml"

From 0bdd3c5b8d8f45cf99078667cdf5e2edfc8bb588 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 09:54:53 +0200
Subject: [PATCH 14/68] add no-arch make targets

---
 Makefile | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index b6b9d017..813f0a82 100644
--- a/Makefile
+++ b/Makefile
@@ -242,6 +242,9 @@ docker-build-bootstrap-%:
 	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$* --build-arg package=./bootstrap/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${BOOTSTRAP_IMG}:${BOOTSTRAP_IMG_TAG}-$*
 docker-build-bootstrap: manager-bootstrap docker-build-bootstrap-amd64 docker-build-bootstrap-arm64
 
+docker-build-bootstrap-no-arch: manager-bootstrap
+	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./bootstrap/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${BOOTSTRAP_IMG}:${BOOTSTRAP_IMG_TAG}
+
 # Push the bootstrap multiarch image
 .PHONY: docker-push-bootstrap
 docker-push-bootstrap-%: docker-build-bootstrap-%
@@ -266,8 +269,8 @@ test-controlplane: envtest generate-controlplane generate-controlplane-conversio
 docker-build-e2e: ## Run docker-build-* targets for all the images with settings to be used for the e2e tests
     # please ensure the generated image name matches image names used in the E2E_CONF_FILE
     # and it also match the image tags in bootstrap/config/default and controlplane/config/default
-	$(MAKE) BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap
-	$(MAKE) CONTROLPLANE_IMG_TAG=dev docker-build-controlplane
+	$(MAKE) BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap-no-arch
+	$(MAKE) CONTROLPLANE_IMG_TAG=dev docker-build-controlplane-no-arch
 
 .PHONY: test-e2e
 test-e2e: $(GINKGO) $(KUSTOMIZE) ## Run the end-to-end tests
@@ -326,6 +329,9 @@ docker-build-controlplane-%:
 	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$* --build-arg package=./controlplane/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${CONTROLPLANE_IMG}:${CONTROLPLANE_IMG_TAG}-$*
 docker-build-controlplane: manager-controlplane docker-build-controlplane-amd64 docker-build-controlplane-arm64
 
+docker-build-controlplane-no-arch: manager-controlplane
+	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=${ARCH} --build-arg package=./controlplane/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${CONTROLPLANE_IMG}:${CONTROLPLANE_IMG_TAG}
+
 # Push the controlplane multiarch image
 .PHONY: docker-push-controlplane
 docker-push-controlplane-%: docker-build-controlplane-%

From 263c4459134c16fde5f631727b5b43c9c9772a40 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 09:56:06 +0200
Subject: [PATCH 15/68] rename no-arch to e2e

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 813f0a82..ad2b7688 100644
--- a/Makefile
+++ b/Makefile
@@ -242,7 +242,7 @@ docker-build-bootstrap-%:
 	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$* --build-arg package=./bootstrap/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${BOOTSTRAP_IMG}:${BOOTSTRAP_IMG_TAG}-$*
 docker-build-bootstrap: manager-bootstrap docker-build-bootstrap-amd64 docker-build-bootstrap-arm64
 
-docker-build-bootstrap-no-arch: manager-bootstrap
+docker-build-bootstrap-e2e: manager-bootstrap
 	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./bootstrap/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${BOOTSTRAP_IMG}:${BOOTSTRAP_IMG_TAG}
 
 # Push the bootstrap multiarch image
@@ -269,8 +269,8 @@ test-controlplane: envtest generate-controlplane generate-controlplane-conversio
 docker-build-e2e: ## Run docker-build-* targets for all the images with settings to be used for the e2e tests
     # please ensure the generated image name matches image names used in the E2E_CONF_FILE
     # and it also match the image tags in bootstrap/config/default and controlplane/config/default
-	$(MAKE) BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap-no-arch
-	$(MAKE) CONTROLPLANE_IMG_TAG=dev docker-build-controlplane-no-arch
+	$(MAKE) BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap-e2e
+	$(MAKE) CONTROLPLANE_IMG_TAG=dev docker-build-controlplane-e2e
 
 .PHONY: test-e2e
 test-e2e: $(GINKGO) $(KUSTOMIZE) ## Run the end-to-end tests
@@ -329,7 +329,7 @@ docker-build-controlplane-%:
 	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$* --build-arg package=./controlplane/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${CONTROLPLANE_IMG}:${CONTROLPLANE_IMG_TAG}-$*
 docker-build-controlplane: manager-controlplane docker-build-controlplane-amd64 docker-build-controlplane-arm64
 
-docker-build-controlplane-no-arch: manager-controlplane
+docker-build-controlplane-e2e: manager-controlplane
 	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=${ARCH} --build-arg package=./controlplane/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${CONTROLPLANE_IMG}:${CONTROLPLANE_IMG_TAG}
 
 # Push the controlplane multiarch image

From 4f530bc4966dc619d9099b866b265fdf1856ec99 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 10:42:29 +0200
Subject: [PATCH 16/68] split GH workflow into build and test jobs

---
 .github/workflows/e2e.yaml | 45 +++++++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 65626ba2..922fee0d 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -7,8 +7,8 @@ permissions:
   contents: read
 
 jobs:
-  e2e-tests:
-    name: Run e2e tests
+  build-e2e-images:
+    name: Build E2E Images
     runs-on: ubuntu-22.04
 
     steps:
@@ -18,18 +18,47 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
-          sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Build images
         run: |
           cd templates/docker
           sudo docker build . -t k8s-snap:dev
-      - name: Free disk space
+      - name: Save provider image
+        run: |
+          sudo docker save -o provider-image.tar $(sudo docker images -q)
+      - name: Save k8s-snap image
         run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /opt/ghc
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo docker save -o k8s-snap-image.tar k8s-snap:dev
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: e2e-images
+          path: |
+            provider-image.tar
+            k8s-snap-image.tar
+
+  run-e2e-tests:
+    name: Run E2E Tests
+    runs-on: ubuntu-22.04
+    needs: build-e2e-images
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Install requirements
+        run: |
+          sudo apt install make
+          sudo snap install go --classic
+          sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: e2e-images
+          path: .
+      - name: Load provider image
+        run: sudo docker load -i provider-image.tar
+      - name: Load k8s-snap image
+        run: sudo docker load -i k8s-snap-image.tar
       - name: Run e2e tests
         run: sudo make test-e2e

From 9b398a8c2e9db9a09a509ba1c67845a72e0413b9 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 12:03:55 +0200
Subject: [PATCH 17/68] cleanup docker to not run into ooo

---
 .github/workflows/e2e.yaml |   7 +
 get-docker.sh              | 741 +++++++++++++++++++++++++++++++++++++
 2 files changed, 748 insertions(+)
 create mode 100644 get-docker.sh

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 922fee0d..f1e01625 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -20,12 +20,19 @@ jobs:
           sudo snap install go --classic
       - name: Build provider images
         run: sudo make docker-build-e2e
+      - name: Cleanup space
+        # Github runners are tight on space and will fail if we don't clean up before.
+        run: sudo docker builder prune -f
       - name: Build images
         run: |
           cd templates/docker
           sudo docker build . -t k8s-snap:dev
+      - name: Cleanup space
+        # Github runners are tight on space and will fail if we don't clean up before.
+        run: sudo docker builder prune -f
       - name: Save provider image
         run: |
+          sudo docker image ls
           sudo docker save -o provider-image.tar $(sudo docker images -q)
       - name: Save k8s-snap image
         run: |
diff --git a/get-docker.sh b/get-docker.sh
new file mode 100644
index 00000000..d037ad9d
--- /dev/null
+++ b/get-docker.sh
@@ -0,0 +1,741 @@
+#!/bin/sh
+set -e
+# Docker Engine for Linux installation script.
+#
+# This script is intended as a convenient way to configure docker's package
+# repositories and to install Docker Engine, This script is not recommended
+# for production environments. Before running this script, make yourself familiar
+# with potential risks and limitations, and refer to the installation manual
+# at https://docs.docker.com/engine/install/ for alternative installation methods.
+#
+# The script:
+#
+# - Requires `root` or `sudo` privileges to run.
+# - Attempts to detect your Linux distribution and version and configure your
+#   package management system for you.
+# - Doesn't allow you to customize most installation parameters.
+# - Installs dependencies and recommendations without asking for confirmation.
+# - Installs the latest stable release (by default) of Docker CLI, Docker Engine,
+#   Docker Buildx, Docker Compose, containerd, and runc. When using this script
+#   to provision a machine, this may result in unexpected major version upgrades
+#   of these packages. Always test upgrades in a test environment before
+#   deploying to your production systems.
+# - Isn't designed to upgrade an existing Docker installation. When using the
+#   script to update an existing installation, dependencies may not be updated
+#   to the expected version, resulting in outdated versions.
+#
+# Source code is available at https://github.com/docker/docker-install/
+#
+# Usage
+# ==============================================================================
+#
+# To install the latest stable versions of Docker CLI, Docker Engine, and their
+# dependencies:
+#
+# 1. download the script
+#
+#   $ curl -fsSL https://get.docker.com -o install-docker.sh
+#
+# 2. verify the script's content
+#
+#   $ cat install-docker.sh
+#
+# 3. run the script with --dry-run to verify the steps it executes
+#
+#   $ sh install-docker.sh --dry-run
+#
+# 4. run the script either as root, or using sudo to perform the installation.
+#
+#   $ sudo sh install-docker.sh
+#
+# Command-line options
+# ==============================================================================
+#
+# --version <VERSION>
+# Use the --version option to install a specific version, for example:
+#
+#   $ sudo sh install-docker.sh --version 23.0
+#
+# --channel <stable|test>
+#
+# Use the --channel option to install from an alternative installation channel.
+# The following example installs the latest versions from the "test" channel,
+# which includes pre-releases (alpha, beta, rc):
+#
+#   $ sudo sh install-docker.sh --channel test
+#
+# Alternatively, use the script at https://test.docker.com, which uses the test
+# channel as default.
+#
+# --mirror <Aliyun|AzureChinaCloud>
+#
+# Use the --mirror option to install from a mirror supported by this script.
+# Available mirrors are "Aliyun" (https://mirrors.aliyun.com/docker-ce), and
+# "AzureChinaCloud" (https://mirror.azure.cn/docker-ce), for example:
+#
+#   $ sudo sh install-docker.sh --mirror AzureChinaCloud
+#
+# ==============================================================================
+
+
+# Git commit from https://github.com/docker/docker-install when
+# the script was uploaded (Should only be modified by upload job):
+SCRIPT_COMMIT_SHA="6d9743e9656cc56f699a64800b098d5ea5a60020"
+
+# strip "v" prefix if present
+VERSION="${VERSION#v}"
+
+# The channel to install from:
+#   * stable
+#   * test
+#   * edge (deprecated)
+#   * nightly (deprecated)
+DEFAULT_CHANNEL_VALUE="stable"
+if [ -z "$CHANNEL" ]; then
+	CHANNEL=$DEFAULT_CHANNEL_VALUE
+fi
+
+DEFAULT_DOWNLOAD_URL="https://download.docker.com"
+if [ -z "$DOWNLOAD_URL" ]; then
+	DOWNLOAD_URL=$DEFAULT_DOWNLOAD_URL
+fi
+
+DEFAULT_REPO_FILE="docker-ce.repo"
+if [ -z "$REPO_FILE" ]; then
+	REPO_FILE="$DEFAULT_REPO_FILE"
+fi
+
+mirror=''
+DRY_RUN=${DRY_RUN:-}
+while [ $# -gt 0 ]; do
+	case "$1" in
+		--channel)
+			CHANNEL="$2"
+			shift
+			;;
+		--dry-run)
+			DRY_RUN=1
+			;;
+		--mirror)
+			mirror="$2"
+			shift
+			;;
+		--version)
+			VERSION="${2#v}"
+			shift
+			;;
+		--*)
+			echo "Illegal option $1"
+			;;
+	esac
+	shift $(( $# > 0 ? 1 : 0 ))
+done
+
+case "$mirror" in
+	Aliyun)
+		DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce"
+		;;
+	AzureChinaCloud)
+		DOWNLOAD_URL="https://mirror.azure.cn/docker-ce"
+		;;
+	"")
+		;;
+	*)
+		>&2 echo "unknown mirror '$mirror': use either 'Aliyun', or 'AzureChinaCloud'."
+		exit 1
+		;;
+esac
+
+case "$CHANNEL" in
+	stable|test)
+		;;
+	edge|nightly)
+		>&2 echo "DEPRECATED: the $CHANNEL channel has been deprecated and is no longer supported by this script."
+		exit 1
+		;;
+	*)
+		>&2 echo "unknown CHANNEL '$CHANNEL': use either stable or test."
+		exit 1
+		;;
+esac
+
+command_exists() {
+	command -v "$@" > /dev/null 2>&1
+}
+
+# version_gte checks if the version specified in $VERSION is at least the given
+# SemVer (Maj.Minor[.Patch]), or CalVer (YY.MM) version.It returns 0 (success)
+# if $VERSION is either unset (=latest) or newer or equal than the specified
+# version, or returns 1 (fail) otherwise.
+#
+# examples:
+#
+# VERSION=23.0
+# version_gte 23.0  // 0 (success)
+# version_gte 20.10 // 0 (success)
+# version_gte 19.03 // 0 (success)
+# version_gte 21.10 // 1 (fail)
+version_gte() {
+	if [ -z "$VERSION" ]; then
+			return 0
+	fi
+	eval version_compare "$VERSION" "$1"
+}
+
+# version_compare compares two version strings (either SemVer (Major.Minor.Path),
+# or CalVer (YY.MM) version strings. It returns 0 (success) if version A is newer
+# or equal than version B, or 1 (fail) otherwise. Patch releases and pre-release
+# (-alpha/-beta) are not taken into account
+#
+# examples:
+#
+# version_compare 23.0.0 20.10 // 0 (success)
+# version_compare 23.0 20.10   // 0 (success)
+# version_compare 20.10 19.03  // 0 (success)
+# version_compare 20.10 20.10  // 0 (success)
+# version_compare 19.03 20.10  // 1 (fail)
+version_compare() (
+	set +x
+
+	yy_a="$(echo "$1" | cut -d'.' -f1)"
+	yy_b="$(echo "$2" | cut -d'.' -f1)"
+	if [ "$yy_a" -lt "$yy_b" ]; then
+		return 1
+	fi
+	if [ "$yy_a" -gt "$yy_b" ]; then
+		return 0
+	fi
+	mm_a="$(echo "$1" | cut -d'.' -f2)"
+	mm_b="$(echo "$2" | cut -d'.' -f2)"
+
+	# trim leading zeros to accommodate CalVer
+	mm_a="${mm_a#0}"
+	mm_b="${mm_b#0}"
+
+	if [ "${mm_a:-0}" -lt "${mm_b:-0}" ]; then
+		return 1
+	fi
+
+	return 0
+)
+
+is_dry_run() {
+	if [ -z "$DRY_RUN" ]; then
+		return 1
+	else
+		return 0
+	fi
+}
+
+is_wsl() {
+	case "$(uname -r)" in
+	*microsoft* ) true ;; # WSL 2
+	*Microsoft* ) true ;; # WSL 1
+	* ) false;;
+	esac
+}
+
+is_darwin() {
+	case "$(uname -s)" in
+	*darwin* ) true ;;
+	*Darwin* ) true ;;
+	* ) false;;
+	esac
+}
+
+deprecation_notice() {
+	distro=$1
+	distro_version=$2
+	echo
+	printf "\033[91;1mDEPRECATION WARNING\033[0m\n"
+	printf "    This Linux distribution (\033[1m%s %s\033[0m) reached end-of-life and is no longer supported by this script.\n" "$distro" "$distro_version"
+	echo   "    No updates or security fixes will be released for this distribution, and users are recommended"
+	echo   "    to upgrade to a currently maintained version of $distro."
+	echo
+	printf   "Press \033[1mCtrl+C\033[0m now to abort this script, or wait for the installation to continue."
+	echo
+	sleep 10
+}
+
+get_distribution() {
+	lsb_dist=""
+	# Every system that we officially support has /etc/os-release
+	if [ -r /etc/os-release ]; then
+		lsb_dist="$(. /etc/os-release && echo "$ID")"
+	fi
+	# Returning an empty string here should be alright since the
+	# case statements don't act unless you provide an actual value
+	echo "$lsb_dist"
+}
+
+echo_docker_as_nonroot() {
+	if is_dry_run; then
+		return
+	fi
+	if command_exists docker && [ -e /var/run/docker.sock ]; then
+		(
+			set -x
+			$sh_c 'docker version'
+		) || true
+	fi
+
+	# intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output
+	echo
+	echo "================================================================================"
+	echo
+	if version_gte "20.10"; then
+		echo "To run Docker as a non-privileged user, consider setting up the"
+		echo "Docker daemon in rootless mode for your user:"
+		echo
+		echo "    dockerd-rootless-setuptool.sh install"
+		echo
+		echo "Visit https://docs.docker.com/go/rootless/ to learn about rootless mode."
+		echo
+	fi
+	echo
+	echo "To run the Docker daemon as a fully privileged service, but granting non-root"
+	echo "users access, refer to https://docs.docker.com/go/daemon-access/"
+	echo
+	echo "WARNING: Access to the remote API on a privileged Docker daemon is equivalent"
+	echo "         to root access on the host. Refer to the 'Docker daemon attack surface'"
+	echo "         documentation for details: https://docs.docker.com/go/attack-surface/"
+	echo
+	echo "================================================================================"
+	echo
+}
+
+# Check if this is a forked Linux distro
+check_forked() {
+
+	# Check for lsb_release command existence, it usually exists in forked distros
+	if command_exists lsb_release; then
+		# Check if the `-u` option is supported
+		set +e
+		lsb_release -a -u > /dev/null 2>&1
+		lsb_release_exit_code=$?
+		set -e
+
+		# Check if the command has exited successfully, it means we're in a forked distro
+		if [ "$lsb_release_exit_code" = "0" ]; then
+			# Print info about current distro
+			cat <<-EOF
+			You're using '$lsb_dist' version '$dist_version'.
+			EOF
+
+			# Get the upstream release info
+			lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]')
+			dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]')
+
+			# Print info about upstream distro
+			cat <<-EOF
+			Upstream release is '$lsb_dist' version '$dist_version'.
+			EOF
+		else
+			if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then
+				if [ "$lsb_dist" = "osmc" ]; then
+					# OSMC runs Raspbian
+					lsb_dist=raspbian
+				else
+					# We're Debian and don't even know it!
+					lsb_dist=debian
+				fi
+				dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
+				case "$dist_version" in
+					12)
+						dist_version="bookworm"
+					;;
+					11)
+						dist_version="bullseye"
+					;;
+					10)
+						dist_version="buster"
+					;;
+					9)
+						dist_version="stretch"
+					;;
+					8)
+						dist_version="jessie"
+					;;
+				esac
+			fi
+		fi
+	fi
+}
+
+do_install() {
+	echo "# Executing docker install script, commit: $SCRIPT_COMMIT_SHA"
+
+	if command_exists docker; then
+		cat >&2 <<-'EOF'
+			Warning: the "docker" command appears to already exist on this system.
+
+			If you already have Docker installed, this script can cause trouble, which is
+			why we're displaying this warning and provide the opportunity to cancel the
+			installation.
+
+			If you installed the current Docker package using this script and are using it
+			again to update Docker, you can safely ignore this message.
+
+			You may press Ctrl+C now to abort this script.
+		EOF
+		( set -x; sleep 20 )
+	fi
+
+	user="$(id -un 2>/dev/null || true)"
+
+	sh_c='sh -c'
+	if [ "$user" != 'root' ]; then
+		if command_exists sudo; then
+			sh_c='sudo -E sh -c'
+		elif command_exists su; then
+			sh_c='su -c'
+		else
+			cat >&2 <<-'EOF'
+			Error: this installer needs the ability to run commands as root.
+			We are unable to find either "sudo" or "su" available to make this happen.
+			EOF
+			exit 1
+		fi
+	fi
+
+	if is_dry_run; then
+		sh_c="echo"
+	fi
+
+	# perform some very rudimentary platform detection
+	lsb_dist=$( get_distribution )
+	lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')"
+
+	if is_wsl; then
+		echo
+		echo "WSL DETECTED: We recommend using Docker Desktop for Windows."
+		echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop/"
+		echo
+		cat >&2 <<-'EOF'
+
+			You may press Ctrl+C now to abort this script.
+		EOF
+		( set -x; sleep 20 )
+	fi
+
+	case "$lsb_dist" in
+
+		ubuntu)
+			if command_exists lsb_release; then
+				dist_version="$(lsb_release --codename | cut -f2)"
+			fi
+			if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then
+				dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")"
+			fi
+		;;
+
+		debian|raspbian)
+			dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
+			case "$dist_version" in
+				12)
+					dist_version="bookworm"
+				;;
+				11)
+					dist_version="bullseye"
+				;;
+				10)
+					dist_version="buster"
+				;;
+				9)
+					dist_version="stretch"
+				;;
+				8)
+					dist_version="jessie"
+				;;
+			esac
+		;;
+
+		centos|rhel)
+			if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
+				dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
+			fi
+		;;
+
+		*)
+			if command_exists lsb_release; then
+				dist_version="$(lsb_release --release | cut -f2)"
+			fi
+			if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
+				dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
+			fi
+		;;
+
+	esac
+
+	# Check if this is a forked Linux distro
+	check_forked
+
+	# Print deprecation warnings for distro versions that recently reached EOL,
+	# but may still be commonly used (especially LTS versions).
+	case "$lsb_dist.$dist_version" in
+		debian.stretch|debian.jessie)
+			deprecation_notice "$lsb_dist" "$dist_version"
+			;;
+		raspbian.stretch|raspbian.jessie)
+			deprecation_notice "$lsb_dist" "$dist_version"
+			;;
+		ubuntu.xenial|ubuntu.trusty)
+			deprecation_notice "$lsb_dist" "$dist_version"
+			;;
+		ubuntu.lunar|ubuntu.kinetic|ubuntu.impish|ubuntu.hirsute|ubuntu.groovy|ubuntu.eoan|ubuntu.disco|ubuntu.cosmic)
+			deprecation_notice "$lsb_dist" "$dist_version"
+			;;
+		fedora.*)
+			if [ "$dist_version" -lt 36 ]; then
+				deprecation_notice "$lsb_dist" "$dist_version"
+			fi
+			;;
+	esac
+
+	# Run setup for each distro accordingly
+	case "$lsb_dist" in
+		ubuntu|debian|raspbian)
+			pre_reqs="apt-transport-https ca-certificates curl"
+			apt_repo="deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] $DOWNLOAD_URL/linux/$lsb_dist $dist_version $CHANNEL"
+			(
+				if ! is_dry_run; then
+					set -x
+				fi
+				$sh_c 'apt-get update -qq >/dev/null'
+				$sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pre_reqs >/dev/null"
+				$sh_c 'install -m 0755 -d /etc/apt/keyrings'
+				$sh_c "curl -fsSL \"$DOWNLOAD_URL/linux/$lsb_dist/gpg\" -o /etc/apt/keyrings/docker.asc"
+				$sh_c "chmod a+r /etc/apt/keyrings/docker.asc"
+				$sh_c "echo \"$apt_repo\" > /etc/apt/sources.list.d/docker.list"
+				$sh_c 'apt-get update -qq >/dev/null'
+			)
+			pkg_version=""
+			if [ -n "$VERSION" ]; then
+				if is_dry_run; then
+					echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
+				else
+					# Will work for incomplete versions IE (17.12), but may not actually grab the "latest" if in the test channel
+					pkg_pattern="$(echo "$VERSION" | sed 's/-ce-/~ce~.*/g' | sed 's/-/.*/g')"
+					search_command="apt-cache madison docker-ce | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3"
+					pkg_version="$($sh_c "$search_command")"
+					echo "INFO: Searching repository for VERSION '$VERSION'"
+					echo "INFO: $search_command"
+					if [ -z "$pkg_version" ]; then
+						echo
+						echo "ERROR: '$VERSION' not found amongst apt-cache madison results"
+						echo
+						exit 1
+					fi
+					if version_gte "18.09"; then
+							search_command="apt-cache madison docker-ce-cli | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3"
+							echo "INFO: $search_command"
+							cli_pkg_version="=$($sh_c "$search_command")"
+					fi
+					pkg_version="=$pkg_version"
+				fi
+			fi
+			(
+				pkgs="docker-ce${pkg_version%=}"
+				if version_gte "18.09"; then
+						# older versions didn't ship the cli and containerd as separate packages
+						pkgs="$pkgs docker-ce-cli${cli_pkg_version%=} containerd.io"
+				fi
+				if version_gte "20.10"; then
+						pkgs="$pkgs docker-compose-plugin docker-ce-rootless-extras$pkg_version"
+				fi
+				if version_gte "23.0"; then
+						pkgs="$pkgs docker-buildx-plugin"
+				fi
+				if ! is_dry_run; then
+					set -x
+				fi
+				$sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pkgs >/dev/null"
+			)
+			echo_docker_as_nonroot
+			exit 0
+			;;
+		centos|fedora|rhel)
+			if [ "$(uname -m)" != "s390x" ] && [ "$lsb_dist" = "rhel" ]; then
+				echo "Packages for RHEL are currently only available for s390x."
+				exit 1
+			fi
+
+			if command_exists dnf; then
+				pkg_manager="dnf"
+				pkg_manager_flags="--best"
+				config_manager="dnf config-manager"
+				enable_channel_flag="--set-enabled"
+				disable_channel_flag="--set-disabled"
+				pre_reqs="dnf-plugins-core"
+			else
+				pkg_manager="yum"
+				pkg_manager_flags=""
+				config_manager="yum-config-manager"
+				enable_channel_flag="--enable"
+				disable_channel_flag="--disable"
+				pre_reqs="yum-utils"
+			fi
+
+			if [ "$lsb_dist" = "fedora" ]; then
+				pkg_suffix="fc$dist_version"
+			else
+				pkg_suffix="el"
+			fi
+			repo_file_url="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE"
+			(
+				if ! is_dry_run; then
+					set -x
+				fi
+				$sh_c "$pkg_manager $pkg_manager_flags install -y -q $pre_reqs"
+				$sh_c "$config_manager --add-repo $repo_file_url"
+
+				if [ "$CHANNEL" != "stable" ]; then
+					$sh_c "$config_manager $disable_channel_flag 'docker-ce-*'"
+					$sh_c "$config_manager $enable_channel_flag 'docker-ce-$CHANNEL'"
+				fi
+				$sh_c "$pkg_manager makecache"
+			)
+			pkg_version=""
+			if [ -n "$VERSION" ]; then
+				if is_dry_run; then
+					echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
+				else
+					pkg_pattern="$(echo "$VERSION" | sed 's/-ce-/\\\\.ce.*/g' | sed 's/-/.*/g').*$pkg_suffix"
+					search_command="$pkg_manager list --showduplicates docker-ce | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'"
+					pkg_version="$($sh_c "$search_command")"
+					echo "INFO: Searching repository for VERSION '$VERSION'"
+					echo "INFO: $search_command"
+					if [ -z "$pkg_version" ]; then
+						echo
+						echo "ERROR: '$VERSION' not found amongst $pkg_manager list results"
+						echo
+						exit 1
+					fi
+					if version_gte "18.09"; then
+						# older versions don't support a cli package
+						search_command="$pkg_manager list --showduplicates docker-ce-cli | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'"
+						cli_pkg_version="$($sh_c "$search_command" | cut -d':' -f 2)"
+					fi
+					# Cut out the epoch and prefix with a '-'
+					pkg_version="-$(echo "$pkg_version" | cut -d':' -f 2)"
+				fi
+			fi
+			(
+				pkgs="docker-ce$pkg_version"
+				if version_gte "18.09"; then
+					# older versions didn't ship the cli and containerd as separate packages
+					if [ -n "$cli_pkg_version" ]; then
+						pkgs="$pkgs docker-ce-cli-$cli_pkg_version containerd.io"
+					else
+						pkgs="$pkgs docker-ce-cli containerd.io"
+					fi
+				fi
+				if version_gte "20.10"; then
+					pkgs="$pkgs docker-compose-plugin docker-ce-rootless-extras$pkg_version"
+				fi
+				if version_gte "23.0"; then
+						pkgs="$pkgs docker-buildx-plugin"
+				fi
+				if ! is_dry_run; then
+					set -x
+				fi
+				$sh_c "$pkg_manager $pkg_manager_flags install -y -q $pkgs"
+			)
+			echo_docker_as_nonroot
+			exit 0
+			;;
+		sles)
+			if [ "$(uname -m)" != "s390x" ]; then
+				echo "Packages for SLES are currently only available for s390x"
+				exit 1
+			fi
+			repo_file_url="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE"
+			pre_reqs="ca-certificates curl libseccomp2 awk"
+			(
+				if ! is_dry_run; then
+					set -x
+				fi
+				$sh_c "zypper install -y $pre_reqs"
+				$sh_c "zypper addrepo $repo_file_url"
+				if ! is_dry_run; then
+						cat >&2 <<-'EOF'
+						WARNING!!
+						openSUSE repository (https://download.opensuse.org/repositories/security:/SELinux) will be enabled now.
+						Do you wish to continue?
+						You may press Ctrl+C now to abort this script.
+						EOF
+						( set -x; sleep 30 )
+				fi
+				opensuse_repo="https://download.opensuse.org/repositories/security:/SELinux/openSUSE_Factory/security:SELinux.repo"
+				$sh_c "zypper addrepo $opensuse_repo"
+				$sh_c "zypper --gpg-auto-import-keys refresh"
+				$sh_c "zypper lr -d"
+			)
+			pkg_version=""
+			if [ -n "$VERSION" ]; then
+				if is_dry_run; then
+					echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
+				else
+					pkg_pattern="$(echo "$VERSION" | sed 's/-ce-/\\\\.ce.*/g' | sed 's/-/.*/g')"
+					search_command="zypper search -s --match-exact 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'"
+					pkg_version="$($sh_c "$search_command")"
+					echo "INFO: Searching repository for VERSION '$VERSION'"
+					echo "INFO: $search_command"
+					if [ -z "$pkg_version" ]; then
+						echo
+						echo "ERROR: '$VERSION' not found amongst zypper list results"
+						echo
+						exit 1
+					fi
+					search_command="zypper search -s --match-exact 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'"
+					# It's okay for cli_pkg_version to be blank, since older versions don't support a cli package
+					cli_pkg_version="$($sh_c "$search_command")"
+					pkg_version="-$pkg_version"
+				fi
+			fi
+			(
+				pkgs="docker-ce$pkg_version"
+				if version_gte "18.09"; then
+					if [ -n "$cli_pkg_version" ]; then
+						# older versions didn't ship the cli and containerd as separate packages
+						pkgs="$pkgs docker-ce-cli-$cli_pkg_version containerd.io"
+					else
+						pkgs="$pkgs docker-ce-cli containerd.io"
+					fi
+				fi
+				if version_gte "20.10"; then
+					pkgs="$pkgs docker-compose-plugin docker-ce-rootless-extras$pkg_version"
+				fi
+				if version_gte "23.0"; then
+						pkgs="$pkgs docker-buildx-plugin"
+				fi
+				if ! is_dry_run; then
+					set -x
+				fi
+				$sh_c "zypper -q install -y $pkgs"
+			)
+			echo_docker_as_nonroot
+			exit 0
+			;;
+		*)
+			if [ -z "$lsb_dist" ]; then
+				if is_darwin; then
+					echo
+					echo "ERROR: Unsupported operating system 'macOS'"
+					echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop"
+					echo
+					exit 1
+				fi
+			fi
+			echo
+			echo "ERROR: Unsupported distribution '$lsb_dist'"
+			echo
+			exit 1
+			;;
+	esac
+	exit 1
+}
+
+# wrapped up in a function so that we have some protection against only getting
+# half the file during "curl | sh"
+do_install

From c6e0fa4e4a724370f1111d898c6aa7814ef60953 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 12:39:55 +0200
Subject: [PATCH 18/68] artifact permissions

---
 .github/workflows/e2e.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index f1e01625..7ea1e304 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -37,8 +37,9 @@ jobs:
       - name: Save k8s-snap image
         run: |
           sudo docker save -o k8s-snap-image.tar k8s-snap:dev
+          ls -la
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: e2e-images
           path: |
@@ -59,7 +60,7 @@ jobs:
           sudo snap install go --classic
           sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true
       - name: Download artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: e2e-images
           path: .

From c0a4b69f9e43c04a246b44a9f699351ef3b6a1d4 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 13:14:52 +0200
Subject: [PATCH 19/68] fix permissions

---
 .github/workflows/e2e.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 7ea1e304..f1c43d00 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -32,18 +32,19 @@ jobs:
         run: sudo docker builder prune -f
       - name: Save provider image
         run: |
-          sudo docker image ls
-          sudo docker save -o provider-image.tar $(sudo docker images -q)
+          sudo docker save -o provider-images.tar $(sudo docker images -q)
+          sudo chmod 775 provider-images.tar
       - name: Save k8s-snap image
         run: |
           sudo docker save -o k8s-snap-image.tar k8s-snap:dev
+          sudo chmod 775 k8s-snap-image.tar
           ls -la
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: e2e-images
           path: |
-            provider-image.tar
+            provider-images.tar
             k8s-snap-image.tar
 
   run-e2e-tests:
@@ -65,7 +66,7 @@ jobs:
           name: e2e-images
           path: .
       - name: Load provider image
-        run: sudo docker load -i provider-image.tar
+        run: sudo docker load -i provider-images.tar
       - name: Load k8s-snap image
         run: sudo docker load -i k8s-snap-image.tar
       - name: Run e2e tests

From f95ab2956f060c9c96ab8e35aa7e29042def4a3b Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 13:44:24 +0200
Subject: [PATCH 20/68] fix provider image artifacts

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index f1c43d00..ffe04bfc 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -32,7 +32,7 @@ jobs:
         run: sudo docker builder prune -f
       - name: Save provider image
         run: |
-          sudo docker save -o provider-images.tar $(sudo docker images -q)
+          sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev
           sudo chmod 775 provider-images.tar
       - name: Save k8s-snap image
         run: |

From eca2cd6151c681fe17be8760991222cc8e7304d4 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 14:20:11 +0200
Subject: [PATCH 21/68] still no space left ...

---
 .github/workflows/e2e.yaml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index ffe04bfc..636b770f 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -53,6 +53,16 @@ jobs:
     needs: build-e2e-images
 
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@v10
+        with:
+          root-reserve-mb: 512
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
       - name: Check out repo
         uses: actions/checkout@v4
       - name: Install requirements
@@ -60,6 +70,7 @@ jobs:
           sudo apt install make
           sudo snap install go --classic
           sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true
+          df -h
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
@@ -69,5 +80,10 @@ jobs:
         run: sudo docker load -i provider-images.tar
       - name: Load k8s-snap image
         run: sudo docker load -i k8s-snap-image.tar
+      - name: Space check
+        run: |
+          df -h
+          rm -rf provider-images.tar k8s-snap-image.tar
+          df -h
       - name: Run e2e tests
         run: sudo make test-e2e

From 1e7861e0197b04fbcb06517f96f52c6e5dd3205b Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 14:59:55 +0200
Subject: [PATCH 22/68] another no space left try...

---
 .github/workflows/e2e.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 636b770f..f06712ad 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -76,14 +76,15 @@ jobs:
         with:
           name: e2e-images
           path: .
-      - name: Load provider image
-        run: sudo docker load -i provider-images.tar
-      - name: Load k8s-snap image
-        run: sudo docker load -i k8s-snap-image.tar
       - name: Space check
         run: |
+          ls
           df -h
           rm -rf provider-images.tar k8s-snap-image.tar
           df -h
+      - name: Load provider image
+        run: sudo docker load -i provider-images.tar
+      - name: Load k8s-snap image
+        run: sudo docker load -i k8s-snap-image.tar
       - name: Run e2e tests
         run: sudo make test-e2e

From f4716c0438adf5437daa0ebddfd9234e06712038 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 15:01:36 +0200
Subject: [PATCH 23/68] try

---
 .github/workflows/e2e.yaml | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index f06712ad..e77ebb13 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -80,11 +80,15 @@ jobs:
         run: |
           ls
           df -h
-          rm -rf provider-images.tar k8s-snap-image.tar
-          df -h
       - name: Load provider image
-        run: sudo docker load -i provider-images.tar
+        run: |
+          sudo docker load -i provider-images.tar
+          sudo rm -rf provider-images.tar
+          df -h
       - name: Load k8s-snap image
-        run: sudo docker load -i k8s-snap-image.tar
+        run: |
+          sudo docker load -i k8s-snap-image.tar
+          sudo rm -rf k8s-snap-image.tar
+          df -h
       - name: Run e2e tests
         run: sudo make test-e2e

From 89643622d1ddd9ee0c365f46fcc42ab6cc739c74 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 15:21:21 +0200
Subject: [PATCH 24/68] still no space...

---
 .github/workflows/e2e.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index e77ebb13..5053ae8d 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -80,6 +80,7 @@ jobs:
         run: |
           ls
           df -h
+          pwd
       - name: Load provider image
         run: |
           sudo docker load -i provider-images.tar

From 12ae2a2e2707c441253d1fe5b6fd2a36384176d7 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 16:19:44 +0200
Subject: [PATCH 25/68] merge steps again

---
 .github/workflows/e2e.yaml | 44 +-------------------------------------
 1 file changed, 1 insertion(+), 43 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 5053ae8d..dbcb8e87 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -30,29 +30,6 @@ jobs:
       - name: Cleanup space
         # Github runners are tight on space and will fail if we don't clean up before.
         run: sudo docker builder prune -f
-      - name: Save provider image
-        run: |
-          sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev
-          sudo chmod 775 provider-images.tar
-      - name: Save k8s-snap image
-        run: |
-          sudo docker save -o k8s-snap-image.tar k8s-snap:dev
-          sudo chmod 775 k8s-snap-image.tar
-          ls -la
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: e2e-images
-          path: |
-            provider-images.tar
-            k8s-snap-image.tar
-
-  run-e2e-tests:
-    name: Run E2E Tests
-    runs-on: ubuntu-22.04
-    needs: build-e2e-images
-
-    steps:
       - name: Maximize build space
         uses: easimon/maximize-build-space@v10
         with:
@@ -63,33 +40,14 @@ jobs:
           remove-haskell: 'true'
           remove-codeql: 'true'
           remove-docker-images: 'true'
-      - name: Check out repo
-        uses: actions/checkout@v4
-      - name: Install requirements
+      - name: Create docker network
         run: |
-          sudo apt install make
-          sudo snap install go --classic
           sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true
           df -h
-      - name: Download artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e-images
-          path: .
       - name: Space check
         run: |
           ls
           df -h
           pwd
-      - name: Load provider image
-        run: |
-          sudo docker load -i provider-images.tar
-          sudo rm -rf provider-images.tar
-          df -h
-      - name: Load k8s-snap image
-        run: |
-          sudo docker load -i k8s-snap-image.tar
-          sudo rm -rf k8s-snap-image.tar
-          df -h
       - name: Run e2e tests
         run: sudo make test-e2e

From fb0bd54bdadf24fcc8df8d20314c1c72760eccad Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 16:51:03 +0200
Subject: [PATCH 26/68] checkout repo

---
 .github/workflows/e2e.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index dbcb8e87..76744266 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -49,5 +49,10 @@ jobs:
           ls
           df -h
           pwd
+      - name: Check out repo
+        uses: actions/checkout@v4
       - name: Run e2e tests
-        run: sudo make test-e2e
+        run: |
+          ls
+          pwd
+          sudo make test-e2e

From 55374ac3c613ebc59b803de7ecd069f5a5194fc2 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 17:12:47 +0200
Subject: [PATCH 27/68] remove unnecessary files

---
 .github/workflows/e2e.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 76744266..448480ce 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -29,7 +29,12 @@ jobs:
           sudo docker build . -t k8s-snap:dev
       - name: Cleanup space
         # Github runners are tight on space and will fail if we don't clean up before.
-        run: sudo docker builder prune -f
+        run: |
+          sudo docker builder prune -f
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Maximize build space
         uses: easimon/maximize-build-space@v10
         with:

From cdceb3fe078b1bc473bd31b1540dc4ca894bd28a Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 17:47:30 +0200
Subject: [PATCH 28/68] use self hosted runners

---
 .github/workflows/e2e.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 448480ce..2c18da17 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -9,8 +9,7 @@ permissions:
 jobs:
   build-e2e-images:
     name: Build E2E Images
-    runs-on: ubuntu-22.04
-
+    runs-on: [self-hosted, linux, x86]
     steps:
       - name: Check out repo
         uses: actions/checkout@v4

From 10890acdaba2cd43fe910c46af88ce1e1f6aa1d3 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 18:11:11 +0200
Subject: [PATCH 29/68] broaden self-hosted tags

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 2c18da17..9092dcc4 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -9,7 +9,7 @@ permissions:
 jobs:
   build-e2e-images:
     name: Build E2E Images
-    runs-on: [self-hosted, linux, x86]
+    runs-on: self-hosted
     steps:
       - name: Check out repo
         uses: actions/checkout@v4

From 1ab80706cc1c581782fa36298453dbdc0df28e84 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 3 Jul 2024 18:15:45 +0200
Subject: [PATCH 30/68] try other tags

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 9092dcc4..31b1b275 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -9,7 +9,7 @@ permissions:
 jobs:
   build-e2e-images:
     name: Build E2E Images
-    runs-on: self-hosted
+    runs-on: [self-hosted, jammy, xlarge]
     steps:
       - name: Check out repo
         uses: actions/checkout@v4

From 24a330cf78326c40fa625164f92be4072828b57f Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 09:58:34 +0200
Subject: [PATCH 31/68] another self-hosted runner config

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 31b1b275..37966a2f 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -9,7 +9,7 @@ permissions:
 jobs:
   build-e2e-images:
     name: Build E2E Images
-    runs-on: [self-hosted, jammy, xlarge]
+    runs-on: [self-hosted, linux, X64, jammy, xlarge]
     steps:
       - name: Check out repo
         uses: actions/checkout@v4

From 4083966306be4dff386882a8125de02b4750abe6 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 10:12:17 +0200
Subject: [PATCH 32/68] use smaller machine

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 37966a2f..f145ec52 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -9,7 +9,7 @@ permissions:
 jobs:
   build-e2e-images:
     name: Build E2E Images
-    runs-on: [self-hosted, linux, X64, jammy, xlarge]
+    runs-on: [self-hosted, linux, X64, jammy, large]
     steps:
       - name: Check out repo
         uses: actions/checkout@v4

From 99e7b9ea17e2bdfec0fe79ddb73f960346184995 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 12:09:02 +0200
Subject: [PATCH 33/68] install docker

---
 .github/workflows/e2e.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index f145ec52..9e7c01d3 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -8,7 +8,7 @@ permissions:
 
 jobs:
   build-e2e-images:
-    name: Build E2E Images
+    name: Build & Run E2E Images
     runs-on: [self-hosted, linux, X64, jammy, large]
     steps:
       - name: Check out repo
@@ -17,6 +17,7 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
+          sudo snap install docker
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Cleanup space

From c2eee34ca40477b07c4d28ca8feec0f0d55ed067 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 12:14:54 +0200
Subject: [PATCH 34/68] install docker via apt

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 9e7c01d3..c9bdb335 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -17,7 +17,7 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
-          sudo snap install docker
+          sudo apt install docker-ce
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Cleanup space

From 2ea46334f5704a8551b8cb88c72f978f3a815f99 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 12:20:16 +0200
Subject: [PATCH 35/68] missing apt update

---
 .github/workflows/e2e.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index c9bdb335..724b9c6f 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -17,6 +17,7 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
+          sudo apt update
           sudo apt install docker-ce
       - name: Build provider images
         run: sudo make docker-build-e2e

From 2128e977be4be97af4a6d952dfad7112860a6c83 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 12:27:36 +0200
Subject: [PATCH 36/68] convenience script

---
 .github/workflows/e2e.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 724b9c6f..4ed11af4 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -17,8 +17,8 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
-          sudo apt update
-          sudo apt install docker-ce
+          curl -fsSL https://get.docker.com -o get-docker.sh
+          sudo sh get-docker.sh
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Cleanup space

From a739a3367f9a6b928ad928b6a581e8a34c433c5e Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 12:46:52 +0200
Subject: [PATCH 37/68] update

---
 .github/workflows/e2e.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 4ed11af4..292541bb 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -17,8 +17,8 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
-          curl -fsSL https://get.docker.com -o get-docker.sh
-          sudo sh get-docker.sh
+          sudo apt update
+          sudo apt install docker-buildx-plugin
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Cleanup space

From df33e37d0543cf7d044418ca59f01b9ba448a5f5 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 12:54:25 +0200
Subject: [PATCH 38/68] update

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 292541bb..c1de802f 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -17,7 +17,7 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
-          sudo apt update
+          sudo apt update && sudo apt upgrade -y
           sudo apt install docker-buildx-plugin
       - name: Build provider images
         run: sudo make docker-build-e2e

From 709c1c2c5617a6730d31b91d442535f6f180407f Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 13:14:33 +0200
Subject: [PATCH 39/68] more docker buildx trickery

---
 .github/workflows/e2e.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index c1de802f..e74b7c61 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -17,8 +17,9 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
-          sudo apt update && sudo apt upgrade -y
-          sudo apt install docker-buildx-plugin
+          curl -Lo $HOME/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.15.1/buildx-v0.15.1.linux-amd64
+          sudo chmod +x ~/.docker/cli-plugins/docker-buildx
+          sudo docker buildx install
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Cleanup space

From 73e58669e264426de85a628a2bc0d9da8e300e88 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 13:21:26 +0200
Subject: [PATCH 40/68] getting there

---
 .github/workflows/e2e.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index e74b7c61..a4d1a46a 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -17,6 +17,7 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
+          mkdir -p $HOME/.docker/cli-plugins
           curl -Lo $HOME/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.15.1/buildx-v0.15.1.linux-amd64
           sudo chmod +x ~/.docker/cli-plugins/docker-buildx
           sudo docker buildx install

From 3cc0f728e31c4189f6e08722f5d0ef9a33563ee9 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 13:25:29 +0200
Subject: [PATCH 41/68] docker

---
 .github/workflows/e2e.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index a4d1a46a..0903f6c9 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -17,10 +17,7 @@ jobs:
         run: |
           sudo apt install make
           sudo snap install go --classic
-          mkdir -p $HOME/.docker/cli-plugins
-          curl -Lo $HOME/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.15.1/buildx-v0.15.1.linux-amd64
-          sudo chmod +x ~/.docker/cli-plugins/docker-buildx
-          sudo docker buildx install
+          sudo apt install docker-buildx
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Cleanup space

From 9626058abe709a7e948a0e01f58471c9b10bd71f Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 14:03:23 +0200
Subject: [PATCH 42/68] remove maximize build space

---
 .github/workflows/e2e.yaml | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 0903f6c9..5eacf1be 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -35,16 +35,6 @@ jobs:
           sudo rm -rf /opt/ghc
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@v10
-        with:
-          root-reserve-mb: 512
-          swap-size-mb: 1024
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-          remove-codeql: 'true'
-          remove-docker-images: 'true'
       - name: Create docker network
         run: |
           sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true

From 34d4e6d5f5be288cd18323ee1617f484e0cbdf81 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 14:31:39 +0200
Subject: [PATCH 43/68] remove double checkout

---
 .github/workflows/e2e.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 5eacf1be..fad62c2c 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -44,8 +44,6 @@ jobs:
           ls
           df -h
           pwd
-      - name: Check out repo
-        uses: actions/checkout@v4
       - name: Run e2e tests
         run: |
           ls

From 363627857df7c126cffda40892fbd7dc458ab206 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 15:01:08 +0200
Subject: [PATCH 44/68] provide kubectl

---
 .github/workflows/e2e.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index fad62c2c..4555c656 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -18,6 +18,7 @@ jobs:
           sudo apt install make
           sudo snap install go --classic
           sudo apt install docker-buildx
+          sudo snap install kubectl --classic
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Cleanup space

From 5d705011c1c70f1ed97f2030f4d931c2bf37252b Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 16:22:48 +0200
Subject: [PATCH 45/68] add tmate session

---
 .github/workflows/e2e.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 4555c656..ec634dc8 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -49,4 +49,7 @@ jobs:
         run: |
           ls
           pwd
-          sudo make test-e2e
+          sudo make test-e2
+      - name: Setup tmate session
+      if: ${{ failure() }}
+      uses: canonical/action-tmate@maine

From bf13e22fe22c59d787d3dc0c83115b426afca4fe Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 17:05:35 +0200
Subject: [PATCH 46/68] fix tmate session

---
 .github/workflows/e2e.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index ec634dc8..40d59080 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -51,5 +51,5 @@ jobs:
           pwd
           sudo make test-e2
       - name: Setup tmate session
-      if: ${{ failure() }}
-      uses: canonical/action-tmate@maine
+        if: ${{ failure() }}
+        uses: canonical/action-tmate@maine

From 1281d5d38a6e56845372402de59edb54acdcdbaa Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 17:07:34 +0200
Subject: [PATCH 47/68] another fix

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 40d59080..7bb61d14 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -52,4 +52,4 @@ jobs:
           sudo make test-e2
       - name: Setup tmate session
         if: ${{ failure() }}
-        uses: canonical/action-tmate@maine
+        uses: canonical/action-tmate@main

From 84766d41f891ccdc1a16a640307eeeaf0b5b4e57 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 17:51:49 +0200
Subject: [PATCH 48/68] fix make target

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 7bb61d14..6e7de9f4 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -49,7 +49,7 @@ jobs:
         run: |
           ls
           pwd
-          sudo make test-e2
+          sudo make test-e2e
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: canonical/action-tmate@main

From 3022b79043914cfe4c7e541f15f1c0fa96e4cbdb Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 19:20:02 +0200
Subject: [PATCH 49/68] more debugging

---
 .github/workflows/e2e.yaml      |   4 +-
 test/e2e/docker_logcollector.go | 195 ++++++++++++++++++++++++++++++++
 test/e2e/e2e_suite_test.go      |   2 +-
 3 files changed, 197 insertions(+), 4 deletions(-)
 create mode 100644 test/e2e/docker_logcollector.go

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 6e7de9f4..c20f5c89 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -47,9 +47,7 @@ jobs:
           pwd
       - name: Run e2e tests
         run: |
-          ls
-          pwd
-          sudo make test-e2e
+          sudo GINKGO_FOCUS="Workload cluster scaling" SKIP_RESOURCE_CLEANUP=true make test-e2e
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: canonical/action-tmate@main
diff --git a/test/e2e/docker_logcollector.go b/test/e2e/docker_logcollector.go
new file mode 100644
index 00000000..2ee5996d
--- /dev/null
+++ b/test/e2e/docker_logcollector.go
@@ -0,0 +1,195 @@
+// Copied from CAPI e2e framework and modified to add pebble logs.
+
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	osExec "os/exec"
+	"path/filepath"
+	"strings"
+
+	kerrors "k8s.io/apimachinery/pkg/util/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kind/pkg/errors"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
+	"sigs.k8s.io/cluster-api/test/infrastructure/container"
+)
+
+// DockerLogCollector collect logs from a CAPD workload cluster.
+type DockerLogCollector struct{}
+
+// machineContainerName return a container name using the same rule used in CAPD.
+// NOTE: if the cluster name is already included in the machine name, the cluster name is not add thus
+// avoiding \"sethostname: invalid argument\"  errors due to container name too long.
+func machineContainerName(cluster, machine string) string {
+	if strings.HasPrefix(machine, cluster) {
+		return machine
+	}
+	return fmt.Sprintf("%s-%s", cluster, machine)
+}
+
+func (k DockerLogCollector) CollectMachineLog(ctx context.Context, _ client.Client, m *clusterv1.Machine, outputPath string) error {
+	containerName := machineContainerName(m.Spec.ClusterName, m.Name)
+	containerRuntime, err := container.NewDockerClient()
+	if err != nil {
+		return err
+	}
+	ctx = container.RuntimeInto(ctx, containerRuntime)
+	return k.collectLogsFromNode(ctx, outputPath, containerName)
+}
+
+func (k DockerLogCollector) CollectMachinePoolLog(ctx context.Context, _ client.Client, m *expv1.MachinePool, outputPath string) error {
+	containerRuntime, err := container.NewDockerClient()
+	if err != nil {
+		return err
+	}
+	ctx = container.RuntimeInto(ctx, containerRuntime)
+
+	var errs []error
+	for _, instance := range m.Status.NodeRefs {
+		containerName := machineContainerName(m.Spec.ClusterName, instance.Name)
+		if err := k.collectLogsFromNode(ctx, filepath.Join(outputPath, instance.Name), containerName); err != nil {
+			// collecting logs is best effort so we proceed to the next instance even if we encounter an error.
+			errs = append(errs, err)
+		}
+	}
+
+	return kerrors.NewAggregate(errs)
+}
+
+func (k DockerLogCollector) CollectInfrastructureLogs(ctx context.Context, _ client.Client, c *clusterv1.Cluster, outputPath string) error {
+	containerRuntime, err := container.NewDockerClient()
+	if err != nil {
+		return err
+	}
+	ctx = container.RuntimeInto(ctx, containerRuntime)
+
+	lbContainerName := fmt.Sprintf("%s-lb", c.GetName())
+
+	f, err := fileOnHost(filepath.Join(outputPath, fmt.Sprintf("%s.log", lbContainerName)))
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	return containerRuntime.ContainerDebugInfo(ctx, lbContainerName, f)
+}
+
+func (k DockerLogCollector) collectLogsFromNode(ctx context.Context, outputPath string, containerName string) error {
+	containerRuntime, err := container.RuntimeFrom(ctx)
+	if err != nil {
+		return errors.Wrap(err, "Failed to collect logs from node")
+	}
+
+	execToPathFn := func(outputFileName, command string, args ...string) func() error {
+		return func() error {
+			f, err := fileOnHost(filepath.Join(outputPath, outputFileName))
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+			execConfig := container.ExecContainerInput{
+				OutputBuffer: f,
+			}
+			return containerRuntime.ExecContainer(ctx, containerName, &execConfig, command, args...)
+		}
+	}
+	copyDirFn := func(containerDir, dirName string) func() error {
+		return func() error {
+			f, err := os.CreateTemp("", containerName)
+			if err != nil {
+				return err
+			}
+
+			tempfileName := f.Name()
+			outputDir := filepath.Join(outputPath, dirName)
+
+			defer os.Remove(tempfileName)
+
+			var execErr string
+			execConfig := container.ExecContainerInput{
+				OutputBuffer: f,
+				ErrorBuffer:  bytes.NewBufferString(execErr),
+			}
+			err = containerRuntime.ExecContainer(
+				ctx,
+				containerName,
+				&execConfig,
+				"tar", "--hard-dereference", "--dereference", "--directory", containerDir, "--create", "--file", "-", ".",
+			)
+			if err != nil {
+				return errors.Wrapf(err, execErr)
+			}
+
+			err = os.MkdirAll(outputDir, os.ModePerm)
+			if err != nil {
+				return err
+			}
+
+			return osExec.Command("tar", "--extract", "--file", tempfileName, "--directory", outputDir).Run() //nolint:gosec // We don't care about command injection here.
+		}
+	}
+	return errors.AggregateConcurrent([]func() error{
+		execToPathFn(
+			"journal.log",
+			"journalctl", "--no-pager", "--output=short-precise",
+		),
+		execToPathFn(
+			"kern.log",
+			"journalctl", "--no-pager", "--output=short-precise", "-k",
+		),
+		execToPathFn(
+			"kubelet-version.txt",
+			"kubelet", "--version",
+		),
+		execToPathFn(
+			"kubelet.log",
+			"journalctl", "--no-pager", "--output=short-precise", "-u", "kubelet.service",
+		),
+		execToPathFn(
+			"containerd-info.txt",
+			"crictl", "info",
+		),
+		execToPathFn(
+			"containerd.log",
+			"journalctl", "--no-pager", "--output=short-precise", "-u", "containerd.service",
+		),
+		execToPathFn(
+			"pebble.log",
+			"pebble", "logs", "-n", "all",
+		),
+		copyDirFn("/var/log/pods", "pods"),
+	})
+}
+
+// fileOnHost is a helper to create a file at path
+// even if the parent directory doesn't exist
+// in which case it will be created with ModePerm.
+func fileOnHost(path string) (*os.File, error) {
+	if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+		return nil, err
+	}
+	return os.Create(path) //nolint:gosec // No security issue: path is safe.
+}
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index 9dd78e93..7db04ab5 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -170,7 +170,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	kubeconfigPath := parts[3]
 
 	e2eConfig = loadE2EConfig(configPath)
-	bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(), framework.WithMachineLogCollector(framework.DockerLogCollector{}))
+	bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(), framework.WithMachineLogCollector(DockerLogCollector{}))
 })
 
 // Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads).

From 5aea85916c399620a24cf7444e026d5ffde80ce3 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 19:56:57 +0200
Subject: [PATCH 50/68] gci

---
 test/e2e/docker_logcollector.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/test/e2e/docker_logcollector.go b/test/e2e/docker_logcollector.go
index 2ee5996d..5c2dd555 100644
--- a/test/e2e/docker_logcollector.go
+++ b/test/e2e/docker_logcollector.go
@@ -28,12 +28,11 @@ import (
 	"strings"
 
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/kind/pkg/errors"
-
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/infrastructure/container"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kind/pkg/errors"
 )
 
 // DockerLogCollector collect logs from a CAPD workload cluster.

From 571edcfd476dc24f8c90190e5cd2627b9b8e9f3f Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 20:22:10 +0200
Subject: [PATCH 51/68] docker

---
 .golangci.yml                   | 1 +
 test/e2e/docker_logcollector.go | 5 ++---
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.golangci.yml b/.golangci.yml
index b5b83e39..317c18df 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -90,6 +90,7 @@ linters-settings:
 
           - github.com/google/uuid
           - github.com/pkg/errors
+          - sigs.k8s.io/kind/pkg/errors
   gci:
     sections:
       - standard
diff --git a/test/e2e/docker_logcollector.go b/test/e2e/docker_logcollector.go
index 5c2dd555..3ac2434c 100644
--- a/test/e2e/docker_logcollector.go
+++ b/test/e2e/docker_logcollector.go
@@ -101,7 +101,6 @@ func (k DockerLogCollector) collectLogsFromNode(ctx context.Context, outputPath
 	if err != nil {
 		return errors.Wrap(err, "Failed to collect logs from node")
 	}
-
 	execToPathFn := func(outputFileName, command string, args ...string) func() error {
 		return func() error {
 			f, err := fileOnHost(filepath.Join(outputPath, outputFileName))
@@ -147,7 +146,7 @@ func (k DockerLogCollector) collectLogsFromNode(ctx context.Context, outputPath
 				return err
 			}
 
-			return osExec.Command("tar", "--extract", "--file", tempfileName, "--directory", outputDir).Run() //nolint:gosec // We don't care about command injection here.
+			return osExec.Command("tar", "--extract", "--file", tempfileName, "--directory", outputDir).Run()
 		}
 	}
 	return errors.AggregateConcurrent([]func() error{
@@ -190,5 +189,5 @@ func fileOnHost(path string) (*os.File, error) {
 	if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
 		return nil, err
 	}
-	return os.Create(path) //nolint:gosec // No security issue: path is safe.
+	return os.Create(path)
 }

From 421c4829e20847ebe052a59a5334853e09c48ac1 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 4 Jul 2024 21:23:00 +0200
Subject: [PATCH 52/68] increase inotifier

---
 .github/workflows/e2e.yaml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index c20f5c89..0fbf6ad8 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -9,7 +9,7 @@ permissions:
 jobs:
   build-e2e-images:
     name: Build & Run E2E Images
-    runs-on: [self-hosted, linux, X64, jammy, large]
+    runs-on: [self-hosted, linux, X64, jammy, xlarge]
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
@@ -45,9 +45,14 @@ jobs:
           ls
           df -h
           pwd
+      - name: Increase inotify watches
+        run: |
+          # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files
+          sysctl fs.inotify.max_user_watches=1048576
+          sysctl fs.inotify.max_user_instances=8192
       - name: Run e2e tests
         run: |
-          sudo GINKGO_FOCUS="Workload cluster scaling" SKIP_RESOURCE_CLEANUP=true make test-e2e
+          sudo make test-e2e
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: canonical/action-tmate@main

From b009865e1dbe1f7b9b04d8669aa26db117622d0d Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Fri, 5 Jul 2024 07:58:01 +0200
Subject: [PATCH 53/68] limit test space

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 0fbf6ad8..4adb7a01 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -52,7 +52,7 @@ jobs:
           sysctl fs.inotify.max_user_instances=8192
       - name: Run e2e tests
         run: |
-          sudo make test-e2e
+          sudo GINKGO_FOCUS="Workload cluster scaling" SKIP_RESOURCE_CLEANUP=true make test-e2e
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: canonical/action-tmate@main

From 1f12d2d10bbe2f84abeef68a55fb59028fcb6f38 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Fri, 5 Jul 2024 09:27:27 +0200
Subject: [PATCH 54/68] sudo for inotifier

---
 .github/workflows/e2e.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 4adb7a01..af41cd50 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -48,11 +48,11 @@ jobs:
       - name: Increase inotify watches
         run: |
           # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files
-          sysctl fs.inotify.max_user_watches=1048576
-          sysctl fs.inotify.max_user_instances=8192
+          sudo sysctl fs.inotify.max_user_watches=1048576
+          sudo sysctl fs.inotify.max_user_instances=8192
       - name: Run e2e tests
         run: |
-          sudo GINKGO_FOCUS="Workload cluster scaling" SKIP_RESOURCE_CLEANUP=true make test-e2e
+          sudo GINKGO_FOCUS="Workload cluster creation" SKIP_RESOURCE_CLEANUP=true make test-e2e
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: canonical/action-tmate@main

From 4a465d2c5ce4b055ee6121bac5e8b053ac590f9d Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Fri, 5 Jul 2024 11:33:46 +0200
Subject: [PATCH 55/68] Run all tests

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index af41cd50..f42c9127 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -52,7 +52,7 @@ jobs:
           sudo sysctl fs.inotify.max_user_instances=8192
       - name: Run e2e tests
         run: |
-          sudo GINKGO_FOCUS="Workload cluster creation" SKIP_RESOURCE_CLEANUP=true make test-e2e
+          sudo SKIP_RESOURCE_CLEANUP=true make test-e2e
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: canonical/action-tmate@main

From ff9110221d470c96bf0d99ec900ff6ff2df57c8b Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Fri, 5 Jul 2024 15:34:50 +0200
Subject: [PATCH 56/68] another round

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index f42c9127..dc5a7b99 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -52,7 +52,7 @@ jobs:
           sudo sysctl fs.inotify.max_user_instances=8192
       - name: Run e2e tests
         run: |
-          sudo SKIP_RESOURCE_CLEANUP=true make test-e2e
+          sudo GINKGO_FOCUS="Workload cluster upgrade" SKIP_RESOURCE_CLEANUP=true make test-e2e
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: canonical/action-tmate@main

From 8b1911fa9c5f2dd58e71e3a886f29a3f84027023 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Mon, 8 Jul 2024 09:49:18 +0200
Subject: [PATCH 57/68] add remediation tests

---
 .github/workflows/e2e.yaml | 68 +++++++++++++++++++++++++++-----------
 1 file changed, 49 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index dc5a7b99..7c924773 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -21,30 +21,60 @@ jobs:
           sudo snap install kubectl --classic
       - name: Build provider images
         run: sudo make docker-build-e2e
-      - name: Cleanup space
-        # Github runners are tight on space and will fail if we don't clean up before.
-        run: sudo docker builder prune -f
-      - name: Build images
+      - name: Build k8s-snap image
         run: |
           cd templates/docker
           sudo docker build . -t k8s-snap:dev
-      - name: Cleanup space
-        # Github runners are tight on space and will fail if we don't clean up before.
-        run: |
-          sudo docker builder prune -f
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /opt/ghc
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+      - name: Save provider image
+        run: |
+          sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev
+          sudo chmod 775 provider-images.tar
+      - name: Save k8s-snap image
+        run: |
+          sudo docker save -o k8s-snap-image.tar k8s-snap:dev
+          sudo chmod 775 k8s-snap-image.tar
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-images
+          path: |
+            provider-images.tar
+            k8s-snap-image.tar
+
+  run-e2e-tests:
+    name: Run E2E Tests
+    runs-on: [self-hosted, linux, X64, jammy, xlarge]
+    needs: build-e2e-images
+    strategy:
+      matrix:
+        ginkgo_focus: [
+          "KCP remediation",
+          "MachineDeployment remediation",
+          "Workload cluster creation",
+          "Workload cluster scaling",
+          "Workload cluster upgrade",
+          ]
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Install requirements
+        run: |
+          sudo apt install make
+          sudo snap install go --classic
+          sudo apt install docker-buildx
+          sudo snap install kubectl --classic
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: e2e-images
+          path: .
+      - name: Load provider image
+        run: sudo docker load -i provider-images.tar
+      - name: Load k8s-snap image
+        run: sudo docker load -i k8s-snap-image.tar
       - name: Create docker network
         run: |
           sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true
-          df -h
-      - name: Space check
-        run: |
-          ls
-          df -h
-          pwd
       - name: Increase inotify watches
         run: |
           # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files
@@ -52,7 +82,7 @@ jobs:
           sudo sysctl fs.inotify.max_user_instances=8192
       - name: Run e2e tests
         run: |
-          sudo GINKGO_FOCUS="Workload cluster upgrade" SKIP_RESOURCE_CLEANUP=true make test-e2e
+          sudo GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: canonical/action-tmate@main

From e5840b6384f053462ba9bdb7b437841c4ac3e18e Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 10 Jul 2024 10:29:47 +0200
Subject: [PATCH 58/68] address PR comments

---
 .github/workflows/e2e.yaml | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 7c924773..53eb2304 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -9,16 +9,19 @@ permissions:
 jobs:
   build-e2e-images:
     name: Build & Run E2E Images
-    runs-on: [self-hosted, linux, X64, jammy, xlarge]
+    runs-on: [self-hosted, linux, X64, jammy, large]
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
       - name: Install requirements
         run: |
           sudo apt install make
-          sudo snap install go --classic
           sudo apt install docker-buildx
-          sudo snap install kubectl --classic
+          sudo snap install kubectl --classic --channel=1.30/stable
       - name: Build provider images
         run: sudo make docker-build-e2e
       - name: Build k8s-snap image
@@ -43,7 +46,7 @@ jobs:
 
   run-e2e-tests:
     name: Run E2E Tests
-    runs-on: [self-hosted, linux, X64, jammy, xlarge]
+    runs-on: [self-hosted, linux, X64, jammy, large]
     needs: build-e2e-images
     strategy:
       matrix:
@@ -57,12 +60,15 @@ jobs:
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
       - name: Install requirements
         run: |
           sudo apt install make
-          sudo snap install go --classic
           sudo apt install docker-buildx
-          sudo snap install kubectl --classic
+          sudo snap install kubectl --classic --channel=1.30/stable
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
@@ -83,6 +89,3 @@ jobs:
       - name: Run e2e tests
         run: |
           sudo GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e
-      - name: Setup tmate session
-        if: ${{ failure() }}
-        uses: canonical/action-tmate@main

From f27d850b7e2fd09b0e2c0d3e2afef1fbfcb3f9b0 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 10 Jul 2024 10:52:02 +0200
Subject: [PATCH 59/68] remove Go action

---
 .github/workflows/e2e.yaml | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 53eb2304..dc328d55 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -13,12 +13,9 @@ jobs:
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
-      - name: Install Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: "1.22"
       - name: Install requirements
         run: |
+          sudo snap install go --classic --channel=1.22/stable
           sudo apt install make
           sudo apt install docker-buildx
           sudo snap install kubectl --classic --channel=1.30/stable
@@ -60,12 +57,9 @@ jobs:
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
-      - name: Install Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: "1.22"
       - name: Install requirements
         run: |
+          sudo snap install go --classic --channel=1.22/stable
           sudo apt install make
           sudo apt install docker-buildx
           sudo snap install kubectl --classic --channel=1.30/stable

From 387dd31ff8c66b88f07ed33b3ac293f5cd1f6a52 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Thu, 11 Jul 2024 14:04:27 +0200
Subject: [PATCH 60/68] remove extra assignment

---
 test/e2e/helpers.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go
index 8c423f5f..1d7cc3c8 100644
--- a/test/e2e/helpers.go
+++ b/test/e2e/helpers.go
@@ -608,7 +608,6 @@ func WaitForNodesReady(ctx context.Context, input WaitForNodesReadyInput) {
 		}
 		nodeReadyCount := 0
 		for _, node := range nodeList.Items {
-			n := node
 			match, err := CompareVersions(node.Status.NodeInfo.KubeletVersion, input.KubernetesVersion, "minor")
 			if err != nil {
 				return false, fmt.Errorf("failed to compare versions: %w", err)
@@ -616,7 +615,7 @@ func WaitForNodesReady(ctx context.Context, input WaitForNodesReadyInput) {
 			if !match {
 				return false, nil
 			}
-			if !noderefutil.IsNodeReady(&n) {
+			if !noderefutil.IsNodeReady(&node) {
 				return false, nil
 			}
 			nodeReadyCount++

From 8ed3b7cd21cd90bdd2f4762b08eda65a96a21e6b Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Tue, 16 Jul 2024 08:27:36 +0200
Subject: [PATCH 61/68] update docker e2e

---
 .github/workflows/e2e.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index dc328d55..41cc5e67 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -17,7 +17,7 @@ jobs:
         run: |
           sudo snap install go --classic --channel=1.22/stable
           sudo apt install make
-          sudo apt install docker-buildx
+          sudo apt install docker-buildx-plugin
           sudo snap install kubectl --classic --channel=1.30/stable
       - name: Build provider images
         run: sudo make docker-build-e2e

From 10393d4327d0704de6923021f389289c601d68d7 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Tue, 16 Jul 2024 08:42:09 +0200
Subject: [PATCH 62/68] docker-buildx

---
 .github/workflows/e2e.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 41cc5e67..e1075611 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -15,9 +15,10 @@ jobs:
         uses: actions/checkout@v4
       - name: Install requirements
         run: |
+          sudo apt update
           sudo snap install go --classic --channel=1.22/stable
           sudo apt install make
-          sudo apt install docker-buildx-plugin
+          sudo apt install docker-buildx
           sudo snap install kubectl --classic --channel=1.30/stable
       - name: Build provider images
         run: sudo make docker-build-e2e

From c309d9c2b784e0ffbc2979b4c6fe2a1fa4406847 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Tue, 16 Jul 2024 09:33:41 +0200
Subject: [PATCH 63/68] docker-buildx

---
 .github/workflows/e2e.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index e1075611..24b84029 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -60,6 +60,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Install requirements
         run: |
+          sudo apt update
           sudo snap install go --classic --channel=1.22/stable
           sudo apt install make
           sudo apt install docker-buildx

From fdfb365e247d0b51061dc26de1ed3122ba35607f Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Tue, 16 Jul 2024 14:53:04 +0200
Subject: [PATCH 64/68] address comments

---
 .github/workflows/e2e.yaml |  13 +-
 README.md                  |  17 +
 get-docker.sh              | 741 -------------------------------------
 3 files changed, 23 insertions(+), 748 deletions(-)
 delete mode 100644 get-docker.sh

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 24b84029..d054c4e1 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -48,13 +48,12 @@ jobs:
     needs: build-e2e-images
     strategy:
       matrix:
-        ginkgo_focus: [
-          "KCP remediation",
-          "MachineDeployment remediation",
-          "Workload cluster creation",
-          "Workload cluster scaling",
-          "Workload cluster upgrade",
-          ]
+        ginkgo_focus:
+          - "KCP remediation"
+          - "MachineDeployment remediation"
+          - "Workload cluster creation"
+          - "Workload cluster scaling"
+          - "Workload cluster upgrade"
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
diff --git a/README.md b/README.md
index 454ae375..2e4bf6d1 100644
--- a/README.md
+++ b/README.md
@@ -5,3 +5,20 @@ This repository contains bootstrap and control plane providers to deploy Canonic
 CABPCK (Cluster API bootstrap provider for Canonical Kubernetes) is responsible for generate cloud-init scripts for generate Machines such that they run Kubernetes nodes. This implementation uses [Canonical Kubernetes](https://github.com/canonical/k8s-snap) to deliver Kubernetes.
 
 CACPCK (Cluster API control plane provider for Canonical Kubernetes) is responsible for managing the lifecycle of machines that host the control plane nodes of a Canonical Kubernetes cluster.
+
+## Tests
+
+Run the unittests with
+
+```shell
+make test-unit
+```
+
+For the e2e tests, run:
+
+```shell
+make docker-build-e2e  # run only once to build the images
+make test-e2e
+```
+
+Visit the [tests README](./test/e2e/README.md) for more information.
diff --git a/get-docker.sh b/get-docker.sh
deleted file mode 100644
index d037ad9d..00000000
--- a/get-docker.sh
+++ /dev/null
@@ -1,741 +0,0 @@
-#!/bin/sh
-set -e
-# Docker Engine for Linux installation script.
-#
-# This script is intended as a convenient way to configure docker's package
-# repositories and to install Docker Engine, This script is not recommended
-# for production environments. Before running this script, make yourself familiar
-# with potential risks and limitations, and refer to the installation manual
-# at https://docs.docker.com/engine/install/ for alternative installation methods.
-#
-# The script:
-#
-# - Requires `root` or `sudo` privileges to run.
-# - Attempts to detect your Linux distribution and version and configure your
-#   package management system for you.
-# - Doesn't allow you to customize most installation parameters.
-# - Installs dependencies and recommendations without asking for confirmation.
-# - Installs the latest stable release (by default) of Docker CLI, Docker Engine,
-#   Docker Buildx, Docker Compose, containerd, and runc. When using this script
-#   to provision a machine, this may result in unexpected major version upgrades
-#   of these packages. Always test upgrades in a test environment before
-#   deploying to your production systems.
-# - Isn't designed to upgrade an existing Docker installation. When using the
-#   script to update an existing installation, dependencies may not be updated
-#   to the expected version, resulting in outdated versions.
-#
-# Source code is available at https://github.com/docker/docker-install/
-#
-# Usage
-# ==============================================================================
-#
-# To install the latest stable versions of Docker CLI, Docker Engine, and their
-# dependencies:
-#
-# 1. download the script
-#
-#   $ curl -fsSL https://get.docker.com -o install-docker.sh
-#
-# 2. verify the script's content
-#
-#   $ cat install-docker.sh
-#
-# 3. run the script with --dry-run to verify the steps it executes
-#
-#   $ sh install-docker.sh --dry-run
-#
-# 4. run the script either as root, or using sudo to perform the installation.
-#
-#   $ sudo sh install-docker.sh
-#
-# Command-line options
-# ==============================================================================
-#
-# --version <VERSION>
-# Use the --version option to install a specific version, for example:
-#
-#   $ sudo sh install-docker.sh --version 23.0
-#
-# --channel <stable|test>
-#
-# Use the --channel option to install from an alternative installation channel.
-# The following example installs the latest versions from the "test" channel,
-# which includes pre-releases (alpha, beta, rc):
-#
-#   $ sudo sh install-docker.sh --channel test
-#
-# Alternatively, use the script at https://test.docker.com, which uses the test
-# channel as default.
-#
-# --mirror <Aliyun|AzureChinaCloud>
-#
-# Use the --mirror option to install from a mirror supported by this script.
-# Available mirrors are "Aliyun" (https://mirrors.aliyun.com/docker-ce), and
-# "AzureChinaCloud" (https://mirror.azure.cn/docker-ce), for example:
-#
-#   $ sudo sh install-docker.sh --mirror AzureChinaCloud
-#
-# ==============================================================================
-
-
-# Git commit from https://github.com/docker/docker-install when
-# the script was uploaded (Should only be modified by upload job):
-SCRIPT_COMMIT_SHA="6d9743e9656cc56f699a64800b098d5ea5a60020"
-
-# strip "v" prefix if present
-VERSION="${VERSION#v}"
-
-# The channel to install from:
-#   * stable
-#   * test
-#   * edge (deprecated)
-#   * nightly (deprecated)
-DEFAULT_CHANNEL_VALUE="stable"
-if [ -z "$CHANNEL" ]; then
-	CHANNEL=$DEFAULT_CHANNEL_VALUE
-fi
-
-DEFAULT_DOWNLOAD_URL="https://download.docker.com"
-if [ -z "$DOWNLOAD_URL" ]; then
-	DOWNLOAD_URL=$DEFAULT_DOWNLOAD_URL
-fi
-
-DEFAULT_REPO_FILE="docker-ce.repo"
-if [ -z "$REPO_FILE" ]; then
-	REPO_FILE="$DEFAULT_REPO_FILE"
-fi
-
-mirror=''
-DRY_RUN=${DRY_RUN:-}
-while [ $# -gt 0 ]; do
-	case "$1" in
-		--channel)
-			CHANNEL="$2"
-			shift
-			;;
-		--dry-run)
-			DRY_RUN=1
-			;;
-		--mirror)
-			mirror="$2"
-			shift
-			;;
-		--version)
-			VERSION="${2#v}"
-			shift
-			;;
-		--*)
-			echo "Illegal option $1"
-			;;
-	esac
-	shift $(( $# > 0 ? 1 : 0 ))
-done
-
-case "$mirror" in
-	Aliyun)
-		DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce"
-		;;
-	AzureChinaCloud)
-		DOWNLOAD_URL="https://mirror.azure.cn/docker-ce"
-		;;
-	"")
-		;;
-	*)
-		>&2 echo "unknown mirror '$mirror': use either 'Aliyun', or 'AzureChinaCloud'."
-		exit 1
-		;;
-esac
-
-case "$CHANNEL" in
-	stable|test)
-		;;
-	edge|nightly)
-		>&2 echo "DEPRECATED: the $CHANNEL channel has been deprecated and is no longer supported by this script."
-		exit 1
-		;;
-	*)
-		>&2 echo "unknown CHANNEL '$CHANNEL': use either stable or test."
-		exit 1
-		;;
-esac
-
-command_exists() {
-	command -v "$@" > /dev/null 2>&1
-}
-
-# version_gte checks if the version specified in $VERSION is at least the given
-# SemVer (Maj.Minor[.Patch]), or CalVer (YY.MM) version.It returns 0 (success)
-# if $VERSION is either unset (=latest) or newer or equal than the specified
-# version, or returns 1 (fail) otherwise.
-#
-# examples:
-#
-# VERSION=23.0
-# version_gte 23.0  // 0 (success)
-# version_gte 20.10 // 0 (success)
-# version_gte 19.03 // 0 (success)
-# version_gte 21.10 // 1 (fail)
-version_gte() {
-	if [ -z "$VERSION" ]; then
-			return 0
-	fi
-	eval version_compare "$VERSION" "$1"
-}
-
-# version_compare compares two version strings (either SemVer (Major.Minor.Path),
-# or CalVer (YY.MM) version strings. It returns 0 (success) if version A is newer
-# or equal than version B, or 1 (fail) otherwise. Patch releases and pre-release
-# (-alpha/-beta) are not taken into account
-#
-# examples:
-#
-# version_compare 23.0.0 20.10 // 0 (success)
-# version_compare 23.0 20.10   // 0 (success)
-# version_compare 20.10 19.03  // 0 (success)
-# version_compare 20.10 20.10  // 0 (success)
-# version_compare 19.03 20.10  // 1 (fail)
-version_compare() (
-	set +x
-
-	yy_a="$(echo "$1" | cut -d'.' -f1)"
-	yy_b="$(echo "$2" | cut -d'.' -f1)"
-	if [ "$yy_a" -lt "$yy_b" ]; then
-		return 1
-	fi
-	if [ "$yy_a" -gt "$yy_b" ]; then
-		return 0
-	fi
-	mm_a="$(echo "$1" | cut -d'.' -f2)"
-	mm_b="$(echo "$2" | cut -d'.' -f2)"
-
-	# trim leading zeros to accommodate CalVer
-	mm_a="${mm_a#0}"
-	mm_b="${mm_b#0}"
-
-	if [ "${mm_a:-0}" -lt "${mm_b:-0}" ]; then
-		return 1
-	fi
-
-	return 0
-)
-
-is_dry_run() {
-	if [ -z "$DRY_RUN" ]; then
-		return 1
-	else
-		return 0
-	fi
-}
-
-is_wsl() {
-	case "$(uname -r)" in
-	*microsoft* ) true ;; # WSL 2
-	*Microsoft* ) true ;; # WSL 1
-	* ) false;;
-	esac
-}
-
-is_darwin() {
-	case "$(uname -s)" in
-	*darwin* ) true ;;
-	*Darwin* ) true ;;
-	* ) false;;
-	esac
-}
-
-deprecation_notice() {
-	distro=$1
-	distro_version=$2
-	echo
-	printf "\033[91;1mDEPRECATION WARNING\033[0m\n"
-	printf "    This Linux distribution (\033[1m%s %s\033[0m) reached end-of-life and is no longer supported by this script.\n" "$distro" "$distro_version"
-	echo   "    No updates or security fixes will be released for this distribution, and users are recommended"
-	echo   "    to upgrade to a currently maintained version of $distro."
-	echo
-	printf   "Press \033[1mCtrl+C\033[0m now to abort this script, or wait for the installation to continue."
-	echo
-	sleep 10
-}
-
-get_distribution() {
-	lsb_dist=""
-	# Every system that we officially support has /etc/os-release
-	if [ -r /etc/os-release ]; then
-		lsb_dist="$(. /etc/os-release && echo "$ID")"
-	fi
-	# Returning an empty string here should be alright since the
-	# case statements don't act unless you provide an actual value
-	echo "$lsb_dist"
-}
-
-echo_docker_as_nonroot() {
-	if is_dry_run; then
-		return
-	fi
-	if command_exists docker && [ -e /var/run/docker.sock ]; then
-		(
-			set -x
-			$sh_c 'docker version'
-		) || true
-	fi
-
-	# intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output
-	echo
-	echo "================================================================================"
-	echo
-	if version_gte "20.10"; then
-		echo "To run Docker as a non-privileged user, consider setting up the"
-		echo "Docker daemon in rootless mode for your user:"
-		echo
-		echo "    dockerd-rootless-setuptool.sh install"
-		echo
-		echo "Visit https://docs.docker.com/go/rootless/ to learn about rootless mode."
-		echo
-	fi
-	echo
-	echo "To run the Docker daemon as a fully privileged service, but granting non-root"
-	echo "users access, refer to https://docs.docker.com/go/daemon-access/"
-	echo
-	echo "WARNING: Access to the remote API on a privileged Docker daemon is equivalent"
-	echo "         to root access on the host. Refer to the 'Docker daemon attack surface'"
-	echo "         documentation for details: https://docs.docker.com/go/attack-surface/"
-	echo
-	echo "================================================================================"
-	echo
-}
-
-# Check if this is a forked Linux distro
-check_forked() {
-
-	# Check for lsb_release command existence, it usually exists in forked distros
-	if command_exists lsb_release; then
-		# Check if the `-u` option is supported
-		set +e
-		lsb_release -a -u > /dev/null 2>&1
-		lsb_release_exit_code=$?
-		set -e
-
-		# Check if the command has exited successfully, it means we're in a forked distro
-		if [ "$lsb_release_exit_code" = "0" ]; then
-			# Print info about current distro
-			cat <<-EOF
-			You're using '$lsb_dist' version '$dist_version'.
-			EOF
-
-			# Get the upstream release info
-			lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]')
-			dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]')
-
-			# Print info about upstream distro
-			cat <<-EOF
-			Upstream release is '$lsb_dist' version '$dist_version'.
-			EOF
-		else
-			if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then
-				if [ "$lsb_dist" = "osmc" ]; then
-					# OSMC runs Raspbian
-					lsb_dist=raspbian
-				else
-					# We're Debian and don't even know it!
-					lsb_dist=debian
-				fi
-				dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
-				case "$dist_version" in
-					12)
-						dist_version="bookworm"
-					;;
-					11)
-						dist_version="bullseye"
-					;;
-					10)
-						dist_version="buster"
-					;;
-					9)
-						dist_version="stretch"
-					;;
-					8)
-						dist_version="jessie"
-					;;
-				esac
-			fi
-		fi
-	fi
-}
-
-do_install() {
-	echo "# Executing docker install script, commit: $SCRIPT_COMMIT_SHA"
-
-	if command_exists docker; then
-		cat >&2 <<-'EOF'
-			Warning: the "docker" command appears to already exist on this system.
-
-			If you already have Docker installed, this script can cause trouble, which is
-			why we're displaying this warning and provide the opportunity to cancel the
-			installation.
-
-			If you installed the current Docker package using this script and are using it
-			again to update Docker, you can safely ignore this message.
-
-			You may press Ctrl+C now to abort this script.
-		EOF
-		( set -x; sleep 20 )
-	fi
-
-	user="$(id -un 2>/dev/null || true)"
-
-	sh_c='sh -c'
-	if [ "$user" != 'root' ]; then
-		if command_exists sudo; then
-			sh_c='sudo -E sh -c'
-		elif command_exists su; then
-			sh_c='su -c'
-		else
-			cat >&2 <<-'EOF'
-			Error: this installer needs the ability to run commands as root.
-			We are unable to find either "sudo" or "su" available to make this happen.
-			EOF
-			exit 1
-		fi
-	fi
-
-	if is_dry_run; then
-		sh_c="echo"
-	fi
-
-	# perform some very rudimentary platform detection
-	lsb_dist=$( get_distribution )
-	lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')"
-
-	if is_wsl; then
-		echo
-		echo "WSL DETECTED: We recommend using Docker Desktop for Windows."
-		echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop/"
-		echo
-		cat >&2 <<-'EOF'
-
-			You may press Ctrl+C now to abort this script.
-		EOF
-		( set -x; sleep 20 )
-	fi
-
-	case "$lsb_dist" in
-
-		ubuntu)
-			if command_exists lsb_release; then
-				dist_version="$(lsb_release --codename | cut -f2)"
-			fi
-			if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then
-				dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")"
-			fi
-		;;
-
-		debian|raspbian)
-			dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
-			case "$dist_version" in
-				12)
-					dist_version="bookworm"
-				;;
-				11)
-					dist_version="bullseye"
-				;;
-				10)
-					dist_version="buster"
-				;;
-				9)
-					dist_version="stretch"
-				;;
-				8)
-					dist_version="jessie"
-				;;
-			esac
-		;;
-
-		centos|rhel)
-			if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
-				dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
-			fi
-		;;
-
-		*)
-			if command_exists lsb_release; then
-				dist_version="$(lsb_release --release | cut -f2)"
-			fi
-			if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
-				dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
-			fi
-		;;
-
-	esac
-
-	# Check if this is a forked Linux distro
-	check_forked
-
-	# Print deprecation warnings for distro versions that recently reached EOL,
-	# but may still be commonly used (especially LTS versions).
-	case "$lsb_dist.$dist_version" in
-		debian.stretch|debian.jessie)
-			deprecation_notice "$lsb_dist" "$dist_version"
-			;;
-		raspbian.stretch|raspbian.jessie)
-			deprecation_notice "$lsb_dist" "$dist_version"
-			;;
-		ubuntu.xenial|ubuntu.trusty)
-			deprecation_notice "$lsb_dist" "$dist_version"
-			;;
-		ubuntu.lunar|ubuntu.kinetic|ubuntu.impish|ubuntu.hirsute|ubuntu.groovy|ubuntu.eoan|ubuntu.disco|ubuntu.cosmic)
-			deprecation_notice "$lsb_dist" "$dist_version"
-			;;
-		fedora.*)
-			if [ "$dist_version" -lt 36 ]; then
-				deprecation_notice "$lsb_dist" "$dist_version"
-			fi
-			;;
-	esac
-
-	# Run setup for each distro accordingly
-	case "$lsb_dist" in
-		ubuntu|debian|raspbian)
-			pre_reqs="apt-transport-https ca-certificates curl"
-			apt_repo="deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] $DOWNLOAD_URL/linux/$lsb_dist $dist_version $CHANNEL"
-			(
-				if ! is_dry_run; then
-					set -x
-				fi
-				$sh_c 'apt-get update -qq >/dev/null'
-				$sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pre_reqs >/dev/null"
-				$sh_c 'install -m 0755 -d /etc/apt/keyrings'
-				$sh_c "curl -fsSL \"$DOWNLOAD_URL/linux/$lsb_dist/gpg\" -o /etc/apt/keyrings/docker.asc"
-				$sh_c "chmod a+r /etc/apt/keyrings/docker.asc"
-				$sh_c "echo \"$apt_repo\" > /etc/apt/sources.list.d/docker.list"
-				$sh_c 'apt-get update -qq >/dev/null'
-			)
-			pkg_version=""
-			if [ -n "$VERSION" ]; then
-				if is_dry_run; then
-					echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
-				else
-					# Will work for incomplete versions IE (17.12), but may not actually grab the "latest" if in the test channel
-					pkg_pattern="$(echo "$VERSION" | sed 's/-ce-/~ce~.*/g' | sed 's/-/.*/g')"
-					search_command="apt-cache madison docker-ce | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3"
-					pkg_version="$($sh_c "$search_command")"
-					echo "INFO: Searching repository for VERSION '$VERSION'"
-					echo "INFO: $search_command"
-					if [ -z "$pkg_version" ]; then
-						echo
-						echo "ERROR: '$VERSION' not found amongst apt-cache madison results"
-						echo
-						exit 1
-					fi
-					if version_gte "18.09"; then
-							search_command="apt-cache madison docker-ce-cli | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3"
-							echo "INFO: $search_command"
-							cli_pkg_version="=$($sh_c "$search_command")"
-					fi
-					pkg_version="=$pkg_version"
-				fi
-			fi
-			(
-				pkgs="docker-ce${pkg_version%=}"
-				if version_gte "18.09"; then
-						# older versions didn't ship the cli and containerd as separate packages
-						pkgs="$pkgs docker-ce-cli${cli_pkg_version%=} containerd.io"
-				fi
-				if version_gte "20.10"; then
-						pkgs="$pkgs docker-compose-plugin docker-ce-rootless-extras$pkg_version"
-				fi
-				if version_gte "23.0"; then
-						pkgs="$pkgs docker-buildx-plugin"
-				fi
-				if ! is_dry_run; then
-					set -x
-				fi
-				$sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pkgs >/dev/null"
-			)
-			echo_docker_as_nonroot
-			exit 0
-			;;
-		centos|fedora|rhel)
-			if [ "$(uname -m)" != "s390x" ] && [ "$lsb_dist" = "rhel" ]; then
-				echo "Packages for RHEL are currently only available for s390x."
-				exit 1
-			fi
-
-			if command_exists dnf; then
-				pkg_manager="dnf"
-				pkg_manager_flags="--best"
-				config_manager="dnf config-manager"
-				enable_channel_flag="--set-enabled"
-				disable_channel_flag="--set-disabled"
-				pre_reqs="dnf-plugins-core"
-			else
-				pkg_manager="yum"
-				pkg_manager_flags=""
-				config_manager="yum-config-manager"
-				enable_channel_flag="--enable"
-				disable_channel_flag="--disable"
-				pre_reqs="yum-utils"
-			fi
-
-			if [ "$lsb_dist" = "fedora" ]; then
-				pkg_suffix="fc$dist_version"
-			else
-				pkg_suffix="el"
-			fi
-			repo_file_url="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE"
-			(
-				if ! is_dry_run; then
-					set -x
-				fi
-				$sh_c "$pkg_manager $pkg_manager_flags install -y -q $pre_reqs"
-				$sh_c "$config_manager --add-repo $repo_file_url"
-
-				if [ "$CHANNEL" != "stable" ]; then
-					$sh_c "$config_manager $disable_channel_flag 'docker-ce-*'"
-					$sh_c "$config_manager $enable_channel_flag 'docker-ce-$CHANNEL'"
-				fi
-				$sh_c "$pkg_manager makecache"
-			)
-			pkg_version=""
-			if [ -n "$VERSION" ]; then
-				if is_dry_run; then
-					echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
-				else
-					pkg_pattern="$(echo "$VERSION" | sed 's/-ce-/\\\\.ce.*/g' | sed 's/-/.*/g').*$pkg_suffix"
-					search_command="$pkg_manager list --showduplicates docker-ce | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'"
-					pkg_version="$($sh_c "$search_command")"
-					echo "INFO: Searching repository for VERSION '$VERSION'"
-					echo "INFO: $search_command"
-					if [ -z "$pkg_version" ]; then
-						echo
-						echo "ERROR: '$VERSION' not found amongst $pkg_manager list results"
-						echo
-						exit 1
-					fi
-					if version_gte "18.09"; then
-						# older versions don't support a cli package
-						search_command="$pkg_manager list --showduplicates docker-ce-cli | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'"
-						cli_pkg_version="$($sh_c "$search_command" | cut -d':' -f 2)"
-					fi
-					# Cut out the epoch and prefix with a '-'
-					pkg_version="-$(echo "$pkg_version" | cut -d':' -f 2)"
-				fi
-			fi
-			(
-				pkgs="docker-ce$pkg_version"
-				if version_gte "18.09"; then
-					# older versions didn't ship the cli and containerd as separate packages
-					if [ -n "$cli_pkg_version" ]; then
-						pkgs="$pkgs docker-ce-cli-$cli_pkg_version containerd.io"
-					else
-						pkgs="$pkgs docker-ce-cli containerd.io"
-					fi
-				fi
-				if version_gte "20.10"; then
-					pkgs="$pkgs docker-compose-plugin docker-ce-rootless-extras$pkg_version"
-				fi
-				if version_gte "23.0"; then
-						pkgs="$pkgs docker-buildx-plugin"
-				fi
-				if ! is_dry_run; then
-					set -x
-				fi
-				$sh_c "$pkg_manager $pkg_manager_flags install -y -q $pkgs"
-			)
-			echo_docker_as_nonroot
-			exit 0
-			;;
-		sles)
-			if [ "$(uname -m)" != "s390x" ]; then
-				echo "Packages for SLES are currently only available for s390x"
-				exit 1
-			fi
-			repo_file_url="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE"
-			pre_reqs="ca-certificates curl libseccomp2 awk"
-			(
-				if ! is_dry_run; then
-					set -x
-				fi
-				$sh_c "zypper install -y $pre_reqs"
-				$sh_c "zypper addrepo $repo_file_url"
-				if ! is_dry_run; then
-						cat >&2 <<-'EOF'
-						WARNING!!
-						openSUSE repository (https://download.opensuse.org/repositories/security:/SELinux) will be enabled now.
-						Do you wish to continue?
-						You may press Ctrl+C now to abort this script.
-						EOF
-						( set -x; sleep 30 )
-				fi
-				opensuse_repo="https://download.opensuse.org/repositories/security:/SELinux/openSUSE_Factory/security:SELinux.repo"
-				$sh_c "zypper addrepo $opensuse_repo"
-				$sh_c "zypper --gpg-auto-import-keys refresh"
-				$sh_c "zypper lr -d"
-			)
-			pkg_version=""
-			if [ -n "$VERSION" ]; then
-				if is_dry_run; then
-					echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
-				else
-					pkg_pattern="$(echo "$VERSION" | sed 's/-ce-/\\\\.ce.*/g' | sed 's/-/.*/g')"
-					search_command="zypper search -s --match-exact 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'"
-					pkg_version="$($sh_c "$search_command")"
-					echo "INFO: Searching repository for VERSION '$VERSION'"
-					echo "INFO: $search_command"
-					if [ -z "$pkg_version" ]; then
-						echo
-						echo "ERROR: '$VERSION' not found amongst zypper list results"
-						echo
-						exit 1
-					fi
-					search_command="zypper search -s --match-exact 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'"
-					# It's okay for cli_pkg_version to be blank, since older versions don't support a cli package
-					cli_pkg_version="$($sh_c "$search_command")"
-					pkg_version="-$pkg_version"
-				fi
-			fi
-			(
-				pkgs="docker-ce$pkg_version"
-				if version_gte "18.09"; then
-					if [ -n "$cli_pkg_version" ]; then
-						# older versions didn't ship the cli and containerd as separate packages
-						pkgs="$pkgs docker-ce-cli-$cli_pkg_version containerd.io"
-					else
-						pkgs="$pkgs docker-ce-cli containerd.io"
-					fi
-				fi
-				if version_gte "20.10"; then
-					pkgs="$pkgs docker-compose-plugin docker-ce-rootless-extras$pkg_version"
-				fi
-				if version_gte "23.0"; then
-						pkgs="$pkgs docker-buildx-plugin"
-				fi
-				if ! is_dry_run; then
-					set -x
-				fi
-				$sh_c "zypper -q install -y $pkgs"
-			)
-			echo_docker_as_nonroot
-			exit 0
-			;;
-		*)
-			if [ -z "$lsb_dist" ]; then
-				if is_darwin; then
-					echo
-					echo "ERROR: Unsupported operating system 'macOS'"
-					echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop"
-					echo
-					exit 1
-				fi
-			fi
-			echo
-			echo "ERROR: Unsupported distribution '$lsb_dist'"
-			echo
-			exit 1
-			;;
-	esac
-	exit 1
-}
-
-# wrapped up in a function so that we have some protection against only getting
-# half the file during "curl | sh"
-do_install

From dd1abfa68ad3d0e6f58cb3e8cd0eb5ee3cadb945 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Tue, 16 Jul 2024 16:41:56 +0200
Subject: [PATCH 65/68] update readme and cleanup outdated sections

---
 README.md          | 17 -----------------
 test/e2e/README.md | 19 +++++++++++--------
 2 files changed, 11 insertions(+), 25 deletions(-)

diff --git a/README.md b/README.md
index 2e4bf6d1..454ae375 100644
--- a/README.md
+++ b/README.md
@@ -5,20 +5,3 @@ This repository contains bootstrap and control plane providers to deploy Canonic
 CABPCK (Cluster API bootstrap provider for Canonical Kubernetes) is responsible for generate cloud-init scripts for generate Machines such that they run Kubernetes nodes. This implementation uses [Canonical Kubernetes](https://github.com/canonical/k8s-snap) to deliver Kubernetes.
 
 CACPCK (Cluster API control plane provider for Canonical Kubernetes) is responsible for managing the lifecycle of machines that host the control plane nodes of a Canonical Kubernetes cluster.
-
-## Tests
-
-Run the unittests with
-
-```shell
-make test-unit
-```
-
-For the e2e tests, run:
-
-```shell
-make docker-build-e2e  # run only once to build the images
-make test-e2e
-```
-
-Visit the [tests README](./test/e2e/README.md) for more information.
diff --git a/test/e2e/README.md b/test/e2e/README.md
index a3e3dce8..849be79c 100644
--- a/test/e2e/README.md
+++ b/test/e2e/README.md
@@ -1,27 +1,29 @@
 # e2e test
+
 The e2e test use the [Cluster API test framework](https://pkg.go.dev/sigs.k8s.io/cluster-api/test/framework?tab=doc) and use the [CAPD](https://github.com/kubernetes-sigs/cluster-api/tree/main/test/infrastructure/docker) as the infrastructure provider. Please make sure you have [Docker](https://docs.docker.com/install/) and [kind](https://kind.sigs.k8s.io/) installed.
 
-You could refer to the [Testing Cluster API](https://cluster-api.sigs.k8s.io/developer/testing) for more information.
+Refer to the [Testing Cluster API](https://cluster-api.sigs.k8s.io/developer/testing) for more information.
 
 ## Run the e2e test
+
 The e2e image will be built with tag `dev`. You should build the image first before running the test.
+
 ```shell
 make docker-build-e2e   # should be run everytime you change the controller code
 make test-e2e   # run all e2e tests
 ```
+
 ### Run a specific e2e test
+
 To run a specific e2e test, such as `[PR-Blocking]`, use the `GINKGO_FOCUS` environment variable as shown below:
+
 ```shell
 make GINKGO_FOCUS="\\[PR-Blocking\\]" test-e2e  # only run e2e test with `[PR-Blocking]` in its spec name
 ```
-### Run the e2e test with tilt
-It is quite useful to run the e2e test with [tilt](https://cluster-api.sigs.k8s.io/developer/tilt), so that you will not need to rebuild docker image with `make docker-build-e2e` everytime. Also you will not need to wait a new cluster creation and setup. If you have set up your tilt cluster and made the current context points to this cluster, you could run:
-```shell
-# running e2e for the cluster pointed by the current context
-make USE_EXISTING_CLUSTER=true test-e2e
-```
+
 ## Develop an e2e test
-You could refer to [Developing E2E tests](https://cluster-api.sigs.k8s.io/developer/e2e) for a complete guide for developing e2e tests.
+
+Refer to [Developing E2E tests](https://cluster-api.sigs.k8s.io/developer/e2e) for a complete guide for developing e2e tests.
 
 A guide for developing a ck8s e2e test:
 
@@ -31,4 +33,5 @@ A guide for developing a ck8s e2e test:
 * If reusing a [cluster-api test spec](https://github.com/kubernetes-sigs/cluster-api/tree/main/test/e2e), note that they assume the use of `KubeadmControlPlane`. For customization, copy code into `test/e2e/helpers.go`.
 
 ## Troubleshooting
+
 * [Cluster API with Docker - "too many open files".](https://cluster-api.sigs.k8s.io/user/troubleshooting.html?highlight=too%20many#cluster-api-with-docker----too-many-open-files)

From 5e2952c992677d58bbb23d2bb023a66421962404 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 17 Jul 2024 11:44:04 +0200
Subject: [PATCH 66/68] use semver package

---
 test/e2e/helpers.go | 62 ++-------------------------------------------
 1 file changed, 2 insertions(+), 60 deletions(-)

diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go
index 1d7cc3c8..5c0cc56a 100644
--- a/test/e2e/helpers.go
+++ b/test/e2e/helpers.go
@@ -22,13 +22,12 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strconv"
-	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
+	"golang.org/x/mod/semver"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -608,11 +607,7 @@ func WaitForNodesReady(ctx context.Context, input WaitForNodesReadyInput) {
 		}
 		nodeReadyCount := 0
 		for _, node := range nodeList.Items {
-			match, err := CompareVersions(node.Status.NodeInfo.KubeletVersion, input.KubernetesVersion, "minor")
-			if err != nil {
-				return false, fmt.Errorf("failed to compare versions: %w", err)
-			}
-			if !match {
+			if !(semver.MajorMinor(node.Status.NodeInfo.KubeletVersion) == semver.MajorMinor(input.KubernetesVersion)) {
 				return false, nil
 			}
 			if !noderefutil.IsNodeReady(&node) {
@@ -633,56 +628,3 @@ func byClusterOptions(name, namespace string) []client.ListOption {
 		},
 	}
 }
-
-func parseVersion(version string) ([]int, error) {
-	// Remove the leading "v" if it exists
-	if strings.HasPrefix(version, "v") {
-		version = version[1:]
-	}
-
-	parts := strings.Split(version, ".")
-	if len(parts) != 3 {
-		return nil, fmt.Errorf("invalid version format")
-	}
-
-	intParts := make([]int, len(parts))
-	for i, part := range parts {
-		num, err := strconv.Atoi(part)
-		if err != nil {
-			return nil, fmt.Errorf("invalid version part: %w", err)
-		}
-		intParts[i] = num
-	}
-
-	return intParts, nil
-}
-
-func compareVersionParts(v1, v2 []int, level string) bool {
-	switch strings.ToLower(level) {
-	case "major":
-		return v1[0] == v2[0]
-	case "minor":
-		return v1[0] == v2[0] && v1[1] == v2[1]
-	case "patch":
-		return v1[0] == v2[0] && v1[1] == v2[1] && v1[2] == v2[2]
-	default:
-		return v1[0] == v2[0] && v1[1] == v2[1] && v1[2] == v2[2]
-	}
-}
-
-// CompareVersions compares two versions and returns true if they are equal at the specified level.
-// The level can be "major", "minor", "patch" or an empty string to compare the full version.
-// Returns an error if the versions are not in the correct format.
-// Example: CompareVersions("1.19.0", "1.19.1", "patch") returns false
-func CompareVersions(version1, version2, level string) (bool, error) {
-	v1, err := parseVersion(version1)
-	if err != nil {
-		return false, fmt.Errorf("failed to parse version1: %w", err)
-	}
-
-	v2, err := parseVersion(version2)
-	if err != nil {
-		return false, fmt.Errorf("failed to parse version2: %w", err)
-	}
-	return compareVersionParts(v1, v2, level), nil
-}

From e34ce8070449e48bfc9565243d0b27acfa214e46 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 17 Jul 2024 11:46:16 +0200
Subject: [PATCH 67/68] update go mod

---
 go.mod | 1 +
 go.sum | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/go.mod b/go.mod
index cad35252..6ce797ce 100644
--- a/go.mod
+++ b/go.mod
@@ -115,6 +115,7 @@ require (
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/mod v0.19.0
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
diff --git a/go.sum b/go.sum
index 1d4b1078..a03034f8 100644
--- a/go.sum
+++ b/go.sum
@@ -337,6 +337,8 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqR
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
+golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

From e10ebb9e69f0f74622c724428f9e62a4b007abdb Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Wed, 17 Jul 2024 13:18:34 +0200
Subject: [PATCH 68/68] remove docker

---
 test/e2e/docker_logcollector.go | 193 --------------------------------
 test/e2e/e2e_suite_test.go      |   2 +-
 2 files changed, 1 insertion(+), 194 deletions(-)
 delete mode 100644 test/e2e/docker_logcollector.go

diff --git a/test/e2e/docker_logcollector.go b/test/e2e/docker_logcollector.go
deleted file mode 100644
index 3ac2434c..00000000
--- a/test/e2e/docker_logcollector.go
+++ /dev/null
@@ -1,193 +0,0 @@
-// Copied from CAPI e2e framework and modified to add pebble logs.
-
-/*
-Copyright 2019 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"os"
-	osExec "os/exec"
-	"path/filepath"
-	"strings"
-
-	kerrors "k8s.io/apimachinery/pkg/util/errors"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
-	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
-	"sigs.k8s.io/cluster-api/test/infrastructure/container"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/kind/pkg/errors"
-)
-
-// DockerLogCollector collect logs from a CAPD workload cluster.
-type DockerLogCollector struct{}
-
-// machineContainerName return a container name using the same rule used in CAPD.
-// NOTE: if the cluster name is already included in the machine name, the cluster name is not add thus
-// avoiding \"sethostname: invalid argument\"  errors due to container name too long.
-func machineContainerName(cluster, machine string) string {
-	if strings.HasPrefix(machine, cluster) {
-		return machine
-	}
-	return fmt.Sprintf("%s-%s", cluster, machine)
-}
-
-func (k DockerLogCollector) CollectMachineLog(ctx context.Context, _ client.Client, m *clusterv1.Machine, outputPath string) error {
-	containerName := machineContainerName(m.Spec.ClusterName, m.Name)
-	containerRuntime, err := container.NewDockerClient()
-	if err != nil {
-		return err
-	}
-	ctx = container.RuntimeInto(ctx, containerRuntime)
-	return k.collectLogsFromNode(ctx, outputPath, containerName)
-}
-
-func (k DockerLogCollector) CollectMachinePoolLog(ctx context.Context, _ client.Client, m *expv1.MachinePool, outputPath string) error {
-	containerRuntime, err := container.NewDockerClient()
-	if err != nil {
-		return err
-	}
-	ctx = container.RuntimeInto(ctx, containerRuntime)
-
-	var errs []error
-	for _, instance := range m.Status.NodeRefs {
-		containerName := machineContainerName(m.Spec.ClusterName, instance.Name)
-		if err := k.collectLogsFromNode(ctx, filepath.Join(outputPath, instance.Name), containerName); err != nil {
-			// collecting logs is best effort so we proceed to the next instance even if we encounter an error.
-			errs = append(errs, err)
-		}
-	}
-
-	return kerrors.NewAggregate(errs)
-}
-
-func (k DockerLogCollector) CollectInfrastructureLogs(ctx context.Context, _ client.Client, c *clusterv1.Cluster, outputPath string) error {
-	containerRuntime, err := container.NewDockerClient()
-	if err != nil {
-		return err
-	}
-	ctx = container.RuntimeInto(ctx, containerRuntime)
-
-	lbContainerName := fmt.Sprintf("%s-lb", c.GetName())
-
-	f, err := fileOnHost(filepath.Join(outputPath, fmt.Sprintf("%s.log", lbContainerName)))
-	if err != nil {
-		return err
-	}
-
-	defer f.Close()
-
-	return containerRuntime.ContainerDebugInfo(ctx, lbContainerName, f)
-}
-
-func (k DockerLogCollector) collectLogsFromNode(ctx context.Context, outputPath string, containerName string) error {
-	containerRuntime, err := container.RuntimeFrom(ctx)
-	if err != nil {
-		return errors.Wrap(err, "Failed to collect logs from node")
-	}
-	execToPathFn := func(outputFileName, command string, args ...string) func() error {
-		return func() error {
-			f, err := fileOnHost(filepath.Join(outputPath, outputFileName))
-			if err != nil {
-				return err
-			}
-			defer f.Close()
-			execConfig := container.ExecContainerInput{
-				OutputBuffer: f,
-			}
-			return containerRuntime.ExecContainer(ctx, containerName, &execConfig, command, args...)
-		}
-	}
-	copyDirFn := func(containerDir, dirName string) func() error {
-		return func() error {
-			f, err := os.CreateTemp("", containerName)
-			if err != nil {
-				return err
-			}
-
-			tempfileName := f.Name()
-			outputDir := filepath.Join(outputPath, dirName)
-
-			defer os.Remove(tempfileName)
-
-			var execErr string
-			execConfig := container.ExecContainerInput{
-				OutputBuffer: f,
-				ErrorBuffer:  bytes.NewBufferString(execErr),
-			}
-			err = containerRuntime.ExecContainer(
-				ctx,
-				containerName,
-				&execConfig,
-				"tar", "--hard-dereference", "--dereference", "--directory", containerDir, "--create", "--file", "-", ".",
-			)
-			if err != nil {
-				return errors.Wrapf(err, execErr)
-			}
-
-			err = os.MkdirAll(outputDir, os.ModePerm)
-			if err != nil {
-				return err
-			}
-
-			return osExec.Command("tar", "--extract", "--file", tempfileName, "--directory", outputDir).Run()
-		}
-	}
-	return errors.AggregateConcurrent([]func() error{
-		execToPathFn(
-			"journal.log",
-			"journalctl", "--no-pager", "--output=short-precise",
-		),
-		execToPathFn(
-			"kern.log",
-			"journalctl", "--no-pager", "--output=short-precise", "-k",
-		),
-		execToPathFn(
-			"kubelet-version.txt",
-			"kubelet", "--version",
-		),
-		execToPathFn(
-			"kubelet.log",
-			"journalctl", "--no-pager", "--output=short-precise", "-u", "kubelet.service",
-		),
-		execToPathFn(
-			"containerd-info.txt",
-			"crictl", "info",
-		),
-		execToPathFn(
-			"containerd.log",
-			"journalctl", "--no-pager", "--output=short-precise", "-u", "containerd.service",
-		),
-		execToPathFn(
-			"pebble.log",
-			"pebble", "logs", "-n", "all",
-		),
-		copyDirFn("/var/log/pods", "pods"),
-	})
-}
-
-// fileOnHost is a helper to create a file at path
-// even if the parent directory doesn't exist
-// in which case it will be created with ModePerm.
-func fileOnHost(path string) (*os.File, error) {
-	if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
-		return nil, err
-	}
-	return os.Create(path)
-}
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index 7db04ab5..9dd78e93 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -170,7 +170,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	kubeconfigPath := parts[3]
 
 	e2eConfig = loadE2EConfig(configPath)
-	bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(), framework.WithMachineLogCollector(DockerLogCollector{}))
+	bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(), framework.WithMachineLogCollector(framework.DockerLogCollector{}))
 })
 
 // Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads).