From 55d9d803c06d29d98eeef36c57dbef851be6f002 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 14:04:06 +0200
Subject: [PATCH 01/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 73 ++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100644 .github/workflows/sonobuoy-test.yaml

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
new file mode 100644
index 000000000..bc82c8345
--- /dev/null
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -0,0 +1,73 @@
+name: Sonobuoy Nightly Latest/Edge Tests
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test-integration:
+    name: Integration Test ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.releases }}
+    strategy:
+      matrix:
+        os: [ "ubuntu:24.04"]
+        arch: ["amd64"]
+        releases: ["latest/edge"]
+
+    runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }}
+
+    steps:
+      - name: Checking out repo
+        uses: actions/checkout@v4
+
+      - name: Install lxd
+        run: |
+          sudo snap refresh lxd --channel 5.21/stable
+          sudo lxd init --auto
+          sudo usermod --append --groups lxd $USER
+          sg lxd -c 'lxc version'
+
+      - name: Install sonobuoy
+        run: |
+          release=$(curl --silent -m 10 --connect-timeout 5 "https://api.github.com/repos/vmware-tanzu/sonobuoy/releases/latest")
+          tag=$(echo "$release" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+          curl -L -o sonobuoy.tar.gz https://github.com/vmware-tanzu/sonobuoy/releases/download/${tag}/sonobuoy_${tag//v}_linux_${{ matrix.arch }}.tar.gz
+          tar -xf sonobuoy.tar.gz
+
+      - name: download $${ matrix.releases }} k8s snap
+        run: |
+          snap download k8s --channel=${{ matrix.releases }} --basename k8s
+
+      - name: create k8s container
+          lxc profile create k8s
+          cat tests/integration/lxd-profile.yaml | lxc profile edit k8s
+          lxc launch -p default -p k8s ${{ matrix.os }} k8s
+          lxc config device add k8s repo disk source=${PWD} path=/repo/
+
+      - name: setup k8s in lxd container
+      - run: |
+          lxc exec k8s -- sudo snap install /repo/k8s_v1.31.1_amd64.snap --dangerous --classic
+          lxc exec k8s -- sudo k8s bootstrap
+          lxc exec k8s -- sudo k8s config > ~/.kube/config
+
+      - name: Run end to end tests
+        run: |
+          ./sonobuoy run --plugin e2e --wait --mode quick
+          ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
+          ./sonobuoy results sonobuoy_e2e.tar.gz
+          tar -xf sonobuoy_e2e.tar.gz --one-top-level
+
+      - name: Upload inspection report artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.os }}_${{ matrix.arch }}
+          path: sonobuoy_e2e.tar.gz
+
+      - name: Test Report
+        uses: dorny/test-reporter@v1
+#        if: success() || failure()    # run this step even if previous step failed
+        with:
+          name: Sonobuoy End to End Tests
+          path: sonobuoy_e2e/plugins/e2e/sonobuoy_results.yaml
+          reporter: jest-junit

From 2f1472d90dbaab52b99c4a23f8fc64abd7106380 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 14:06:00 +0200
Subject: [PATCH 02/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index bc82c8345..3b0ad5714 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -40,6 +40,7 @@ jobs:
           snap download k8s --channel=${{ matrix.releases }} --basename k8s
 
       - name: create k8s container
+        run: |
           lxc profile create k8s
           cat tests/integration/lxd-profile.yaml | lxc profile edit k8s
           lxc launch -p default -p k8s ${{ matrix.os }} k8s

From 46970fa9ba18ff8efe96022192addce9a5fb7b16 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 14:10:03 +0200
Subject: [PATCH 03/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 3b0ad5714..222f1fa24 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -47,7 +47,7 @@ jobs:
           lxc config device add k8s repo disk source=${PWD} path=/repo/
 
       - name: setup k8s in lxd container
-      - run: |
+        run: |
           lxc exec k8s -- sudo snap install /repo/k8s_v1.31.1_amd64.snap --dangerous --classic
           lxc exec k8s -- sudo k8s bootstrap
           lxc exec k8s -- sudo k8s config > ~/.kube/config

From b29871e9cfde649b92c8783cefff565ff8182bcd Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 14:11:34 +0200
Subject: [PATCH 04/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 222f1fa24..db7d98c2e 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -2,6 +2,9 @@ name: Sonobuoy Nightly Latest/Edge Tests
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - KU-269/sonobuoy-e2e
 
 permissions:
   contents: read

From 17681f0bf89752b544d0a9454eb591d5d28547bf Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 14:16:00 +0200
Subject: [PATCH 05/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index db7d98c2e..ec5421ddd 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -44,10 +44,10 @@ jobs:
 
       - name: create k8s container
         run: |
-          lxc profile create k8s
-          cat tests/integration/lxd-profile.yaml | lxc profile edit k8s
-          lxc launch -p default -p k8s ${{ matrix.os }} k8s
-          lxc config device add k8s repo disk source=${PWD} path=/repo/
+          sg lxd -c 'lxc profile create k8s'
+          sg lxd -c 'cat tests/integration/lxd-profile.yaml | lxc profile edit k8s'
+          sg lxd -c 'lxc launch -p default -p k8s ${{ matrix.os }} k8s'
+          sg lxd -c 'lxc config device add k8s repo disk source=${PWD} path=/repo/'
 
       - name: setup k8s in lxd container
         run: |

From 7fbd273ba81b6a832001ee3f5dd39505194464cb Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 14:18:28 +0200
Subject: [PATCH 06/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index ec5421ddd..85ea764da 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -51,9 +51,9 @@ jobs:
 
       - name: setup k8s in lxd container
         run: |
-          lxc exec k8s -- sudo snap install /repo/k8s_v1.31.1_amd64.snap --dangerous --classic
-          lxc exec k8s -- sudo k8s bootstrap
-          lxc exec k8s -- sudo k8s config > ~/.kube/config
+          sg lxd -c 'lxc exec k8s -- sudo snap install /repo/k8s_v1.31.1_amd64.snap --dangerous --classic'
+          sg lxd -c 'lxc exec k8s -- sudo k8s bootstrap'
+          sg lxd -c 'lxc exec k8s -- sudo k8s config' > ~/.kube/config
 
       - name: Run end to end tests
         run: |

From b9b5e8391aa07e9a91ea5191af6c4c83586ba795 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 14:20:43 +0200
Subject: [PATCH 07/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 85ea764da..fb3718634 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -51,7 +51,7 @@ jobs:
 
       - name: setup k8s in lxd container
         run: |
-          sg lxd -c 'lxc exec k8s -- sudo snap install /repo/k8s_v1.31.1_amd64.snap --dangerous --classic'
+          sg lxd -c 'lxc exec k8s -- sudo snap install /repo/k8s.snap --dangerous --classic'
           sg lxd -c 'lxc exec k8s -- sudo k8s bootstrap'
           sg lxd -c 'lxc exec k8s -- sudo k8s config' > ~/.kube/config
 

From 8484ea3d549f6e511d4e6ef283f7828d5a1f7872 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 14:56:54 +0200
Subject: [PATCH 08/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index fb3718634..2ba20b201 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -51,9 +51,9 @@ jobs:
 
       - name: setup k8s in lxd container
         run: |
-          sg lxd -c 'lxc exec k8s -- sudo snap install /repo/k8s.snap --dangerous --classic'
-          sg lxd -c 'lxc exec k8s -- sudo k8s bootstrap'
-          sg lxd -c 'lxc exec k8s -- sudo k8s config' > ~/.kube/config
+          sg lxd -c 'lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic'
+          sg lxd -c 'lxc exec k8s -- k8s bootstrap'
+          sg lxd -c 'lxc exec k8s -- k8s config' > ~/.kube/config
 
       - name: Run end to end tests
         run: |

From b604ffd8ea1ab25f71ee53e2e04cf6bb3e0eeb2d Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 15:12:42 +0200
Subject: [PATCH 09/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 2ba20b201..14f693ca0 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -51,6 +51,7 @@ jobs:
 
       - name: setup k8s in lxd container
         run: |
+          sg lxd -c 'lxc exec k8s -- service snapd start'
           sg lxd -c 'lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic'
           sg lxd -c 'lxc exec k8s -- k8s bootstrap'
           sg lxd -c 'lxc exec k8s -- k8s config' > ~/.kube/config

From 040652cc22f7cd9f6dbd19c641ee9a78649369f0 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 15:15:56 +0200
Subject: [PATCH 10/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 14f693ca0..c6c01a51c 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -54,6 +54,7 @@ jobs:
           sg lxd -c 'lxc exec k8s -- service snapd start'
           sg lxd -c 'lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic'
           sg lxd -c 'lxc exec k8s -- k8s bootstrap'
+          mkdir ~/.kube
           sg lxd -c 'lxc exec k8s -- k8s config' > ~/.kube/config
 
       - name: Run end to end tests

From d54af78fce9e5c455e5316cc246b521f3d0fd1ac Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 15:37:24 +0200
Subject: [PATCH 11/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index c6c01a51c..8f446d4ee 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -54,6 +54,7 @@ jobs:
           sg lxd -c 'lxc exec k8s -- service snapd start'
           sg lxd -c 'lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic'
           sg lxd -c 'lxc exec k8s -- k8s bootstrap'
+          sg lxd -c 'lxc exec k8s -- sudo k8s kubectl get all'
           mkdir ~/.kube
           sg lxd -c 'lxc exec k8s -- k8s config' > ~/.kube/config
 

From 215d57459b17733dca06d8d91d8f868014c7eac5 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 15:43:11 +0200
Subject: [PATCH 12/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 8f446d4ee..40d678814 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -54,7 +54,7 @@ jobs:
           sg lxd -c 'lxc exec k8s -- service snapd start'
           sg lxd -c 'lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic'
           sg lxd -c 'lxc exec k8s -- k8s bootstrap'
-          sg lxd -c 'lxc exec k8s -- sudo k8s kubectl get all'
+          sg lxd -c 'lxc exec k8s -- k8s status'
           mkdir ~/.kube
           sg lxd -c 'lxc exec k8s -- k8s config' > ~/.kube/config
 

From 64592ec63979c851c0716039d73705d3521a4bb1 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 15:54:20 +0200
Subject: [PATCH 13/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 40d678814..a2b32f01f 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -54,7 +54,7 @@ jobs:
           sg lxd -c 'lxc exec k8s -- service snapd start'
           sg lxd -c 'lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic'
           sg lxd -c 'lxc exec k8s -- k8s bootstrap'
-          sg lxd -c 'lxc exec k8s -- k8s status'
+          sg lxd -c 'lxc exec k8s -- k8s status --wait-ready'
           mkdir ~/.kube
           sg lxd -c 'lxc exec k8s -- k8s config' > ~/.kube/config
 

From 934fb203a6d40ae8b4933ad45a23dc287d60412d Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 16:16:39 +0200
Subject: [PATCH 14/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index a2b32f01f..0788da46d 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -68,7 +68,7 @@ jobs:
       - name: Upload inspection report artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.os }}_${{ matrix.arch }}
+          name: report_${{ matrix.arch }}
           path: sonobuoy_e2e.tar.gz
 
       - name: Test Report

From b32b134337f2fc162745bb5774b93f439fe3663a Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 16:23:15 +0200
Subject: [PATCH 15/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 0788da46d..236012345 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -76,5 +76,5 @@ jobs:
 #        if: success() || failure()    # run this step even if previous step failed
         with:
           name: Sonobuoy End to End Tests
-          path: sonobuoy_e2e/plugins/e2e/sonobuoy_results.yaml
+          path: sonobuoy_e2e/plugins/e2e/results/global/*.xml
           reporter: jest-junit

From d0958b60532fc99fffb4520f01e447a6099c8ef7 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 16:31:11 +0200
Subject: [PATCH 16/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 236012345..ebcd7a14b 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -8,6 +8,8 @@ on:
 
 permissions:
   contents: read
+  actions: read
+  checks: write
 
 jobs:
   test-integration:
@@ -30,6 +32,7 @@ jobs:
           sudo lxd init --auto
           sudo usermod --append --groups lxd $USER
           sg lxd -c 'lxc version'
+          echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
 
       - name: Install sonobuoy
         run: |
@@ -68,8 +71,8 @@ jobs:
       - name: Upload inspection report artifact
         uses: actions/upload-artifact@v4
         with:
-          name: report_${{ matrix.arch }}
-          path: sonobuoy_e2e.tar.gz
+          name: report_sonobuoy_e2e_${{ github.os_dash }}_${{ matrix.arch }}
+          path: sonobuoy_e2e
 
       - name: Test Report
         uses: dorny/test-reporter@v1

From 41bdf6f601a3fc0969e727dcae34cb1f0e2500b7 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 16:38:26 +0200
Subject: [PATCH 17/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index ebcd7a14b..59db3859e 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -75,7 +75,7 @@ jobs:
           path: sonobuoy_e2e
 
       - name: Test Report
-        uses: dorny/test-reporter@v1
+        uses: phoenix-actions/test-reporting@v8
 #        if: success() || failure()    # run this step even if previous step failed
         with:
           name: Sonobuoy End to End Tests

From 8c291d91916bc13120bd20e91665bb916c7ab952 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 17:03:43 +0200
Subject: [PATCH 18/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 59db3859e..17030d26e 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -41,7 +41,7 @@ jobs:
           curl -L -o sonobuoy.tar.gz https://github.com/vmware-tanzu/sonobuoy/releases/download/${tag}/sonobuoy_${tag//v}_linux_${{ matrix.arch }}.tar.gz
           tar -xf sonobuoy.tar.gz
 
-      - name: download $${ matrix.releases }} k8s snap
+      - name: download ${ matrix.releases }} k8s snap
         run: |
           snap download k8s --channel=${{ matrix.releases }} --basename k8s
 
@@ -81,3 +81,15 @@ jobs:
           name: Sonobuoy End to End Tests
           path: sonobuoy_e2e/plugins/e2e/results/global/*.xml
           reporter: jest-junit
+      - name: Test Summary
+        uses: test-summary/action@v2
+        with:
+          paths: "sonobuoy_e2e/plugins/e2e/results/global/*.xml"
+          output: test-summary.md
+        if: always()
+      - name: Upload test summary
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-summary
+          path: test-summary.md
+        if: always()

From fd4158cb91e4f772b32c8e9e62bc71b0e8ceca04 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 17:11:56 +0200
Subject: [PATCH 19/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 17030d26e..8fa2a4e70 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -85,11 +85,3 @@ jobs:
         uses: test-summary/action@v2
         with:
           paths: "sonobuoy_e2e/plugins/e2e/results/global/*.xml"
-          output: test-summary.md
-        if: always()
-      - name: Upload test summary
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-summary
-          path: test-summary.md
-        if: always()

From 5af23f5eee31dd1036bba23a7eb116785ff206bf Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 4 Oct 2024 17:32:49 +0200
Subject: [PATCH 20/59] Sonobuoy Nightly Tests

---
 .github/workflows/sonobuoy-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 8fa2a4e70..ce88f9c0d 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -63,7 +63,7 @@ jobs:
 
       - name: Run end to end tests
         run: |
-          ./sonobuoy run --plugin e2e --wait --mode quick
+          ./sonobuoy run --plugin e2e --wait
           ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
           ./sonobuoy results sonobuoy_e2e.tar.gz
           tar -xf sonobuoy_e2e.tar.gz --one-top-level

From 4178e7ebafe0720a0fe5e77c68762e9a5d8f2663 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 7 Oct 2024 09:03:10 +0200
Subject: [PATCH 21/59] remove summary

---
 .github/workflows/sonobuoy-test.yaml | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index ce88f9c0d..7ac942a85 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -8,8 +8,6 @@ on:
 
 permissions:
   contents: read
-  actions: read
-  checks: write
 
 jobs:
   test-integration:
@@ -73,15 +71,3 @@ jobs:
         with:
           name: report_sonobuoy_e2e_${{ github.os_dash }}_${{ matrix.arch }}
           path: sonobuoy_e2e
-
-      - name: Test Report
-        uses: phoenix-actions/test-reporting@v8
-#        if: success() || failure()    # run this step even if previous step failed
-        with:
-          name: Sonobuoy End to End Tests
-          path: sonobuoy_e2e/plugins/e2e/results/global/*.xml
-          reporter: jest-junit
-      - name: Test Summary
-        uses: test-summary/action@v2
-        with:
-          paths: "sonobuoy_e2e/plugins/e2e/results/global/*.xml"

From 35d0e3f56f37c3d9dee839e2d0e83ae992120582 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 7 Oct 2024 10:26:46 +0200
Subject: [PATCH 22/59] switch to schedule

---
 .github/workflows/sonobuoy-test.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 7ac942a85..73c61a6dc 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -1,10 +1,8 @@
 name: Sonobuoy Nightly Latest/Edge Tests
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - KU-269/sonobuoy-e2e
+  schedule:
+    - cron: '0 0 * * *' # Runs every midnight
 
 permissions:
   contents: read

From 14129900bb82aabc458a46ec249db1889a5deefa Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 7 Oct 2024 10:51:05 +0200
Subject: [PATCH 23/59] make run weekly

---
 .github/workflows/sonobuoy-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
index 73c61a6dc..4440ff991 100644
--- a/.github/workflows/sonobuoy-test.yaml
+++ b/.github/workflows/sonobuoy-test.yaml
@@ -2,7 +2,7 @@ name: Sonobuoy Nightly Latest/Edge Tests
 
 on:
   schedule:
-    - cron: '0 0 * * *' # Runs every midnight
+    - cron: '0 0 * * 1' # Runs every monday midnight
 
 permissions:
   contents: read

From 6d19b7be7412c8ed58fcbaa34a92f3c37885a49a Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 7 Oct 2024 15:21:43 +0200
Subject: [PATCH 24/59] move do shell script

---
 .github/workflows/cncf-conformance-test.yaml | 76 +++++++++++++++++++
 .github/workflows/sonobuoy-test.yaml         | 71 ------------------
 build-scripts/hack/sonobuoy.sh               | 77 ++++++++++++++++++++
 3 files changed, 153 insertions(+), 71 deletions(-)
 create mode 100644 .github/workflows/cncf-conformance-test.yaml
 delete mode 100644 .github/workflows/sonobuoy-test.yaml
 create mode 100755 build-scripts/hack/sonobuoy.sh

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
new file mode 100644
index 000000000..df150a352
--- /dev/null
+++ b/.github/workflows/cncf-conformance-test.yaml
@@ -0,0 +1,76 @@
+name: CNCF Conformance Nightly Latest/Edge Tests
+
+on:
+  schedule:
+    - cron: '0 0 * * 1' # Runs every monday midnight
+  pull_request:
+permissions:
+  contents: read
+
+jobs:
+  test-cncf-conformance:
+    name: Integration Test ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.releases }}
+    strategy:
+      matrix:
+        os: [ "ubuntu:24.04"]
+#        os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
+        arch: ["amd64"]
+        releases: ["latest/edge"]
+
+    runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }}
+
+    steps:
+      - name: Checking out repo
+        uses: actions/checkout@v4
+
+      - name: Install lxd
+        run: |
+          sudo snap refresh lxd --channel 5.21/stable
+          sudo lxd init --auto
+          sudo usermod --append --groups lxd $USER
+          sg lxd -c 'lxc version'
+          echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
+
+      - name: Install sonobuoy
+        run: |
+          build-scripts/hack/sonobuoy.sh download ${{ matrix.arch }}
+#          release=$(curl --silent -m 10 --connect-timeout 5 "https://api.github.com/repos/vmware-tanzu/sonobuoy/releases/latest")
+#          tag=$(echo "$release" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+#          curl -L -o sonobuoy.tar.gz https://github.com/vmware-tanzu/sonobuoy/releases/download/${tag}/sonobuoy_${tag//v}_linux_${{ matrix.arch }}.tar.gz
+#          tar -xf sonobuoy.tar.gz
+
+      - name: download ${{ matrix.releases }} k8s snap
+        run: |
+          snap download k8s --channel=${{ matrix.releases }} --basename k8s
+
+      - name: create k8s container
+        run: |
+          sg lxd -c 'build-scripts/hack/sonobuoy.sh create_container ${{ matrix.os }}'
+#          sg lxd -c 'lxc profile create k8s'
+#          sg lxd -c 'cat tests/integration/lxd-profile.yaml | lxc profile edit k8s'
+#          sg lxd -c 'lxc launch -p default -p k8s ${{ matrix.os }} k8s'
+#          sg lxd -c 'lxc config device add k8s repo disk source=${PWD} path=/repo/'
+
+      - name: setup k8s in lxd container
+        run: |
+          sg lxd -c 'build-scripts/hack/sonobuoy.sh setup_k8s'
+#          sg lxd -c 'lxc exec k8s -- service snapd start'
+#          sg lxd -c 'lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic'
+#          sg lxd -c 'lxc exec k8s -- k8s bootstrap'
+#          sg lxd -c 'lxc exec k8s -- k8s status --wait-ready'
+#          mkdir ~/.kube
+#          sg lxd -c 'lxc exec k8s -- k8s config' > ~/.kube/config
+
+      - name: Run end to end tests
+        run: |
+          build-scripts/hack/sonobuoy.sh run_e2e
+#          ./sonobuoy run --plugin e2e --wait
+#          ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
+#          ./sonobuoy results sonobuoy_e2e.tar.gz
+#          tar -xf sonobuoy_e2e.tar.gz --one-top-level
+
+      - name: Upload inspection report artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: report_sonobuoy_e2e_${{ github.os_dash }}_${{ matrix.arch }}
+          path: sonobuoy_e2e
diff --git a/.github/workflows/sonobuoy-test.yaml b/.github/workflows/sonobuoy-test.yaml
deleted file mode 100644
index 4440ff991..000000000
--- a/.github/workflows/sonobuoy-test.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
-name: Sonobuoy Nightly Latest/Edge Tests
-
-on:
-  schedule:
-    - cron: '0 0 * * 1' # Runs every monday midnight
-
-permissions:
-  contents: read
-
-jobs:
-  test-integration:
-    name: Integration Test ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.releases }}
-    strategy:
-      matrix:
-        os: [ "ubuntu:24.04"]
-        arch: ["amd64"]
-        releases: ["latest/edge"]
-
-    runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }}
-
-    steps:
-      - name: Checking out repo
-        uses: actions/checkout@v4
-
-      - name: Install lxd
-        run: |
-          sudo snap refresh lxd --channel 5.21/stable
-          sudo lxd init --auto
-          sudo usermod --append --groups lxd $USER
-          sg lxd -c 'lxc version'
-          echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
-
-      - name: Install sonobuoy
-        run: |
-          release=$(curl --silent -m 10 --connect-timeout 5 "https://api.github.com/repos/vmware-tanzu/sonobuoy/releases/latest")
-          tag=$(echo "$release" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
-          curl -L -o sonobuoy.tar.gz https://github.com/vmware-tanzu/sonobuoy/releases/download/${tag}/sonobuoy_${tag//v}_linux_${{ matrix.arch }}.tar.gz
-          tar -xf sonobuoy.tar.gz
-
-      - name: download ${ matrix.releases }} k8s snap
-        run: |
-          snap download k8s --channel=${{ matrix.releases }} --basename k8s
-
-      - name: create k8s container
-        run: |
-          sg lxd -c 'lxc profile create k8s'
-          sg lxd -c 'cat tests/integration/lxd-profile.yaml | lxc profile edit k8s'
-          sg lxd -c 'lxc launch -p default -p k8s ${{ matrix.os }} k8s'
-          sg lxd -c 'lxc config device add k8s repo disk source=${PWD} path=/repo/'
-
-      - name: setup k8s in lxd container
-        run: |
-          sg lxd -c 'lxc exec k8s -- service snapd start'
-          sg lxd -c 'lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic'
-          sg lxd -c 'lxc exec k8s -- k8s bootstrap'
-          sg lxd -c 'lxc exec k8s -- k8s status --wait-ready'
-          mkdir ~/.kube
-          sg lxd -c 'lxc exec k8s -- k8s config' > ~/.kube/config
-
-      - name: Run end to end tests
-        run: |
-          ./sonobuoy run --plugin e2e --wait
-          ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
-          ./sonobuoy results sonobuoy_e2e.tar.gz
-          tar -xf sonobuoy_e2e.tar.gz --one-top-level
-
-      - name: Upload inspection report artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: report_sonobuoy_e2e_${{ github.os_dash }}_${{ matrix.arch }}
-          path: sonobuoy_e2e
diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
new file mode 100755
index 000000000..65aab7cb3
--- /dev/null
+++ b/build-scripts/hack/sonobuoy.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+set -e
+
+function download() {
+    local architecture=$1
+    local release=$(curl --silent -m 10 --connect-timeout 5 "https://api.github.com/repos/vmware-tanzu/sonobuoy/releases/latest")
+    local tag=$(echo "$release" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+    curl -L -o sonobuoy.tar.gz https://github.com/vmware-tanzu/sonobuoy/releases/download/${tag}/sonobuoy_${tag//v}_linux_${architecture}.tar.gz
+    tar -xf sonobuoy.tar.gz
+    rm sonobuoy.tar.gz
+}
+
+function create_container() {
+  local os=$1
+  lxc profile create k8s
+  lxc profile create k8s
+  cat tests/integration/lxd-profile.yaml | lxc profile edit k8s
+  lxc launch -p default -p k8s ${os} k8s
+  lxc config device add k8s repo disk source=${PWD} path=/repo/
+}
+
+function setup_k8s() {
+  lxc exec k8s -- service snapd start
+  lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic
+  lxc exec k8s -- k8s bootstrap
+  lxc exec k8s -- k8s status --wait-ready
+  mkdir -p ~/.kube
+  lxc exec k8s -- k8s config > ~/.kube/config
+}
+
+function run_e2e() {
+#  ./sonobuoy run --plugin e2e --wait
+  ./sonobuoy run --plugin e2e --wait --mode quick
+  ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
+  ./sonobuoy results sonobuoy_e2e.tar.gz
+  tar -xf sonobuoy_e2e.tar.gz --one-top-level
+}
+
+function main() {
+    local command=$1
+    shift
+    case $command in
+        "download")
+            download "${@}"
+            ;;
+        "create_container")
+            create_container "${@}"
+            ;;
+        "setup_k8s")
+            setup_k8s "${@}"
+            ;;
+        "run_e2e")
+            run_e2e "${@}"
+            ;;
+        *)
+            cat << EOF
+Unknown command: $1
+
+usage: $0 <command>
+
+Commands:
+    download <amd64|arm64|386|ppc64le|s390x>   Download sonobuoy for given architecture.
+
+    create_container <os>                      Creates lxd container for given operating system.
+
+    setup_k8s                                  Install k8s in k8s lxd container.
+
+    run_e2e                                    Runs sonobuoy end-to-end tests and saves results in
+                                               sonobuoy_e2e.tar.gz and sonobuoy_e2e directory
+EOF
+            ;;
+    esac
+}
+
+if [[ $sourced -ne 1 ]]; then
+    main $@
+fi

From d055eb53ae38267945333ccd4ff33d4584206158 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 7 Oct 2024 15:26:10 +0200
Subject: [PATCH 25/59] profile fix

---
 build-scripts/hack/sonobuoy.sh | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
index 65aab7cb3..ff6e859ba 100755
--- a/build-scripts/hack/sonobuoy.sh
+++ b/build-scripts/hack/sonobuoy.sh
@@ -12,20 +12,19 @@ function download() {
 
 function create_container() {
   local os=$1
-  lxc profile create k8s
-  lxc profile create k8s
-  cat tests/integration/lxd-profile.yaml | lxc profile edit k8s
-  lxc launch -p default -p k8s ${os} k8s
-  lxc config device add k8s repo disk source=${PWD} path=/repo/
+  lxc profile create k8s-e2e
+  cat tests/integration/lxd-profile.yaml | lxc profile edit k8s-e2e
+  lxc launch -p default -p k8s-e2e ${os} k8s-e2e
+  lxc config device add k8s-e2e repo disk source=${PWD} path=/repo/
 }
 
 function setup_k8s() {
-  lxc exec k8s -- service snapd start
-  lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic
-  lxc exec k8s -- k8s bootstrap
-  lxc exec k8s -- k8s status --wait-ready
+  lxc exec k8s-e2e -- service snapd start
+  lxc exec k8s-e2e -- snap install /repo/k8s.snap --dangerous --classic
+  lxc exec k8s-e2e -- k8s bootstrap
+  lxc exec k8s-e2e -- k8s status --wait-ready
   mkdir -p ~/.kube
-  lxc exec k8s -- k8s config > ~/.kube/config
+  lxc exec k8s-e2e -- k8s config > ~/.kube/config
 }
 
 function run_e2e() {

From dfc6f9518521c6a19e95f385c27c5e92fb2755a6 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 7 Oct 2024 16:16:05 +0200
Subject: [PATCH 26/59] remove comments

---
 .github/workflows/cncf-conformance-test.yaml | 23 ++------------------
 1 file changed, 2 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
index df150a352..7290c66a7 100644
--- a/.github/workflows/cncf-conformance-test.yaml
+++ b/.github/workflows/cncf-conformance-test.yaml
@@ -9,11 +9,10 @@ permissions:
 
 jobs:
   test-cncf-conformance:
-    name: Integration Test ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.releases }}
+    name: CNCF Conformance ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.releases }}
     strategy:
       matrix:
-        os: [ "ubuntu:24.04"]
-#        os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
+        os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
         arch: ["amd64"]
         releases: ["latest/edge"]
 
@@ -34,10 +33,6 @@ jobs:
       - name: Install sonobuoy
         run: |
           build-scripts/hack/sonobuoy.sh download ${{ matrix.arch }}
-#          release=$(curl --silent -m 10 --connect-timeout 5 "https://api.github.com/repos/vmware-tanzu/sonobuoy/releases/latest")
-#          tag=$(echo "$release" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
-#          curl -L -o sonobuoy.tar.gz https://github.com/vmware-tanzu/sonobuoy/releases/download/${tag}/sonobuoy_${tag//v}_linux_${{ matrix.arch }}.tar.gz
-#          tar -xf sonobuoy.tar.gz
 
       - name: download ${{ matrix.releases }} k8s snap
         run: |
@@ -46,28 +41,14 @@ jobs:
       - name: create k8s container
         run: |
           sg lxd -c 'build-scripts/hack/sonobuoy.sh create_container ${{ matrix.os }}'
-#          sg lxd -c 'lxc profile create k8s'
-#          sg lxd -c 'cat tests/integration/lxd-profile.yaml | lxc profile edit k8s'
-#          sg lxd -c 'lxc launch -p default -p k8s ${{ matrix.os }} k8s'
-#          sg lxd -c 'lxc config device add k8s repo disk source=${PWD} path=/repo/'
 
       - name: setup k8s in lxd container
         run: |
           sg lxd -c 'build-scripts/hack/sonobuoy.sh setup_k8s'
-#          sg lxd -c 'lxc exec k8s -- service snapd start'
-#          sg lxd -c 'lxc exec k8s -- snap install /repo/k8s.snap --dangerous --classic'
-#          sg lxd -c 'lxc exec k8s -- k8s bootstrap'
-#          sg lxd -c 'lxc exec k8s -- k8s status --wait-ready'
-#          mkdir ~/.kube
-#          sg lxd -c 'lxc exec k8s -- k8s config' > ~/.kube/config
 
       - name: Run end to end tests
         run: |
           build-scripts/hack/sonobuoy.sh run_e2e
-#          ./sonobuoy run --plugin e2e --wait
-#          ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
-#          ./sonobuoy results sonobuoy_e2e.tar.gz
-#          tar -xf sonobuoy_e2e.tar.gz --one-top-level
 
       - name: Upload inspection report artifact
         uses: actions/upload-artifact@v4

From df4f77111dd842596b05013b6fd638753e777aca Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 7 Oct 2024 16:26:40 +0200
Subject: [PATCH 27/59] test1

---
 .github/workflows/cncf-conformance-test.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
index 7290c66a7..e6b7df413 100644
--- a/.github/workflows/cncf-conformance-test.yaml
+++ b/.github/workflows/cncf-conformance-test.yaml
@@ -12,7 +12,8 @@ jobs:
     name: CNCF Conformance ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.releases }}
     strategy:
       matrix:
-        os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
+#        os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
+        os: [ "ubuntu:22.04", "ubuntu:24.04"]
         arch: ["amd64"]
         releases: ["latest/edge"]
 

From b1ecb009b4e23711aff1737cc63df2bec0fe01c8 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 7 Oct 2024 16:38:34 +0200
Subject: [PATCH 28/59] test2

---
 .github/workflows/cncf-conformance-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
index e6b7df413..2c9371bb8 100644
--- a/.github/workflows/cncf-conformance-test.yaml
+++ b/.github/workflows/cncf-conformance-test.yaml
@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
 #        os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
-        os: [ "ubuntu:22.04", "ubuntu:24.04"]
+        os: ["ubuntu:24.04"]
         arch: ["amd64"]
         releases: ["latest/edge"]
 

From c56c14973d96fbb8aed011649f92c51068574ffb Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 7 Oct 2024 17:14:23 +0200
Subject: [PATCH 29/59] run full e2e

---
 build-scripts/hack/sonobuoy.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
index ff6e859ba..41163f357 100755
--- a/build-scripts/hack/sonobuoy.sh
+++ b/build-scripts/hack/sonobuoy.sh
@@ -28,8 +28,7 @@ function setup_k8s() {
 }
 
 function run_e2e() {
-#  ./sonobuoy run --plugin e2e --wait
-  ./sonobuoy run --plugin e2e --wait --mode quick
+  ./sonobuoy run --plugin e2e --wait
   ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
   ./sonobuoy results sonobuoy_e2e.tar.gz
   tar -xf sonobuoy_e2e.tar.gz --one-top-level

From e1922fa4a479d2283070c3147d5fccb5e73b29fa Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Wed, 9 Oct 2024 15:03:15 +0200
Subject: [PATCH 30/59] review fixes

---
 build-scripts/hack/sonobuoy.sh | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
index 41163f357..052b186b3 100755
--- a/build-scripts/hack/sonobuoy.sh
+++ b/build-scripts/hack/sonobuoy.sh
@@ -11,27 +11,29 @@ function download() {
 }
 
 function create_container() {
-  local os=$1
-  lxc profile create k8s-e2e
-  cat tests/integration/lxd-profile.yaml | lxc profile edit k8s-e2e
-  lxc launch -p default -p k8s-e2e ${os} k8s-e2e
-  lxc config device add k8s-e2e repo disk source=${PWD} path=/repo/
+    local os=$1
+    lxc profile create k8s-e2e
+    cat tests/integration/lxd-profile.yaml | lxc profile edit k8s-e2e
+    lxc launch -p default -p k8s-e2e ${os} k8s-e2e
+    lxc config device add k8s-e2e repo disk source=${PWD} path=/repo/
 }
 
 function setup_k8s() {
-  lxc exec k8s-e2e -- service snapd start
-  lxc exec k8s-e2e -- snap install /repo/k8s.snap --dangerous --classic
-  lxc exec k8s-e2e -- k8s bootstrap
-  lxc exec k8s-e2e -- k8s status --wait-ready
-  mkdir -p ~/.kube
-  lxc exec k8s-e2e -- k8s config > ~/.kube/config
+    lxc exec k8s-e2e -- service snapd start
+    lxc exec k8s-e2e -- snap install /repo/k8s.snap --dangerous --classic
+    lxc exec k8s-e2e -- k8s bootstrap
+    lxc exec k8s-e2e -- k8s status --wait-ready
+    mkdir -p ~/.kube
+    lxc exec k8s-e2e -- k8s config > ~/.kube/config
 }
 
 function run_e2e() {
-  ./sonobuoy run --plugin e2e --wait
-  ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
-  ./sonobuoy results sonobuoy_e2e.tar.gz
-  tar -xf sonobuoy_e2e.tar.gz --one-top-level
+    ./sonobuoy run --plugin e2e --wait
+    ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
+    ./sonobuoy results sonobuoy_e2e.tar.gz
+    tar -xf sonobuoy_e2e.tar.gz --one-top-level
+    ./sonobuoy results sonobuoy_e2e.tar.gz | grep -E "^Failed: 0$"
+    exit $?
 }
 
 function main() {

From 117f5ba2b7185b0e16cc3ed77bd4047dbcdca3d9 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Sun, 13 Oct 2024 18:10:16 +0200
Subject: [PATCH 31/59] review fixes

---
 .github/workflows/cncf-conformance-test.yaml | 33 ++++++-------
 build-scripts/hack/sonobuoy.sh               | 49 +++++++-------------
 2 files changed, 35 insertions(+), 47 deletions(-)

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
index 2c9371bb8..8f4d5151f 100644
--- a/.github/workflows/cncf-conformance-test.yaml
+++ b/.github/workflows/cncf-conformance-test.yaml
@@ -12,10 +12,9 @@ jobs:
     name: CNCF Conformance ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.releases }}
     strategy:
       matrix:
-#        os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
         os: ["ubuntu:24.04"]
         arch: ["amd64"]
-        releases: ["latest/edge"]
+        releases: ["latest/edge/moonray"]
 
     runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }}
 
@@ -31,26 +30,28 @@ jobs:
           sg lxd -c 'lxc version'
           echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
 
-      - name: Install sonobuoy
-        run: |
-          build-scripts/hack/sonobuoy.sh download ${{ matrix.arch }}
-
       - name: download ${{ matrix.releases }} k8s snap
         run: |
           snap download k8s --channel=${{ matrix.releases }} --basename k8s
 
-      - name: create k8s container
-        run: |
-          sg lxd -c 'build-scripts/hack/sonobuoy.sh create_container ${{ matrix.os }}'
-
-      - name: setup k8s in lxd container
-        run: |
-          sg lxd -c 'build-scripts/hack/sonobuoy.sh setup_k8s'
-
+#      - name: Install sonobuoy
+#        run: |
+#          build-scripts/hack/sonobuoy.sh download ${{ matrix.arch }}
+#
+#      - name: create k8s container
+#        run: |
+#          sg lxd -c 'build-scripts/hack/sonobuoy.sh create_container ${{ matrix.os }}'
+#
+#      - name: setup k8s in lxd container
+#        run: |
+#          sg lxd -c 'build-scripts/hack/sonobuoy.sh setup_k8s'
+#
+#      - name: Run end to end tests
+#        run: |
+#          build-scripts/hack/sonobuoy.sh run_e2e
       - name: Run end to end tests
         run: |
-          build-scripts/hack/sonobuoy.sh run_e2e
-
+          sg lxd -c 'build-scripts/hack/sonobuoy.sh ${{ matrix.arch }} ${{ matrix.os }}'
       - name: Upload inspection report artifact
         uses: actions/upload-artifact@v4
         with:
diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
index 052b186b3..6d0f891cd 100755
--- a/build-scripts/hack/sonobuoy.sh
+++ b/build-scripts/hack/sonobuoy.sh
@@ -28,48 +28,35 @@ function setup_k8s() {
 }
 
 function run_e2e() {
-    ./sonobuoy run --plugin e2e --wait
+#    ./sonobuoy run --plugin e2e --wait
+    ./sonobuoy run --plugin e2e --wait --mode quick
     ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
     ./sonobuoy results sonobuoy_e2e.tar.gz
     tar -xf sonobuoy_e2e.tar.gz --one-top-level
+    set +e
     ./sonobuoy results sonobuoy_e2e.tar.gz | grep -E "^Failed: 0$"
-    exit $?
+    return $?
 }
 
 function main() {
-    local command=$1
-    shift
-    case $command in
-        "download")
-            download "${@}"
-            ;;
-        "create_container")
-            create_container "${@}"
-            ;;
-        "setup_k8s")
-            setup_k8s "${@}"
-            ;;
-        "run_e2e")
-            run_e2e "${@}"
-            ;;
-        *)
-            cat << EOF
-Unknown command: $1
-
-usage: $0 <command>
+    if [ "$#" -ne 2 ]; then
+      cat << EOF
+Expected 2 arguments, provided: $@
 
-Commands:
-    download <amd64|arm64|386|ppc64le|s390x>   Download sonobuoy for given architecture.
+usage: $0 <architecture> <os>
+EOF
+      exit 255
+    fi
 
-    create_container <os>                      Creates lxd container for given operating system.
+    local architecture=$1
+    local os=$2
 
-    setup_k8s                                  Install k8s in k8s lxd container.
+    download ${architecture}
+    create_container ${os}
+    setup_k8s
+    run_e2e
+    exit $?
 
-    run_e2e                                    Runs sonobuoy end-to-end tests and saves results in
-                                               sonobuoy_e2e.tar.gz and sonobuoy_e2e directory
-EOF
-            ;;
-    esac
 }
 
 if [[ $sourced -ne 1 ]]; then

From 90845a27b96b69b86a4b58e596ffe882b0f8db48 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 14 Oct 2024 09:06:42 +0200
Subject: [PATCH 32/59] review fixes

---
 .github/workflows/cncf-conformance-test.yaml | 21 +++-----------------
 build-scripts/hack/sonobuoy.sh               |  1 -
 2 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
index 8f4d5151f..b87cfb2a1 100644
--- a/.github/workflows/cncf-conformance-test.yaml
+++ b/.github/workflows/cncf-conformance-test.yaml
@@ -14,8 +14,7 @@ jobs:
       matrix:
         os: ["ubuntu:24.04"]
         arch: ["amd64"]
-        releases: ["latest/edge/moonray"]
-
+        releases: ["latest/edge/moonray"] # TODO add latest/edge after upgrade Cilium to 1.17
     runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }}
 
     steps:
@@ -34,24 +33,10 @@ jobs:
         run: |
           snap download k8s --channel=${{ matrix.releases }} --basename k8s
 
-#      - name: Install sonobuoy
-#        run: |
-#          build-scripts/hack/sonobuoy.sh download ${{ matrix.arch }}
-#
-#      - name: create k8s container
-#        run: |
-#          sg lxd -c 'build-scripts/hack/sonobuoy.sh create_container ${{ matrix.os }}'
-#
-#      - name: setup k8s in lxd container
-#        run: |
-#          sg lxd -c 'build-scripts/hack/sonobuoy.sh setup_k8s'
-#
-#      - name: Run end to end tests
-#        run: |
-#          build-scripts/hack/sonobuoy.sh run_e2e
-      - name: Run end to end tests
+      - name: Run end to end tests on ${{ matrix.arch }} ${{ matrix.os }}
         run: |
           sg lxd -c 'build-scripts/hack/sonobuoy.sh ${{ matrix.arch }} ${{ matrix.os }}'
+
       - name: Upload inspection report artifact
         uses: actions/upload-artifact@v4
         with:
diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
index 6d0f891cd..62469c943 100755
--- a/build-scripts/hack/sonobuoy.sh
+++ b/build-scripts/hack/sonobuoy.sh
@@ -56,7 +56,6 @@ EOF
     setup_k8s
     run_e2e
     exit $?
-
 }
 
 if [[ $sourced -ne 1 ]]; then

From 4c4ed97bfcc7ee2b36f7918c19a251b5f7735e9c Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 14 Oct 2024 09:40:22 +0200
Subject: [PATCH 33/59] full tests

---
 build-scripts/hack/sonobuoy.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
index 62469c943..5022ecbca 100755
--- a/build-scripts/hack/sonobuoy.sh
+++ b/build-scripts/hack/sonobuoy.sh
@@ -28,8 +28,7 @@ function setup_k8s() {
 }
 
 function run_e2e() {
-#    ./sonobuoy run --plugin e2e --wait
-    ./sonobuoy run --plugin e2e --wait --mode quick
+    ./sonobuoy run --plugin e2e --wait
     ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
     ./sonobuoy results sonobuoy_e2e.tar.gz
     tar -xf sonobuoy_e2e.tar.gz --one-top-level

From a37f187d0e1271732136013f2d8c570ca43e9f97 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 14 Oct 2024 13:24:33 +0200
Subject: [PATCH 34/59] remove pr run

---
 .github/workflows/cncf-conformance-test.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
index b87cfb2a1..3342fa67d 100644
--- a/.github/workflows/cncf-conformance-test.yaml
+++ b/.github/workflows/cncf-conformance-test.yaml
@@ -3,7 +3,6 @@ name: CNCF Conformance Nightly Latest/Edge Tests
 on:
   schedule:
     - cron: '0 0 * * 1' # Runs every monday midnight
-  pull_request:
 permissions:
   contents: read
 

From 2d5c44b777fd835768acff169fd788c249e2d06d Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 14 Oct 2024 16:13:14 +0200
Subject: [PATCH 35/59] add pr run

---
 .github/workflows/cncf-conformance-test.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
index 3342fa67d..e926cb09e 100644
--- a/.github/workflows/cncf-conformance-test.yaml
+++ b/.github/workflows/cncf-conformance-test.yaml
@@ -3,6 +3,7 @@ name: CNCF Conformance Nightly Latest/Edge Tests
 on:
   schedule:
     - cron: '0 0 * * 1' # Runs every monday midnight
+  pull_request:
 permissions:
   contents: read
 
@@ -38,6 +39,7 @@ jobs:
 
       - name: Upload inspection report artifact
         uses: actions/upload-artifact@v4
+        if: always()
         with:
           name: report_sonobuoy_e2e_${{ github.os_dash }}_${{ matrix.arch }}
           path: sonobuoy_e2e

From 8938d6280ff99777cb8e81c4044f9c7a35d1364b Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 14 Oct 2024 20:30:11 +0200
Subject: [PATCH 36/59] WIP

---
 build-scripts/hack/sonobuoy.sh | 55 ++++++++++++++++++++++------------
 1 file changed, 36 insertions(+), 19 deletions(-)

diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
index 5022ecbca..28f1ca6e7 100755
--- a/build-scripts/hack/sonobuoy.sh
+++ b/build-scripts/hack/sonobuoy.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
-set -e
-
+set -ex
+token=""
 function download() {
     local architecture=$1
     local release=$(curl --silent -m 10 --connect-timeout 5 "https://api.github.com/repos/vmware-tanzu/sonobuoy/releases/latest")
@@ -11,20 +11,32 @@ function download() {
 }
 
 function create_container() {
-    local os=$1
-    lxc profile create k8s-e2e
-    cat tests/integration/lxd-profile.yaml | lxc profile edit k8s-e2e
-    lxc launch -p default -p k8s-e2e ${os} k8s-e2e
-    lxc config device add k8s-e2e repo disk source=${PWD} path=/repo/
+    local container_name=$1
+    local os=$2
+#    lxc profile create ${container_name}
+#    cat tests/integration/lxd-profile.yaml | lxc profile edit ${container_name}
+    lxc launch -p default -p ${container_name} ${os} ${container_name}
+    lxc config device add ${container_name} repo disk source=${PWD} path=/repo/
 }
 
 function setup_k8s() {
-    lxc exec k8s-e2e -- service snapd start
-    lxc exec k8s-e2e -- snap install /repo/k8s.snap --dangerous --classic
-    lxc exec k8s-e2e -- k8s bootstrap
-    lxc exec k8s-e2e -- k8s status --wait-ready
+    local container_name=$1
+    lxc exec ${container_name} -- service snapd start
+    lxc exec ${container_name} -- snap install /repo/k8s.snap --dangerous --classic
+    lxc exec ${container_name} -- k8s bootstrap
+    lxc exec ${container_name} -- k8s status --wait-ready
     mkdir -p ~/.kube
-    lxc exec k8s-e2e -- k8s config > ~/.kube/config
+    lxc exec ${container_name} -- k8s config >> ~/.kube/config
+    lxc exec ${container_name} -- k8s get-join-token worker
+}
+
+function add_k8s_node() {
+    local container_name=$1
+    local token=$2
+    lxc exec ${container_name} -- service snapd start
+    lxc exec ${container_name} -- snap install /repo/k8s.snap --dangerous --classic
+#    lxc exec ${container_name} -- k8s join-cluster ${token}
+#    lxc exec ${container_name} -- k8s status --wait-ready
 }
 
 function run_e2e() {
@@ -46,15 +58,20 @@ usage: $0 <architecture> <os>
 EOF
       exit 255
     fi
-
     local architecture=$1
     local os=$2
-
-    download ${architecture}
-    create_container ${os}
-    setup_k8s
-    run_e2e
-    exit $?
+    local main="main"
+    local fallow="fallow"
+#    download ${architecture}
+    for container_name in  ${main} ${fallow}; do
+        create_container ${container_name} ${os} &
+    done
+    wait
+    local token=$(setup_k8s ${main})
+    echo "token: $token"
+    add_k8s_node ${fallow}
+#    run_e2e
+#    exit $?
 }
 
 if [[ $sourced -ne 1 ]]; then

From fc0813adc0b26b168607980f2c3c941701a89ab0 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Wed, 16 Oct 2024 08:39:13 +0200
Subject: [PATCH 37/59] wip

---
 build-scripts/hack/sonobuoy.sh | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
index 28f1ca6e7..552614167 100755
--- a/build-scripts/hack/sonobuoy.sh
+++ b/build-scripts/hack/sonobuoy.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 set -ex
-token=""
+# /home/maciek/canon/k8s-snap/build-scripts/hack/sonobuoy.sh sdf ubuntu:24.04
+
 function download() {
     local architecture=$1
     local release=$(curl --silent -m 10 --connect-timeout 5 "https://api.github.com/repos/vmware-tanzu/sonobuoy/releases/latest")
@@ -27,16 +28,16 @@ function setup_k8s() {
     lxc exec ${container_name} -- k8s status --wait-ready
     mkdir -p ~/.kube
     lxc exec ${container_name} -- k8s config >> ~/.kube/config
-    lxc exec ${container_name} -- k8s get-join-token worker
+    export token=$(lxc exec ${container_name} -- k8s get-join-token second)
 }
 
 function add_k8s_node() {
     local container_name=$1
-    local token=$2
+    local token1=$2
     lxc exec ${container_name} -- service snapd start
     lxc exec ${container_name} -- snap install /repo/k8s.snap --dangerous --classic
-#    lxc exec ${container_name} -- k8s join-cluster ${token}
-#    lxc exec ${container_name} -- k8s status --wait-ready
+    lxc exec ${container_name} -- k8s join-cluster ${token1}
+    lxc exec ${container_name} -- k8s status --wait-ready
 }
 
 function run_e2e() {
@@ -65,11 +66,12 @@ EOF
 #    download ${architecture}
     for container_name in  ${main} ${fallow}; do
         create_container ${container_name} ${os} &
+#        create_container ${container_name} ${os}
     done
     wait
-    local token=$(setup_k8s ${main})
+    setup_k8s ${main}
     echo "token: $token"
-    add_k8s_node ${fallow}
+    add_k8s_node ${fallow} ${token}
 #    run_e2e
 #    exit $?
 }

From 855cff69687bc95b7d268e76e5ed070229325aa1 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 18 Oct 2024 15:31:45 +0200
Subject: [PATCH 38/59] move tests to python

---
 .github/workflows/cncf-conformance-test.yaml  |  28 ++++-
 .../integration/tests/test_cncf_conformace.py | 100 ++++++++++++++++++
 tests/integration/tox.ini                     |  25 ++++-
 3 files changed, 148 insertions(+), 5 deletions(-)
 create mode 100644 tests/integration/tests/test_cncf_conformace.py

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
index e926cb09e..4b6a4b8ec 100644
--- a/.github/workflows/cncf-conformance-test.yaml
+++ b/.github/workflows/cncf-conformance-test.yaml
@@ -18,9 +18,16 @@ jobs:
     runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }}
 
     steps:
-      - name: Checking out repo
+      - name: Check out code
         uses: actions/checkout@v4
 
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.8'
+      - name: Install tox
+        run: pip install tox
+
       - name: Install lxd
         run: |
           sudo snap refresh lxd --channel 5.21/stable
@@ -33,13 +40,26 @@ jobs:
         run: |
           snap download k8s --channel=${{ matrix.releases }} --basename k8s
 
-      - name: Run end to end tests on ${{ matrix.arch }} ${{ matrix.os }}
+      - name: Run end to end tests
+        env:
+          TEST_SNAP: ${{ github.workspace }}/k8s.snap
+          TEST_SUBSTRATE: lxd
+          TEST_LXD_IMAGE: ${{ matrix.os }}
+          TEST_CNCF_E2E: true
+          TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
+        working-directory: tests/integration
+        run: |
+          sg lxd -c 'tox -e e2e'
+
+      - name: Extract report
+        if: always()
+        working-directory: tests/integration
         run: |
-          sg lxd -c 'build-scripts/hack/sonobuoy.sh ${{ matrix.arch }} ${{ matrix.os }}'
+          tar -xf sonobuoy_e2e.tar.gz --one-top-level
 
       - name: Upload inspection report artifact
         uses: actions/upload-artifact@v4
         if: always()
         with:
           name: report_sonobuoy_e2e_${{ github.os_dash }}_${{ matrix.arch }}
-          path: sonobuoy_e2e
+          path: tests/integration/sonobuoy_e2e
diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
new file mode 100644
index 000000000..b46c5028c
--- /dev/null
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -0,0 +1,100 @@
+#
+# Copyright 2024 Canonical, Ltd.
+#
+import logging
+import os
+import platform
+from typing import List
+
+import pytest
+from test_util import harness, util
+
+LOG = logging.getLogger(__name__)
+
+ARCH = platform.machine()
+ARCH_MAP = {"aarch64": "arm64", "x86_64": "amd64"}
+SONOBUOY_VERSION = "v0.57.2"
+SONOBUOY_TAR_GZ = f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{ARCH_MAP.get(ARCH)}.tar.gz"
+
+
+@pytest.mark.skipif(
+    os.getenv("TEST_CNCF_E2E") in ["false", None],
+    reason="Test is long and should be run nightly",
+)
+@pytest.mark.node_count(2)
+def test_cncf_conformance(instances: List[harness.Instance]):
+    cluster_node = cluster_setup(instances)
+    install_sonobuoy(cluster_node)
+
+    resp = cluster_node.exec(
+        ["./sonobuoy"],
+        capture_output=True,
+    )
+    assert resp.returncode == 0
+
+    resp = cluster_node.exec(
+        ["./sonobuoy", "run", "--plugin", "e2e", "--wait", "--mode", "quick"],
+        capture_output=True,
+    )
+    assert resp.returncode == 0
+
+    resp = cluster_node.exec(
+        ["./sonobuoy", "retrieve", "-f", "sonobuoy_e2e.tar.gz"],
+        capture_output=True,
+    )
+    assert resp.returncode == 0
+
+    resp = cluster_node.exec(
+        ["tar", "-xf", "sonobuoy_e2e.tar.gz", "--one-top-level"],
+        capture_output=True,
+    )
+    assert resp.returncode == 0
+
+    resp = cluster_node.exec(
+        ["./sonobuoy", "results", "sonobuoy_e2e.tar.gz"],
+        capture_output=True,
+    )
+    assert resp.returncode == 0
+    pull_report(cluster_node)
+    assert "Failed: 0" in resp.stdout.decode()
+
+
+def cluster_setup(instances):
+    cluster_node = instances[0]
+    joining_node = instances[1]
+
+    join_token = util.get_join_token(cluster_node, joining_node)
+    util.join_cluster(joining_node, join_token)
+
+    util.wait_until_k8s_ready(cluster_node, instances)
+
+    nodes = util.ready_nodes(cluster_node)
+    assert len(nodes) == 2, "node should have joined cluster"
+    assert "control-plane" in util.get_local_node_status(cluster_node)
+    assert "control-plane" in util.get_local_node_status(joining_node)
+
+    # note: this workaround shlex issue when using string ">>"
+    util.run(
+        [
+            "lxc",
+            "shell",
+            cluster_node.id,
+            "--",
+            "bash",
+            "-c",
+            "k8s config >> /root/.kube/config",
+        ]
+    )
+    return cluster_node
+
+
+def pull_report(instance: harness.Instance):
+    instance.pull_file("/root/sonobuoy_e2e.tar.gz", "sonobuoy_e2e.tar.gz")
+
+
+def install_sonobuoy(instance: harness.Instance):
+
+    # Download cilium-cli
+    instance.exec(["curl", "-L", SONOBUOY_TAR_GZ, "-o", "sonobuoy.tar.gz"])
+    instance.exec(["tar", "xvzf", "sonobuoy.tar.gz"])
+    instance.exec(["./sonobuoy", "version"])
diff --git a/tests/integration/tox.ini b/tests/integration/tox.ini
index b59d696d7..1653fe46b 100644
--- a/tests/integration/tox.ini
+++ b/tests/integration/tox.ini
@@ -1,7 +1,7 @@
 [tox]
 skipsdist = True
 skip_missing_interpreters = True
-env_list = format, lint, integration
+env_list = format, lint, integration, e2e
 
 [testenv]
 set_env =
@@ -43,6 +43,29 @@ commands =
 passenv =
     TEST_*
 
+[testenv:e2e]
+; TODO remove after development
+; set_env =
+;   TEST_SNAP=/home/maciek/canon/k8s-snap/k8s_v1.31.1_amd64.snap
+;   TEST_SUBSTRATE=lxd
+;   TEST_LXD_IMAGE=ubuntu:24.04
+;   TEST_CNCF_E2E=true
+;   TEST_SKIP_CLEANUP=1
+description = Run integration tests
+deps =
+    -r {tox_root}/requirements-test.txt
+
+commands =
+    pytest -v \
+        --maxfail 1 \
+        --tb native \
+        --log-cli-level DEBUG \
+        --disable-warnings \
+        {posargs} \
+        {tox_root}/tests/ -k "test_cncf_conformance"
+passenv =
+    TEST_*
+
 [flake8]
 max-line-length = 120
 select = E,W,F,C,N

From cb68dcf1698344feeef7b654f9b49f7ff584bc3e Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 18 Oct 2024 15:59:43 +0200
Subject: [PATCH 39/59] full test

---
 tests/integration/tests/test_cncf_conformace.py | 6 ++----
 tests/integration/tox.ini                       | 9 +--------
 2 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index b46c5028c..9d1a91dea 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -14,7 +14,7 @@
 ARCH = platform.machine()
 ARCH_MAP = {"aarch64": "arm64", "x86_64": "amd64"}
 SONOBUOY_VERSION = "v0.57.2"
-SONOBUOY_TAR_GZ = f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{ARCH_MAP.get(ARCH)}.tar.gz"
+SONOBUOY_TAR_GZ = f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{ARCH_MAP.get(ARCH)}.tar.gz"  # noqa
 
 
 @pytest.mark.skipif(
@@ -33,7 +33,7 @@ def test_cncf_conformance(instances: List[harness.Instance]):
     assert resp.returncode == 0
 
     resp = cluster_node.exec(
-        ["./sonobuoy", "run", "--plugin", "e2e", "--wait", "--mode", "quick"],
+        ["./sonobuoy", "run", "--plugin", "e2e", "--wait"],
         capture_output=True,
     )
     assert resp.returncode == 0
@@ -93,8 +93,6 @@ def pull_report(instance: harness.Instance):
 
 
 def install_sonobuoy(instance: harness.Instance):
-
-    # Download cilium-cli
     instance.exec(["curl", "-L", SONOBUOY_TAR_GZ, "-o", "sonobuoy.tar.gz"])
     instance.exec(["tar", "xvzf", "sonobuoy.tar.gz"])
     instance.exec(["./sonobuoy", "version"])
diff --git a/tests/integration/tox.ini b/tests/integration/tox.ini
index 1653fe46b..cfa36930d 100644
--- a/tests/integration/tox.ini
+++ b/tests/integration/tox.ini
@@ -44,14 +44,7 @@ passenv =
     TEST_*
 
 [testenv:e2e]
-; TODO remove after development
-; set_env =
-;   TEST_SNAP=/home/maciek/canon/k8s-snap/k8s_v1.31.1_amd64.snap
-;   TEST_SUBSTRATE=lxd
-;   TEST_LXD_IMAGE=ubuntu:24.04
-;   TEST_CNCF_E2E=true
-;   TEST_SKIP_CLEANUP=1
-description = Run integration tests
+description = Run cncf conformance tests
 deps =
     -r {tox_root}/requirements-test.txt
 

From 7d4103e89fbe6527da8498b5af3453f54bf6219b Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 18 Oct 2024 20:06:33 +0200
Subject: [PATCH 40/59] add types

---
 tests/integration/tests/test_cncf_conformace.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index 9d1a91dea..dcfe561d4 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -59,7 +59,7 @@ def test_cncf_conformance(instances: List[harness.Instance]):
     assert "Failed: 0" in resp.stdout.decode()
 
 
-def cluster_setup(instances):
+def cluster_setup(instances: List[harness.Instance]) -> harness.Instance:
     cluster_node = instances[0]
     joining_node = instances[1]
 

From a5d81c36240dc978c65e590eda63297a5cf7d79f Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Sun, 20 Oct 2024 10:13:34 +0200
Subject: [PATCH 41/59] added print of status report

---
 tests/integration/tests/test_cncf_conformace.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index dcfe561d4..c43403536 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -53,10 +53,12 @@ def test_cncf_conformance(instances: List[harness.Instance]):
     resp = cluster_node.exec(
         ["./sonobuoy", "results", "sonobuoy_e2e.tar.gz"],
         capture_output=True,
+        text=True,
     )
     assert resp.returncode == 0
+    LOG.info(resp.stdout())
     pull_report(cluster_node)
-    assert "Failed: 0" in resp.stdout.decode()
+    assert "Failed: 0" in resp.stdout()
 
 
 def cluster_setup(instances: List[harness.Instance]) -> harness.Instance:

From 0447898234775acb0070e2ed9e3f705a963a25f2 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Sun, 20 Oct 2024 21:10:42 +0200
Subject: [PATCH 42/59] added print of status report

---
 tests/integration/tests/test_cncf_conformace.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index c43403536..25b296b9e 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -53,12 +53,11 @@ def test_cncf_conformance(instances: List[harness.Instance]):
     resp = cluster_node.exec(
         ["./sonobuoy", "results", "sonobuoy_e2e.tar.gz"],
         capture_output=True,
-        text=True,
     )
     assert resp.returncode == 0
-    LOG.info(resp.stdout())
+    LOG.info(resp.stdout.decode())
     pull_report(cluster_node)
-    assert "Failed: 0" in resp.stdout()
+    assert "Failed: 0" in resp.stdout.decode()
 
 
 def cluster_setup(instances: List[harness.Instance]) -> harness.Instance:

From a476b6dac4eb9cdacfe1d024c620ea07bd8b25ef Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 21 Oct 2024 18:51:12 +0200
Subject: [PATCH 43/59] review fixes

---
 build-scripts/hack/sonobuoy.sh                | 81 -------------------
 .../integration/tests/test_cncf_conformace.py | 48 +++--------
 tests/integration/tests/test_util/config.py   | 16 ++++
 3 files changed, 26 insertions(+), 119 deletions(-)
 delete mode 100755 build-scripts/hack/sonobuoy.sh

diff --git a/build-scripts/hack/sonobuoy.sh b/build-scripts/hack/sonobuoy.sh
deleted file mode 100755
index 552614167..000000000
--- a/build-scripts/hack/sonobuoy.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/bash
-set -ex
-# /home/maciek/canon/k8s-snap/build-scripts/hack/sonobuoy.sh sdf ubuntu:24.04
-
-function download() {
-    local architecture=$1
-    local release=$(curl --silent -m 10 --connect-timeout 5 "https://api.github.com/repos/vmware-tanzu/sonobuoy/releases/latest")
-    local tag=$(echo "$release" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
-    curl -L -o sonobuoy.tar.gz https://github.com/vmware-tanzu/sonobuoy/releases/download/${tag}/sonobuoy_${tag//v}_linux_${architecture}.tar.gz
-    tar -xf sonobuoy.tar.gz
-    rm sonobuoy.tar.gz
-}
-
-function create_container() {
-    local container_name=$1
-    local os=$2
-#    lxc profile create ${container_name}
-#    cat tests/integration/lxd-profile.yaml | lxc profile edit ${container_name}
-    lxc launch -p default -p ${container_name} ${os} ${container_name}
-    lxc config device add ${container_name} repo disk source=${PWD} path=/repo/
-}
-
-function setup_k8s() {
-    local container_name=$1
-    lxc exec ${container_name} -- service snapd start
-    lxc exec ${container_name} -- snap install /repo/k8s.snap --dangerous --classic
-    lxc exec ${container_name} -- k8s bootstrap
-    lxc exec ${container_name} -- k8s status --wait-ready
-    mkdir -p ~/.kube
-    lxc exec ${container_name} -- k8s config >> ~/.kube/config
-    export token=$(lxc exec ${container_name} -- k8s get-join-token second)
-}
-
-function add_k8s_node() {
-    local container_name=$1
-    local token1=$2
-    lxc exec ${container_name} -- service snapd start
-    lxc exec ${container_name} -- snap install /repo/k8s.snap --dangerous --classic
-    lxc exec ${container_name} -- k8s join-cluster ${token1}
-    lxc exec ${container_name} -- k8s status --wait-ready
-}
-
-function run_e2e() {
-    ./sonobuoy run --plugin e2e --wait
-    ./sonobuoy retrieve -f sonobuoy_e2e.tar.gz
-    ./sonobuoy results sonobuoy_e2e.tar.gz
-    tar -xf sonobuoy_e2e.tar.gz --one-top-level
-    set +e
-    ./sonobuoy results sonobuoy_e2e.tar.gz | grep -E "^Failed: 0$"
-    return $?
-}
-
-function main() {
-    if [ "$#" -ne 2 ]; then
-      cat << EOF
-Expected 2 arguments, provided: $@
-
-usage: $0 <architecture> <os>
-EOF
-      exit 255
-    fi
-    local architecture=$1
-    local os=$2
-    local main="main"
-    local fallow="fallow"
-#    download ${architecture}
-    for container_name in  ${main} ${fallow}; do
-        create_container ${container_name} ${os} &
-#        create_container ${container_name} ${os}
-    done
-    wait
-    setup_k8s ${main}
-    echo "token: $token"
-    add_k8s_node ${fallow} ${token}
-#    run_e2e
-#    exit $?
-}
-
-if [[ $sourced -ne 1 ]]; then
-    main $@
-fi
diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index 25b296b9e..5625a214b 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -3,19 +3,13 @@
 #
 import logging
 import os
-import platform
 from typing import List
 
 import pytest
-from test_util import harness, util
+from test_util import config, harness, util
 
 LOG = logging.getLogger(__name__)
 
-ARCH = platform.machine()
-ARCH_MAP = {"aarch64": "arm64", "x86_64": "amd64"}
-SONOBUOY_VERSION = "v0.57.2"
-SONOBUOY_TAR_GZ = f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{ARCH_MAP.get(ARCH)}.tar.gz"  # noqa
-
 
 @pytest.mark.skipif(
     os.getenv("TEST_CNCF_E2E") in ["false", None],
@@ -26,37 +20,28 @@ def test_cncf_conformance(instances: List[harness.Instance]):
     cluster_node = cluster_setup(instances)
     install_sonobuoy(cluster_node)
 
-    resp = cluster_node.exec(
+    cluster_node.exec(
         ["./sonobuoy"],
         capture_output=True,
     )
-    assert resp.returncode == 0
-
-    resp = cluster_node.exec(
+    cluster_node.exec(
         ["./sonobuoy", "run", "--plugin", "e2e", "--wait"],
         capture_output=True,
     )
-    assert resp.returncode == 0
-
-    resp = cluster_node.exec(
+    cluster_node.exec(
         ["./sonobuoy", "retrieve", "-f", "sonobuoy_e2e.tar.gz"],
         capture_output=True,
     )
-    assert resp.returncode == 0
-
-    resp = cluster_node.exec(
+    cluster_node.exec(
         ["tar", "-xf", "sonobuoy_e2e.tar.gz", "--one-top-level"],
         capture_output=True,
     )
-    assert resp.returncode == 0
-
     resp = cluster_node.exec(
         ["./sonobuoy", "results", "sonobuoy_e2e.tar.gz"],
         capture_output=True,
     )
-    assert resp.returncode == 0
     LOG.info(resp.stdout.decode())
-    pull_report(cluster_node)
+    cluster_node.pull_file("/root/sonobuoy_e2e.tar.gz", "sonobuoy_e2e.tar.gz")
     assert "Failed: 0" in resp.stdout.decode()
 
 
@@ -74,26 +59,13 @@ def cluster_setup(instances: List[harness.Instance]) -> harness.Instance:
     assert "control-plane" in util.get_local_node_status(cluster_node)
     assert "control-plane" in util.get_local_node_status(joining_node)
 
-    # note: this workaround shlex issue when using string ">>"
-    util.run(
-        [
-            "lxc",
-            "shell",
-            cluster_node.id,
-            "--",
-            "bash",
-            "-c",
-            "k8s config >> /root/.kube/config",
-        ]
-    )
-    return cluster_node
-
+    config = cluster_node.exec(["k8s", "config"], capture_output=True)
+    cluster_node.exec(["dd", "of=/root/.kube/config"], input=config.stdout)
 
-def pull_report(instance: harness.Instance):
-    instance.pull_file("/root/sonobuoy_e2e.tar.gz", "sonobuoy_e2e.tar.gz")
+    return cluster_node
 
 
 def install_sonobuoy(instance: harness.Instance):
-    instance.exec(["curl", "-L", SONOBUOY_TAR_GZ, "-o", "sonobuoy.tar.gz"])
+    instance.exec(["curl", "-L", config.SONOBUOY_TAR_GZ, "-o", "sonobuoy.tar.gz"])
     instance.exec(["tar", "xvzf", "sonobuoy.tar.gz"])
     instance.exec(["./sonobuoy", "version"])
diff --git a/tests/integration/tests/test_util/config.py b/tests/integration/tests/test_util/config.py
index 45c9ea5e6..5bd1641aa 100644
--- a/tests/integration/tests/test_util/config.py
+++ b/tests/integration/tests/test_util/config.py
@@ -2,6 +2,7 @@
 # Copyright 2024 Canonical, Ltd.
 #
 import os
+import platform
 from pathlib import Path
 
 DIR = Path(__file__).absolute().parent
@@ -101,3 +102,18 @@
 VERSION_UPGRADE_CHANNELS = (
     os.environ.get("TEST_VERSION_UPGRADE_CHANNELS", "").strip().split()
 )
+
+# ARCH_MAP is a map changing python architecture naming to common one
+ARCH_MAP = {"aarch64": "arm64", "x86_64": "amd64"}
+
+# ARCHITECTURE is an architecture of current machine
+ARCHITECTURE = os.getenv("TEST_ARCHITECTURE") or ARCH_MAP.get(platform.machine())
+
+# SONOBUOY_VERSION is version of sonobuoy to use
+SONOBUOY_VERSION = os.getenv("TEST_SONOBUOY_VERSION") or "v0.57.2"
+
+# SONOBUOY_VERSION is a full path of sonobuoy to download
+SONOBUOY_TAR_GZ = (
+    os.getenv("TEST_SONOBUOY_TAR_GZ")
+    or f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{ARCHITECTURE}.tar.gz"  # noqa
+)

From 7f89bf1a2dbbb94b5e4b721836070267135259d5 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Tue, 22 Oct 2024 17:57:02 +0200
Subject: [PATCH 44/59] review fixes

---
 .github/workflows/cncf-conformance-test.yaml  | 65 -------------------
 .github/workflows/nightly-test.yaml           | 14 ++++
 .../integration/tests/test_cncf_conformace.py |  4 --
 tests/integration/tests/test_util/config.py   |  2 +-
 tests/integration/tox.ini                     | 16 -----
 5 files changed, 15 insertions(+), 86 deletions(-)
 delete mode 100644 .github/workflows/cncf-conformance-test.yaml

diff --git a/.github/workflows/cncf-conformance-test.yaml b/.github/workflows/cncf-conformance-test.yaml
deleted file mode 100644
index 4b6a4b8ec..000000000
--- a/.github/workflows/cncf-conformance-test.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-name: CNCF Conformance Nightly Latest/Edge Tests
-
-on:
-  schedule:
-    - cron: '0 0 * * 1' # Runs every monday midnight
-  pull_request:
-permissions:
-  contents: read
-
-jobs:
-  test-cncf-conformance:
-    name: CNCF Conformance ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.releases }}
-    strategy:
-      matrix:
-        os: ["ubuntu:24.04"]
-        arch: ["amd64"]
-        releases: ["latest/edge/moonray"] # TODO add latest/edge after upgrade Cilium to 1.17
-    runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }}
-
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.8'
-      - name: Install tox
-        run: pip install tox
-
-      - name: Install lxd
-        run: |
-          sudo snap refresh lxd --channel 5.21/stable
-          sudo lxd init --auto
-          sudo usermod --append --groups lxd $USER
-          sg lxd -c 'lxc version'
-          echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
-
-      - name: download ${{ matrix.releases }} k8s snap
-        run: |
-          snap download k8s --channel=${{ matrix.releases }} --basename k8s
-
-      - name: Run end to end tests
-        env:
-          TEST_SNAP: ${{ github.workspace }}/k8s.snap
-          TEST_SUBSTRATE: lxd
-          TEST_LXD_IMAGE: ${{ matrix.os }}
-          TEST_CNCF_E2E: true
-          TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
-        working-directory: tests/integration
-        run: |
-          sg lxd -c 'tox -e e2e'
-
-      - name: Extract report
-        if: always()
-        working-directory: tests/integration
-        run: |
-          tar -xf sonobuoy_e2e.tar.gz --one-top-level
-
-      - name: Upload inspection report artifact
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: report_sonobuoy_e2e_${{ github.os_dash }}_${{ matrix.arch }}
-          path: tests/integration/sonobuoy_e2e
diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
index b4a57a9de..f1c77fc99 100644
--- a/.github/workflows/nightly-test.yaml
+++ b/.github/workflows/nightly-test.yaml
@@ -3,6 +3,7 @@ name: Nightly Latest/Edge Tests
 on:
   schedule:
     - cron: '0 0 * * *' # Runs every midnight
+  pull_request: # todo remove before merge
 
 permissions:
   contents: read
@@ -42,6 +43,7 @@ jobs:
           TEST_SUBSTRATE: lxd
           TEST_LXD_IMAGE: ${{ matrix.os }}
           TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
+          TEST_CNCF_E2E: true
           # Test the latest (up to) 6 releases for the flavour
           # TODO(ben): upgrade nightly to run all flavours
           TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic"
@@ -63,3 +65,15 @@ jobs:
         if: ${{ failure() && github.event_name == 'pull_request' }}
         uses: mxschmitt/action-tmate@v3
         timeout-minutes: 10
+      - name: Extract CNCF conformance report
+        if: failure()
+        working-directory: tests/integration
+        run: |
+         tar -xf sonobuoy_e2e.tar.gz --one-top-level
+         echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
+      - name: Upload CNCF conformance report artifact
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+         name: report_sonobuoy_e2e_${{ github.os_dash }}_${{ matrix.arch }}
+         path: tests/integration/sonobuoy_e2e
diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index 5625a214b..ac15bff8f 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -22,19 +22,15 @@ def test_cncf_conformance(instances: List[harness.Instance]):
 
     cluster_node.exec(
         ["./sonobuoy"],
-        capture_output=True,
     )
     cluster_node.exec(
         ["./sonobuoy", "run", "--plugin", "e2e", "--wait"],
-        capture_output=True,
     )
     cluster_node.exec(
         ["./sonobuoy", "retrieve", "-f", "sonobuoy_e2e.tar.gz"],
-        capture_output=True,
     )
     cluster_node.exec(
         ["tar", "-xf", "sonobuoy_e2e.tar.gz", "--one-top-level"],
-        capture_output=True,
     )
     resp = cluster_node.exec(
         ["./sonobuoy", "results", "sonobuoy_e2e.tar.gz"],
diff --git a/tests/integration/tests/test_util/config.py b/tests/integration/tests/test_util/config.py
index 5bd1641aa..011f8e573 100644
--- a/tests/integration/tests/test_util/config.py
+++ b/tests/integration/tests/test_util/config.py
@@ -112,7 +112,7 @@
 # SONOBUOY_VERSION is version of sonobuoy to use
 SONOBUOY_VERSION = os.getenv("TEST_SONOBUOY_VERSION") or "v0.57.2"
 
-# SONOBUOY_VERSION is a full path of sonobuoy to download
+# SONOBUOY_TAR_GZ is a full path of sonobuoy to download
 SONOBUOY_TAR_GZ = (
     os.getenv("TEST_SONOBUOY_TAR_GZ")
     or f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{ARCHITECTURE}.tar.gz"  # noqa
diff --git a/tests/integration/tox.ini b/tests/integration/tox.ini
index cfa36930d..5abdcd627 100644
--- a/tests/integration/tox.ini
+++ b/tests/integration/tox.ini
@@ -43,22 +43,6 @@ commands =
 passenv =
     TEST_*
 
-[testenv:e2e]
-description = Run cncf conformance tests
-deps =
-    -r {tox_root}/requirements-test.txt
-
-commands =
-    pytest -v \
-        --maxfail 1 \
-        --tb native \
-        --log-cli-level DEBUG \
-        --disable-warnings \
-        {posargs} \
-        {tox_root}/tests/ -k "test_cncf_conformance"
-passenv =
-    TEST_*
-
 [flake8]
 max-line-length = 120
 select = E,W,F,C,N

From ccf15bf76e17e4aee81ad8593891590f480f25dc Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Wed, 23 Oct 2024 09:00:33 +0200
Subject: [PATCH 45/59] artifact name fix

---
 .github/workflows/nightly-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
index 7d75fd98b..963ff0447 100644
--- a/.github/workflows/nightly-test.yaml
+++ b/.github/workflows/nightly-test.yaml
@@ -76,5 +76,5 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-         name: report_sonobuoy_e2e_${{ github.os_dash }}_${{ matrix.arch }}
+         name: report_sonobuoy_e2e_${{ env.os_dash }}_${{ matrix.arch }}
          path: tests/integration/sonobuoy_e2e

From 68e70048d0a983119ff7cbb4aa5e937655df61ad Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Thu, 24 Oct 2024 17:31:49 +0200
Subject: [PATCH 46/59] workflow quick test

---
 .github/workflows/nightly-test.yaml             | 11 +++++++----
 tests/integration/tests/test_cncf_conformace.py |  3 ++-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
index 963ff0447..fd30b797d 100644
--- a/.github/workflows/nightly-test.yaml
+++ b/.github/workflows/nightly-test.yaml
@@ -15,7 +15,7 @@ jobs:
       matrix:
         os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
         arch: ["amd64", "arm64"]
-        release: ["latest/edge"]
+        release: ["latest/edge/moonray", "latest/edge/classic", "latest/edge/strict"]
       fail-fast: false # TODO: remove once arm64 works
 
     runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }}
@@ -43,12 +43,15 @@ jobs:
           TEST_SUBSTRATE: lxd
           TEST_LXD_IMAGE: ${{ matrix.os }}
           TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
-          TEST_CNCF_E2E: true
           # Test the latest (up to) 6 releases for the flavour
           # TODO(ben): upgrade nightly to run all flavours
           TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic"
           TEST_STRICT_INTERFACE_CHANNELS: "recent 6 strict"
         run: |
+          # IPv6-only is only supported on moonray
+          if [[ "${{ matrix.release }}" == "latest/edge/moonray" ]]; then
+            export TEST_CNCF_E2E="true"
+          fi
           export PATH="/home/runner/.local/bin:$PATH"
           cd tests/integration && sg lxd -c 'tox -vve integration'
       - name: Prepare inspection reports
@@ -67,14 +70,14 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         timeout-minutes: 10
       - name: Extract CNCF conformance report
-        if: failure()
+        if: ${{ failure() && matrix.release == 'latest/edge/moonray' }}
         working-directory: tests/integration
         run: |
          tar -xf sonobuoy_e2e.tar.gz --one-top-level
          echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
       - name: Upload CNCF conformance report artifact
         uses: actions/upload-artifact@v4
-        if: failure()
+        if: ${{ failure() && matrix.release == 'latest/edge/moonray' }}
         with:
          name: report_sonobuoy_e2e_${{ env.os_dash }}_${{ matrix.arch }}
          path: tests/integration/sonobuoy_e2e
diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index ac15bff8f..14bf41524 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -24,7 +24,8 @@ def test_cncf_conformance(instances: List[harness.Instance]):
         ["./sonobuoy"],
     )
     cluster_node.exec(
-        ["./sonobuoy", "run", "--plugin", "e2e", "--wait"],
+        ["./sonobuoy", "run", "--plugin", "e2e", "--wait", "--mode", "quick"],
+        # ["./sonobuoy", "run", "--plugin", "e2e", "--wait"],
     )
     cluster_node.exec(
         ["./sonobuoy", "retrieve", "-f", "sonobuoy_e2e.tar.gz"],

From 26289fe1823b91af6dfc746d2f28e09d0031c292 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 25 Oct 2024 10:47:39 +0200
Subject: [PATCH 47/59] add release on cncf report

---
 .github/workflows/nightly-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
index fd30b797d..bb766e22e 100644
--- a/.github/workflows/nightly-test.yaml
+++ b/.github/workflows/nightly-test.yaml
@@ -79,5 +79,5 @@ jobs:
         uses: actions/upload-artifact@v4
         if: ${{ failure() && matrix.release == 'latest/edge/moonray' }}
         with:
-         name: report_sonobuoy_e2e_${{ env.os_dash }}_${{ matrix.arch }}
+         name: report_sonobuoy_e2e_${{ env.os_dash }}_${{ matrix.release }}_${{ matrix.arch }}
          path: tests/integration/sonobuoy_e2e

From 0107d4aab06518afd55b6f6e66aaf1286bdd7eec Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 28 Oct 2024 09:02:55 +0100
Subject: [PATCH 48/59] update cncf reports

---
 .github/workflows/nightly-test.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
index bb766e22e..e4333995c 100644
--- a/.github/workflows/nightly-test.yaml
+++ b/.github/workflows/nightly-test.yaml
@@ -75,9 +75,10 @@ jobs:
         run: |
          tar -xf sonobuoy_e2e.tar.gz --one-top-level
          echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
+         echo "release_dash=${{  matrix.release }}" | sed 's#/#-#g'>> $GITHUB_ENV
       - name: Upload CNCF conformance report artifact
         uses: actions/upload-artifact@v4
         if: ${{ failure() && matrix.release == 'latest/edge/moonray' }}
         with:
-         name: report_sonobuoy_e2e_${{ env.os_dash }}_${{ matrix.release }}_${{ matrix.arch }}
+         name: report_sonobuoy_e2e_${{ env.os_dash }}_${{ env.release_dash }}_${{ matrix.arch }}
          path: tests/integration/sonobuoy_e2e

From 35484729d277b17c69ea1d6942bb140951132c03 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 1 Nov 2024 20:05:41 +0100
Subject: [PATCH 49/59] test skip test_mixed_version_join

---
 .github/workflows/nightly-test.yaml             | 3 ++-
 tests/integration/tests/test_clustering.py      | 6 +++++-
 tests/integration/tests/test_cncf_conformace.py | 1 -
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
index e4333995c..97ada0e3b 100644
--- a/.github/workflows/nightly-test.yaml
+++ b/.github/workflows/nightly-test.yaml
@@ -42,13 +42,14 @@ jobs:
           TEST_SNAP: ${{ github.workspace }}/build/k8s.snap
           TEST_SUBSTRATE: lxd
           TEST_LXD_IMAGE: ${{ matrix.os }}
+          TEST_SNAP_RELEASE: ${{ matrix.release }}
           TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
           # Test the latest (up to) 6 releases for the flavour
           # TODO(ben): upgrade nightly to run all flavours
           TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic"
           TEST_STRICT_INTERFACE_CHANNELS: "recent 6 strict"
         run: |
-          # IPv6-only is only supported on moonray
+          # e2e works with Cilium to 1.17 remove if after upgrade
           if [[ "${{ matrix.release }}" == "latest/edge/moonray" ]]; then
             export TEST_CNCF_E2E="true"
           fi
diff --git a/tests/integration/tests/test_clustering.py b/tests/integration/tests/test_clustering.py
index a77e3f9c5..fcd5fdaf1 100644
--- a/tests/integration/tests/test_clustering.py
+++ b/tests/integration/tests/test_clustering.py
@@ -3,6 +3,7 @@
 #
 import logging
 from typing import List
+import os
 
 import pytest
 from test_util import config, harness, util
@@ -32,7 +33,10 @@ def test_control_plane_nodes(instances: List[harness.Instance]):
         nodes[0]["metadata"]["name"] == cluster_node.id
     ), f"only {cluster_node.id} should be left in cluster"
 
-
+@pytest.mark.skipif(
+    os.getenv("TEST_SNAP_RELEASE") in ["latest/edge/moonray", "latest/edge/strict"],
+    reason="Test is breaks on moonray and strict",
+    )
 @pytest.mark.node_count(2)
 @pytest.mark.snap_versions([util.previous_track(config.SNAP), config.SNAP])
 def test_mixed_version_join(instances: List[harness.Instance]):
diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index 14bf41524..1183312ee 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -25,7 +25,6 @@ def test_cncf_conformance(instances: List[harness.Instance]):
     )
     cluster_node.exec(
         ["./sonobuoy", "run", "--plugin", "e2e", "--wait", "--mode", "quick"],
-        # ["./sonobuoy", "run", "--plugin", "e2e", "--wait"],
     )
     cluster_node.exec(
         ["./sonobuoy", "retrieve", "-f", "sonobuoy_e2e.tar.gz"],

From f6758164d8d0ff1b9c5f8973e6bdf82cce741952 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Sat, 2 Nov 2024 09:02:36 +0100
Subject: [PATCH 50/59] fix inspection reports filename

---
 .github/workflows/nightly-test.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
index 97ada0e3b..4820475a7 100644
--- a/.github/workflows/nightly-test.yaml
+++ b/.github/workflows/nightly-test.yaml
@@ -37,6 +37,9 @@ jobs:
         run: |
           cd build
           snap download k8s --channel=${{ matrix.release }} --basename k8s
+          #create env variables for artifacts
+          echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
+          echo "release_dash=${{  matrix.release }}" | sed 's#/#-#g'>> $GITHUB_ENV
       - name: Run end to end tests # tox path needs to be specified for arm64
         env:
           TEST_SNAP: ${{ github.workspace }}/build/k8s.snap
@@ -65,7 +68,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.artifact_name }}
-          path: ${{ github.workspace }}/inspection-reports.tar.gz
+          path: ${{ github.workspace }}/inspection-reports_${{ env.os_dash }}_${{ env.release_dash }}_${{ matrix.arch }}.tar.gz
       - name: Tmate debugging session
         if: ${{ failure() && github.event_name == 'pull_request' }}
         uses: mxschmitt/action-tmate@v3
@@ -75,8 +78,6 @@ jobs:
         working-directory: tests/integration
         run: |
          tar -xf sonobuoy_e2e.tar.gz --one-top-level
-         echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
-         echo "release_dash=${{  matrix.release }}" | sed 's#/#-#g'>> $GITHUB_ENV
       - name: Upload CNCF conformance report artifact
         uses: actions/upload-artifact@v4
         if: ${{ failure() && matrix.release == 'latest/edge/moonray' }}

From f071e9205740a19859b906bfa8a44eecb8141c1a Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 4 Nov 2024 09:24:13 +0100
Subject: [PATCH 51/59] test1

---
 k8s/hack/network-requirements.sh           | 6 +++---
 tests/integration/tests/test_clustering.py | 9 +++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/k8s/hack/network-requirements.sh b/k8s/hack/network-requirements.sh
index 4c724fa16..7a8bed45f 100755
--- a/k8s/hack/network-requirements.sh
+++ b/k8s/hack/network-requirements.sh
@@ -32,9 +32,9 @@ sudo sysctl -a | grep '\.rp_filter' | awk '{print $1" = 0"}' | sudo tee -a /etc/
 sudo sysctl --system
 
 # TODO: Remove after https://bugs.launchpad.net/snapd/+bug/2047798
-sudo sed -i -e 's/\/run\/netns\/ r/\/run\/netns\/ rk/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
-sudo sed -i -e 's/userns,/userns rwk,/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd # Tempfix
-sudo apparmor_parser -r /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
+#sudo sed -i -e 's/\/run\/netns\/ r/\/run\/netns\/ rk/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
+#sudo sed -i -e 's/userns,/userns rwk,/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd # Tempfix
+#sudo apparmor_parser -r /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
 
 # TODO: Remove after https://bugs.launchpad.net/snapd/+bug/2053271
 sudo sed -i -e '/mountinfo/aowner @{PROC}\/@{pid}\/task\/@{tid}\/mountinfo r,' /var/lib/snapd/apparmor/profiles/snap.k8s.k8sd
diff --git a/tests/integration/tests/test_clustering.py b/tests/integration/tests/test_clustering.py
index fcd5fdaf1..c3a1005fb 100644
--- a/tests/integration/tests/test_clustering.py
+++ b/tests/integration/tests/test_clustering.py
@@ -33,10 +33,11 @@ def test_control_plane_nodes(instances: List[harness.Instance]):
         nodes[0]["metadata"]["name"] == cluster_node.id
     ), f"only {cluster_node.id} should be left in cluster"
 
-@pytest.mark.skipif(
-    os.getenv("TEST_SNAP_RELEASE") in ["latest/edge/moonray", "latest/edge/strict"],
-    reason="Test is breaks on moonray and strict",
-    )
+
+# @pytest.mark.skipif(
+#     os.getenv("TEST_SNAP_RELEASE") in ["latest/edge/moonray", "latest/edge/strict"],
+#     reason="Test is breaks on moonray and strict",
+#     )
 @pytest.mark.node_count(2)
 @pytest.mark.snap_versions([util.previous_track(config.SNAP), config.SNAP])
 def test_mixed_version_join(instances: List[harness.Instance]):

From d9f7501d053674f659347857cbfe4ac577c754af Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 4 Nov 2024 15:14:34 +0100
Subject: [PATCH 52/59] test2

---
 k8s/hack/network-requirements.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/k8s/hack/network-requirements.sh b/k8s/hack/network-requirements.sh
index 7a8bed45f..9b838c65d 100755
--- a/k8s/hack/network-requirements.sh
+++ b/k8s/hack/network-requirements.sh
@@ -32,9 +32,9 @@ sudo sysctl -a | grep '\.rp_filter' | awk '{print $1" = 0"}' | sudo tee -a /etc/
 sudo sysctl --system
 
 # TODO: Remove after https://bugs.launchpad.net/snapd/+bug/2047798
-#sudo sed -i -e 's/\/run\/netns\/ r/\/run\/netns\/ rk/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
+sudo sed -i -e 's/\/run\/netns\/ r/\/run\/netns\/ rk/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
 #sudo sed -i -e 's/userns,/userns rwk,/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd # Tempfix
-#sudo apparmor_parser -r /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
+sudo apparmor_parser -r /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
 
 # TODO: Remove after https://bugs.launchpad.net/snapd/+bug/2053271
 sudo sed -i -e '/mountinfo/aowner @{PROC}\/@{pid}\/task\/@{tid}\/mountinfo r,' /var/lib/snapd/apparmor/profiles/snap.k8s.k8sd

From 53195ac51a5f37856144f290e267ab521cced0fa Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 4 Nov 2024 20:15:48 +0100
Subject: [PATCH 53/59] test3

---
 tests/integration/tests/test_clustering.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/tests/test_clustering.py b/tests/integration/tests/test_clustering.py
index c3a1005fb..ac70cbe03 100644
--- a/tests/integration/tests/test_clustering.py
+++ b/tests/integration/tests/test_clustering.py
@@ -34,10 +34,10 @@ def test_control_plane_nodes(instances: List[harness.Instance]):
     ), f"only {cluster_node.id} should be left in cluster"
 
 
-# @pytest.mark.skipif(
-#     os.getenv("TEST_SNAP_RELEASE") in ["latest/edge/moonray", "latest/edge/strict"],
-#     reason="Test is breaks on moonray and strict",
-#     )
+@pytest.mark.skipif(
+    os.getenv("TEST_SNAP_RELEASE") in ["latest/edge/moonray", "latest/edge/strict"],
+    reason="Test is breaks on moonray and strict",
+    )
 @pytest.mark.node_count(2)
 @pytest.mark.snap_versions([util.previous_track(config.SNAP), config.SNAP])
 def test_mixed_version_join(instances: List[harness.Instance]):

From 4e66c59954780c535f52fc19f8e67bb746d89c13 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Wed, 6 Nov 2024 15:21:26 +0100
Subject: [PATCH 54/59] linter fix

---
 tests/integration/tests/test_clustering.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/tests/test_clustering.py b/tests/integration/tests/test_clustering.py
index ac70cbe03..ce2d1c180 100644
--- a/tests/integration/tests/test_clustering.py
+++ b/tests/integration/tests/test_clustering.py
@@ -37,7 +37,7 @@ def test_control_plane_nodes(instances: List[harness.Instance]):
 @pytest.mark.skipif(
     os.getenv("TEST_SNAP_RELEASE") in ["latest/edge/moonray", "latest/edge/strict"],
     reason="Test is breaks on moonray and strict",
-    )
+)
 @pytest.mark.node_count(2)
 @pytest.mark.snap_versions([util.previous_track(config.SNAP), config.SNAP])
 def test_mixed_version_join(instances: List[harness.Instance]):

From 6b38a489bce2688fac4394194612dc052adee57e Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Wed, 6 Nov 2024 15:25:11 +0100
Subject: [PATCH 55/59] full cncf conformance

---
 tests/integration/tests/test_cncf_conformace.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index 1183312ee..ac15bff8f 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -24,7 +24,7 @@ def test_cncf_conformance(instances: List[harness.Instance]):
         ["./sonobuoy"],
     )
     cluster_node.exec(
-        ["./sonobuoy", "run", "--plugin", "e2e", "--wait", "--mode", "quick"],
+        ["./sonobuoy", "run", "--plugin", "e2e", "--wait"],
     )
     cluster_node.exec(
         ["./sonobuoy", "retrieve", "-f", "sonobuoy_e2e.tar.gz"],

From b71f703c3ea6cf5f8aa5933a53e77644270d132c Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 8 Nov 2024 17:20:30 +0100
Subject: [PATCH 56/59] review fixes

---
 .github/workflows/nightly-test.yaml           | 19 ++++++++++---------
 k8s/hack/network-requirements.sh              |  2 +-
 tests/integration/tests/test_clustering.py    |  4 ++--
 .../integration/tests/test_cncf_conformace.py | 11 ++++++++---
 tests/integration/tests/test_util/config.py   | 17 +++++------------
 tests/integration/tox.ini                     |  2 +-
 6 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
index 4820475a7..1fe9078fa 100644
--- a/.github/workflows/nightly-test.yaml
+++ b/.github/workflows/nightly-test.yaml
@@ -11,6 +11,9 @@ permissions:
 jobs:
   test-integration:
     name: Integration Test ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.release }}
+    env:
+      # todo(mac) e2e works with Cilium to 1.17 remove if after upgrade
+      TEST_CNCF_E2E: ${{ matrix.release == 'latest/edge/moonray' }}
     strategy:
       matrix:
         os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
@@ -37,9 +40,6 @@ jobs:
         run: |
           cd build
           snap download k8s --channel=${{ matrix.release }} --basename k8s
-          #create env variables for artifacts
-          echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
-          echo "release_dash=${{  matrix.release }}" | sed 's#/#-#g'>> $GITHUB_ENV
       - name: Run end to end tests # tox path needs to be specified for arm64
         env:
           TEST_SNAP: ${{ github.workspace }}/build/k8s.snap
@@ -52,12 +52,13 @@ jobs:
           TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic"
           TEST_STRICT_INTERFACE_CHANNELS: "recent 6 strict"
         run: |
-          # e2e works with Cilium to 1.17 remove if after upgrade
-          if [[ "${{ matrix.release }}" == "latest/edge/moonray" ]]; then
-            export TEST_CNCF_E2E="true"
-          fi
           export PATH="/home/runner/.local/bin:$PATH"
           cd tests/integration && sg lxd -c 'tox -vve integration'
+      - name: Create env variables for artifacts
+        if: failure()
+        run: |
+          echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
+          echo "release_dash=${{  matrix.release }}" | sed 's#/#-#g'>> $GITHUB_ENV
       - name: Prepare inspection reports
         if: failure()
         run: |
@@ -74,13 +75,13 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         timeout-minutes: 10
       - name: Extract CNCF conformance report
-        if: ${{ failure() && matrix.release == 'latest/edge/moonray' }}
+        if: ${{ failure() && ( env.TEST_CNCF_E2E == 'true' ) }}
         working-directory: tests/integration
         run: |
          tar -xf sonobuoy_e2e.tar.gz --one-top-level
       - name: Upload CNCF conformance report artifact
         uses: actions/upload-artifact@v4
-        if: ${{ failure() && matrix.release == 'latest/edge/moonray' }}
+        if: ${{ failure() && ( env.TEST_CNCF_E2E == 'true' ) }}
         with:
          name: report_sonobuoy_e2e_${{ env.os_dash }}_${{ env.release_dash }}_${{ matrix.arch }}
          path: tests/integration/sonobuoy_e2e
diff --git a/k8s/hack/network-requirements.sh b/k8s/hack/network-requirements.sh
index 9b838c65d..4c724fa16 100755
--- a/k8s/hack/network-requirements.sh
+++ b/k8s/hack/network-requirements.sh
@@ -33,7 +33,7 @@ sudo sysctl --system
 
 # TODO: Remove after https://bugs.launchpad.net/snapd/+bug/2047798
 sudo sed -i -e 's/\/run\/netns\/ r/\/run\/netns\/ rk/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
-#sudo sed -i -e 's/userns,/userns rwk,/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd # Tempfix
+sudo sed -i -e 's/userns,/userns rwk,/g' /var/lib/snapd/apparmor/profiles/snap.k8s.containerd # Tempfix
 sudo apparmor_parser -r /var/lib/snapd/apparmor/profiles/snap.k8s.containerd
 
 # TODO: Remove after https://bugs.launchpad.net/snapd/+bug/2053271
diff --git a/tests/integration/tests/test_clustering.py b/tests/integration/tests/test_clustering.py
index ce2d1c180..b897f6369 100644
--- a/tests/integration/tests/test_clustering.py
+++ b/tests/integration/tests/test_clustering.py
@@ -35,8 +35,8 @@ def test_control_plane_nodes(instances: List[harness.Instance]):
 
 
 @pytest.mark.skipif(
-    os.getenv("TEST_SNAP_RELEASE") in ["latest/edge/moonray", "latest/edge/strict"],
-    reason="Test is breaks on moonray and strict",
+    os.getenv("TEST_SNAP_RELEASE") in ["latest/edge/classic", "latest/edge/strict"],
+    reason="Test is breaks on classic and strict",
 )
 @pytest.mark.node_count(2)
 @pytest.mark.snap_versions([util.previous_track(config.SNAP), config.SNAP])
diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index ac15bff8f..3b68554aa 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -3,6 +3,7 @@
 #
 import logging
 import os
+import re
 from typing import List
 
 import pytest
@@ -36,9 +37,13 @@ def test_cncf_conformance(instances: List[harness.Instance]):
         ["./sonobuoy", "results", "sonobuoy_e2e.tar.gz"],
         capture_output=True,
     )
-    LOG.info(resp.stdout.decode())
+
     cluster_node.pull_file("/root/sonobuoy_e2e.tar.gz", "sonobuoy_e2e.tar.gz")
-    assert "Failed: 0" in resp.stdout.decode()
+
+    output = resp.stdout.decode()
+    LOG.info(output)
+    failed_tests = int(re.search("Failed: (\\d+)", output).group(1))
+    assert failed_tests == 0, f"{failed_tests} tests failed"
 
 
 def cluster_setup(instances: List[harness.Instance]) -> harness.Instance:
@@ -62,6 +67,6 @@ def cluster_setup(instances: List[harness.Instance]) -> harness.Instance:
 
 
 def install_sonobuoy(instance: harness.Instance):
-    instance.exec(["curl", "-L", config.SONOBUOY_TAR_GZ, "-o", "sonobuoy.tar.gz"])
+    instance.exec(["curl", "-L", config.sonobuoy_tar_gz(instance.arch), "-o", "sonobuoy.tar.gz"])
     instance.exec(["tar", "xvzf", "sonobuoy.tar.gz"])
     instance.exec(["./sonobuoy", "version"])
diff --git a/tests/integration/tests/test_util/config.py b/tests/integration/tests/test_util/config.py
index b65fc998c..740a647a4 100644
--- a/tests/integration/tests/test_util/config.py
+++ b/tests/integration/tests/test_util/config.py
@@ -2,7 +2,6 @@
 # Copyright 2024 Canonical, Ltd.
 #
 import os
-import platform
 from pathlib import Path
 
 DIR = Path(__file__).absolute().parent
@@ -126,17 +125,11 @@
     os.environ.get("TEST_STRICT_INTERFACE_CHANNELS", "").strip().split()
 )
 
-# ARCH_MAP is a map changing python architecture naming to common one
-ARCH_MAP = {"aarch64": "arm64", "x86_64": "amd64"}
-
-# ARCHITECTURE is an architecture of current machine
-ARCHITECTURE = os.getenv("TEST_ARCHITECTURE") or ARCH_MAP.get(platform.machine())
-
 # SONOBUOY_VERSION is version of sonobuoy to use
 SONOBUOY_VERSION = os.getenv("TEST_SONOBUOY_VERSION") or "v0.57.2"
 
-# SONOBUOY_TAR_GZ is a full path of sonobuoy to download
-SONOBUOY_TAR_GZ = (
-    os.getenv("TEST_SONOBUOY_TAR_GZ")
-    or f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{ARCHITECTURE}.tar.gz"  # noqa
-)
+
+# sonobuoy_tar_gz is a full path of sonobuoy to download
+def sonobuoy_tar_gz(architecture: str) -> str:
+    os.getenv("TEST_SONOBUOY_VERSION") or "v0.57.2"
+    return  f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{architecture}.tar.gz" # noqa
diff --git a/tests/integration/tox.ini b/tests/integration/tox.ini
index 5abdcd627..b59d696d7 100644
--- a/tests/integration/tox.ini
+++ b/tests/integration/tox.ini
@@ -1,7 +1,7 @@
 [tox]
 skipsdist = True
 skip_missing_interpreters = True
-env_list = format, lint, integration, e2e
+env_list = format, lint, integration
 
 [testenv]
 set_env =

From 8d41794068e6f8d4446c5b41d8cc4cb007dc9bca Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Fri, 8 Nov 2024 17:41:37 +0100
Subject: [PATCH 57/59] merge with main

---
 tests/integration/tests/test_clustering.py      | 1 -
 tests/integration/tests/test_cncf_conformace.py | 4 +++-
 tests/integration/tests/test_util/config.py     | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/integration/tests/test_clustering.py b/tests/integration/tests/test_clustering.py
index 0e08d617d..2dabeb215 100644
--- a/tests/integration/tests/test_clustering.py
+++ b/tests/integration/tests/test_clustering.py
@@ -7,7 +7,6 @@
 import subprocess
 import tempfile
 from typing import List
-import os
 
 import pytest
 from cryptography import x509
diff --git a/tests/integration/tests/test_cncf_conformace.py b/tests/integration/tests/test_cncf_conformace.py
index 3b68554aa..5232ba20d 100644
--- a/tests/integration/tests/test_cncf_conformace.py
+++ b/tests/integration/tests/test_cncf_conformace.py
@@ -67,6 +67,8 @@ def cluster_setup(instances: List[harness.Instance]) -> harness.Instance:
 
 
 def install_sonobuoy(instance: harness.Instance):
-    instance.exec(["curl", "-L", config.sonobuoy_tar_gz(instance.arch), "-o", "sonobuoy.tar.gz"])
+    instance.exec(
+        ["curl", "-L", config.sonobuoy_tar_gz(instance.arch), "-o", "sonobuoy.tar.gz"]
+    )
     instance.exec(["tar", "xvzf", "sonobuoy.tar.gz"])
     instance.exec(["./sonobuoy", "version"])
diff --git a/tests/integration/tests/test_util/config.py b/tests/integration/tests/test_util/config.py
index bfbc0fe90..e3dd73ff0 100644
--- a/tests/integration/tests/test_util/config.py
+++ b/tests/integration/tests/test_util/config.py
@@ -137,4 +137,4 @@
 # sonobuoy_tar_gz is a full path of sonobuoy to download
 def sonobuoy_tar_gz(architecture: str) -> str:
     os.getenv("TEST_SONOBUOY_VERSION") or "v0.57.2"
-    return  f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{architecture}.tar.gz" # noqa
+    return f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{architecture}.tar.gz"  # noqa

From e1d207c20081ce0f8cb738bbdc4a9bb965514f72 Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Mon, 11 Nov 2024 09:17:26 +0100
Subject: [PATCH 58/59] test cncf only

---
 tests/integration/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/tox.ini b/tests/integration/tox.ini
index 1b33bcda9..8bec27121 100644
--- a/tests/integration/tox.ini
+++ b/tests/integration/tox.ini
@@ -39,7 +39,7 @@ commands =
         --log-cli-level DEBUG \
         --disable-warnings \
         {posargs} \
-        {toxinidir}/tests
+        {toxinidir}/tests -k "test_cncf_conformance"
 passenv =
     TEST_*
 

From 79d650ce6bce26cda462b64d56a9eea3b96ad9fb Mon Sep 17 00:00:00 2001
From: Maciej Golaszewski <maciej.golaszewski@canonical.com>
Date: Tue, 12 Nov 2024 15:24:46 +0100
Subject: [PATCH 59/59] test cncf only

---
 .github/workflows/nightly-test.yaml | 2 +-
 tests/integration/tox.ini           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
index 010bc208a..321a37482 100644
--- a/.github/workflows/nightly-test.yaml
+++ b/.github/workflows/nightly-test.yaml
@@ -18,7 +18,7 @@ jobs:
       matrix:
         os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
         arch: ["amd64", "arm64"]
-        release: ["latest/edge/moonray", "latest/edge/classic", "latest/edge/strict"]
+        release: ["latest/edge/moonray", "latest/edge/classic"]
       fail-fast: false # TODO: remove once arm64 works
 
     runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }}
diff --git a/tests/integration/tox.ini b/tests/integration/tox.ini
index 8bec27121..1b33bcda9 100644
--- a/tests/integration/tox.ini
+++ b/tests/integration/tox.ini
@@ -39,7 +39,7 @@ commands =
         --log-cli-level DEBUG \
         --disable-warnings \
         {posargs} \
-        {toxinidir}/tests -k "test_cncf_conformance"
+        {toxinidir}/tests
 passenv =
     TEST_*