added CVE scan on Sundays after daily build, cleanup some stuff

Author: mattkjames7

.github/workflows/daily_build.yml

@@ -1,7 +1,7 @@
 name: 'Create Daily Build'
-# concurrency:
-#   group: ${{ github.workflow }}-${{ github.ref_name }}
-#   cancel-in-progress: true
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
 
 on:
   repository_dispatch:
@@ -14,7 +14,6 @@ on:
             description: "Date to build packages for (YYYYMMDD)"
             required: false
 
-
 jobs:
   DailyBuildVariables:
     runs-on: ["self-hosted"]
@@ -56,7 +55,7 @@ jobs:
             payload="{\"date\": ${build_date}}"
           fi
           echo "Received payload: $payload"
-          read mage_version memgraph_version memgraph_commit build_date < <(python3 daily_build_vars.py "$payload")
+          read mage_version memgraph_version memgraph_commit build_date < <(python3 scripts/daily_build_vars.py "$payload")
           echo "mage_version=${mage_version}" >> $GITHUB_OUTPUT
           echo "memgraph_version=${memgraph_version}" >> $GITHUB_OUTPUT
           echo "memgraph_commit=${memgraph_commit}" >> $GITHUB_OUTPUT
@@ -143,7 +142,7 @@ jobs:
         run: |
           echo "TEST_RESULT: $TEST_RESULT"
           echo "Package Date: $CURRENT_BUILD_DATE"
-          echo "BUILD_TEST_RESULTS=$(python3 aggregate_build_tests.py)" >> $GITHUB_ENV
+          echo "BUILD_TEST_RESULTS=$(python3 scripts/aggregate_build_tests.py)" >> $GITHUB_ENV
 
       - name: Trigger Daily Builds Page Update
         env: 
@@ -157,4 +156,65 @@ jobs:
             -H "Authorization: token $GITHUB_TOKEN" \
             https://api.github.com/repos/memgraph/daily-builds/dispatches \
             -d "$payload"    
-  
+
+  SetupCVEScan:
+    name: Setup CVE Scan
+    runs-on: ubuntu-latest
+    needs: [DailyBuildVariables, DailyBuildArtifact]
+    outputs: 
+      amd_url: ${{ steps.image_urls.outputs.AMD_URL }}
+      arm_url: ${{ steps.image_urls.outputs.ARM_URL }}
+      run_scan: ${{ steps.sunday.outputs.RUN_SCAN }}
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: recursive
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Is today Sunday? 🤔
+        id: sunday
+        run: |
+          day="$(date -u +%A)"
+          if [[ "$day" == "Sunday" ]]; then
+            echo "RUN_SCAN=true" >> $GITHUB_OUTPUT
+          else
+            echo "RUN_SCAN=false" >> $GITHUB_OUTPUT
+          fi
+          
+
+      - name: Fetch URLS
+        id: image_urls
+        env:
+          CURRENT_BUILD_DATE: ${{ needs.DailyBuildVariables.outputs.build_date }}
+        run: |
+          arm_url="$(python3 scripts/get_cve_image_url 'arm64')"
+          amd_url="$(python3 scripts/get_cve_image_url 'amd64')"
+          
+          echo "arm64 URL: $arm_url"
+          echo "amd64 URL: $amd_url"
+
+          echo "ARM_URL=$arm_url" >> $GITHUB_OUTPUT
+          echo "AMD_URL=$amd_url" >> $GITHUB_OUTPUT
+
+  RunCVEScan:
+    name: Run CVE Scan
+    needs: [SetupCVEScan]
+    if: ${{ needs.SetupCVEScan.outputs.run_scan == 'true' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: arm64
+            image_url: needs.SetupCVEScan.outputs.arm_url
+          - arch: amd64
+            image_url: needs.SetupCVEScan.outputs.amd_url
+    uses: ./.github/workflows/reusable_cve_scan.yml
+    with:
+      arch: ${{ matrix.arch }}
+      image_url: ${{ matrix.image_url }}
+      run_trivy: true
+      run_grype: true
+      run_cbt: true
+    secrets: inherit

.github/workflows/reusable_smoke_tests.yml

@@ -42,10 +42,10 @@ jobs:
 
       - name: Determine Input Types
         run: |
-          read next_type next_image < <(python3 smoke-release-testing/workflow_image_setup.py "${{ inputs.next_version }}" "${{ inputs.arch }}" "${{ inputs.malloc }}")
+          read next_type next_image < <(python3 scripts/workflow_image_setup.py "${{ inputs.next_version }}" "${{ inputs.arch }}" "${{ inputs.malloc }}")
           echo "NEXT_IMAGE=${next_image}" >> $GITHUB_ENV
           echo "NEXT_TYPE=${next_type}" >> $GITHUB_ENV
-          read last_type last_image < <(python3 smoke-release-testing/workflow_image_setup.py "${{ inputs.last_version }}" "${{ inputs.arch }}" "${{ inputs.malloc }}")
+          read last_type last_image < <(python3 scripts/workflow_image_setup.py "${{ inputs.last_version }}" "${{ inputs.arch }}" "${{ inputs.malloc }}")
           echo "LAST_IMAGE=${last_image}" >> $GITHUB_ENV
           echo "LAST_TYPE=${last_type}" >> $GITHUB_ENV
 

.github/workflows/test.yml

@@ -66,7 +66,7 @@ jobs:
       - name: Compute variables using Python script
         id: compute
         run: |
-          read mage_version memgraph_version memgraph_commit build_date < <(python3 daily_build_vars.py)
+          read mage_version memgraph_version memgraph_commit build_date < <(python3 scripts/daily_build_vars.py)
           echo "mage_version=${mage_version}" >> $GITHUB_OUTPUT
           echo "memgraph_version=${memgraph_version}" >> $GITHUB_OUTPUT
           echo "memgraph_commit=${memgraph_commit}" >> $GITHUB_OUTPUT

aggregate_build_tests.py renamed to scripts/aggregate_build_tests.py

@@ -0,0 +1,26 @@
+from aggregate_build_tests import list_daily_release_packages
+import os
+import argparse
+
+
+def main() -> None:
+    """
+    return the relevant image URL to be scanned for CVEs
+    """
+    date = int(os.getenv("CURRENT_BUILD_DATE"))
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("arch", type=str)
+    args = parser.parse_args()
+
+    # translate to dict key
+    key, arch = ("Docker (arm64)", "arm64") if args.arch == "arm64" else ("Docker (x86_64)", "x86_64")
+
+    packages = list_daily_release_packages(date)
+    url = packages[key][arch]
+
+    print(url)
+
+
+if __name__ == "__main__":
+    main()

daily_build_vars.py renamed to scripts/daily_build_vars.py

@@ -1,15 +1,6 @@
-import sys
-import os
 import re
 import argparse
-# add mage root dir to python path to find other functions
-# TODO(matt): refactor workflow helper scripts like this into subdirectory
-sys.path.append(
-    os.path.dirname(
-        os.path.dirname(__file__)
-    )
-)
-from aggregate_build_tests import list_daily_release_packages  # noqa: E402
+from aggregate_build_tests import list_daily_release_packages
 
 # Compile regex patterns
 URL_PATTERN = re.compile(