feat: updated karpenter compatibility and added scraper (#2838)

Author: smyja

static/compatibilities.yaml

@@ -1741,29 +1741,32 @@ addons:
   release_url: https://github.com/aws/karpenter-provider-aws/releases/tag/v{vsn}
   helm_repository_url: https://charts.karpenter.sh
   versions:
+  - version: 1.8.0
+    kube: ['1.34', '1.33', '1.32', '1.31', '1.30', '1.29', '1.28']
+    requirements: []
+    incompatibilities: []
+  - version: 1.5.0
+    kube: ['1.33', '1.32', '1.31', '1.30', '1.29', '1.28']
+    requirements: []
+    incompatibilities: []
   - version: 1.2.0
-    kube: ['1.32', '1.31', '1.30', '1.29', '1.28', '1.27', '1.26', '1.25', '1.24',
-      '1.23', '1.22', '1.21', '1.20', '1.19']
+    kube: ['1.32', '1.31', '1.30', '1.29', '1.28']
     requirements: []
     incompatibilities: []
   - version: 1.0.5
-    kube: ['1.31', '1.30', '1.29', '1.28', '1.27', '1.26', '1.25', '1.24', '1.23',
-      '1.22', '1.21', '1.20', '1.19']
+    kube: ['1.31', '1.30', '1.29', '1.28']
     requirements: []
     incompatibilities: []
   - version: 0.37.0
-    kube: ['1.30', '1.29', '1.28', '1.27', '1.26', '1.25', '1.24', '1.23', '1.22',
-      '1.21', '1.20', '1.19']
+    kube: ['1.30', '1.29', '1.28']
     requirements: []
     incompatibilities: []
   - version: 0.34.0
-    kube: ['1.29', '1.28', '1.27', '1.26', '1.25', '1.24', '1.23', '1.22', '1.21',
-      '1.20', '1.19']
+    kube: ['1.29', '1.28']
     requirements: []
     incompatibilities: []
   - version: 0.31.0
-    kube: ['1.28', '1.27', '1.26', '1.25', '1.24', '1.23', '1.22', '1.21', '1.20',
-      '1.19']
+    kube: ['1.28']
     requirements: []
     incompatibilities: []
   - version: 0.28.0

static/compatibilities/karpenter.yaml

@@ -3,28 +3,32 @@ git_url: https://github.com/aws/karpenter-provider-aws
 release_url: https://github.com/aws/karpenter-provider-aws/releases/tag/v{vsn}
 helm_repository_url: https://charts.karpenter.sh
 versions:
+- version: 1.8.0
+  kube: ['1.34', '1.33', '1.32', '1.31', '1.30', '1.29', '1.28']
+  requirements: []
+  incompatibilities: []
+- version: 1.5.0
+  kube: ['1.33', '1.32', '1.31', '1.30', '1.29', '1.28']
+  requirements: []
+  incompatibilities: []
 - version: 1.2.0
-  kube: ['1.32', '1.31', '1.30', '1.29', '1.28', '1.27', '1.26', '1.25', '1.24', '1.23',
-    '1.22', '1.21', '1.20', '1.19']
+  kube: ['1.32', '1.31', '1.30', '1.29', '1.28']
   requirements: []
   incompatibilities: []
 - version: 1.0.5
-  kube: ['1.31', '1.30', '1.29', '1.28', '1.27', '1.26', '1.25', '1.24', '1.23', '1.22',
-    '1.21', '1.20', '1.19']
+  kube: ['1.31', '1.30', '1.29', '1.28']
   requirements: []
   incompatibilities: []
 - version: 0.37.0
-  kube: ['1.30', '1.29', '1.28', '1.27', '1.26', '1.25', '1.24', '1.23', '1.22', '1.21',
-    '1.20', '1.19']
+  kube: ['1.30', '1.29', '1.28']
   requirements: []
   incompatibilities: []
 - version: 0.34.0
-  kube: ['1.29', '1.28', '1.27', '1.26', '1.25', '1.24', '1.23', '1.22', '1.21', '1.20',
-    '1.19']
+  kube: ['1.29', '1.28']
   requirements: []
   incompatibilities: []
 - version: 0.31.0
-  kube: ['1.28', '1.27', '1.26', '1.25', '1.24', '1.23', '1.22', '1.21', '1.20', '1.19']
+  kube: ['1.28']
   requirements: []
   incompatibilities: []
 - version: 0.28.0

utils/compatibility/scrapers/karpenter.py

@@ -1,85 +1,93 @@
-# scrapers/cert_manager.py
-
 from bs4 import BeautifulSoup
 from collections import OrderedDict
+from packaging.version import Version
 from utils import (
     print_error,
     fetch_page,
     update_compatibility_info,
     get_chart_versions,
     validate_semver,
-    expand_kube_versions,
 )
 
 app_name = "karpenter"
 compatibility_url = "https://karpenter.sh/preview/upgrading/compatibility/"
 
 
-def parse_page(content):
+def find_compatibility_table(content):
     soup = BeautifulSoup(content, "html.parser")
-    sections = soup.find_all("h2")
-    for section in sections:
-        if section.text == "Compatibility Matrix":
-            return section.find_next("table")
-    return sections
-
-
-def find_target_tables(sections):
-    target_tables = []
-    table = []
-
-    for section in sections:
-        for text in section.stripped_strings:
-            lines = [line.strip() for line in text.split("\n") if line.strip()]
-
-            # If "KUBERNETES" or "karpenter" is found, start a new table
-            if lines and (lines[0] in ["KUBERNETES", "karpenter"]):
-                if (
-                    table
-                ):  # If there's an existing table, add it to target_tables
-                    target_tables.append(table)
-                table = lines  # Start a new table
-            else:
-                # Add the lines to the current table
-                table.extend(lines)
-
-    # Add the last table if it exists
-    if table:
-        target_tables.append(table)
-
-    return target_tables
-
-
-def extract_table_data(target_tables, chart_versions):
-    if len(target_tables) < 2:
-        print_error("Insufficient data in target tables.")
-        return []
-
-    k8s_versions = target_tables[0][1:]  # Starting from the second element
-    kar_versions = target_tables[1][1:]  # Starting from the second element
-
-    rows = []
-    for k8s_ver, kar_ver in zip(k8s_versions, kar_versions):
-        expanded_k8s_ver = expand_kube_versions("1.19", k8s_ver)
-        kar_ver = kar_ver.split(" ")[1].strip()
-        kar_ver = validate_semver(kar_ver)
-
-        if kar_ver:
-            ver = str(kar_ver)
-            chart_version = chart_versions.get(ver)
-            if not chart_version:
-                continue
-            version_info = OrderedDict(
-                {
-                    "version": ver,
-                    "kube": expanded_k8s_ver,
-                    "chart_version": chart_version,
-                    "images": [],
-                    "requirements": [],
-                    "incompatibilities": [],
-                }
-            )
-            rows.append(version_info)
+
+    header = soup.find(id="compatibility-matrix")
+    if header:
+        table = header.find_next("table")
+        if table:
+            return table
+
+    for h2 in soup.find_all("h2"):
+        if "Compatibility Matrix" in h2.get_text(strip=True):
+            table = h2.find_next("table")
+            if table:
+                return table
+
+    return None
+
+
+def parse_table(table):
+    rows = table.find_all("tr")
+    if len(rows) < 2:
+        return {}
+
+    kube_headers = [c.get_text(strip=True).lstrip("v") for c in rows[0].find_all(["th", "td"])][1:]
+    requirements = [c.get_text(strip=True) for c in rows[1].find_all(["th", "td"])][1:]
+
+    min_map = {}
+    for kube, required in zip(kube_headers, requirements):
+        normalized = required.replace(">=", "").replace("v", "").strip()
+        semver = validate_semver(normalized)
+        if semver:
+            min_map[kube] = str(semver)
+
+    return min_map
+
+
+def build_rows_from_table(table, chart_versions):
+    min_map = parse_table(table)
+    if not min_map:
+        return {}
+
+    version_sets = {}
+    versions = sorted({v for v in min_map.values()}, key=lambda s: validate_semver(s))
+
+    for ver in versions:
+        ver_sem = validate_semver(ver)
+        if not ver_sem:
+            continue
+
+        kube_versions = []
+        for kube, required in min_map.items():
+            req_sem = validate_semver(required)
+            if req_sem and req_sem <= ver_sem:
+                kube_versions.append(kube)
+
+        if kube_versions:
+            version_sets.setdefault(ver, set()).update(kube_versions)
+
+    rows = {}
+    for ver, kube_set in version_sets.items():
+        chart_version = chart_versions.get(ver)
+        kube_sorted = sorted(kube_set, key=lambda k: Version(k), reverse=True)
+        ordered_items = [
+            ("version", ver),
+            ("kube", kube_sorted),
+        ]
+        if chart_version:
+            ordered_items.append(("chart_version", chart_version))
+        ordered_items.extend(
+            [
+                ("requirements", []),
+                ("incompatibilities", []),
+            ]
+        )
+        rows[ver] = OrderedDict(ordered_items)
 
     return rows
 
@@ -90,14 +98,18 @@ def scrape():
     if not page_content:
         return
 
-    sections = parse_page(page_content)
-    target_tables = find_target_tables(sections)
-    if target_tables.__len__() >= 1:
-        chart_versions = get_chart_versions(app_name)
-        rows = extract_table_data(target_tables, chart_versions)
-        update_compatibility_info(
-            f"../../static/compatibilities/{app_name}.yaml", rows
-        )
-    else:
+    chart_versions = get_chart_versions(app_name)
+
+    table = find_compatibility_table(page_content)
+    if table is None:
         print_error("No compatibility information found.")
+        return
+
+    page_rows = build_rows_from_table(table, chart_versions)
+    if not page_rows:
+        print_error("No compatibility information found.")
+        return
 
+    update_compatibility_info(
+        f"../../static/compatibilities/{app_name}.yaml", list(page_rows.values())
+    )