Extend pyversions to include dependencies

Relenv has been tracking what python verions are available for some time
now. It'll be great to track dependency versions too, this will reduce
the burdon of maintaining things movnig forward.

Author: dwoz

relenv/build/common.py

@@ -204,11 +204,14 @@ def print_ui(
 
 def verify_checksum(file: PathLike, checksum: Optional[str]) -> bool:
     """
-    Verify the checksum of a files.
+    Verify the checksum of a file.
+
+    Supports both SHA-1 (40 hex chars) and SHA-256 (64 hex chars) checksums.
+    The hash algorithm is auto-detected based on checksum length.
 
     :param file: The path to the file to check.
     :type file: str
-    :param checksum: The checksum to verify against
+    :param checksum: The checksum to verify against (SHA-1 or SHA-256)
     :type checksum: str
 
     :raises RelenvException: If the checksum verification failed
@@ -219,11 +222,26 @@ def verify_checksum(file: PathLike, checksum: Optional[str]) -> bool:
     if checksum is None:
         log.error("Can't verify checksum because none was given")
         return False
+
+    # Auto-detect hash type based on length
+    # SHA-1: 40 hex chars, SHA-256: 64 hex chars
+    if len(checksum) == 64:
+        hash_algo = hashlib.sha256()
+        hash_name = "sha256"
+    elif len(checksum) == 40:
+        hash_algo = hashlib.sha1()
+        hash_name = "sha1"
+    else:
+        raise RelenvException(
+            f"Invalid checksum length {len(checksum)}. Expected 40 (SHA-1) or 64 (SHA-256)"
+        )
+
     with open(file, "rb") as fp:
-        file_checksum = hashlib.sha1(fp.read()).hexdigest()
+        hash_algo.update(fp.read())
+        file_checksum = hash_algo.hexdigest()
         if checksum != file_checksum:
             raise RelenvException(
-                f"sha1 checksum verification failed. expected={checksum} found={file_checksum}"
+                f"{hash_name} checksum verification failed. expected={checksum} found={file_checksum}"
             )
     return True
 
@@ -541,6 +559,52 @@ def uuid_version(href: str) -> Optional[str]:
     return None
 
 
+def get_dependency_version(name: str, platform: str) -> Optional[Dict[str, str]]:
+    """
+    Get dependency version and metadata from python-versions.json.
+
+    Returns dict with keys: version, url, sha256, and any extra fields (e.g., sqliteversion)
+    Returns None if dependency not found.
+
+    :param name: Dependency name (openssl, sqlite, xz)
+    :param platform: Platform name (linux, darwin, win32)
+    :return: Dict with version, url, sha256, and extra fields, or None
+    """
+    versions_file = MODULE_DIR / "python-versions.json"
+    if not versions_file.exists():
+        return None
+
+    import json
+
+    data = json.loads(versions_file.read_text())
+    dependencies = data.get("dependencies", {})
+
+    if name not in dependencies:
+        return None
+
+    # Get the latest version for this dependency that supports the platform
+    dep_versions = dependencies[name]
+    for version, info in sorted(
+        dep_versions.items(),
+        key=lambda x: [int(n) for n in x[0].split(".")],
+        reverse=True,
+    ):
+        if platform in info.get("platforms", []):
+            # Build result dict with version, url, sha256, and any extra fields
+            result = {
+                "version": version,
+                "url": info["url"],
+                "sha256": info.get("sha256", ""),
+            }
+            # Add any extra fields (like sqliteversion for SQLite)
+            for key, value in info.items():
+                if key not in ["url", "sha256", "platforms"]:
+                    result[key] = value
+            return result
+
+    return None
+
+
 def parse_links(text: str) -> List[str]:
     class HrefParser(HTMLParser):
         def __init__(self) -> None:

relenv/build/darwin.py

@@ -10,7 +10,15 @@
 from typing import IO, MutableMapping
 
 from ..common import DARWIN, MACOS_DEVELOPMENT_TARGET, arches
-from .common import Dirs, build_openssl, build_sqlite, builds, finalize, runcmd
+from .common import (
+    Dirs,
+    build_openssl,
+    build_sqlite,
+    builds,
+    finalize,
+    get_dependency_version,
+    runcmd,
+)
 
 ARCHES = arches[DARWIN]
 
@@ -38,6 +46,54 @@ def populate_env(env: MutableMapping[str, str], dirs: Dirs) -> None:
     env["CFLAGS"] = " ".join(cflags).format(prefix=dirs.prefix)
 
 
+def update_expat(dirs: Dirs, env: MutableMapping[str, str]) -> None:
+    """
+    Update the bundled expat library to the latest version.
+
+    Python ships with an older bundled expat. This function updates it
+    to the latest version for security and bug fixes.
+    """
+    import pathlib
+    import shutil
+    import glob
+    import urllib.request
+    import tarfile
+
+    # Get version from JSON
+    expat_info = get_dependency_version("expat", "darwin")
+    if not expat_info:
+        # No update needed, use bundled version
+        return
+
+    version = expat_info["version"]
+    version_tag = version.replace(".", "_")
+    url = f"https://github.com/libexpat/libexpat/releases/download/R_{version_tag}/expat-{version}.tar.xz"
+
+    expat_dir = pathlib.Path(dirs.source) / "Modules" / "expat"
+    if not expat_dir.exists():
+        # No expat directory, skip
+        return
+
+    # Download expat tarball
+    tmpbuild = pathlib.Path(dirs.tmpbuild)
+    tarball_path = tmpbuild / f"expat-{version}.tar.xz"
+    urllib.request.urlretrieve(url, str(tarball_path))
+
+    # Extract tarball
+    with tarfile.open(tarball_path) as tar:
+        tar.extractall(path=str(tmpbuild))
+
+    # Copy source files to Modules/expat/
+    expat_source_dir = tmpbuild / f"expat-{version}" / "lib"
+    for source_file in ["*.h", "*.c"]:
+        for file_path in glob.glob(str(expat_source_dir / source_file)):
+            target_file = expat_dir / pathlib.Path(file_path).name
+            # Remove old file if it exists
+            if target_file.exists():
+                target_file.unlink()
+            shutil.copy2(file_path, str(expat_dir))
+
+
 def build_python(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> None:
     """
     Run the commands to build Python.
@@ -49,6 +105,9 @@ def build_python(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> N
     :param logfp: A handle for the log file
     :type logfp: file
     """
+    # Update bundled expat to latest version
+    update_expat(dirs, env)
+
     env["LDFLAGS"] = "-Wl,-rpath,{prefix}/lib {ldflags}".format(
         prefix=dirs.prefix, ldflags=env["LDFLAGS"]
     )
@@ -77,33 +136,66 @@ def build_python(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> N
 
 build = builds.add("darwin", populate_env=populate_env)
 
+# Get dependency versions from JSON (with fallback to hardcoded values)
+openssl_info = get_dependency_version("openssl", "darwin")
+if openssl_info:
+    openssl_version = openssl_info["version"]
+    openssl_url = openssl_info["url"]
+    openssl_checksum = openssl_info["sha256"]
+else:
+    openssl_version = "3.5.4"
+    openssl_url = "https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz"
+    openssl_checksum = "b75daac8e10f189abe28a076ba5905d363e4801f"
+
 build.add(
     "openssl",
     build_func=build_openssl,
     download={
-        "url": "https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz",
-        "version": "3.5.4",
-        "checksum": "b75daac8e10f189abe28a076ba5905d363e4801f",
+        "url": openssl_url,
+        "version": openssl_version,
+        "checksum": openssl_checksum,
     },
 )
 
+# Get XZ version from JSON
+xz_info = get_dependency_version("xz", "darwin")
+if xz_info:
+    xz_version = xz_info["version"]
+    xz_url = xz_info["url"]
+    xz_checksum = xz_info["sha256"]
+else:
+    xz_version = "5.8.1"
+    xz_url = "http://tukaani.org/xz/xz-{version}.tar.gz"
+    xz_checksum = "ed4d5589c4cfe84e1697bd02a9954b76af336931"
+
 build.add(
     "XZ",
     download={
-        "url": "http://tukaani.org/xz/xz-{version}.tar.gz",
-        "version": "5.8.1",
-        "checksum": "ed4d5589c4cfe84e1697bd02a9954b76af336931",
+        "url": xz_url,
+        "version": xz_version,
+        "checksum": xz_checksum,
     },
 )
 
+# Get SQLite version from JSON
+sqlite_info = get_dependency_version("sqlite", "darwin")
+if sqlite_info:
+    sqlite_url = sqlite_info["url"]
+    sqlite_checksum = sqlite_info["sha256"]
+    sqlite_version_num = sqlite_info.get("sqliteversion", "3500400")
+else:
+    sqlite_version_num = "3500400"
+    sqlite_url = "https://sqlite.org/2025/sqlite-autoconf-{version}.tar.gz"
+    sqlite_checksum = "145048005c777796dd8494aa1cfed304e8c34283"
+
 build.add(
     name="SQLite",
     build_func=build_sqlite,
     download={
-        "url": "https://sqlite.org/2025/sqlite-autoconf-{version}.tar.gz",
+        "url": sqlite_url,
         "fallback_url": "https://woz.io/relenv/dependencies/sqlite-autoconf-{version}.tar.gz",
-        "version": "3500400",
-        "checksum": "145048005c777796dd8494aa1cfed304e8c34283",
+        "version": sqlite_version_num,
+        "checksum": sqlite_checksum,
     },
 )