From 2968f2ef422bcd5fa2d7da87d66f3442f49afbd6 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 3 Nov 2025 22:34:17 -0700 Subject: [PATCH 1/7] update expat for all versions --- relenv/build/windows.py | 1 - 1 file changed, 1 deletion(-) diff --git a/relenv/build/windows.py b/relenv/build/windows.py index a88f41db..2dd712ea 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -265,7 +265,6 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: for target_file in updated_files: os.utime(target_file, (now, now)) - # Update SBOM with correct checksums for updated expat files # Map SBOM file names to actual file paths files_to_update = {} for target_file in updated_files: From 9d930ce6a8bb176859b13b25d9467cb31242ce29 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 4 Nov 2025 00:02:57 -0700 Subject: [PATCH 2/7] Add tests to verify our version updates are working --- relenv/build/darwin.py | 8 +++++--- relenv/build/linux.py | 6 ++++-- relenv/build/windows.py | 8 ++++++-- tests/test_verify_build.py | 14 ++++++-------- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index 2bf0c019..8f70e3a8 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -8,11 +8,9 @@ import glob import io -import os import pathlib import shutil import tarfile -import time import urllib.request from typing import IO, MutableMapping @@ -24,7 +22,6 @@ builds, finalize, get_dependency_version, - update_sbom_checksums, ) ARCHES = arches[DARWIN] @@ -98,11 +95,16 @@ def update_expat(dirs: Dirs, env: MutableMapping[str, str]) -> None: # Touch all updated files to ensure make rebuilds them # (The tarball may contain files with newer timestamps) + import time + import os + now = time.time() for target_file in updated_files: os.utime(target_file, (now, now)) # Update SBOM with correct checksums for updated expat files + from relenv.build.common import update_sbom_checksums + files_to_update = {} for target_file in updated_files: # SBOM uses relative paths from Python source root diff --git a/relenv/build/linux.py b/relenv/build/linux.py index 5a0024c5..0f252d58 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -13,7 +13,6 @@ import shutil import tarfile import tempfile -import time import urllib.request from typing import IO, MutableMapping @@ -25,7 +24,6 @@ builds, finalize, get_dependency_version, - update_sbom_checksums, ) from ..common import LINUX, Version, arches, runcmd @@ -410,11 +408,15 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: # Touch all updated files to ensure make rebuilds them # (The tarball may contain files with newer timestamps) + import time + now = time.time() for target_file in updated_files: os.utime(target_file, (now, now)) # Update SBOM with correct checksums for updated expat files + from relenv.build.common import update_sbom_checksums + files_to_update = {} for target_file in updated_files: # SBOM uses relative paths from Python source root diff --git a/relenv/build/windows.py b/relenv/build/windows.py index 2dd712ea..c29458b8 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -14,7 +14,6 @@ import shutil import sys import tarfile -import time from typing import IO, MutableMapping, Union from .common import ( @@ -25,7 +24,6 @@ install_runtime, patch_file, update_ensurepip, - update_sbom_checksums, ) from ..common import ( WIN32, @@ -261,11 +259,17 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: # Touch all updated files to ensure MSBuild rebuilds them # (The original files may have newer timestamps) + import time + now = time.time() for target_file in updated_files: os.utime(target_file, (now, now)) # Map SBOM file names to actual file paths + # Update SBOM with correct checksums for updated expat files + # Map SBOM file names to actual file paths + from relenv.build.common import update_sbom_checksums + files_to_update = {} for target_file in updated_files: # SBOM uses relative paths from Python source root diff --git a/tests/test_verify_build.py b/tests/test_verify_build.py index 3ed7a33a..6e26093b 100644 --- a/tests/test_verify_build.py +++ b/tests/test_verify_build.py @@ -2142,10 +2142,9 @@ def test_expat_version(pyexec): # Convert "expat_2_7_3" -> "2.7.3" actual_version = actual_version_str.replace("expat_", "").replace("_", ".") - assert actual_version == expected_version, ( - f"Expat version mismatch on {platform}: expected {expected_version}, " - f"found {actual_version} (from {actual_version_str})" - ) + assert ( + actual_version == expected_version + ), f"Expat version mismatch on {platform}: expected {expected_version}, " def test_sqlite_version(pyexec): @@ -2245,10 +2244,9 @@ def test_openssl_version(pyexec): actual_version = match.group(1) - assert actual_version == expected_version, ( - f"OpenSSL version mismatch on {platform}: expected {expected_version}, " - f"found {actual_version} (from {actual_version_str})" - ) + assert ( + actual_version == expected_version + ), f"found {actual_version} (from {actual_version_str})" def test_xz_lzma_functionality(pyexec): From 67c378114a8f3c1485a4699819e34c235e9d6134 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 4 Nov 2025 00:44:07 -0700 Subject: [PATCH 3/7] Fix up imports --- relenv/build/darwin.py | 5 ----- relenv/build/linux.py | 4 ---- relenv/build/windows.py | 6 ++---- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index 8f70e3a8..ee215723 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -95,16 +95,11 @@ def update_expat(dirs: Dirs, env: MutableMapping[str, str]) -> None: # Touch all updated files to ensure make rebuilds them # (The tarball may contain files with newer timestamps) - import time - import os - now = time.time() for target_file in updated_files: os.utime(target_file, (now, now)) # Update SBOM with correct checksums for updated expat files - from relenv.build.common import update_sbom_checksums - files_to_update = {} for target_file in updated_files: # SBOM uses relative paths from Python source root diff --git a/relenv/build/linux.py b/relenv/build/linux.py index 0f252d58..b192eb08 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -408,15 +408,11 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: # Touch all updated files to ensure make rebuilds them # (The tarball may contain files with newer timestamps) - import time - now = time.time() for target_file in updated_files: os.utime(target_file, (now, now)) # Update SBOM with correct checksums for updated expat files - from relenv.build.common import update_sbom_checksums - files_to_update = {} for target_file in updated_files: # SBOM uses relative paths from Python source root diff --git a/relenv/build/windows.py b/relenv/build/windows.py index c29458b8..5ccf96e6 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -14,6 +14,7 @@ import shutil import sys import tarfile +import time from typing import IO, MutableMapping, Union from .common import ( @@ -24,6 +25,7 @@ install_runtime, patch_file, update_ensurepip, + update_sbom_checksums, ) from ..common import ( WIN32, @@ -259,8 +261,6 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: # Touch all updated files to ensure MSBuild rebuilds them # (The original files may have newer timestamps) - import time - now = time.time() for target_file in updated_files: os.utime(target_file, (now, now)) @@ -268,8 +268,6 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: # Map SBOM file names to actual file paths # Update SBOM with correct checksums for updated expat files # Map SBOM file names to actual file paths - from relenv.build.common import update_sbom_checksums - files_to_update = {} for target_file in updated_files: # SBOM uses relative paths from Python source root From e1d4286d165b28f03e78fd1624eaddfe93b6e4a7 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 4 Nov 2025 02:14:37 -0700 Subject: [PATCH 4/7] wip --- relenv/build/common.py | 1901 ++++++++++++++++++++++++++++++++++++++++ tests/test_build.py | 118 +++ 2 files changed, 2019 insertions(+) create mode 100644 relenv/build/common.py diff --git a/relenv/build/common.py b/relenv/build/common.py new file mode 100644 index 00000000..89483ac8 --- /dev/null +++ b/relenv/build/common.py @@ -0,0 +1,1901 @@ +# Copyright 2022-2025 Broadcom. +# SPDX-License-Identifier: Apache-2.0 +""" +Build process common methods. +""" +from __future__ import annotations + +import fnmatch +import hashlib +import io +import logging +import multiprocessing +import os +import os.path +import pathlib +import pprint +import random +import re +import shutil +import subprocess +import sys +import tempfile +import time +import tarfile +from types import ModuleType +from typing import ( + Any, + Callable, + Dict, + IO, + List, + MutableMapping, + Optional, + Sequence, + Tuple, + Union, + cast, +) + +from typing import TYPE_CHECKING, TypedDict + +if TYPE_CHECKING: + from multiprocessing.synchronize import Event as SyncEvent +else: + SyncEvent = Any + +from relenv.common import ( + DATA_DIR, + LINUX, + MODULE_DIR, + RelenvException, + build_arch, + download_url, + extract_archive, + format_shebang, + get_download_location, + get_toolchain, + get_triplet, + runcmd, + work_dirs, + Version, + WorkDirs, +) +import relenv.relocate + + +PathLike = Union[str, os.PathLike[str]] + +log = logging.getLogger(__name__) + + +GREEN = "\033[0;32m" +YELLOW = "\033[1;33m" +RED = "\033[0;31m" +END = "\033[0m" +MOVEUP = "\033[F" + + +CICD = "CI" in os.environ +NODOWLOAD = False + + +RELENV_PTH = ( + "import os; " + "import sys; " + "from importlib import util; " + "from pathlib import Path; " + "spec = util.spec_from_file_location(" + "'relenv.runtime', str(Path(__file__).parent / 'site-packages' / 'relenv' / 'runtime.py')" + "); " + "mod = util.module_from_spec(spec); " + "sys.modules['relenv.runtime'] = mod; " + "spec.loader.exec_module(mod); mod.bootstrap();" +) + + +SYSCONFIGDATA = """ +import pathlib, sys, platform, os, logging + +log = logging.getLogger(__name__) + +def build_arch(): + machine = platform.machine() + return machine.lower() + +def get_triplet(machine=None, plat=None): + if not plat: + plat = sys.platform + if not machine: + machine = build_arch() + if plat == "darwin": + return f"{machine}-macos" + elif plat == "win32": + return f"{machine}-win" + elif plat == "linux": + return f"{machine}-linux-gnu" + else: + raise RelenvException("Unknown platform {}".format(platform)) + + + +pydir = pathlib.Path(__file__).resolve().parent +if sys.platform == "win32": + DEFAULT_DATA_DIR = pathlib.Path.home() / "AppData" / "Local" / "relenv" +else: + DEFAULT_DATA_DIR = pathlib.Path.home() / ".local" / "relenv" + +if "RELENV_DATA" in os.environ: + DATA_DIR = pathlib.Path(os.environ["RELENV_DATA"]).resolve() +else: + DATA_DIR = DEFAULT_DATA_DIR + +buildroot = pydir.parent.parent + +toolchain = DATA_DIR / "toolchain" / get_triplet() + +build_time_vars = {} +for key in _build_time_vars: + val = _build_time_vars[key] + orig = val + if isinstance(val, str): + val = val.format( + BUILDROOT=buildroot, + TOOLCHAIN=toolchain, + ) + build_time_vars[key] = val +""" + + +def print_ui( + events: MutableMapping[str, "multiprocessing.synchronize.Event"], + processes: MutableMapping[str, multiprocessing.Process], + fails: Sequence[str], + flipstat: Optional[Dict[str, Tuple[int, float]]] = None, +) -> None: + """ + Prints the UI during the relenv building process. + + :param events: A dictionary of events that are updated during the build process + :type events: dict + :param processes: A dictionary of build processes + :type processes: dict + :param fails: A list of processes that have failed + :type fails: list + :param flipstat: A dictionary of process statuses, defaults to {} + :type flipstat: dict, optional + """ + if flipstat is None: + flipstat = {} + if CICD: + sys.stdout.flush() + return + uiline = [] + for name in events: + if not events[name].is_set(): + status = " {}.".format(YELLOW) + elif name in processes: + now = time.time() + if name not in flipstat: + flipstat[name] = (0, now) + if flipstat[name][1] < now: + flipstat[name] = (1 - flipstat[name][0], now + random.random()) + status = " {}{}".format(GREEN, " " if flipstat[name][0] == 1 else ".") + elif name in fails: + status = " {}\u2718".format(RED) + else: + status = " {}\u2718".format(GREEN) + uiline.append(status) + uiline.append(" " + END) + sys.stdout.write("\r") + sys.stdout.write("".join(uiline)) + sys.stdout.flush() + + +def verify_checksum(file: PathLike, checksum: Optional[str]) -> bool: + """ + Verify the checksum of a file. + + Supports both SHA-1 (40 hex chars) and SHA-256 (64 hex chars) checksums. + The hash algorithm is auto-detected based on checksum length. + + :param file: The path to the file to check. + :type file: str + :param checksum: The checksum to verify against (SHA-1 or SHA-256) + :type checksum: str + + :raises RelenvException: If the checksum verification failed + + :return: True if it succeeded, or False if the checksum was None + :rtype: bool + """ + if checksum is None: + log.error("Can't verify checksum because none was given") + return False + + # Auto-detect hash type based on length + # SHA-1: 40 hex chars, SHA-256: 64 hex chars + if len(checksum) == 64: + hash_algo = hashlib.sha256() + hash_name = "sha256" + elif len(checksum) == 40: + hash_algo = hashlib.sha1() + hash_name = "sha1" + else: + raise RelenvException( + f"Invalid checksum length {len(checksum)}. Expected 40 (SHA-1) or 64 (SHA-256)" + ) + + with open(file, "rb") as fp: + hash_algo.update(fp.read()) + file_checksum = hash_algo.hexdigest() + if checksum != file_checksum: + raise RelenvException( + f"{hash_name} checksum verification failed. expected={checksum} found={file_checksum}" + ) + return True + + +def compute_checksums(file: PathLike) -> Dict[str, str]: + """ + Compute SHA-1 and SHA-256 checksums for a file. + + Returns a dictionary with 'sha1' and 'sha256' keys. + + :param file: The path to the file to checksum + :type file: str or PathLike + + :return: Dictionary with 'sha1' and 'sha256' checksums + :rtype: dict + """ + sha1 = hashlib.sha1() + sha256 = hashlib.sha256() + + with open(file, "rb") as fp: + data = fp.read() + sha1.update(data) + sha256.update(data) + + return { + "sha1": sha1.hexdigest(), + "sha256": sha256.hexdigest(), + } + + +def update_sbom_checksums( + source_dir: PathLike, files_to_update: Dict[str, PathLike] +) -> None: + """ + Update checksums in Python's SBOM file for modified source files. + + This function updates Misc/sbom.spdx.json with correct checksums for + files that have been replaced (e.g., updated expat library files). + + :param source_dir: The Python source directory containing Misc/sbom.spdx.json + :type source_dir: PathLike + :param files_to_update: Dict mapping SBOM file names to actual file paths + :type files_to_update: dict + """ + import json + import pathlib + + spdx_json = pathlib.Path(source_dir) / "Misc" / "sbom.spdx.json" + if not spdx_json.exists(): + # Python < 3.12 doesn't have SBOM files + return + + with open(str(spdx_json), "r") as f: + data = json.load(f) + + # Update checksums for each file in the SBOM + for file_entry in data.get("files", []): + file_name = file_entry.get("fileName", "") + if file_name in files_to_update: + file_path = files_to_update[file_name] + checksums = compute_checksums(file_path) + file_entry["checksums"] = [ + {"algorithm": "SHA1", "checksumValue": checksums["sha1"]}, + {"algorithm": "SHA256", "checksumValue": checksums["sha256"]}, + ] + + with open(str(spdx_json), "w") as f: + json.dump(data, f, indent=2) + + +def all_dirs(root: PathLike, recurse: bool = True) -> List[str]: + """ + Get all directories under and including the given root. + + :param root: The root directory to traverse + :type root: str + :param recurse: Whether to recursively search for directories, defaults to True + :type recurse: bool, optional + + :return: A list of directories found + :rtype: list + """ + root_str = os.fspath(root) + paths: List[str] = [root_str] + for current_root, dirs, _files in os.walk(root_str): + if not recurse and current_root != root_str: + continue + for name in dirs: + paths.append(os.path.join(current_root, name)) + return paths + + +def populate_env(env: MutableMapping[str, str], dirs: "Dirs") -> None: + """Populate environment variables for a build step. + + This default implementation intentionally does nothing; specific steps may + provide their own implementation via the ``populate_env`` hook. + """ + _ = env + _ = dirs + + +def build_default(env: MutableMapping[str, str], dirs: "Dirs", logfp: IO[str]) -> None: + """ + The default build function if none is given during the build process. + + :param env: The environment dictionary + :type env: dict + :param dirs: The working directories + :type dirs: ``relenv.build.common.Dirs`` + :param logfp: A handle for the log file + :type logfp: file + """ + cmd = [ + "./configure", + "--prefix={}".format(dirs.prefix), + ] + if env["RELENV_HOST"].find("linux") > -1: + cmd += [ + "--build={}".format(env["RELENV_BUILD"]), + "--host={}".format(env["RELENV_HOST"]), + ] + runcmd(cmd, env=env, stderr=logfp, stdout=logfp) + runcmd(["make", "-j8"], env=env, stderr=logfp, stdout=logfp) + runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) + + +def build_openssl_fips( + env: MutableMapping[str, str], dirs: "Dirs", logfp: IO[str] +) -> None: + return build_openssl(env, dirs, logfp, fips=True) + + +def build_openssl( + env: MutableMapping[str, str], + dirs: "Dirs", + logfp: IO[str], + fips: bool = False, +) -> None: + """ + Build openssl. + + :param env: The environment dictionary + :type env: dict + :param dirs: The working directories + :type dirs: ``relenv.build.common.Dirs`` + :param logfp: A handle for the log file + :type logfp: file + """ + arch = "aarch64" + if sys.platform == "darwin": + plat = "darwin64" + if env["RELENV_HOST_ARCH"] == "x86_64": + arch = "x86_64-cc" + elif env["RELENV_HOST_ARCH"] == "arm64": + arch = "arm64-cc" + else: + raise RelenvException(f"Unable to build {env['RELENV_HOST_ARCH']}") + extended_cmd = [] + else: + plat = "linux" + if env["RELENV_HOST_ARCH"] == "x86_64": + arch = "x86_64" + elif env["RELENV_HOST_ARCH"] == "aarch64": + arch = "aarch64" + else: + raise RelenvException(f"Unable to build {env['RELENV_HOST_ARCH']}") + extended_cmd = [ + "-Wl,-z,noexecstack", + ] + if fips: + extended_cmd.append("enable-fips") + cmd = [ + "./Configure", + f"{plat}-{arch}", + f"--prefix={dirs.prefix}", + "--openssldir=/etc/ssl", + "--libdir=lib", + "--api=1.1.1", + "--shared", + "--with-rand-seed=os,egd", + "enable-md2", + "enable-egd", + "no-idea", + ] + cmd.extend(extended_cmd) + runcmd( + cmd, + env=env, + stderr=logfp, + stdout=logfp, + ) + runcmd(["make", "-j8"], env=env, stderr=logfp, stdout=logfp) + if fips: + shutil.copy( + pathlib.Path("providers") / "fips.so", + pathlib.Path(dirs.prefix) / "lib" / "ossl-modules", + ) + else: + runcmd(["make", "install_sw"], env=env, stderr=logfp, stdout=logfp) + + +def build_sqlite(env: MutableMapping[str, str], dirs: "Dirs", logfp: IO[str]) -> None: + """ + Build sqlite. + + :param env: The environment dictionary + :type env: dict + :param dirs: The working directories + :type dirs: ``relenv.build.common.Dirs`` + :param logfp: A handle for the log file + :type logfp: file + """ + # extra_cflags=('-Os ' + # '-DSQLITE_ENABLE_FTS5 ' + # '-DSQLITE_ENABLE_FTS4 ' + # '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' + # '-DSQLITE_ENABLE_JSON1 ' + # '-DSQLITE_ENABLE_RTREE ' + # '-DSQLITE_TCL=0 ' + # ) + # configure_pre=[ + # '--enable-threadsafe', + # '--enable-shared=no', + # '--enable-static=yes', + # '--disable-readline', + # '--disable-dependency-tracking', + # ] + cmd = [ + "./configure", + # "--with-shared", + # "--without-static", + "--enable-threadsafe", + "--disable-readline", + "--disable-dependency-tracking", + "--prefix={}".format(dirs.prefix), + # "--enable-add-ons=nptl,ports", + ] + if env["RELENV_HOST"].find("linux") > -1: + cmd += [ + "--build={}".format(env["RELENV_BUILD_ARCH"]), + "--host={}".format(env["RELENV_HOST"]), + ] + runcmd(cmd, env=env, stderr=logfp, stdout=logfp) + runcmd(["make", "-j8"], env=env, stderr=logfp, stdout=logfp) + runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) + + +def update_ensurepip(directory: pathlib.Path) -> None: + """ + Update bundled dependencies for ensurepip (pip & setuptools). + """ + # ensurepip bundle location + bundle_dir = directory / "ensurepip" / "_bundled" + + # Make sure the destination directory exists + bundle_dir.mkdir(parents=True, exist_ok=True) + + # Detect existing whl. Later versions of python don't include setuptools. We + # only want to update whl files that python expects to be there + pip_version = "25.2" + setuptools_version = "80.9.0" + update_pip = False + update_setuptools = False + for file in bundle_dir.glob("*.whl"): + + log.debug("Checking whl: %s", str(file)) + if file.name.startswith("pip-"): + found_version = file.name.split("-")[1] + log.debug("Found version %s", found_version) + if Version(found_version) >= Version(pip_version): + log.debug("Found correct pip version or newer: %s", found_version) + else: + file.unlink() + update_pip = True + if file.name.startswith("setuptools-"): + found_version = file.name.split("-")[1] + log.debug("Found version %s", found_version) + if Version(found_version) >= Version(setuptools_version): + log.debug( + "Found correct setuptools version or newer: %s", found_version + ) + else: + file.unlink() + update_setuptools = True + + # Download whl files and update __init__.py + init_file = directory / "ensurepip" / "__init__.py" + if update_pip: + whl = f"pip-{pip_version}-py3-none-any.whl" + whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" + url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" + download_url(url=url, dest=bundle_dir) + assert (bundle_dir / whl).exists() + + # Update __init__.py + old = "^_PIP_VERSION.*" + new = f'_PIP_VERSION = "{pip_version}"' + patch_file(path=init_file, old=old, new=new) + + # setuptools + if update_setuptools: + whl = f"setuptools-{setuptools_version}-py3-none-any.whl" + whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" + url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" + download_url(url=url, dest=bundle_dir) + assert (bundle_dir / whl).exists() + + # setuptools + old = "^_SETUPTOOLS_VERSION.*" + new = f'_SETUPTOOLS_VERSION = "{setuptools_version}"' + patch_file(path=init_file, old=old, new=new) + + log.debug("ensurepip __init__.py contents:") + log.debug(init_file.read_text()) + + +def patch_file(path: PathLike, old: str, new: str) -> None: + """ + Search a file line by line for a string to replace. + + :param path: Location of the file to search + :type path: str + :param old: The value that will be replaced + :type path: str + :param new: The value that will replace the 'old' value. + :type path: str + """ + log.debug("Patching file: %s", path) + with open(path, "r") as fp: + content = fp.read() + new_content = "" + for line in content.splitlines(): + line = re.sub(old, new, line) + new_content += line + "\n" + with open(path, "w") as fp: + fp.write(new_content) + + +def get_dependency_version(name: str, platform: str) -> Optional[Dict[str, str]]: + """ + Get dependency version and metadata from python-versions.json. + + Returns dict with keys: version, url, sha256, and any extra fields (e.g., sqliteversion) + Returns None if dependency not found. + + :param name: Dependency name (openssl, sqlite, xz) + :param platform: Platform name (linux, darwin, win32) + :return: Dict with version, url, sha256, and extra fields, or None + """ + versions_file = MODULE_DIR / "python-versions.json" + if not versions_file.exists(): + return None + + import json + + data = json.loads(versions_file.read_text()) + dependencies = data.get("dependencies", {}) + + if name not in dependencies: + return None + + # Get the latest version for this dependency that supports the platform + dep_versions = dependencies[name] + for version, info in sorted( + dep_versions.items(), + key=lambda x: [int(n) for n in x[0].split(".")], + reverse=True, + ): + if platform in info.get("platforms", []): + # Build result dict with version, url, sha256, and any extra fields + result = { + "version": version, + "url": info["url"], + "sha256": info.get("sha256", ""), + } + # Add any extra fields (like sqliteversion for SQLite) + for key, value in info.items(): + if key not in ["url", "sha256", "platforms"]: + result[key] = value + return result + + return None + + +class Download: + """ + A utility that holds information about content to be downloaded. + + :param name: The name of the download + :type name: str + :param url: The url of the download + :type url: str + :param signature: The signature of the download, defaults to None + :type signature: str + :param destination: The path to download the file to + :type destination: str + :param version: The version of the content to download + :type version: str + :param sha1: The sha1 sum of the download + :type sha1: str + + """ + + def __init__( + self, + name: str, + url: str, + fallback_url: Optional[str] = None, + signature: Optional[str] = None, + destination: PathLike = "", + version: str = "", + checksum: Optional[str] = None, + ) -> None: + self.name = name + self.url_tpl = url + self.fallback_url_tpl = fallback_url + self.signature_tpl = signature + self._destination: pathlib.Path = pathlib.Path() + if destination: + self._destination = pathlib.Path(destination) + self.version = version + self.checksum = checksum + + def copy(self) -> "Download": + return Download( + self.name, + self.url_tpl, + self.fallback_url_tpl, + self.signature_tpl, + self.destination, + self.version, + self.checksum, + ) + + @property + def destination(self) -> pathlib.Path: + return self._destination + + @destination.setter + def destination(self, value: Optional[PathLike]) -> None: + if value: + self._destination = pathlib.Path(value) + else: + self._destination = pathlib.Path() + + @property + def url(self) -> str: + return self.url_tpl.format(version=self.version) + + @property + def fallback_url(self) -> Optional[str]: + if self.fallback_url_tpl: + return self.fallback_url_tpl.format(version=self.version) + return None + + @property + def signature_url(self) -> str: + if self.signature_tpl is None: + raise RelenvException("Signature template not configured") + return self.signature_tpl.format(version=self.version) + + @property + def filepath(self) -> pathlib.Path: + _, name = self.url.rsplit("/", 1) + return self.destination / name + + @property + def formatted_url(self) -> str: + return self.url_tpl.format(version=self.version) + + def fetch_file(self) -> Tuple[str, bool]: + """ + Download the file. + + :return: The path to the downloaded content, and whether it was downloaded. + :rtype: tuple(str, bool) + """ + try: + return download_url(self.url, self.destination, CICD), True + except Exception as exc: + fallback = self.fallback_url + if fallback: + print(f"Download failed {self.url} ({exc}); trying fallback url") + return download_url(fallback, self.destination, CICD), True + raise + + def fetch_signature(self, version: Optional[str] = None) -> Tuple[str, bool]: + """ + Download the file signature. + + :return: The path to the downloaded signature. + :rtype: str + """ + return download_url(self.signature_url, self.destination, CICD), True + + def exists(self) -> bool: + """ + True when the artifact already exists on disk. + + :return: True when the artifact already exists on disk + :rtype: bool + """ + return self.filepath.exists() + + def valid_hash(self) -> None: + pass + + @staticmethod + def validate_signature(archive: PathLike, signature: Optional[PathLike]) -> bool: + """ + True when the archive's signature is valid. + + :param archive: The path to the archive to validate + :type archive: str + :param signature: The path to the signature to validate against + :type signature: str + + :return: True if it validated properly, else False + :rtype: bool + """ + if signature is None: + log.error("Can't check signature because none was given") + return False + try: + runcmd( + ["gpg", "--verify", signature, archive], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + return True + except RelenvException as exc: + log.error("Signature validation failed on %s: %s", archive, exc) + return False + + @staticmethod + def validate_checksum(archive: PathLike, checksum: Optional[str]) -> bool: + """ + True when when the archive matches the sha1 hash. + + :param archive: The path to the archive to validate + :type archive: str + :param checksum: The sha1 sum to validate against + :type checksum: str + :return: True if the sums matched, else False + :rtype: bool + """ + try: + verify_checksum(archive, checksum) + return True + except RelenvException as exc: + log.error("sha1 validation failed on %s: %s", archive, exc) + return False + + def __call__( + self, + force_download: bool = False, + show_ui: bool = False, + exit_on_failure: bool = False, + ) -> bool: + """ + Downloads the url and validates the signature and sha1 sum. + + :return: Whether or not validation succeeded + :rtype: bool + """ + os.makedirs(self.filepath.parent, exist_ok=True) + + downloaded = False + if force_download: + _, downloaded = self.fetch_file() + else: + file_is_valid = False + dest = get_download_location(self.url, self.destination) + if self.checksum and os.path.exists(dest): + file_is_valid = self.validate_checksum(dest, self.checksum) + if file_is_valid: + log.debug("%s already downloaded, skipping.", self.url) + else: + _, downloaded = self.fetch_file() + valid = True + if downloaded: + if self.signature_tpl is not None: + sig, _ = self.fetch_signature() + valid_sig = self.validate_signature(self.filepath, sig) + valid = valid and valid_sig + if self.checksum is not None: + valid_checksum = self.validate_checksum(self.filepath, self.checksum) + valid = valid and valid_checksum + + if not valid: + log.warning("Checksum did not match %s: %s", self.name, self.checksum) + if show_ui: + sys.stderr.write( + f"\nChecksum did not match {self.name}: {self.checksum}\n" + ) + sys.stderr.flush() + if exit_on_failure and not valid: + sys.exit(1) + return valid + + +class Recipe(TypedDict): + """Typed description of a build recipe entry.""" + + build_func: Callable[[MutableMapping[str, str], "Dirs", IO[str]], None] + wait_on: List[str] + download: Optional[Download] + + +class Dirs: + """ + A container for directories during build time. + + :param dirs: A collection of working directories + :type dirs: ``relenv.common.WorkDirs`` + :param name: The name of this collection + :type name: str + :param arch: The architecture being worked with + :type arch: str + """ + + def __init__(self, dirs: WorkDirs, name: str, arch: str, version: str) -> None: + # XXX name is the specific to a step where as everything + # else here is generalized to the entire build + self.name = name + self.version = version + self.arch = arch + self.root = dirs.root + self.build = dirs.build + self.downloads = dirs.download + self.logs = dirs.logs + self.sources = dirs.src + self.tmpbuild = tempfile.mkdtemp(prefix="{}_build".format(name)) + self.source: Optional[pathlib.Path] = None + + @property + def toolchain(self) -> Optional[pathlib.Path]: + if sys.platform == "darwin": + return get_toolchain(root=self.root) + elif sys.platform == "win32": + return get_toolchain(root=self.root) + else: + return get_toolchain(self.arch, self.root) + + @property + def _triplet(self) -> str: + if sys.platform == "darwin": + return "{}-macos".format(self.arch) + elif sys.platform == "win32": + return "{}-win".format(self.arch) + else: + return "{}-linux-gnu".format(self.arch) + + @property + def prefix(self) -> pathlib.Path: + return self.build / f"{self.version}-{self._triplet}" + + def __getstate__(self) -> Dict[str, Any]: + """ + Return an object used for pickling. + + :return: The picklable state + """ + return { + "name": self.name, + "arch": self.arch, + "root": self.root, + "build": self.build, + "downloads": self.downloads, + "logs": self.logs, + "sources": self.sources, + "tmpbuild": self.tmpbuild, + } + + def __setstate__(self, state: Dict[str, Any]) -> None: + """ + Unwrap the object returned from unpickling. + + :param state: The state to unpickle + :type state: dict + """ + self.name = state["name"] + self.arch = state["arch"] + self.root = state["root"] + self.downloads = state["downloads"] + self.logs = state["logs"] + self.sources = state["sources"] + self.build = state["build"] + self.tmpbuild = state["tmpbuild"] + + def to_dict(self) -> Dict[str, Any]: + """ + Get a dictionary representation of the directories in this collection. + + :return: A dictionary of all the directories + :rtype: dict + """ + return { + x: getattr(self, x) + for x in [ + "root", + "prefix", + "downloads", + "logs", + "sources", + "build", + "toolchain", + ] + } + + +class Builder: + """ + Utility that handles the build process. + + :param root: The root of the working directories for this build + :type root: str + :param recipies: The instructions for the build steps + :type recipes: list + :param build_default: The default build function, defaults to ``build_default`` + :type build_default: types.FunctionType + :param populate_env: The default function to populate the build environment, defaults to ``populate_env`` + :type populate_env: types.FunctionType + :param force_download: If True, forces downloading the archives even if they exist, defaults to False + :type force_download: bool + :param arch: The architecture being built + :type arch: str + """ + + def __init__( + self, + root: Optional[PathLike] = None, + recipies: Optional[Dict[str, Recipe]] = None, + build_default: Callable[ + [MutableMapping[str, str], "Dirs", IO[str]], None + ] = build_default, + populate_env: Callable[[MutableMapping[str, str], "Dirs"], None] = populate_env, + arch: str = "x86_64", + version: str = "", + ) -> None: + self.root = root + self.dirs: WorkDirs = work_dirs(root) + self.build_arch = build_arch() + self.build_triplet = get_triplet(self.build_arch) + self.arch = arch + self.sources = self.dirs.src + self.downloads = self.dirs.download + + if recipies is None: + self.recipies: Dict[str, Recipe] = {} + else: + self.recipies = recipies + + self.build_default = build_default + self.populate_env = populate_env + self.version = version + self.set_arch(self.arch) + + def copy(self, version: str, checksum: Optional[str]) -> "Builder": + recipies: Dict[str, Recipe] = {} + for name in self.recipies: + recipe = self.recipies[name] + recipies[name] = { + "build_func": recipe["build_func"], + "wait_on": list(recipe["wait_on"]), + "download": recipe["download"].copy() if recipe["download"] else None, + } + build = Builder( + self.root, + recipies, + self.build_default, + self.populate_env, + self.arch, + version, + ) + python_download = build.recipies["python"].get("download") + if python_download is None: + raise RelenvException("Python recipe is missing a download entry") + python_download.version = version + python_download.checksum = checksum + return build + + def set_arch(self, arch: str) -> None: + """ + Set the architecture for the build. + + :param arch: The arch to build + :type arch: str + """ + self.arch = arch + self._toolchain: Optional[pathlib.Path] = None + + @property + def toolchain(self) -> Optional[pathlib.Path]: + """Lazily fetch toolchain only when needed.""" + if self._toolchain is None and sys.platform == "linux": + self._toolchain = get_toolchain(self.arch, self.dirs.root) + return self._toolchain + + @property + def triplet(self) -> str: + return get_triplet(self.arch) + + @property + def prefix(self) -> pathlib.Path: + return self.dirs.build / f"{self.version}-{self.triplet}" + + @property + def _triplet(self) -> str: + if sys.platform == "darwin": + return "{}-macos".format(self.arch) + elif sys.platform == "win32": + return "{}-win".format(self.arch) + else: + return "{}-linux-gnu".format(self.arch) + + def add( + self, + name: str, + build_func: Optional[Callable[..., Any]] = None, + wait_on: Optional[Sequence[str]] = None, + download: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Add a step to the build process. + + :param name: The name of the step + :type name: str + :param build_func: The function that builds this step, defaults to None + :type build_func: types.FunctionType, optional + :param wait_on: Processes to wait on before running this step, defaults to None + :type wait_on: list, optional + :param download: A dictionary of download information, defaults to None + :type download: dict, optional + """ + if wait_on is None: + wait_on_list: List[str] = [] + else: + wait_on_list = list(wait_on) + if build_func is None: + build_func = self.build_default + download_obj: Optional[Download] = None + if download is not None: + download_obj = Download(name, destination=self.downloads, **download) + self.recipies[name] = { + "build_func": build_func, + "wait_on": wait_on_list, + "download": download_obj, + } + + def run( + self, + name: str, + event: "multiprocessing.synchronize.Event", + build_func: Callable[..., Any], + download: Optional[Download], + show_ui: bool = False, + log_level: str = "WARNING", + ) -> Any: + """ + Run a build step. + + :param name: The name of the step to run + :type name: str + :param event: An event to track this process' status and alert waiting steps + :type event: ``multiprocessing.Event`` + :param build_func: The function to use to build this step + :type build_func: types.FunctionType + :param download: The ``Download`` instance for this step + :type download: ``Download`` + + :return: The output of the build function + """ + root_log = logging.getLogger(None) + if sys.platform == "win32": + if not show_ui: + handler = logging.StreamHandler() + handler.setLevel(logging.getLevelName(log_level)) + root_log.addHandler(handler) + + for handler in root_log.handlers: + if isinstance(handler, logging.StreamHandler): + handler.setFormatter( + logging.Formatter(f"%(asctime)s {name} %(message)s") + ) + + if not self.dirs.build.exists(): + os.makedirs(self.dirs.build, exist_ok=True) + + dirs = Dirs(self.dirs, name, self.arch, self.version) + os.makedirs(dirs.sources, exist_ok=True) + os.makedirs(dirs.logs, exist_ok=True) + os.makedirs(dirs.prefix, exist_ok=True) + + while event.is_set() is False: + time.sleep(0.3) + + logfp = io.open(os.path.join(dirs.logs, "{}.log".format(name)), "w") + handler = logging.FileHandler(dirs.logs / f"{name}.log") + root_log.addHandler(handler) + root_log.setLevel(logging.NOTSET) + + # DEBUG: Uncomment to debug + # logfp = sys.stdout + + cwd = os.getcwd() + if download: + extract_archive(dirs.sources, str(download.filepath)) + dirs.source = dirs.sources / download.filepath.name.split(".tar")[0] + os.chdir(dirs.source) + else: + os.chdir(dirs.prefix) + + if sys.platform == "win32": + env = os.environ.copy() + else: + env = { + "PATH": os.environ["PATH"], + } + env["RELENV_DEBUG"] = "1" + env["RELENV_BUILDENV"] = "1" + env["RELENV_HOST"] = self.triplet + env["RELENV_HOST_ARCH"] = self.arch + env["RELENV_BUILD"] = self.build_triplet + env["RELENV_BUILD_ARCH"] = self.build_arch + python_download = self.recipies["python"].get("download") + if python_download is None: + raise RelenvException("Python recipe is missing download configuration") + env["RELENV_PY_VERSION"] = python_download.version + env["RELENV_PY_MAJOR_VERSION"] = env["RELENV_PY_VERSION"].rsplit(".", 1)[0] + if "RELENV_DATA" in os.environ: + env["RELENV_DATA"] = os.environ["RELENV_DATA"] + if self.build_arch != self.arch: + native_root = DATA_DIR / "native" + env["RELENV_NATIVE_PY"] = str(native_root / "bin" / "python3") + + self.populate_env(env, dirs) + + _ = dirs.to_dict() + for k in _: + log.info("Directory %s %s", k, _[k]) + for k in env: + log.info("Environment %s %s", k, env[k]) + try: + return build_func(env, dirs, logfp) + except Exception: + log.exception("Build failure") + sys.exit(1) + finally: + os.chdir(cwd) + log.removeHandler(handler) + logfp.close() + + def cleanup(self) -> None: + """ + Clean up the build directories. + """ + shutil.rmtree(self.prefix) + + def clean(self) -> None: + """ + Completely clean up the remnants of a relenv build. + """ + # Clean directories + for _ in [self.prefix, self.sources]: + try: + shutil.rmtree(_) + except PermissionError: + sys.stderr.write(f"Unable to remove directory: {_}") + except FileNotFoundError: + pass + # Clean files + archive = f"{self.prefix}.tar.xz" + for _ in [archive]: + try: + os.remove(_) + except FileNotFoundError: + pass + + def download_files( + self, + steps: Optional[Sequence[str]] = None, + force_download: bool = False, + show_ui: bool = False, + ) -> None: + """ + Download all of the needed archives. + + :param steps: The steps to download archives for, defaults to None + :type steps: list, optional + """ + step_names = list(steps) if steps is not None else list(self.recipies) + + fails: List[str] = [] + processes: Dict[str, multiprocessing.Process] = {} + events: Dict[str, SyncEvent] = {} + if show_ui: + sys.stdout.write("Starting downloads \n") + log.info("Starting downloads") + if show_ui: + print_ui(events, processes, fails) + for name in step_names: + download = self.recipies[name]["download"] + if download is None: + continue + event = multiprocessing.Event() + event.set() + events[name] = event + proc = multiprocessing.Process( + name=name, + target=download, + kwargs={ + "force_download": force_download, + "show_ui": show_ui, + "exit_on_failure": True, + }, + ) + proc.start() + processes[name] = proc + + while processes: + for proc in list(processes.values()): + proc.join(0.3) + # DEBUG: Comment to debug + if show_ui: + print_ui(events, processes, fails) + if proc.exitcode is None: + continue + processes.pop(proc.name) + if proc.exitcode != 0: + fails.append(proc.name) + if show_ui: + print_ui(events, processes, fails) + sys.stdout.write("\n") + if fails and False: + if show_ui: + print_ui(events, processes, fails) + sys.stderr.write("The following failures were reported\n") + for fail in fails: + sys.stderr.write(fail + "\n") + sys.stderr.flush() + sys.exit(1) + + def build( + self, + steps: Optional[Sequence[str]] = None, + cleanup: bool = True, + show_ui: bool = False, + log_level: str = "WARNING", + ) -> None: + """ + Build! + + :param steps: The steps to run, defaults to None + :type steps: list, optional + :param cleanup: Whether to clean up or not, defaults to True + :type cleanup: bool, optional + """ # noqa: D400 + fails: List[str] = [] + events: Dict[str, SyncEvent] = {} + waits: Dict[str, List[str]] = {} + processes: Dict[str, multiprocessing.Process] = {} + + if show_ui: + sys.stdout.write("Starting builds\n") + # DEBUG: Comment to debug + print_ui(events, processes, fails) + log.info("Starting builds") + + step_names = list(steps) if steps is not None else list(self.recipies) + + for name in step_names: + event = multiprocessing.Event() + events[name] = event + recipe = self.recipies[name] + kwargs = dict(recipe) + kwargs["show_ui"] = show_ui + kwargs["log_level"] = log_level + + # Determine needed dependency recipies. + wait_on_seq = cast(List[str], kwargs.pop("wait_on", [])) + wait_on_list = list(wait_on_seq) + for dependency in wait_on_list[:]: + if dependency not in step_names: + wait_on_list.remove(dependency) + + waits[name] = wait_on_list + if not waits[name]: + event.set() + + proc = multiprocessing.Process( + name=name, target=self.run, args=(name, event), kwargs=kwargs + ) + proc.start() + processes[name] = proc + + # Wait for the processes to finish and check if we should send any + # dependency events. + while processes: + for proc in list(processes.values()): + proc.join(0.3) + if show_ui: + # DEBUG: Comment to debug + print_ui(events, processes, fails) + if proc.exitcode is None: + continue + processes.pop(proc.name) + if proc.exitcode != 0: + fails.append(proc.name) + is_failure = True + else: + is_failure = False + for name in waits: + if proc.name in waits[name]: + if is_failure: + if name in processes: + processes[name].terminate() + time.sleep(0.1) + waits[name].remove(proc.name) + if not waits[name] and not events[name].is_set(): + events[name].set() + + if fails: + sys.stderr.write("The following failures were reported\n") + last_outs = {} + for fail in fails: + log_file = self.dirs.logs / f"{fail}.log" + try: + with io.open(log_file) as fp: + fp.seek(0, 2) + end = fp.tell() + ind = end - 4096 + if ind > 0: + fp.seek(ind) + else: + fp.seek(0) + last_out = fp.read() + if show_ui: + sys.stderr.write("=" * 20 + f" {fail} " + "=" * 20 + "\n") + sys.stderr.write(fp.read() + "\n\n") + except FileNotFoundError: + last_outs[fail] = f"Log file not found: {log_file}" + log.error("Build step %s has failed", fail) + log.error(last_out) + if show_ui: + sys.stderr.flush() + if cleanup: + log.debug("Performing cleanup.") + self.cleanup() + sys.exit(1) + if show_ui: + time.sleep(0.3) + print_ui(events, processes, fails) + sys.stdout.write("\n") + sys.stdout.flush() + if cleanup: + log.debug("Performing cleanup.") + self.cleanup() + + def check_prereqs(self) -> List[str]: + """ + Check pre-requsists for build. + + This method verifies all requrements for a successful build are satisfied. + + :return: Returns a list of string describing failed checks + :rtype: list + """ + fail: List[str] = [] + if sys.platform == "linux": + if not self.toolchain or not self.toolchain.exists(): + fail.append( + f"Toolchain for {self.arch} does not exist. Please pip install ppbt." + ) + return fail + + def __call__( + self, + steps: Optional[Sequence[str]] = None, + arch: Optional[str] = None, + clean: bool = True, + cleanup: bool = True, + force_download: bool = False, + download_only: bool = False, + show_ui: bool = False, + log_level: str = "WARNING", + ) -> None: + """ + Set the architecture, define the steps, clean if needed, download what is needed, and build. + + :param steps: The steps to run, defaults to None + :type steps: list, optional + :param arch: The architecture to build, defaults to None + :type arch: str, optional + :param clean: If true, cleans the directories first, defaults to True + :type clean: bool, optional + :param cleanup: Cleans up after build if true, defaults to True + :type cleanup: bool, optional + :param force_download: Whether or not to download the content if it already exists, defaults to True + :type force_download: bool, optional + """ + log = logging.getLogger(None) + log.setLevel(logging.NOTSET) + + stream_handler: Optional[logging.Handler] = None + if not show_ui: + stream_handler = logging.StreamHandler() + stream_handler.setLevel(logging.getLevelName(log_level)) + log.addHandler(stream_handler) + + os.makedirs(self.dirs.logs, exist_ok=True) + file_handler = logging.FileHandler(self.dirs.logs / "build.log") + file_handler.setLevel(logging.INFO) + log.addHandler(file_handler) + + if arch: + self.set_arch(arch) + + step_names = list(steps) if steps is not None else list(self.recipies) + + failures = self.check_prereqs() + if not download_only and failures: + for _ in failures: + sys.stderr.write(f"{_}\n") + sys.stderr.flush() + sys.exit(1) + + if clean: + self.clean() + + if self.build_arch != self.arch: + native_root = DATA_DIR / "native" + if not native_root.exists(): + if "RELENV_NATIVE_PY_VERSION" in os.environ: + version = os.environ["RELENV_NATIVE_PY_VERSION"] + else: + version = self.version + from relenv.create import create + + create("native", DATA_DIR, version=version) + + # Start a process for each build passing it an event used to notify each + # process if it's dependencies have finished. + try: + self.download_files( + step_names, force_download=force_download, show_ui=show_ui + ) + if download_only: + return + self.build(step_names, cleanup, show_ui=show_ui, log_level=log_level) + finally: + log.removeHandler(file_handler) + if stream_handler is not None: + log.removeHandler(stream_handler) + + +class Builds: + """Collection of platform-specific builders.""" + + def __init__(self) -> None: + self.builds: Dict[str, Builder] = {} + + def add(self, platform: str, *args: Any, **kwargs: Any) -> Builder: + if "builder" in kwargs: + build_candidate = kwargs.pop("builder") + if args or kwargs: + raise RuntimeError( + "builder keyword can not be used with other kwargs or args" + ) + build = cast(Builder, build_candidate) + else: + build = Builder(*args, **kwargs) + self.builds[platform] = build + return build + + +builds = Builds() + + +def patch_shebang(path: PathLike, old: str, new: str) -> bool: + """ + Replace a file's shebang. + + :param path: The path of the file to patch + :type path: str + :param old: The old shebang, will only patch when this is found + :type old: str + :param name: The new shebang to be written + :type name: str + """ + with open(path, "rb") as fp: + try: + data = fp.read(len(old.encode())).decode() + except UnicodeError: + return False + except Exception as exc: + log.warning("Unhandled exception: %r", exc) + return False + if data != old: + log.warning("Shebang doesn't match: %s %r != %r", path, old, data) + return False + data = fp.read().decode() + with open(path, "w") as fp: + fp.write(new) + fp.write(data) + with open(path, "r") as fp: + data = fp.read() + log.info("Patched shebang of %s => %r", path, data) + return True + + +def patch_shebangs(path: PathLike, old: str, new: str) -> None: + """ + Traverse directory and patch shebangs. + + :param path: The of the directory to traverse + :type path: str + :param old: The old shebang, will only patch when this is found + :type old: str + :param name: The new shebang to be written + :type name: str + """ + for root, _dirs, files in os.walk(str(path)): + for file in files: + patch_shebang(os.path.join(root, file), old, new) + + +def install_sysdata( + mod: ModuleType, + destfile: PathLike, + buildroot: PathLike, + toolchain: Optional[PathLike], +) -> None: + """ + Create a Relenv Python environment's sysconfigdata. + + Helper method used by the `finalize` build method to create a Relenv + Python environment's sysconfigdata. + + :param mod: The module to operate on + :type mod: ``types.ModuleType`` + :param destfile: Path to the file to write the data to + :type destfile: str + :param buildroot: Path to the root of the build + :type buildroot: str + :param toolchain: Path to the root of the toolchain + :type toolchain: str + """ + data = {} + + def fbuildroot(s: str) -> str: + return s.replace(str(buildroot), "{BUILDROOT}") + + def ftoolchain(s: str) -> str: + return s.replace(str(toolchain), "{TOOLCHAIN}") + + # XXX: keymap is not used, remove it? + # keymap = { + # "BINDIR": (fbuildroot,), + # "BINLIBDEST": (fbuildroot,), + # "CFLAGS": (fbuildroot, ftoolchain), + # "CPPLAGS": (fbuildroot, ftoolchain), + # "CXXFLAGS": (fbuildroot, ftoolchain), + # "datarootdir": (fbuildroot,), + # "exec_prefix": (fbuildroot,), + # "LDFLAGS": (fbuildroot, ftoolchain), + # "LDSHARED": (fbuildroot, ftoolchain), + # "LIBDEST": (fbuildroot,), + # "prefix": (fbuildroot,), + # "SCRIPTDIR": (fbuildroot,), + # } + for key in sorted(mod.build_time_vars): + val = mod.build_time_vars[key] + if isinstance(val, str): + for _ in (fbuildroot, ftoolchain): + val = _(val) + log.info("SYSCONFIG [%s] %s => %s", key, mod.build_time_vars[key], val) + data[key] = val + + with open(destfile, "w", encoding="utf8") as f: + f.write( + "# system configuration generated and used by" " the relenv at runtime\n" + ) + f.write("_build_time_vars = ") + pprint.pprint(data, stream=f) + f.write(SYSCONFIGDATA) + + +def find_sysconfigdata(pymodules: PathLike) -> str: + """ + Find sysconfigdata directory for python installation. + + :param pymodules: Path to python modules (e.g. lib/python3.10) + :type pymodules: str + + :return: The name of the sysconig data module + :rtype: str + """ + for root, dirs, files in os.walk(pymodules): + for file in files: + if file.find("sysconfigdata") > -1 and file.endswith(".py"): + return file[:-3] + raise RelenvException("Unable to locate sysconfigdata module") + + +def install_runtime(sitepackages: PathLike) -> None: + """ + Install a base relenv runtime. + """ + site_dir = pathlib.Path(sitepackages) + relenv_pth = site_dir / "relenv.pth" + with io.open(str(relenv_pth), "w") as fp: + fp.write(RELENV_PTH) + + # Lay down relenv.runtime, we'll pip install the rest later + relenv = site_dir / "relenv" + os.makedirs(relenv, exist_ok=True) + + for name in [ + "runtime.py", + "relocate.py", + "common.py", + "buildenv.py", + "__init__.py", + ]: + src = MODULE_DIR / name + dest = relenv / name + with io.open(src, "r") as rfp: + with io.open(dest, "w") as wfp: + wfp.write(rfp.read()) + + +def copy_sbom_files(dirs: Dirs) -> None: + """ + Copy SBOM files from Python source to the prefix directory. + + SBOM files (Software Bill of Materials) document the build dependencies + and source file checksums. These files are available in Python 3.12+. + + :param dirs: The working directories + :type dirs: ``relenv.build.common.Dirs`` + """ + # Find the Python source directory in dirs.sources + python_source = None + if dirs.sources.exists(): + # Look for Python-{version} directory + for entry in dirs.sources.iterdir(): + if entry.is_dir() and entry.name.startswith("Python-"): + python_source = entry + break + + if python_source: + sbom_files = ["sbom.spdx.json", "externals.spdx.json"] + source_misc_dir = python_source / "Misc" + for sbom_file in sbom_files: + source_sbom = source_misc_dir / sbom_file + if source_sbom.exists(): + dest_sbom = pathlib.Path(dirs.prefix) / sbom_file + shutil.copy2(str(source_sbom), str(dest_sbom)) + log.info("Copied %s to archive", sbom_file) + else: + log.debug("SBOM file %s not found (Python < 3.12?)", sbom_file) + + +def finalize( + env: MutableMapping[str, str], + dirs: Dirs, + logfp: IO[str], +) -> None: + """ + Run after we've fully built python. + + This method enhances the newly created python with Relenv's runtime hacks. + + :param env: The environment dictionary + :type env: dict + :param dirs: The working directories + :type dirs: ``relenv.build.common.Dirs`` + :param logfp: A handle for the log file + :type logfp: file + """ + # Run relok8 to make sure the rpaths are relocatable. + relenv.relocate.main(dirs.prefix, log_file_name=str(dirs.logs / "relocate.py.log")) + # Install relenv-sysconfigdata module + libdir = pathlib.Path(dirs.prefix) / "lib" + + def find_pythonlib(libdir: pathlib.Path) -> Optional[str]: + for _root, dirs, _files in os.walk(libdir): + for entry in dirs: + if entry.startswith("python"): + return entry + return None + + python_lib = find_pythonlib(libdir) + if python_lib is None: + raise RelenvException("Unable to locate python library directory") + + pymodules = libdir / python_lib + + # update ensurepip + update_ensurepip(pymodules) + + cwd = os.getcwd() + modname = find_sysconfigdata(pymodules) + path = sys.path + sys.path = [str(pymodules)] + try: + mod = __import__(str(modname)) + finally: + os.chdir(cwd) + sys.path = path + + dest = pymodules / f"{modname}.py" + install_sysdata(mod, dest, dirs.prefix, dirs.toolchain) + + # Lay down site customize + bindir = pathlib.Path(dirs.prefix) / "bin" + sitepackages = pymodules / "site-packages" + install_runtime(sitepackages) + + # Install pip + python_exe = str(dirs.prefix / "bin" / "python3") + if env["RELENV_HOST_ARCH"] != env["RELENV_BUILD_ARCH"]: + env["RELENV_CROSS"] = str(dirs.prefix) + python_exe = env["RELENV_NATIVE_PY"] + logfp.write("\nRUN ENSURE PIP\n") + + env.pop("RELENV_BUILDENV") + + runcmd( + [python_exe, "-m", "ensurepip"], + env=env, + stderr=logfp, + stdout=logfp, + ) + + # Fix the shebangs in the scripts python layed down. Order matters. + shebangs = [ + "#!{}".format(bindir / f"python{env['RELENV_PY_MAJOR_VERSION']}"), + "#!{}".format( + bindir / f"python{env['RELENV_PY_MAJOR_VERSION'].split('.', 1)[0]}" + ), + ] + newshebang = format_shebang("/python3") + for shebang in shebangs: + log.info("Patch shebang %r with %r", shebang, newshebang) + patch_shebangs( + str(pathlib.Path(dirs.prefix) / "bin"), + shebang, + newshebang, + ) + + if sys.platform == "linux": + pyconf = f"config-{env['RELENV_PY_MAJOR_VERSION']}-{env['RELENV_HOST']}" + patch_shebang( + str(pymodules / pyconf / "python-config.py"), + "#!{}".format(str(bindir / f"python{env['RELENV_PY_MAJOR_VERSION']}")), + format_shebang("../../../bin/python3"), + ) + + toolchain_path = dirs.toolchain + if toolchain_path is None: + raise RelenvException("Toolchain path is required for linux builds") + shutil.copy( + pathlib.Path(toolchain_path) + / env["RELENV_HOST"] + / "sysroot" + / "lib" + / "libstdc++.so.6", + libdir, + ) + + # Moved in python 3.13 or removed? + if (pymodules / "cgi.py").exists(): + patch_shebang( + str(pymodules / "cgi.py"), + "#! /usr/local/bin/python", + format_shebang("../../bin/python3"), + ) + + def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: + logfp.write(f"\nRUN PIP {pkg} {upgrade}\n") + target: Optional[pathlib.Path] = None + python_exe = str(dirs.prefix / "bin" / "python3") + if sys.platform == LINUX: + if env["RELENV_HOST_ARCH"] != env["RELENV_BUILD_ARCH"]: + target = pymodules / "site-packages" + python_exe = env["RELENV_NATIVE_PY"] + cmd = [ + python_exe, + "-m", + "pip", + "install", + str(pkg), + ] + if upgrade: + cmd.append("--upgrade") + if target: + cmd.append("--target={}".format(target)) + runcmd(cmd, env=env, stderr=logfp, stdout=logfp) + + runpip("wheel") + # This needs to handle running from the root of the git repo and also from + # an installed Relenv + if (MODULE_DIR.parent / ".git").exists(): + runpip(MODULE_DIR.parent, upgrade=True) + else: + runpip("relenv", upgrade=True) + + copy_sbom_files(dirs) + + globs = [ + "/bin/python*", + "/bin/pip*", + "/bin/relenv", + "/lib/python*/ensurepip/*", + "/lib/python*/site-packages/*", + "/include/*", + "*.so", + "/lib/*.so.*", + "*.py", + "*.spdx.json", # Include SBOM files + # Mac specific, factor this out + "*.dylib", + ] + archive = f"{ dirs.prefix }.tar.xz" + log.info("Archive is %s", archive) + with tarfile.open(archive, mode="w:xz") as fp: + create_archive(fp, dirs.prefix, globs, logfp) + + +def create_archive( + tarfp: tarfile.TarFile, + toarchive: PathLike, + globs: Sequence[str], + logfp: Optional[IO[str]] = None, +) -> None: + """ + Create an archive. + + :param tarfp: A pointer to the archive to be created + :type tarfp: file + :param toarchive: The path to the directory to archive + :type toarchive: str + :param globs: A list of filtering patterns to match against files to be added + :type globs: list + :param logfp: A pointer to the log file + :type logfp: file + """ + log.debug("Current directory %s", os.getcwd()) + log.debug("Creating archive %s", tarfp.name) + for root, _dirs, files in os.walk(toarchive): + relroot = pathlib.Path(root).relative_to(toarchive) + for f in files: + relpath = relroot / f + matches = False + for g in globs: + candidate = pathlib.Path("/") / relpath + if fnmatch.fnmatch(str(candidate), g): + matches = True + break + if matches: + log.debug("Adding %s", relpath) + tarfp.add(relpath, arcname=str(relpath), recursive=False) + else: + log.debug("Skipping %s", relpath) diff --git a/tests/test_build.py b/tests/test_build.py index 055c8e18..e113d44e 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -458,3 +458,121 @@ def test_download_destination_setter() -> None: # Set to None d.destination = None assert d.destination == pathlib.Path() + + +def test_sbom_copy_python_312(tmp_path: pathlib.Path) -> None: + """Test that SBOM files are copied from Python 3.12+ source to prefix directory.""" + import json + + from relenv.build.common import Dirs, copy_sbom_files + from relenv.common import WorkDirs + + # Create mock directory structure + root = tmp_path / "relenv_root" + src_dir = root / "src" + build_dir = root / "build" + + # Create Python source directory with SBOM files + python_src = src_dir / "Python-3.12.8" + misc_dir = python_src / "Misc" + misc_dir.mkdir(parents=True) + + # Create mock SBOM files + sbom_content = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "CPython-3.12.8", + } + externals_content = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "CPython-3.12.8-externals", + } + + (misc_dir / "sbom.spdx.json").write_text(json.dumps(sbom_content)) + (misc_dir / "externals.spdx.json").write_text(json.dumps(externals_content)) + + # Create WorkDirs and Dirs objects + work_dirs = WorkDirs(root=root) + # Override the default paths to use our mock structure + work_dirs.build = build_dir + work_dirs.src = src_dir + work_dirs.logs = root / "logs" + + dirs = Dirs(work_dirs, "test", "x86_64", "3.12.8") + + # Create prefix directory with minimal required structure + # (dirs.prefix is computed from dirs.build / version-triplet) + prefix_dir = dirs.prefix + prefix_dir.mkdir(parents=True) + bin_dir = prefix_dir / "bin" + bin_dir.mkdir() + lib_dir = prefix_dir / "lib" + lib_dir.mkdir() + + # Call copy_sbom_files (this should copy SBOM files) + copy_sbom_files(dirs) + + # Verify SBOM files were copied to prefix directory + dest_sbom = prefix_dir / "sbom.spdx.json" + dest_externals = prefix_dir / "externals.spdx.json" + + assert dest_sbom.exists(), "sbom.spdx.json should be copied to prefix directory" + assert ( + dest_externals.exists() + ), "externals.spdx.json should be copied to prefix directory" + + # Verify content is correct + assert json.loads(dest_sbom.read_text())["name"] == "CPython-3.12.8" + assert json.loads(dest_externals.read_text())["name"] == "CPython-3.12.8-externals" + + +def test_sbom_copy_python_310(tmp_path: pathlib.Path) -> None: + """Test that SBOM files are gracefully skipped for Python < 3.12.""" + from relenv.build.common import Dirs, copy_sbom_files + from relenv.common import WorkDirs + + # Create mock directory structure + root = tmp_path / "relenv_root" + src_dir = root / "src" + build_dir = root / "build" + + # Create Python source directory WITHOUT SBOM files (Python 3.10) + python_src = src_dir / "Python-3.10.18" + misc_dir = python_src / "Misc" + misc_dir.mkdir(parents=True) + + # Create some other files but NOT SBOM files + (misc_dir / "README").write_text("Python 3.10 Misc directory") + + # Create WorkDirs and Dirs objects + work_dirs = WorkDirs(root=root) + # Override the default paths to use our mock structure + work_dirs.build = build_dir + work_dirs.src = src_dir + work_dirs.logs = root / "logs" + + dirs = Dirs(work_dirs, "test", "x86_64", "3.10.18") + + # Create prefix directory with minimal required structure + # (dirs.prefix is computed from dirs.build / version-triplet) + prefix_dir = dirs.prefix + prefix_dir.mkdir(parents=True) + bin_dir = prefix_dir / "bin" + bin_dir.mkdir() + lib_dir = prefix_dir / "lib" + lib_dir.mkdir() + + # Call copy_sbom_files (this should gracefully skip SBOM files) + copy_sbom_files(dirs) + + # Verify SBOM files were NOT copied (they don't exist in source) + dest_sbom = prefix_dir / "sbom.spdx.json" + dest_externals = prefix_dir / "externals.spdx.json" + + assert not dest_sbom.exists(), "sbom.spdx.json should not exist for Python < 3.12" + assert ( + not dest_externals.exists() + ), "externals.spdx.json should not exist for Python < 3.12" From 0080295347b9373014075c907ce57d1ee636939b Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 4 Nov 2025 04:10:15 -0700 Subject: [PATCH 5/7] Include python's sbom files in our builds when >3.12 --- relenv/build/common/__init__.py | 2 ++ relenv/build/common/install.py | 36 +++++++++++++++++++++++++++++++++ relenv/build/darwin.py | 3 +++ relenv/build/linux.py | 2 ++ 4 files changed, 43 insertions(+) diff --git a/relenv/build/common/__init__.py b/relenv/build/common/__init__.py index 0c66f912..0a07a3fd 100644 --- a/relenv/build/common/__init__.py +++ b/relenv/build/common/__init__.py @@ -21,6 +21,7 @@ create_archive, patch_file, update_sbom_checksums, + copy_sbom_files, ) from .builder import ( @@ -43,6 +44,7 @@ "update_ensurepip", "patch_file", "update_sbom_checksums", + "copy_sbom_files", # Builders (specific build functions) "build_openssl", "build_openssl_fips", diff --git a/relenv/build/common/install.py b/relenv/build/common/install.py index 4ab30d37..bebe361a 100644 --- a/relenv/build/common/install.py +++ b/relenv/build/common/install.py @@ -409,6 +409,38 @@ def install_runtime(sitepackages: PathLike) -> None: wfp.write(rfp.read()) +def copy_sbom_files(dirs: Dirs) -> None: + """ + Copy SBOM files from Python source to the prefix directory. + + SBOM files (Software Bill of Materials) document the build dependencies + and source file checksums. These files are available in Python 3.12+. + + :param dirs: The working directories + :type dirs: ``relenv.build.common.Dirs`` + """ + # Find the Python source directory in dirs.sources + python_source = None + if dirs.sources.exists(): + # Look for Python-{version} directory + for entry in dirs.sources.iterdir(): + if entry.is_dir() and entry.name.startswith("Python-"): + python_source = entry + break + + if python_source: + sbom_files = ["sbom.spdx.json", "externals.spdx.json"] + source_misc_dir = python_source / "Misc" + for sbom_file in sbom_files: + source_sbom = source_misc_dir / sbom_file + if source_sbom.exists(): + dest_sbom = pathlib.Path(dirs.prefix) / sbom_file + shutil.copy2(str(source_sbom), str(dest_sbom)) + log.info("Copied %s to archive", sbom_file) + else: + log.debug("SBOM file %s not found (Python < 3.12?)", sbom_file) + + def finalize( env: MutableMapping[str, str], dirs: Dirs, @@ -553,6 +585,9 @@ def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: runpip(MODULE_DIR.parent, upgrade=True) else: runpip("relenv", upgrade=True) + + copy_sbom_files(dirs) + globs = [ "/bin/python*", "/bin/pip*", @@ -563,6 +598,7 @@ def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: "*.so", "/lib/*.so.*", "*.py", + "*.spdx.json", # Include SBOM files # Mac specific, factor this out "*.dylib", ] diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index ee215723..2bf0c019 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -8,9 +8,11 @@ import glob import io +import os import pathlib import shutil import tarfile +import time import urllib.request from typing import IO, MutableMapping @@ -22,6 +24,7 @@ builds, finalize, get_dependency_version, + update_sbom_checksums, ) ARCHES = arches[DARWIN] diff --git a/relenv/build/linux.py b/relenv/build/linux.py index b192eb08..5a0024c5 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -13,6 +13,7 @@ import shutil import tarfile import tempfile +import time import urllib.request from typing import IO, MutableMapping @@ -24,6 +25,7 @@ builds, finalize, get_dependency_version, + update_sbom_checksums, ) from ..common import LINUX, Version, arches, runcmd From 63cae3d6a2e625e79ea7dabb902551a303c68875 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 4 Nov 2025 14:33:43 -0700 Subject: [PATCH 6/7] Initial sbom support --- relenv/__main__.py | 3 +- relenv/build/common.py | 1901 ------------------------------- relenv/build/common/__init__.py | 4 +- relenv/build/common/install.py | 283 ++++- relenv/sbom.py | 245 ++++ requirements/tests.txt | 1 + tests/test_build.py | 118 -- tests/test_sbom.py | 339 ++++++ tests/test_verify_build.py | 194 ++++ 9 files changed, 1040 insertions(+), 2048 deletions(-) delete mode 100644 relenv/build/common.py create mode 100644 relenv/sbom.py create mode 100644 tests/test_sbom.py diff --git a/relenv/__main__.py b/relenv/__main__.py index 4869370f..fa577c01 100644 --- a/relenv/__main__.py +++ b/relenv/__main__.py @@ -10,7 +10,7 @@ from argparse import ArgumentParser from types import ModuleType -from . import build, buildenv, check, create, fetch, pyversions, toolchain +from . import build, buildenv, check, create, fetch, pyversions, sbom, toolchain from .common import __version__ @@ -41,6 +41,7 @@ def setup_cli() -> ArgumentParser: check, buildenv, pyversions, + sbom, ] for mod in modules_to_setup: mod.setup_parser(subparsers) diff --git a/relenv/build/common.py b/relenv/build/common.py deleted file mode 100644 index 89483ac8..00000000 --- a/relenv/build/common.py +++ /dev/null @@ -1,1901 +0,0 @@ -# Copyright 2022-2025 Broadcom. -# SPDX-License-Identifier: Apache-2.0 -""" -Build process common methods. -""" -from __future__ import annotations - -import fnmatch -import hashlib -import io -import logging -import multiprocessing -import os -import os.path -import pathlib -import pprint -import random -import re -import shutil -import subprocess -import sys -import tempfile -import time -import tarfile -from types import ModuleType -from typing import ( - Any, - Callable, - Dict, - IO, - List, - MutableMapping, - Optional, - Sequence, - Tuple, - Union, - cast, -) - -from typing import TYPE_CHECKING, TypedDict - -if TYPE_CHECKING: - from multiprocessing.synchronize import Event as SyncEvent -else: - SyncEvent = Any - -from relenv.common import ( - DATA_DIR, - LINUX, - MODULE_DIR, - RelenvException, - build_arch, - download_url, - extract_archive, - format_shebang, - get_download_location, - get_toolchain, - get_triplet, - runcmd, - work_dirs, - Version, - WorkDirs, -) -import relenv.relocate - - -PathLike = Union[str, os.PathLike[str]] - -log = logging.getLogger(__name__) - - -GREEN = "\033[0;32m" -YELLOW = "\033[1;33m" -RED = "\033[0;31m" -END = "\033[0m" -MOVEUP = "\033[F" - - -CICD = "CI" in os.environ -NODOWLOAD = False - - -RELENV_PTH = ( - "import os; " - "import sys; " - "from importlib import util; " - "from pathlib import Path; " - "spec = util.spec_from_file_location(" - "'relenv.runtime', str(Path(__file__).parent / 'site-packages' / 'relenv' / 'runtime.py')" - "); " - "mod = util.module_from_spec(spec); " - "sys.modules['relenv.runtime'] = mod; " - "spec.loader.exec_module(mod); mod.bootstrap();" -) - - -SYSCONFIGDATA = """ -import pathlib, sys, platform, os, logging - -log = logging.getLogger(__name__) - -def build_arch(): - machine = platform.machine() - return machine.lower() - -def get_triplet(machine=None, plat=None): - if not plat: - plat = sys.platform - if not machine: - machine = build_arch() - if plat == "darwin": - return f"{machine}-macos" - elif plat == "win32": - return f"{machine}-win" - elif plat == "linux": - return f"{machine}-linux-gnu" - else: - raise RelenvException("Unknown platform {}".format(platform)) - - - -pydir = pathlib.Path(__file__).resolve().parent -if sys.platform == "win32": - DEFAULT_DATA_DIR = pathlib.Path.home() / "AppData" / "Local" / "relenv" -else: - DEFAULT_DATA_DIR = pathlib.Path.home() / ".local" / "relenv" - -if "RELENV_DATA" in os.environ: - DATA_DIR = pathlib.Path(os.environ["RELENV_DATA"]).resolve() -else: - DATA_DIR = DEFAULT_DATA_DIR - -buildroot = pydir.parent.parent - -toolchain = DATA_DIR / "toolchain" / get_triplet() - -build_time_vars = {} -for key in _build_time_vars: - val = _build_time_vars[key] - orig = val - if isinstance(val, str): - val = val.format( - BUILDROOT=buildroot, - TOOLCHAIN=toolchain, - ) - build_time_vars[key] = val -""" - - -def print_ui( - events: MutableMapping[str, "multiprocessing.synchronize.Event"], - processes: MutableMapping[str, multiprocessing.Process], - fails: Sequence[str], - flipstat: Optional[Dict[str, Tuple[int, float]]] = None, -) -> None: - """ - Prints the UI during the relenv building process. - - :param events: A dictionary of events that are updated during the build process - :type events: dict - :param processes: A dictionary of build processes - :type processes: dict - :param fails: A list of processes that have failed - :type fails: list - :param flipstat: A dictionary of process statuses, defaults to {} - :type flipstat: dict, optional - """ - if flipstat is None: - flipstat = {} - if CICD: - sys.stdout.flush() - return - uiline = [] - for name in events: - if not events[name].is_set(): - status = " {}.".format(YELLOW) - elif name in processes: - now = time.time() - if name not in flipstat: - flipstat[name] = (0, now) - if flipstat[name][1] < now: - flipstat[name] = (1 - flipstat[name][0], now + random.random()) - status = " {}{}".format(GREEN, " " if flipstat[name][0] == 1 else ".") - elif name in fails: - status = " {}\u2718".format(RED) - else: - status = " {}\u2718".format(GREEN) - uiline.append(status) - uiline.append(" " + END) - sys.stdout.write("\r") - sys.stdout.write("".join(uiline)) - sys.stdout.flush() - - -def verify_checksum(file: PathLike, checksum: Optional[str]) -> bool: - """ - Verify the checksum of a file. - - Supports both SHA-1 (40 hex chars) and SHA-256 (64 hex chars) checksums. - The hash algorithm is auto-detected based on checksum length. - - :param file: The path to the file to check. - :type file: str - :param checksum: The checksum to verify against (SHA-1 or SHA-256) - :type checksum: str - - :raises RelenvException: If the checksum verification failed - - :return: True if it succeeded, or False if the checksum was None - :rtype: bool - """ - if checksum is None: - log.error("Can't verify checksum because none was given") - return False - - # Auto-detect hash type based on length - # SHA-1: 40 hex chars, SHA-256: 64 hex chars - if len(checksum) == 64: - hash_algo = hashlib.sha256() - hash_name = "sha256" - elif len(checksum) == 40: - hash_algo = hashlib.sha1() - hash_name = "sha1" - else: - raise RelenvException( - f"Invalid checksum length {len(checksum)}. Expected 40 (SHA-1) or 64 (SHA-256)" - ) - - with open(file, "rb") as fp: - hash_algo.update(fp.read()) - file_checksum = hash_algo.hexdigest() - if checksum != file_checksum: - raise RelenvException( - f"{hash_name} checksum verification failed. expected={checksum} found={file_checksum}" - ) - return True - - -def compute_checksums(file: PathLike) -> Dict[str, str]: - """ - Compute SHA-1 and SHA-256 checksums for a file. - - Returns a dictionary with 'sha1' and 'sha256' keys. - - :param file: The path to the file to checksum - :type file: str or PathLike - - :return: Dictionary with 'sha1' and 'sha256' checksums - :rtype: dict - """ - sha1 = hashlib.sha1() - sha256 = hashlib.sha256() - - with open(file, "rb") as fp: - data = fp.read() - sha1.update(data) - sha256.update(data) - - return { - "sha1": sha1.hexdigest(), - "sha256": sha256.hexdigest(), - } - - -def update_sbom_checksums( - source_dir: PathLike, files_to_update: Dict[str, PathLike] -) -> None: - """ - Update checksums in Python's SBOM file for modified source files. - - This function updates Misc/sbom.spdx.json with correct checksums for - files that have been replaced (e.g., updated expat library files). - - :param source_dir: The Python source directory containing Misc/sbom.spdx.json - :type source_dir: PathLike - :param files_to_update: Dict mapping SBOM file names to actual file paths - :type files_to_update: dict - """ - import json - import pathlib - - spdx_json = pathlib.Path(source_dir) / "Misc" / "sbom.spdx.json" - if not spdx_json.exists(): - # Python < 3.12 doesn't have SBOM files - return - - with open(str(spdx_json), "r") as f: - data = json.load(f) - - # Update checksums for each file in the SBOM - for file_entry in data.get("files", []): - file_name = file_entry.get("fileName", "") - if file_name in files_to_update: - file_path = files_to_update[file_name] - checksums = compute_checksums(file_path) - file_entry["checksums"] = [ - {"algorithm": "SHA1", "checksumValue": checksums["sha1"]}, - {"algorithm": "SHA256", "checksumValue": checksums["sha256"]}, - ] - - with open(str(spdx_json), "w") as f: - json.dump(data, f, indent=2) - - -def all_dirs(root: PathLike, recurse: bool = True) -> List[str]: - """ - Get all directories under and including the given root. - - :param root: The root directory to traverse - :type root: str - :param recurse: Whether to recursively search for directories, defaults to True - :type recurse: bool, optional - - :return: A list of directories found - :rtype: list - """ - root_str = os.fspath(root) - paths: List[str] = [root_str] - for current_root, dirs, _files in os.walk(root_str): - if not recurse and current_root != root_str: - continue - for name in dirs: - paths.append(os.path.join(current_root, name)) - return paths - - -def populate_env(env: MutableMapping[str, str], dirs: "Dirs") -> None: - """Populate environment variables for a build step. - - This default implementation intentionally does nothing; specific steps may - provide their own implementation via the ``populate_env`` hook. - """ - _ = env - _ = dirs - - -def build_default(env: MutableMapping[str, str], dirs: "Dirs", logfp: IO[str]) -> None: - """ - The default build function if none is given during the build process. - - :param env: The environment dictionary - :type env: dict - :param dirs: The working directories - :type dirs: ``relenv.build.common.Dirs`` - :param logfp: A handle for the log file - :type logfp: file - """ - cmd = [ - "./configure", - "--prefix={}".format(dirs.prefix), - ] - if env["RELENV_HOST"].find("linux") > -1: - cmd += [ - "--build={}".format(env["RELENV_BUILD"]), - "--host={}".format(env["RELENV_HOST"]), - ] - runcmd(cmd, env=env, stderr=logfp, stdout=logfp) - runcmd(["make", "-j8"], env=env, stderr=logfp, stdout=logfp) - runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) - - -def build_openssl_fips( - env: MutableMapping[str, str], dirs: "Dirs", logfp: IO[str] -) -> None: - return build_openssl(env, dirs, logfp, fips=True) - - -def build_openssl( - env: MutableMapping[str, str], - dirs: "Dirs", - logfp: IO[str], - fips: bool = False, -) -> None: - """ - Build openssl. - - :param env: The environment dictionary - :type env: dict - :param dirs: The working directories - :type dirs: ``relenv.build.common.Dirs`` - :param logfp: A handle for the log file - :type logfp: file - """ - arch = "aarch64" - if sys.platform == "darwin": - plat = "darwin64" - if env["RELENV_HOST_ARCH"] == "x86_64": - arch = "x86_64-cc" - elif env["RELENV_HOST_ARCH"] == "arm64": - arch = "arm64-cc" - else: - raise RelenvException(f"Unable to build {env['RELENV_HOST_ARCH']}") - extended_cmd = [] - else: - plat = "linux" - if env["RELENV_HOST_ARCH"] == "x86_64": - arch = "x86_64" - elif env["RELENV_HOST_ARCH"] == "aarch64": - arch = "aarch64" - else: - raise RelenvException(f"Unable to build {env['RELENV_HOST_ARCH']}") - extended_cmd = [ - "-Wl,-z,noexecstack", - ] - if fips: - extended_cmd.append("enable-fips") - cmd = [ - "./Configure", - f"{plat}-{arch}", - f"--prefix={dirs.prefix}", - "--openssldir=/etc/ssl", - "--libdir=lib", - "--api=1.1.1", - "--shared", - "--with-rand-seed=os,egd", - "enable-md2", - "enable-egd", - "no-idea", - ] - cmd.extend(extended_cmd) - runcmd( - cmd, - env=env, - stderr=logfp, - stdout=logfp, - ) - runcmd(["make", "-j8"], env=env, stderr=logfp, stdout=logfp) - if fips: - shutil.copy( - pathlib.Path("providers") / "fips.so", - pathlib.Path(dirs.prefix) / "lib" / "ossl-modules", - ) - else: - runcmd(["make", "install_sw"], env=env, stderr=logfp, stdout=logfp) - - -def build_sqlite(env: MutableMapping[str, str], dirs: "Dirs", logfp: IO[str]) -> None: - """ - Build sqlite. - - :param env: The environment dictionary - :type env: dict - :param dirs: The working directories - :type dirs: ``relenv.build.common.Dirs`` - :param logfp: A handle for the log file - :type logfp: file - """ - # extra_cflags=('-Os ' - # '-DSQLITE_ENABLE_FTS5 ' - # '-DSQLITE_ENABLE_FTS4 ' - # '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' - # '-DSQLITE_ENABLE_JSON1 ' - # '-DSQLITE_ENABLE_RTREE ' - # '-DSQLITE_TCL=0 ' - # ) - # configure_pre=[ - # '--enable-threadsafe', - # '--enable-shared=no', - # '--enable-static=yes', - # '--disable-readline', - # '--disable-dependency-tracking', - # ] - cmd = [ - "./configure", - # "--with-shared", - # "--without-static", - "--enable-threadsafe", - "--disable-readline", - "--disable-dependency-tracking", - "--prefix={}".format(dirs.prefix), - # "--enable-add-ons=nptl,ports", - ] - if env["RELENV_HOST"].find("linux") > -1: - cmd += [ - "--build={}".format(env["RELENV_BUILD_ARCH"]), - "--host={}".format(env["RELENV_HOST"]), - ] - runcmd(cmd, env=env, stderr=logfp, stdout=logfp) - runcmd(["make", "-j8"], env=env, stderr=logfp, stdout=logfp) - runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) - - -def update_ensurepip(directory: pathlib.Path) -> None: - """ - Update bundled dependencies for ensurepip (pip & setuptools). - """ - # ensurepip bundle location - bundle_dir = directory / "ensurepip" / "_bundled" - - # Make sure the destination directory exists - bundle_dir.mkdir(parents=True, exist_ok=True) - - # Detect existing whl. Later versions of python don't include setuptools. We - # only want to update whl files that python expects to be there - pip_version = "25.2" - setuptools_version = "80.9.0" - update_pip = False - update_setuptools = False - for file in bundle_dir.glob("*.whl"): - - log.debug("Checking whl: %s", str(file)) - if file.name.startswith("pip-"): - found_version = file.name.split("-")[1] - log.debug("Found version %s", found_version) - if Version(found_version) >= Version(pip_version): - log.debug("Found correct pip version or newer: %s", found_version) - else: - file.unlink() - update_pip = True - if file.name.startswith("setuptools-"): - found_version = file.name.split("-")[1] - log.debug("Found version %s", found_version) - if Version(found_version) >= Version(setuptools_version): - log.debug( - "Found correct setuptools version or newer: %s", found_version - ) - else: - file.unlink() - update_setuptools = True - - # Download whl files and update __init__.py - init_file = directory / "ensurepip" / "__init__.py" - if update_pip: - whl = f"pip-{pip_version}-py3-none-any.whl" - whl_path = "b7/3f/945ef7ab14dc4f9d7f40288d2df998d1837ee0888ec3659c813487572faa" - url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" - download_url(url=url, dest=bundle_dir) - assert (bundle_dir / whl).exists() - - # Update __init__.py - old = "^_PIP_VERSION.*" - new = f'_PIP_VERSION = "{pip_version}"' - patch_file(path=init_file, old=old, new=new) - - # setuptools - if update_setuptools: - whl = f"setuptools-{setuptools_version}-py3-none-any.whl" - whl_path = "a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772" - url = f"https://files.pythonhosted.org/packages/{whl_path}/{whl}" - download_url(url=url, dest=bundle_dir) - assert (bundle_dir / whl).exists() - - # setuptools - old = "^_SETUPTOOLS_VERSION.*" - new = f'_SETUPTOOLS_VERSION = "{setuptools_version}"' - patch_file(path=init_file, old=old, new=new) - - log.debug("ensurepip __init__.py contents:") - log.debug(init_file.read_text()) - - -def patch_file(path: PathLike, old: str, new: str) -> None: - """ - Search a file line by line for a string to replace. - - :param path: Location of the file to search - :type path: str - :param old: The value that will be replaced - :type path: str - :param new: The value that will replace the 'old' value. - :type path: str - """ - log.debug("Patching file: %s", path) - with open(path, "r") as fp: - content = fp.read() - new_content = "" - for line in content.splitlines(): - line = re.sub(old, new, line) - new_content += line + "\n" - with open(path, "w") as fp: - fp.write(new_content) - - -def get_dependency_version(name: str, platform: str) -> Optional[Dict[str, str]]: - """ - Get dependency version and metadata from python-versions.json. - - Returns dict with keys: version, url, sha256, and any extra fields (e.g., sqliteversion) - Returns None if dependency not found. - - :param name: Dependency name (openssl, sqlite, xz) - :param platform: Platform name (linux, darwin, win32) - :return: Dict with version, url, sha256, and extra fields, or None - """ - versions_file = MODULE_DIR / "python-versions.json" - if not versions_file.exists(): - return None - - import json - - data = json.loads(versions_file.read_text()) - dependencies = data.get("dependencies", {}) - - if name not in dependencies: - return None - - # Get the latest version for this dependency that supports the platform - dep_versions = dependencies[name] - for version, info in sorted( - dep_versions.items(), - key=lambda x: [int(n) for n in x[0].split(".")], - reverse=True, - ): - if platform in info.get("platforms", []): - # Build result dict with version, url, sha256, and any extra fields - result = { - "version": version, - "url": info["url"], - "sha256": info.get("sha256", ""), - } - # Add any extra fields (like sqliteversion for SQLite) - for key, value in info.items(): - if key not in ["url", "sha256", "platforms"]: - result[key] = value - return result - - return None - - -class Download: - """ - A utility that holds information about content to be downloaded. - - :param name: The name of the download - :type name: str - :param url: The url of the download - :type url: str - :param signature: The signature of the download, defaults to None - :type signature: str - :param destination: The path to download the file to - :type destination: str - :param version: The version of the content to download - :type version: str - :param sha1: The sha1 sum of the download - :type sha1: str - - """ - - def __init__( - self, - name: str, - url: str, - fallback_url: Optional[str] = None, - signature: Optional[str] = None, - destination: PathLike = "", - version: str = "", - checksum: Optional[str] = None, - ) -> None: - self.name = name - self.url_tpl = url - self.fallback_url_tpl = fallback_url - self.signature_tpl = signature - self._destination: pathlib.Path = pathlib.Path() - if destination: - self._destination = pathlib.Path(destination) - self.version = version - self.checksum = checksum - - def copy(self) -> "Download": - return Download( - self.name, - self.url_tpl, - self.fallback_url_tpl, - self.signature_tpl, - self.destination, - self.version, - self.checksum, - ) - - @property - def destination(self) -> pathlib.Path: - return self._destination - - @destination.setter - def destination(self, value: Optional[PathLike]) -> None: - if value: - self._destination = pathlib.Path(value) - else: - self._destination = pathlib.Path() - - @property - def url(self) -> str: - return self.url_tpl.format(version=self.version) - - @property - def fallback_url(self) -> Optional[str]: - if self.fallback_url_tpl: - return self.fallback_url_tpl.format(version=self.version) - return None - - @property - def signature_url(self) -> str: - if self.signature_tpl is None: - raise RelenvException("Signature template not configured") - return self.signature_tpl.format(version=self.version) - - @property - def filepath(self) -> pathlib.Path: - _, name = self.url.rsplit("/", 1) - return self.destination / name - - @property - def formatted_url(self) -> str: - return self.url_tpl.format(version=self.version) - - def fetch_file(self) -> Tuple[str, bool]: - """ - Download the file. - - :return: The path to the downloaded content, and whether it was downloaded. - :rtype: tuple(str, bool) - """ - try: - return download_url(self.url, self.destination, CICD), True - except Exception as exc: - fallback = self.fallback_url - if fallback: - print(f"Download failed {self.url} ({exc}); trying fallback url") - return download_url(fallback, self.destination, CICD), True - raise - - def fetch_signature(self, version: Optional[str] = None) -> Tuple[str, bool]: - """ - Download the file signature. - - :return: The path to the downloaded signature. - :rtype: str - """ - return download_url(self.signature_url, self.destination, CICD), True - - def exists(self) -> bool: - """ - True when the artifact already exists on disk. - - :return: True when the artifact already exists on disk - :rtype: bool - """ - return self.filepath.exists() - - def valid_hash(self) -> None: - pass - - @staticmethod - def validate_signature(archive: PathLike, signature: Optional[PathLike]) -> bool: - """ - True when the archive's signature is valid. - - :param archive: The path to the archive to validate - :type archive: str - :param signature: The path to the signature to validate against - :type signature: str - - :return: True if it validated properly, else False - :rtype: bool - """ - if signature is None: - log.error("Can't check signature because none was given") - return False - try: - runcmd( - ["gpg", "--verify", signature, archive], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - return True - except RelenvException as exc: - log.error("Signature validation failed on %s: %s", archive, exc) - return False - - @staticmethod - def validate_checksum(archive: PathLike, checksum: Optional[str]) -> bool: - """ - True when when the archive matches the sha1 hash. - - :param archive: The path to the archive to validate - :type archive: str - :param checksum: The sha1 sum to validate against - :type checksum: str - :return: True if the sums matched, else False - :rtype: bool - """ - try: - verify_checksum(archive, checksum) - return True - except RelenvException as exc: - log.error("sha1 validation failed on %s: %s", archive, exc) - return False - - def __call__( - self, - force_download: bool = False, - show_ui: bool = False, - exit_on_failure: bool = False, - ) -> bool: - """ - Downloads the url and validates the signature and sha1 sum. - - :return: Whether or not validation succeeded - :rtype: bool - """ - os.makedirs(self.filepath.parent, exist_ok=True) - - downloaded = False - if force_download: - _, downloaded = self.fetch_file() - else: - file_is_valid = False - dest = get_download_location(self.url, self.destination) - if self.checksum and os.path.exists(dest): - file_is_valid = self.validate_checksum(dest, self.checksum) - if file_is_valid: - log.debug("%s already downloaded, skipping.", self.url) - else: - _, downloaded = self.fetch_file() - valid = True - if downloaded: - if self.signature_tpl is not None: - sig, _ = self.fetch_signature() - valid_sig = self.validate_signature(self.filepath, sig) - valid = valid and valid_sig - if self.checksum is not None: - valid_checksum = self.validate_checksum(self.filepath, self.checksum) - valid = valid and valid_checksum - - if not valid: - log.warning("Checksum did not match %s: %s", self.name, self.checksum) - if show_ui: - sys.stderr.write( - f"\nChecksum did not match {self.name}: {self.checksum}\n" - ) - sys.stderr.flush() - if exit_on_failure and not valid: - sys.exit(1) - return valid - - -class Recipe(TypedDict): - """Typed description of a build recipe entry.""" - - build_func: Callable[[MutableMapping[str, str], "Dirs", IO[str]], None] - wait_on: List[str] - download: Optional[Download] - - -class Dirs: - """ - A container for directories during build time. - - :param dirs: A collection of working directories - :type dirs: ``relenv.common.WorkDirs`` - :param name: The name of this collection - :type name: str - :param arch: The architecture being worked with - :type arch: str - """ - - def __init__(self, dirs: WorkDirs, name: str, arch: str, version: str) -> None: - # XXX name is the specific to a step where as everything - # else here is generalized to the entire build - self.name = name - self.version = version - self.arch = arch - self.root = dirs.root - self.build = dirs.build - self.downloads = dirs.download - self.logs = dirs.logs - self.sources = dirs.src - self.tmpbuild = tempfile.mkdtemp(prefix="{}_build".format(name)) - self.source: Optional[pathlib.Path] = None - - @property - def toolchain(self) -> Optional[pathlib.Path]: - if sys.platform == "darwin": - return get_toolchain(root=self.root) - elif sys.platform == "win32": - return get_toolchain(root=self.root) - else: - return get_toolchain(self.arch, self.root) - - @property - def _triplet(self) -> str: - if sys.platform == "darwin": - return "{}-macos".format(self.arch) - elif sys.platform == "win32": - return "{}-win".format(self.arch) - else: - return "{}-linux-gnu".format(self.arch) - - @property - def prefix(self) -> pathlib.Path: - return self.build / f"{self.version}-{self._triplet}" - - def __getstate__(self) -> Dict[str, Any]: - """ - Return an object used for pickling. - - :return: The picklable state - """ - return { - "name": self.name, - "arch": self.arch, - "root": self.root, - "build": self.build, - "downloads": self.downloads, - "logs": self.logs, - "sources": self.sources, - "tmpbuild": self.tmpbuild, - } - - def __setstate__(self, state: Dict[str, Any]) -> None: - """ - Unwrap the object returned from unpickling. - - :param state: The state to unpickle - :type state: dict - """ - self.name = state["name"] - self.arch = state["arch"] - self.root = state["root"] - self.downloads = state["downloads"] - self.logs = state["logs"] - self.sources = state["sources"] - self.build = state["build"] - self.tmpbuild = state["tmpbuild"] - - def to_dict(self) -> Dict[str, Any]: - """ - Get a dictionary representation of the directories in this collection. - - :return: A dictionary of all the directories - :rtype: dict - """ - return { - x: getattr(self, x) - for x in [ - "root", - "prefix", - "downloads", - "logs", - "sources", - "build", - "toolchain", - ] - } - - -class Builder: - """ - Utility that handles the build process. - - :param root: The root of the working directories for this build - :type root: str - :param recipies: The instructions for the build steps - :type recipes: list - :param build_default: The default build function, defaults to ``build_default`` - :type build_default: types.FunctionType - :param populate_env: The default function to populate the build environment, defaults to ``populate_env`` - :type populate_env: types.FunctionType - :param force_download: If True, forces downloading the archives even if they exist, defaults to False - :type force_download: bool - :param arch: The architecture being built - :type arch: str - """ - - def __init__( - self, - root: Optional[PathLike] = None, - recipies: Optional[Dict[str, Recipe]] = None, - build_default: Callable[ - [MutableMapping[str, str], "Dirs", IO[str]], None - ] = build_default, - populate_env: Callable[[MutableMapping[str, str], "Dirs"], None] = populate_env, - arch: str = "x86_64", - version: str = "", - ) -> None: - self.root = root - self.dirs: WorkDirs = work_dirs(root) - self.build_arch = build_arch() - self.build_triplet = get_triplet(self.build_arch) - self.arch = arch - self.sources = self.dirs.src - self.downloads = self.dirs.download - - if recipies is None: - self.recipies: Dict[str, Recipe] = {} - else: - self.recipies = recipies - - self.build_default = build_default - self.populate_env = populate_env - self.version = version - self.set_arch(self.arch) - - def copy(self, version: str, checksum: Optional[str]) -> "Builder": - recipies: Dict[str, Recipe] = {} - for name in self.recipies: - recipe = self.recipies[name] - recipies[name] = { - "build_func": recipe["build_func"], - "wait_on": list(recipe["wait_on"]), - "download": recipe["download"].copy() if recipe["download"] else None, - } - build = Builder( - self.root, - recipies, - self.build_default, - self.populate_env, - self.arch, - version, - ) - python_download = build.recipies["python"].get("download") - if python_download is None: - raise RelenvException("Python recipe is missing a download entry") - python_download.version = version - python_download.checksum = checksum - return build - - def set_arch(self, arch: str) -> None: - """ - Set the architecture for the build. - - :param arch: The arch to build - :type arch: str - """ - self.arch = arch - self._toolchain: Optional[pathlib.Path] = None - - @property - def toolchain(self) -> Optional[pathlib.Path]: - """Lazily fetch toolchain only when needed.""" - if self._toolchain is None and sys.platform == "linux": - self._toolchain = get_toolchain(self.arch, self.dirs.root) - return self._toolchain - - @property - def triplet(self) -> str: - return get_triplet(self.arch) - - @property - def prefix(self) -> pathlib.Path: - return self.dirs.build / f"{self.version}-{self.triplet}" - - @property - def _triplet(self) -> str: - if sys.platform == "darwin": - return "{}-macos".format(self.arch) - elif sys.platform == "win32": - return "{}-win".format(self.arch) - else: - return "{}-linux-gnu".format(self.arch) - - def add( - self, - name: str, - build_func: Optional[Callable[..., Any]] = None, - wait_on: Optional[Sequence[str]] = None, - download: Optional[Dict[str, Any]] = None, - ) -> None: - """ - Add a step to the build process. - - :param name: The name of the step - :type name: str - :param build_func: The function that builds this step, defaults to None - :type build_func: types.FunctionType, optional - :param wait_on: Processes to wait on before running this step, defaults to None - :type wait_on: list, optional - :param download: A dictionary of download information, defaults to None - :type download: dict, optional - """ - if wait_on is None: - wait_on_list: List[str] = [] - else: - wait_on_list = list(wait_on) - if build_func is None: - build_func = self.build_default - download_obj: Optional[Download] = None - if download is not None: - download_obj = Download(name, destination=self.downloads, **download) - self.recipies[name] = { - "build_func": build_func, - "wait_on": wait_on_list, - "download": download_obj, - } - - def run( - self, - name: str, - event: "multiprocessing.synchronize.Event", - build_func: Callable[..., Any], - download: Optional[Download], - show_ui: bool = False, - log_level: str = "WARNING", - ) -> Any: - """ - Run a build step. - - :param name: The name of the step to run - :type name: str - :param event: An event to track this process' status and alert waiting steps - :type event: ``multiprocessing.Event`` - :param build_func: The function to use to build this step - :type build_func: types.FunctionType - :param download: The ``Download`` instance for this step - :type download: ``Download`` - - :return: The output of the build function - """ - root_log = logging.getLogger(None) - if sys.platform == "win32": - if not show_ui: - handler = logging.StreamHandler() - handler.setLevel(logging.getLevelName(log_level)) - root_log.addHandler(handler) - - for handler in root_log.handlers: - if isinstance(handler, logging.StreamHandler): - handler.setFormatter( - logging.Formatter(f"%(asctime)s {name} %(message)s") - ) - - if not self.dirs.build.exists(): - os.makedirs(self.dirs.build, exist_ok=True) - - dirs = Dirs(self.dirs, name, self.arch, self.version) - os.makedirs(dirs.sources, exist_ok=True) - os.makedirs(dirs.logs, exist_ok=True) - os.makedirs(dirs.prefix, exist_ok=True) - - while event.is_set() is False: - time.sleep(0.3) - - logfp = io.open(os.path.join(dirs.logs, "{}.log".format(name)), "w") - handler = logging.FileHandler(dirs.logs / f"{name}.log") - root_log.addHandler(handler) - root_log.setLevel(logging.NOTSET) - - # DEBUG: Uncomment to debug - # logfp = sys.stdout - - cwd = os.getcwd() - if download: - extract_archive(dirs.sources, str(download.filepath)) - dirs.source = dirs.sources / download.filepath.name.split(".tar")[0] - os.chdir(dirs.source) - else: - os.chdir(dirs.prefix) - - if sys.platform == "win32": - env = os.environ.copy() - else: - env = { - "PATH": os.environ["PATH"], - } - env["RELENV_DEBUG"] = "1" - env["RELENV_BUILDENV"] = "1" - env["RELENV_HOST"] = self.triplet - env["RELENV_HOST_ARCH"] = self.arch - env["RELENV_BUILD"] = self.build_triplet - env["RELENV_BUILD_ARCH"] = self.build_arch - python_download = self.recipies["python"].get("download") - if python_download is None: - raise RelenvException("Python recipe is missing download configuration") - env["RELENV_PY_VERSION"] = python_download.version - env["RELENV_PY_MAJOR_VERSION"] = env["RELENV_PY_VERSION"].rsplit(".", 1)[0] - if "RELENV_DATA" in os.environ: - env["RELENV_DATA"] = os.environ["RELENV_DATA"] - if self.build_arch != self.arch: - native_root = DATA_DIR / "native" - env["RELENV_NATIVE_PY"] = str(native_root / "bin" / "python3") - - self.populate_env(env, dirs) - - _ = dirs.to_dict() - for k in _: - log.info("Directory %s %s", k, _[k]) - for k in env: - log.info("Environment %s %s", k, env[k]) - try: - return build_func(env, dirs, logfp) - except Exception: - log.exception("Build failure") - sys.exit(1) - finally: - os.chdir(cwd) - log.removeHandler(handler) - logfp.close() - - def cleanup(self) -> None: - """ - Clean up the build directories. - """ - shutil.rmtree(self.prefix) - - def clean(self) -> None: - """ - Completely clean up the remnants of a relenv build. - """ - # Clean directories - for _ in [self.prefix, self.sources]: - try: - shutil.rmtree(_) - except PermissionError: - sys.stderr.write(f"Unable to remove directory: {_}") - except FileNotFoundError: - pass - # Clean files - archive = f"{self.prefix}.tar.xz" - for _ in [archive]: - try: - os.remove(_) - except FileNotFoundError: - pass - - def download_files( - self, - steps: Optional[Sequence[str]] = None, - force_download: bool = False, - show_ui: bool = False, - ) -> None: - """ - Download all of the needed archives. - - :param steps: The steps to download archives for, defaults to None - :type steps: list, optional - """ - step_names = list(steps) if steps is not None else list(self.recipies) - - fails: List[str] = [] - processes: Dict[str, multiprocessing.Process] = {} - events: Dict[str, SyncEvent] = {} - if show_ui: - sys.stdout.write("Starting downloads \n") - log.info("Starting downloads") - if show_ui: - print_ui(events, processes, fails) - for name in step_names: - download = self.recipies[name]["download"] - if download is None: - continue - event = multiprocessing.Event() - event.set() - events[name] = event - proc = multiprocessing.Process( - name=name, - target=download, - kwargs={ - "force_download": force_download, - "show_ui": show_ui, - "exit_on_failure": True, - }, - ) - proc.start() - processes[name] = proc - - while processes: - for proc in list(processes.values()): - proc.join(0.3) - # DEBUG: Comment to debug - if show_ui: - print_ui(events, processes, fails) - if proc.exitcode is None: - continue - processes.pop(proc.name) - if proc.exitcode != 0: - fails.append(proc.name) - if show_ui: - print_ui(events, processes, fails) - sys.stdout.write("\n") - if fails and False: - if show_ui: - print_ui(events, processes, fails) - sys.stderr.write("The following failures were reported\n") - for fail in fails: - sys.stderr.write(fail + "\n") - sys.stderr.flush() - sys.exit(1) - - def build( - self, - steps: Optional[Sequence[str]] = None, - cleanup: bool = True, - show_ui: bool = False, - log_level: str = "WARNING", - ) -> None: - """ - Build! - - :param steps: The steps to run, defaults to None - :type steps: list, optional - :param cleanup: Whether to clean up or not, defaults to True - :type cleanup: bool, optional - """ # noqa: D400 - fails: List[str] = [] - events: Dict[str, SyncEvent] = {} - waits: Dict[str, List[str]] = {} - processes: Dict[str, multiprocessing.Process] = {} - - if show_ui: - sys.stdout.write("Starting builds\n") - # DEBUG: Comment to debug - print_ui(events, processes, fails) - log.info("Starting builds") - - step_names = list(steps) if steps is not None else list(self.recipies) - - for name in step_names: - event = multiprocessing.Event() - events[name] = event - recipe = self.recipies[name] - kwargs = dict(recipe) - kwargs["show_ui"] = show_ui - kwargs["log_level"] = log_level - - # Determine needed dependency recipies. - wait_on_seq = cast(List[str], kwargs.pop("wait_on", [])) - wait_on_list = list(wait_on_seq) - for dependency in wait_on_list[:]: - if dependency not in step_names: - wait_on_list.remove(dependency) - - waits[name] = wait_on_list - if not waits[name]: - event.set() - - proc = multiprocessing.Process( - name=name, target=self.run, args=(name, event), kwargs=kwargs - ) - proc.start() - processes[name] = proc - - # Wait for the processes to finish and check if we should send any - # dependency events. - while processes: - for proc in list(processes.values()): - proc.join(0.3) - if show_ui: - # DEBUG: Comment to debug - print_ui(events, processes, fails) - if proc.exitcode is None: - continue - processes.pop(proc.name) - if proc.exitcode != 0: - fails.append(proc.name) - is_failure = True - else: - is_failure = False - for name in waits: - if proc.name in waits[name]: - if is_failure: - if name in processes: - processes[name].terminate() - time.sleep(0.1) - waits[name].remove(proc.name) - if not waits[name] and not events[name].is_set(): - events[name].set() - - if fails: - sys.stderr.write("The following failures were reported\n") - last_outs = {} - for fail in fails: - log_file = self.dirs.logs / f"{fail}.log" - try: - with io.open(log_file) as fp: - fp.seek(0, 2) - end = fp.tell() - ind = end - 4096 - if ind > 0: - fp.seek(ind) - else: - fp.seek(0) - last_out = fp.read() - if show_ui: - sys.stderr.write("=" * 20 + f" {fail} " + "=" * 20 + "\n") - sys.stderr.write(fp.read() + "\n\n") - except FileNotFoundError: - last_outs[fail] = f"Log file not found: {log_file}" - log.error("Build step %s has failed", fail) - log.error(last_out) - if show_ui: - sys.stderr.flush() - if cleanup: - log.debug("Performing cleanup.") - self.cleanup() - sys.exit(1) - if show_ui: - time.sleep(0.3) - print_ui(events, processes, fails) - sys.stdout.write("\n") - sys.stdout.flush() - if cleanup: - log.debug("Performing cleanup.") - self.cleanup() - - def check_prereqs(self) -> List[str]: - """ - Check pre-requsists for build. - - This method verifies all requrements for a successful build are satisfied. - - :return: Returns a list of string describing failed checks - :rtype: list - """ - fail: List[str] = [] - if sys.platform == "linux": - if not self.toolchain or not self.toolchain.exists(): - fail.append( - f"Toolchain for {self.arch} does not exist. Please pip install ppbt." - ) - return fail - - def __call__( - self, - steps: Optional[Sequence[str]] = None, - arch: Optional[str] = None, - clean: bool = True, - cleanup: bool = True, - force_download: bool = False, - download_only: bool = False, - show_ui: bool = False, - log_level: str = "WARNING", - ) -> None: - """ - Set the architecture, define the steps, clean if needed, download what is needed, and build. - - :param steps: The steps to run, defaults to None - :type steps: list, optional - :param arch: The architecture to build, defaults to None - :type arch: str, optional - :param clean: If true, cleans the directories first, defaults to True - :type clean: bool, optional - :param cleanup: Cleans up after build if true, defaults to True - :type cleanup: bool, optional - :param force_download: Whether or not to download the content if it already exists, defaults to True - :type force_download: bool, optional - """ - log = logging.getLogger(None) - log.setLevel(logging.NOTSET) - - stream_handler: Optional[logging.Handler] = None - if not show_ui: - stream_handler = logging.StreamHandler() - stream_handler.setLevel(logging.getLevelName(log_level)) - log.addHandler(stream_handler) - - os.makedirs(self.dirs.logs, exist_ok=True) - file_handler = logging.FileHandler(self.dirs.logs / "build.log") - file_handler.setLevel(logging.INFO) - log.addHandler(file_handler) - - if arch: - self.set_arch(arch) - - step_names = list(steps) if steps is not None else list(self.recipies) - - failures = self.check_prereqs() - if not download_only and failures: - for _ in failures: - sys.stderr.write(f"{_}\n") - sys.stderr.flush() - sys.exit(1) - - if clean: - self.clean() - - if self.build_arch != self.arch: - native_root = DATA_DIR / "native" - if not native_root.exists(): - if "RELENV_NATIVE_PY_VERSION" in os.environ: - version = os.environ["RELENV_NATIVE_PY_VERSION"] - else: - version = self.version - from relenv.create import create - - create("native", DATA_DIR, version=version) - - # Start a process for each build passing it an event used to notify each - # process if it's dependencies have finished. - try: - self.download_files( - step_names, force_download=force_download, show_ui=show_ui - ) - if download_only: - return - self.build(step_names, cleanup, show_ui=show_ui, log_level=log_level) - finally: - log.removeHandler(file_handler) - if stream_handler is not None: - log.removeHandler(stream_handler) - - -class Builds: - """Collection of platform-specific builders.""" - - def __init__(self) -> None: - self.builds: Dict[str, Builder] = {} - - def add(self, platform: str, *args: Any, **kwargs: Any) -> Builder: - if "builder" in kwargs: - build_candidate = kwargs.pop("builder") - if args or kwargs: - raise RuntimeError( - "builder keyword can not be used with other kwargs or args" - ) - build = cast(Builder, build_candidate) - else: - build = Builder(*args, **kwargs) - self.builds[platform] = build - return build - - -builds = Builds() - - -def patch_shebang(path: PathLike, old: str, new: str) -> bool: - """ - Replace a file's shebang. - - :param path: The path of the file to patch - :type path: str - :param old: The old shebang, will only patch when this is found - :type old: str - :param name: The new shebang to be written - :type name: str - """ - with open(path, "rb") as fp: - try: - data = fp.read(len(old.encode())).decode() - except UnicodeError: - return False - except Exception as exc: - log.warning("Unhandled exception: %r", exc) - return False - if data != old: - log.warning("Shebang doesn't match: %s %r != %r", path, old, data) - return False - data = fp.read().decode() - with open(path, "w") as fp: - fp.write(new) - fp.write(data) - with open(path, "r") as fp: - data = fp.read() - log.info("Patched shebang of %s => %r", path, data) - return True - - -def patch_shebangs(path: PathLike, old: str, new: str) -> None: - """ - Traverse directory and patch shebangs. - - :param path: The of the directory to traverse - :type path: str - :param old: The old shebang, will only patch when this is found - :type old: str - :param name: The new shebang to be written - :type name: str - """ - for root, _dirs, files in os.walk(str(path)): - for file in files: - patch_shebang(os.path.join(root, file), old, new) - - -def install_sysdata( - mod: ModuleType, - destfile: PathLike, - buildroot: PathLike, - toolchain: Optional[PathLike], -) -> None: - """ - Create a Relenv Python environment's sysconfigdata. - - Helper method used by the `finalize` build method to create a Relenv - Python environment's sysconfigdata. - - :param mod: The module to operate on - :type mod: ``types.ModuleType`` - :param destfile: Path to the file to write the data to - :type destfile: str - :param buildroot: Path to the root of the build - :type buildroot: str - :param toolchain: Path to the root of the toolchain - :type toolchain: str - """ - data = {} - - def fbuildroot(s: str) -> str: - return s.replace(str(buildroot), "{BUILDROOT}") - - def ftoolchain(s: str) -> str: - return s.replace(str(toolchain), "{TOOLCHAIN}") - - # XXX: keymap is not used, remove it? - # keymap = { - # "BINDIR": (fbuildroot,), - # "BINLIBDEST": (fbuildroot,), - # "CFLAGS": (fbuildroot, ftoolchain), - # "CPPLAGS": (fbuildroot, ftoolchain), - # "CXXFLAGS": (fbuildroot, ftoolchain), - # "datarootdir": (fbuildroot,), - # "exec_prefix": (fbuildroot,), - # "LDFLAGS": (fbuildroot, ftoolchain), - # "LDSHARED": (fbuildroot, ftoolchain), - # "LIBDEST": (fbuildroot,), - # "prefix": (fbuildroot,), - # "SCRIPTDIR": (fbuildroot,), - # } - for key in sorted(mod.build_time_vars): - val = mod.build_time_vars[key] - if isinstance(val, str): - for _ in (fbuildroot, ftoolchain): - val = _(val) - log.info("SYSCONFIG [%s] %s => %s", key, mod.build_time_vars[key], val) - data[key] = val - - with open(destfile, "w", encoding="utf8") as f: - f.write( - "# system configuration generated and used by" " the relenv at runtime\n" - ) - f.write("_build_time_vars = ") - pprint.pprint(data, stream=f) - f.write(SYSCONFIGDATA) - - -def find_sysconfigdata(pymodules: PathLike) -> str: - """ - Find sysconfigdata directory for python installation. - - :param pymodules: Path to python modules (e.g. lib/python3.10) - :type pymodules: str - - :return: The name of the sysconig data module - :rtype: str - """ - for root, dirs, files in os.walk(pymodules): - for file in files: - if file.find("sysconfigdata") > -1 and file.endswith(".py"): - return file[:-3] - raise RelenvException("Unable to locate sysconfigdata module") - - -def install_runtime(sitepackages: PathLike) -> None: - """ - Install a base relenv runtime. - """ - site_dir = pathlib.Path(sitepackages) - relenv_pth = site_dir / "relenv.pth" - with io.open(str(relenv_pth), "w") as fp: - fp.write(RELENV_PTH) - - # Lay down relenv.runtime, we'll pip install the rest later - relenv = site_dir / "relenv" - os.makedirs(relenv, exist_ok=True) - - for name in [ - "runtime.py", - "relocate.py", - "common.py", - "buildenv.py", - "__init__.py", - ]: - src = MODULE_DIR / name - dest = relenv / name - with io.open(src, "r") as rfp: - with io.open(dest, "w") as wfp: - wfp.write(rfp.read()) - - -def copy_sbom_files(dirs: Dirs) -> None: - """ - Copy SBOM files from Python source to the prefix directory. - - SBOM files (Software Bill of Materials) document the build dependencies - and source file checksums. These files are available in Python 3.12+. - - :param dirs: The working directories - :type dirs: ``relenv.build.common.Dirs`` - """ - # Find the Python source directory in dirs.sources - python_source = None - if dirs.sources.exists(): - # Look for Python-{version} directory - for entry in dirs.sources.iterdir(): - if entry.is_dir() and entry.name.startswith("Python-"): - python_source = entry - break - - if python_source: - sbom_files = ["sbom.spdx.json", "externals.spdx.json"] - source_misc_dir = python_source / "Misc" - for sbom_file in sbom_files: - source_sbom = source_misc_dir / sbom_file - if source_sbom.exists(): - dest_sbom = pathlib.Path(dirs.prefix) / sbom_file - shutil.copy2(str(source_sbom), str(dest_sbom)) - log.info("Copied %s to archive", sbom_file) - else: - log.debug("SBOM file %s not found (Python < 3.12?)", sbom_file) - - -def finalize( - env: MutableMapping[str, str], - dirs: Dirs, - logfp: IO[str], -) -> None: - """ - Run after we've fully built python. - - This method enhances the newly created python with Relenv's runtime hacks. - - :param env: The environment dictionary - :type env: dict - :param dirs: The working directories - :type dirs: ``relenv.build.common.Dirs`` - :param logfp: A handle for the log file - :type logfp: file - """ - # Run relok8 to make sure the rpaths are relocatable. - relenv.relocate.main(dirs.prefix, log_file_name=str(dirs.logs / "relocate.py.log")) - # Install relenv-sysconfigdata module - libdir = pathlib.Path(dirs.prefix) / "lib" - - def find_pythonlib(libdir: pathlib.Path) -> Optional[str]: - for _root, dirs, _files in os.walk(libdir): - for entry in dirs: - if entry.startswith("python"): - return entry - return None - - python_lib = find_pythonlib(libdir) - if python_lib is None: - raise RelenvException("Unable to locate python library directory") - - pymodules = libdir / python_lib - - # update ensurepip - update_ensurepip(pymodules) - - cwd = os.getcwd() - modname = find_sysconfigdata(pymodules) - path = sys.path - sys.path = [str(pymodules)] - try: - mod = __import__(str(modname)) - finally: - os.chdir(cwd) - sys.path = path - - dest = pymodules / f"{modname}.py" - install_sysdata(mod, dest, dirs.prefix, dirs.toolchain) - - # Lay down site customize - bindir = pathlib.Path(dirs.prefix) / "bin" - sitepackages = pymodules / "site-packages" - install_runtime(sitepackages) - - # Install pip - python_exe = str(dirs.prefix / "bin" / "python3") - if env["RELENV_HOST_ARCH"] != env["RELENV_BUILD_ARCH"]: - env["RELENV_CROSS"] = str(dirs.prefix) - python_exe = env["RELENV_NATIVE_PY"] - logfp.write("\nRUN ENSURE PIP\n") - - env.pop("RELENV_BUILDENV") - - runcmd( - [python_exe, "-m", "ensurepip"], - env=env, - stderr=logfp, - stdout=logfp, - ) - - # Fix the shebangs in the scripts python layed down. Order matters. - shebangs = [ - "#!{}".format(bindir / f"python{env['RELENV_PY_MAJOR_VERSION']}"), - "#!{}".format( - bindir / f"python{env['RELENV_PY_MAJOR_VERSION'].split('.', 1)[0]}" - ), - ] - newshebang = format_shebang("/python3") - for shebang in shebangs: - log.info("Patch shebang %r with %r", shebang, newshebang) - patch_shebangs( - str(pathlib.Path(dirs.prefix) / "bin"), - shebang, - newshebang, - ) - - if sys.platform == "linux": - pyconf = f"config-{env['RELENV_PY_MAJOR_VERSION']}-{env['RELENV_HOST']}" - patch_shebang( - str(pymodules / pyconf / "python-config.py"), - "#!{}".format(str(bindir / f"python{env['RELENV_PY_MAJOR_VERSION']}")), - format_shebang("../../../bin/python3"), - ) - - toolchain_path = dirs.toolchain - if toolchain_path is None: - raise RelenvException("Toolchain path is required for linux builds") - shutil.copy( - pathlib.Path(toolchain_path) - / env["RELENV_HOST"] - / "sysroot" - / "lib" - / "libstdc++.so.6", - libdir, - ) - - # Moved in python 3.13 or removed? - if (pymodules / "cgi.py").exists(): - patch_shebang( - str(pymodules / "cgi.py"), - "#! /usr/local/bin/python", - format_shebang("../../bin/python3"), - ) - - def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: - logfp.write(f"\nRUN PIP {pkg} {upgrade}\n") - target: Optional[pathlib.Path] = None - python_exe = str(dirs.prefix / "bin" / "python3") - if sys.platform == LINUX: - if env["RELENV_HOST_ARCH"] != env["RELENV_BUILD_ARCH"]: - target = pymodules / "site-packages" - python_exe = env["RELENV_NATIVE_PY"] - cmd = [ - python_exe, - "-m", - "pip", - "install", - str(pkg), - ] - if upgrade: - cmd.append("--upgrade") - if target: - cmd.append("--target={}".format(target)) - runcmd(cmd, env=env, stderr=logfp, stdout=logfp) - - runpip("wheel") - # This needs to handle running from the root of the git repo and also from - # an installed Relenv - if (MODULE_DIR.parent / ".git").exists(): - runpip(MODULE_DIR.parent, upgrade=True) - else: - runpip("relenv", upgrade=True) - - copy_sbom_files(dirs) - - globs = [ - "/bin/python*", - "/bin/pip*", - "/bin/relenv", - "/lib/python*/ensurepip/*", - "/lib/python*/site-packages/*", - "/include/*", - "*.so", - "/lib/*.so.*", - "*.py", - "*.spdx.json", # Include SBOM files - # Mac specific, factor this out - "*.dylib", - ] - archive = f"{ dirs.prefix }.tar.xz" - log.info("Archive is %s", archive) - with tarfile.open(archive, mode="w:xz") as fp: - create_archive(fp, dirs.prefix, globs, logfp) - - -def create_archive( - tarfp: tarfile.TarFile, - toarchive: PathLike, - globs: Sequence[str], - logfp: Optional[IO[str]] = None, -) -> None: - """ - Create an archive. - - :param tarfp: A pointer to the archive to be created - :type tarfp: file - :param toarchive: The path to the directory to archive - :type toarchive: str - :param globs: A list of filtering patterns to match against files to be added - :type globs: list - :param logfp: A pointer to the log file - :type logfp: file - """ - log.debug("Current directory %s", os.getcwd()) - log.debug("Creating archive %s", tarfp.name) - for root, _dirs, files in os.walk(toarchive): - relroot = pathlib.Path(root).relative_to(toarchive) - for f in files: - relpath = relroot / f - matches = False - for g in globs: - candidate = pathlib.Path("/") / relpath - if fnmatch.fnmatch(str(candidate), g): - matches = True - break - if matches: - log.debug("Adding %s", relpath) - tarfp.add(relpath, arcname=str(relpath), recursive=False) - else: - log.debug("Skipping %s", relpath) diff --git a/relenv/build/common/__init__.py b/relenv/build/common/__init__.py index 0a07a3fd..d9a5a723 100644 --- a/relenv/build/common/__init__.py +++ b/relenv/build/common/__init__.py @@ -21,7 +21,7 @@ create_archive, patch_file, update_sbom_checksums, - copy_sbom_files, + generate_relenv_sbom, ) from .builder import ( @@ -44,7 +44,7 @@ "update_ensurepip", "patch_file", "update_sbom_checksums", - "copy_sbom_files", + "generate_relenv_sbom", # Builders (specific build functions) "build_openssl", "build_openssl_fips", diff --git a/relenv/build/common/install.py b/relenv/build/common/install.py index bebe361a..a938b1a6 100644 --- a/relenv/build/common/install.py +++ b/relenv/build/common/install.py @@ -18,8 +18,19 @@ import shutil import sys import tarfile +import time from types import ModuleType -from typing import IO, MutableMapping, Optional, Sequence, Union, TYPE_CHECKING +from typing import ( + Any, + Dict, + IO, + List, + MutableMapping, + Optional, + Sequence, + Union, + TYPE_CHECKING, +) from relenv.common import ( LINUX, @@ -409,36 +420,255 @@ def install_runtime(sitepackages: PathLike) -> None: wfp.write(rfp.read()) -def copy_sbom_files(dirs: Dirs) -> None: +def generate_relenv_sbom(env: MutableMapping[str, str], dirs: Dirs) -> None: """ - Copy SBOM files from Python source to the prefix directory. + Generate the authoritative relenv-sbom.spdx.json for this build. + + This is the single, comprehensive SBOM that documents: + - Python itself (the CPython interpreter) + - All build dependencies we compiled (openssl, sqlite, ncurses, etc.) + - All pip-installed runtime packages (relenv, pip, setuptools, wheel, etc.) + + This replaces copying Python's native SBOM files (sbom.spdx.json and + externals.spdx.json) which contain incomplete/inaccurate information for + relenv builds (e.g., they list OpenSSL 3.0.15 but we build 3.6.0). - SBOM files (Software Bill of Materials) document the build dependencies - and source file checksums. These files are available in Python 3.12+. + Generates SBOM for all Python versions (3.10+). + :param env: The environment dictionary + :type env: dict :param dirs: The working directories :type dirs: ``relenv.build.common.Dirs`` """ - # Find the Python source directory in dirs.sources - python_source = None - if dirs.sources.exists(): - # Look for Python-{version} directory - for entry in dirs.sources.iterdir(): - if entry.is_dir() and entry.name.startswith("Python-"): - python_source = entry - break - - if python_source: - sbom_files = ["sbom.spdx.json", "externals.spdx.json"] - source_misc_dir = python_source / "Misc" - for sbom_file in sbom_files: - source_sbom = source_misc_dir / sbom_file - if source_sbom.exists(): - dest_sbom = pathlib.Path(dirs.prefix) / sbom_file - shutil.copy2(str(source_sbom), str(dest_sbom)) - log.info("Copied %s to archive", sbom_file) - else: - log.debug("SBOM file %s not found (Python < 3.12?)", sbom_file) + from .builder import get_dependency_version + import relenv + + python_version = dirs.version + + platform_map = { + "linux": "linux", + "darwin": "darwin", + "win32": "win32", + } + platform = platform_map.get(sys.platform, sys.platform) + + # Build dependency list - get versions from python-versions.json + packages: List[Dict[str, Any]] = [] + + # Add Python itself as the primary package + python_package: Dict[str, Any] = { + "SPDXID": "SPDXRef-PACKAGE-Python", + "name": "Python", + "versionInfo": python_version, + "downloadLocation": f"https://www.python.org/ftp/python/{python_version}/Python-{python_version}.tar.xz", + "filesAnalyzed": False, + "primaryPackagePurpose": "APPLICATION", + "licenseConcluded": "Python-2.0", + "comment": "CPython interpreter - the core component of this relenv build", + } + packages.append(python_package) + + # Define dependencies we build (these are the ones relenv compiles) + # Order matters - list them in a logical grouping + relenv_deps = [ + # Compression libraries + ("bzip2", "https://sourceware.org/pub/bzip2/bzip2-{version}.tar.gz"), + ( + "xz", + "https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz", + ), + ( + "zlib", + "https://github.com/madler/zlib/releases/download/v{version}/zlib-{version}.tar.gz", + ), + # Crypto and security + ( + "openssl", + "https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz", + ), + ( + "libxcrypt", + "https://github.com/besser82/libxcrypt/releases/download/v{version}/libxcrypt-{version}.tar.xz", + ), + # Database + ("sqlite", "https://sqlite.org/{year}/sqlite-autoconf-{sqliteversion}.tar.gz"), + ("gdbm", "https://ftp.gnu.org/gnu/gdbm/gdbm-{version}.tar.gz"), + # Terminal libraries + ("ncurses", "https://ftp.gnu.org/gnu/ncurses/ncurses-{version}.tar.gz"), + ("readline", "https://ftp.gnu.org/gnu/readline/readline-{version}.tar.gz"), + # Other libraries + ( + "libffi", + "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz", + ), + ( + "uuid", + "https://sourceforge.net/projects/libuuid/files/libuuid-{version}.tar.gz", + ), + # XML parser (bundled in Python source, updated by relenv) + ( + "expat", + "https://github.com/libexpat/libexpat/releases/download/R_{version_tag}/expat-{version}.tar.xz", + ), + ] + + # Linux-specific dependencies + if sys.platform == "linux": + relenv_deps.extend( + [ + ( + "tirpc", + "https://downloads.sourceforge.net/project/libtirpc/" + "libtirpc/{version}/libtirpc-{version}.tar.bz2", + ), + ( + "krb5", + "https://kerberos.org/dist/krb5/{major_minor}/krb5-{version}.tar.gz", + ), + ] + ) + + for dep_name, url_template in relenv_deps: + dep_info = get_dependency_version(dep_name, platform) + if dep_info: + version = dep_info["version"] + url = dep_info.get("url", url_template).format( + version=version, + sqliteversion=dep_info.get("sqliteversion", ""), + year=dep_info.get("year", "2025"), + major_minor=".".join(version.split(".")[:2]), + version_tag=version.replace(".", "_"), + ) + checksum = dep_info.get("sha256", "") + + package: Dict[str, Any] = { + "SPDXID": f"SPDXRef-PACKAGE-{dep_name}", + "name": dep_name, + "versionInfo": version, + "downloadLocation": url, + "filesAnalyzed": False, + "primaryPackagePurpose": "SOURCE", + "licenseConcluded": "NOASSERTION", + } + + if checksum: + package["checksums"] = [ + { + "algorithm": "SHA256", + "checksumValue": checksum, + } + ] + + packages.append(package) + + # Add Python runtime packages installed via pip + # These are determined at finalize time after pip install + python_lib = pathlib.Path(dirs.prefix) / "lib" + for entry in python_lib.glob("python*/site-packages/*.dist-info"): + # Parse package name and version from dist-info directory + # Format: package-version.dist-info + dist_name = entry.name.replace(".dist-info", "") + if "-" in dist_name: + parts = dist_name.rsplit("-", 1) + if len(parts) == 2: + pkg_name, pkg_version = parts + package2: Dict[str, Any] = { + "SPDXID": f"SPDXRef-PACKAGE-python-{pkg_name}", + "name": pkg_name, + "versionInfo": pkg_version, + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "primaryPackagePurpose": "LIBRARY", + "licenseConcluded": "NOASSERTION", + "comment": "Python package installed via pip", + } + packages.append(package2) + + # Add Python's bundled dependencies that we don't build separately + # These are embedded in Python's source tree and compiled into Python + # For Python 3.12+, we can extract versions from Python's own SBOM + bundled_deps = [] + + # Try to read Python's SBOM to get accurate versions of bundled components + python_sbom_path = pathlib.Path(str(dirs.source)) / "Misc" / "sbom.spdx.json" + python_bundled_versions: Dict[str, Dict[str, Any]] = {} + if python_sbom_path.exists(): + try: + with io.open(python_sbom_path, "r") as fp: + python_sbom = json.load(fp) + for pkg in python_sbom.get("packages", []): + pkg_name = pkg.get("name") + if pkg_name: + python_bundled_versions[pkg_name] = pkg + except Exception: + # If we can't read Python's SBOM, skip bundled deps + pass + + # Document bundled dependencies if we have version information + if python_bundled_versions: + # Define bundled components we want to include (excluding expat since we handle it separately) + bundled_components = { + "mpdecimal": "Bundled in Python source (Modules/_decimal/libmpdec) - decimal arithmetic", + "hacl-star": "Bundled in Python source (Modules/_hacl) - cryptographic primitives", + "libb2": "Bundled in Python source (Modules/_blake2) - BLAKE2 cryptographic hash", + "macholib": "Bundled in Python source (Lib/ctypes/macholib) - Mach-O binary parsing", + } + + for comp_name, comp_desc in bundled_components.items(): + if comp_name in python_bundled_versions: + src_pkg = python_bundled_versions[comp_name] + bundled_pkg: Dict[str, Any] = { + "SPDXID": f"SPDXRef-PACKAGE-{comp_name}", + "name": comp_name, + "versionInfo": src_pkg.get("versionInfo", "NOASSERTION"), + "downloadLocation": src_pkg.get("downloadLocation", "NOASSERTION"), + "filesAnalyzed": False, + "primaryPackagePurpose": "SOURCE", + "licenseConcluded": src_pkg.get("licenseConcluded", "NOASSERTION"), + "comment": comp_desc, + } + # Copy checksums if present + if "checksums" in src_pkg: + bundled_pkg["checksums"] = src_pkg["checksums"] + # Copy externalRefs (CPE identifiers) if present + if "externalRefs" in src_pkg: + bundled_pkg["externalRefs"] = src_pkg["externalRefs"] + bundled_deps.append(bundled_pkg) + + packages.extend(bundled_deps) + + # Create the SBOM document + # Generate unique document namespace (required by SPDX 2.3) + timestamp = time.strftime("%Y%m%d%H%M%S", time.gmtime()) + doc_name = f"relenv-{env.get('RELENV_PY_VERSION', 'unknown')}-{env.get('RELENV_HOST', 'unknown')}" + + sbom = { + "SPDXID": "SPDXRef-DOCUMENT", + "spdxVersion": "SPDX-2.3", + "name": doc_name, + "documentNamespace": f"https://github.com/saltstack/relenv/spdx/{doc_name}-{timestamp}", + "dataLicense": "CC0-1.0", + "creationInfo": { + "created": f"{time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}", + "creators": [ + f"Tool: relenv-{relenv.__version__}", + ], + "comment": "Authoritative SBOM for this relenv build. Documents all compiled build " + "dependencies and installed runtime packages. This is the single source of truth for " + "vulnerability scanning and compliance.", + }, + "packages": packages, + } + + # Write the SBOM file + sbom_path = pathlib.Path(dirs.prefix) / "relenv-sbom.spdx.json" + with io.open(sbom_path, "w") as fp: + json.dump(sbom, fp, indent=2) + log.info( + "Generated relenv-sbom.spdx.json with %d packages (Python %s + dependencies + pip packages)", + len(packages), + python_version, + ) def finalize( @@ -586,7 +816,8 @@ def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: else: runpip("relenv", upgrade=True) - copy_sbom_files(dirs) + # Generate single comprehensive SBOM (replaces copying Python's multiple SBOMs) + generate_relenv_sbom(env, dirs) globs = [ "/bin/python*", diff --git a/relenv/sbom.py b/relenv/sbom.py new file mode 100644 index 00000000..6913e0c6 --- /dev/null +++ b/relenv/sbom.py @@ -0,0 +1,245 @@ +# Copyright 2022-2025 Broadcom. +# SPDX-License-Identifier: Apache-2.0 +""" +SBOM (Software Bill of Materials) management for relenv. +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import sys +import time +from typing import Any, Dict, List, Optional, Tuple + + +def get_python_version(relenv_root: pathlib.Path) -> Optional[Tuple[int, int, int]]: + """ + Get the Python version of a relenv environment. + + :param relenv_root: Path to relenv environment root + :return: Tuple of (major, minor, micro) version numbers, or None if cannot determine + """ + python_exe = relenv_root / "bin" / "python3" + if not python_exe.exists(): + python_exe = relenv_root / "bin" / "python3.exe" + + if not python_exe.exists(): + return None + + try: + import subprocess + + result = subprocess.run( + [ + str(python_exe), + "-c", + "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}')", + ], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + version_str = result.stdout.strip() + parts = version_str.split(".") + if len(parts) >= 3: + return (int(parts[0]), int(parts[1]), int(parts[2])) + except (subprocess.SubprocessError, ValueError, FileNotFoundError): + pass + + return None + + +def find_relenv_root(start_path: pathlib.Path) -> pathlib.Path: + """ + Find the root of a relenv environment. + + Looks for indicators like bin/python3 and relenv-sbom.spdx.json or sbom.spdx.json. + + :param start_path: Starting path to search from + :return: Path to relenv root + :raises FileNotFoundError: If not a relenv environment + """ + # Normalize the path + path = start_path.resolve() + + # Check if we're already at the root + if (path / "bin" / "python3").exists() or (path / "bin" / "python3.exe").exists(): + return path + + # Check if we're inside a relenv environment (e.g., in bin/) + if (path.parent / "bin" / "python3").exists(): + return path.parent + + # Not a relenv environment + raise FileNotFoundError( + f"Not a relenv environment: {start_path}\n" + f"Expected to find bin/python3 or bin/python3.exe" + ) + + +def scan_installed_packages(relenv_root: pathlib.Path) -> List[Dict[str, Any]]: + """ + Scan for installed Python packages in a relenv environment. + + :param relenv_root: Path to relenv environment root + :return: List of package dicts with SPDX metadata + """ + packages: List[Dict[str, Any]] = [] + + # Find the Python site-packages directory + lib_dir = relenv_root / "lib" + if not lib_dir.exists(): + return packages + + # Scan for .dist-info directories + for entry in lib_dir.glob("python*/site-packages/*.dist-info"): + # Parse package name and version from dist-info directory + # Format: package-version.dist-info + dist_name = entry.name.replace(".dist-info", "") + if "-" in dist_name: + parts = dist_name.rsplit("-", 1) + if len(parts) == 2: + pkg_name, pkg_version = parts + package: Dict[str, Any] = { + "SPDXID": f"SPDXRef-PACKAGE-python-{pkg_name}", + "name": pkg_name, + "versionInfo": pkg_version, + "downloadLocation": "NOASSERTION", + "primaryPackagePurpose": "LIBRARY", + "licenseConcluded": "NOASSERTION", + "comment": "Python package installed via pip", + } + packages.append(package) + + return packages + + +def update_sbom(relenv_root: pathlib.Path) -> None: + """ + Update relenv-sbom.spdx.json with currently installed packages. + + This updates only the Python packages section, preserving the build + dependencies section from the original SBOM. + + Only works for Python 3.12+ environments (when Python started including SBOM files). + + :param relenv_root: Path to relenv environment root + :raises RuntimeError: If Python version is less than 3.12 + """ + import relenv + + # Check Python version + py_version = get_python_version(relenv_root) + if py_version is None: + raise RuntimeError(f"Could not determine Python version for {relenv_root}") + + major, minor, micro = py_version + if major < 3 or (major == 3 and minor < 12): + raise RuntimeError( + f"SBOM generation is only supported for Python 3.12+. " + f"This environment is Python {major}.{minor}.{micro}" + ) + + sbom_path = relenv_root / "relenv-sbom.spdx.json" + + # Load existing SBOM if it exists + if sbom_path.exists(): + with open(sbom_path, "r") as f: + sbom = json.load(f) + else: + # Create new SBOM if it doesn't exist + sbom = { + "SPDXID": "SPDXRef-DOCUMENT", + "spdxVersion": "SPDX-2.3", + "name": f"relenv-{relenv_root.name}", + "dataLicense": "CC0-1.0", + "creationInfo": { + "created": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "creators": [f"Tool: relenv-{relenv.__version__}"], + }, + "packages": [], + } + + # Separate build dependencies from Python packages + build_deps = [ + pkg + for pkg in sbom.get("packages", []) + if not pkg.get("SPDXID", "").startswith("SPDXRef-PACKAGE-python-") + ] + + # Scan for currently installed packages + python_packages = scan_installed_packages(relenv_root) + + # Combine build deps + current Python packages + sbom["packages"] = build_deps + python_packages + + # Update creation time + if "creationInfo" not in sbom: + sbom["creationInfo"] = {} + sbom["creationInfo"]["created"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + sbom["creationInfo"]["creators"] = [f"Tool: relenv-{relenv.__version__}"] + + # Write updated SBOM + with open(sbom_path, "w") as f: + json.dump(sbom, f, indent=2) + + print(f"Updated {sbom_path}") + print(f" Build dependencies: {len(build_deps)}") + print(f" Python packages: {len(python_packages)}") + + +def setup_parser( + subparsers: argparse._SubParsersAction[argparse.ArgumentParser], +) -> None: + """ + Setup argument parser for sbom-update command. + + :param subparsers: Subparser action from argparse + """ + parser = subparsers.add_parser( + "sbom-update", + description="Update relenv-sbom.spdx.json with currently installed packages", + help="Update SBOM with installed packages", + ) + parser.add_argument( + "path", + nargs="?", + default=".", + help="Path to relenv environment (default: current directory)", + ) + parser.set_defaults(func=main) + + +def main(args: argparse.Namespace) -> int: + """ + Main entry point for sbom-update command. + + :param args: Parsed command-line arguments + :return: Exit code (0 for success, 1 for error) + """ + try: + # Find the relenv root + start_path = pathlib.Path(args.path) + relenv_root = find_relenv_root(start_path) + + print(f"Found relenv environment at: {relenv_root}") + + # Update the SBOM + update_sbom(relenv_root) + + return 0 + + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except RuntimeError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except Exception as e: + print(f"Error updating SBOM: {e}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 diff --git a/requirements/tests.txt b/requirements/tests.txt index c31bda0d..ae9f713f 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -5,3 +5,4 @@ pytest pytest-skip-markers swig ppbt; sys_platform == 'linux' +spdx-tools # For SBOM validation in tests diff --git a/tests/test_build.py b/tests/test_build.py index e113d44e..055c8e18 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -458,121 +458,3 @@ def test_download_destination_setter() -> None: # Set to None d.destination = None assert d.destination == pathlib.Path() - - -def test_sbom_copy_python_312(tmp_path: pathlib.Path) -> None: - """Test that SBOM files are copied from Python 3.12+ source to prefix directory.""" - import json - - from relenv.build.common import Dirs, copy_sbom_files - from relenv.common import WorkDirs - - # Create mock directory structure - root = tmp_path / "relenv_root" - src_dir = root / "src" - build_dir = root / "build" - - # Create Python source directory with SBOM files - python_src = src_dir / "Python-3.12.8" - misc_dir = python_src / "Misc" - misc_dir.mkdir(parents=True) - - # Create mock SBOM files - sbom_content = { - "spdxVersion": "SPDX-2.3", - "dataLicense": "CC0-1.0", - "SPDXID": "SPDXRef-DOCUMENT", - "name": "CPython-3.12.8", - } - externals_content = { - "spdxVersion": "SPDX-2.3", - "dataLicense": "CC0-1.0", - "SPDXID": "SPDXRef-DOCUMENT", - "name": "CPython-3.12.8-externals", - } - - (misc_dir / "sbom.spdx.json").write_text(json.dumps(sbom_content)) - (misc_dir / "externals.spdx.json").write_text(json.dumps(externals_content)) - - # Create WorkDirs and Dirs objects - work_dirs = WorkDirs(root=root) - # Override the default paths to use our mock structure - work_dirs.build = build_dir - work_dirs.src = src_dir - work_dirs.logs = root / "logs" - - dirs = Dirs(work_dirs, "test", "x86_64", "3.12.8") - - # Create prefix directory with minimal required structure - # (dirs.prefix is computed from dirs.build / version-triplet) - prefix_dir = dirs.prefix - prefix_dir.mkdir(parents=True) - bin_dir = prefix_dir / "bin" - bin_dir.mkdir() - lib_dir = prefix_dir / "lib" - lib_dir.mkdir() - - # Call copy_sbom_files (this should copy SBOM files) - copy_sbom_files(dirs) - - # Verify SBOM files were copied to prefix directory - dest_sbom = prefix_dir / "sbom.spdx.json" - dest_externals = prefix_dir / "externals.spdx.json" - - assert dest_sbom.exists(), "sbom.spdx.json should be copied to prefix directory" - assert ( - dest_externals.exists() - ), "externals.spdx.json should be copied to prefix directory" - - # Verify content is correct - assert json.loads(dest_sbom.read_text())["name"] == "CPython-3.12.8" - assert json.loads(dest_externals.read_text())["name"] == "CPython-3.12.8-externals" - - -def test_sbom_copy_python_310(tmp_path: pathlib.Path) -> None: - """Test that SBOM files are gracefully skipped for Python < 3.12.""" - from relenv.build.common import Dirs, copy_sbom_files - from relenv.common import WorkDirs - - # Create mock directory structure - root = tmp_path / "relenv_root" - src_dir = root / "src" - build_dir = root / "build" - - # Create Python source directory WITHOUT SBOM files (Python 3.10) - python_src = src_dir / "Python-3.10.18" - misc_dir = python_src / "Misc" - misc_dir.mkdir(parents=True) - - # Create some other files but NOT SBOM files - (misc_dir / "README").write_text("Python 3.10 Misc directory") - - # Create WorkDirs and Dirs objects - work_dirs = WorkDirs(root=root) - # Override the default paths to use our mock structure - work_dirs.build = build_dir - work_dirs.src = src_dir - work_dirs.logs = root / "logs" - - dirs = Dirs(work_dirs, "test", "x86_64", "3.10.18") - - # Create prefix directory with minimal required structure - # (dirs.prefix is computed from dirs.build / version-triplet) - prefix_dir = dirs.prefix - prefix_dir.mkdir(parents=True) - bin_dir = prefix_dir / "bin" - bin_dir.mkdir() - lib_dir = prefix_dir / "lib" - lib_dir.mkdir() - - # Call copy_sbom_files (this should gracefully skip SBOM files) - copy_sbom_files(dirs) - - # Verify SBOM files were NOT copied (they don't exist in source) - dest_sbom = prefix_dir / "sbom.spdx.json" - dest_externals = prefix_dir / "externals.spdx.json" - - assert not dest_sbom.exists(), "sbom.spdx.json should not exist for Python < 3.12" - assert ( - not dest_externals.exists() - ), "externals.spdx.json should not exist for Python < 3.12" diff --git a/tests/test_sbom.py b/tests/test_sbom.py new file mode 100644 index 00000000..b1dbd777 --- /dev/null +++ b/tests/test_sbom.py @@ -0,0 +1,339 @@ +# Copyright 2022-2025 Broadcom. +# SPDX-License-Identifier: Apache-2.0 +""" +Tests for SBOM (Software Bill of Materials) functionality. +""" +from __future__ import annotations + +import json +import pathlib +from unittest import mock + +import pytest + +from relenv import sbom + + +def test_find_relenv_root_from_root(tmp_path: pathlib.Path) -> None: + """Test finding relenv root when starting at root directory.""" + # Create a fake relenv environment structure + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + python_exe = bin_dir / "python3" + python_exe.touch() + + # Should find the root + root = sbom.find_relenv_root(tmp_path) + assert root == tmp_path + + +def test_find_relenv_root_from_subdir(tmp_path: pathlib.Path) -> None: + """Test finding relenv root when starting from subdirectory.""" + # Create a fake relenv environment structure + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + python_exe = bin_dir / "python3" + python_exe.touch() + + # Should find the root from bin directory + root = sbom.find_relenv_root(bin_dir) + assert root == tmp_path + + +def test_find_relenv_root_not_found(tmp_path: pathlib.Path) -> None: + """Test finding relenv root when not in a relenv environment.""" + # Empty directory - should raise + with pytest.raises(FileNotFoundError, match="Not a relenv environment"): + sbom.find_relenv_root(tmp_path) + + +def test_get_python_version(tmp_path: pathlib.Path) -> None: + """Test getting Python version from a relenv environment.""" + # Create fake Python executable + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + python_exe = bin_dir / "python3" + python_exe.write_text("#!/bin/bash\necho '3.12.1'") + python_exe.chmod(0o755) + + # Mock subprocess to return version + with mock.patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = "3.12.1\n" + + version = sbom.get_python_version(tmp_path) + assert version == (3, 12, 1) + + +def test_get_python_version_not_found(tmp_path: pathlib.Path) -> None: + """Test getting Python version when python3 doesn't exist.""" + version = sbom.get_python_version(tmp_path) + assert version is None + + +def test_scan_installed_packages_empty(tmp_path: pathlib.Path) -> None: + """Test scanning packages when none are installed.""" + packages = sbom.scan_installed_packages(tmp_path) + assert packages == [] + + +def test_scan_installed_packages(tmp_path: pathlib.Path) -> None: + """Test scanning installed packages from dist-info directories.""" + # Create fake site-packages with dist-info + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + + # Create some fake dist-info directories + (site_packages / "pip-23.0.1.dist-info").mkdir() + (site_packages / "setuptools-68.0.0.dist-info").mkdir() + (site_packages / "relenv-0.21.2.dist-info").mkdir() + (site_packages / "cowsay-5.0.dist-info").mkdir() + + # Scan packages + packages = sbom.scan_installed_packages(tmp_path) + + # Should find all 4 packages + assert len(packages) == 4 + + # Check structure of first package + pip_pkg = next(p for p in packages if p["name"] == "pip") + assert pip_pkg["SPDXID"] == "SPDXRef-PACKAGE-python-pip" + assert pip_pkg["name"] == "pip" + assert pip_pkg["versionInfo"] == "23.0.1" + assert pip_pkg["downloadLocation"] == "NOASSERTION" + assert pip_pkg["primaryPackagePurpose"] == "LIBRARY" + assert pip_pkg["licenseConcluded"] == "NOASSERTION" + assert pip_pkg["comment"] == "Python package installed via pip" + + # Check all packages are present + pkg_names = {p["name"] for p in packages} + assert pkg_names == {"pip", "setuptools", "relenv", "cowsay"} + + +def test_update_sbom_create_new(tmp_path: pathlib.Path) -> None: + """Test creating a new SBOM when none exists.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + (site_packages / "pip-23.0.1.dist-info").mkdir() + (site_packages / "relenv-0.21.2.dist-info").mkdir() + + # Mock Python version to be 3.12 + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 12, 1)): + # Update SBOM (should create new one) + sbom.update_sbom(tmp_path) + + # Verify SBOM was created + sbom_path = tmp_path / "relenv-sbom.spdx.json" + assert sbom_path.exists() + + # Load and verify structure + with open(sbom_path) as f: + sbom_data = json.load(f) + + assert sbom_data["SPDXID"] == "SPDXRef-DOCUMENT" + assert sbom_data["spdxVersion"] == "SPDX-2.3" + assert sbom_data["dataLicense"] == "CC0-1.0" + assert "creationInfo" in sbom_data + assert "created" in sbom_data["creationInfo"] + assert len(sbom_data["creationInfo"]["creators"]) == 1 + assert sbom_data["creationInfo"]["creators"][0].startswith("Tool: relenv-") + + # Should have 2 Python packages + packages = sbom_data["packages"] + assert len(packages) == 2 + pkg_names = {p["name"] for p in packages} + assert pkg_names == {"pip", "relenv"} + + +def test_update_sbom_preserve_build_deps(tmp_path: pathlib.Path) -> None: + """Test that updating SBOM preserves build dependencies.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + (site_packages / "pip-23.0.1.dist-info").mkdir() + + # Mock Python version + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 12, 1)): + # Create initial SBOM with build deps + sbom_path = tmp_path / "relenv-sbom.spdx.json" + initial_sbom = { + "SPDXID": "SPDXRef-DOCUMENT", + "spdxVersion": "SPDX-2.3", + "name": "relenv-test", + "dataLicense": "CC0-1.0", + "creationInfo": { + "created": "2025-01-01T00:00:00Z", + "creators": ["Tool: relenv-0.21.0"], + }, + "packages": [ + { + "SPDXID": "SPDXRef-PACKAGE-openssl", + "name": "openssl", + "versionInfo": "3.6.0", + "downloadLocation": "https://example.com/openssl.tar.gz", + "primaryPackagePurpose": "SOURCE", + "licenseConcluded": "NOASSERTION", + }, + { + "SPDXID": "SPDXRef-PACKAGE-sqlite", + "name": "sqlite", + "versionInfo": "3.50.4.0", + "downloadLocation": "https://example.com/sqlite.tar.gz", + "primaryPackagePurpose": "SOURCE", + "licenseConcluded": "NOASSERTION", + }, + { + "SPDXID": "SPDXRef-PACKAGE-python-wheel", + "name": "wheel", + "versionInfo": "0.42.0", + "downloadLocation": "NOASSERTION", + "primaryPackagePurpose": "LIBRARY", + "licenseConcluded": "NOASSERTION", + "comment": "Python package installed via pip", + }, + ], + } + + with open(sbom_path, "w") as f: + json.dump(initial_sbom, f, indent=2) + + # Update SBOM (should preserve build deps, update Python packages) + sbom.update_sbom(tmp_path) + + # Load and verify + with open(sbom_path) as f: + updated_sbom = json.load(f) + + packages = updated_sbom["packages"] + + # Should have 2 build deps + 1 new Python package + assert len(packages) == 3 + + # Build deps should be preserved + openssl = next((p for p in packages if p["name"] == "openssl"), None) + assert openssl is not None + assert openssl["versionInfo"] == "3.6.0" + + sqlite = next((p for p in packages if p["name"] == "sqlite"), None) + assert sqlite is not None + assert sqlite["versionInfo"] == "3.50.4.0" + + # Old wheel package should be removed, new pip package should be present + pip_pkg = next((p for p in packages if p["name"] == "pip"), None) + assert pip_pkg is not None + assert pip_pkg["versionInfo"] == "23.0.1" + + wheel = next((p for p in packages if p["name"] == "wheel"), None) + assert wheel is None + + +def test_update_sbom_replaces_python_packages(tmp_path: pathlib.Path) -> None: + """Test that updating SBOM replaces Python packages with current scan.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + + # Mock Python version + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 12, 1)): + # Initially install pip and cowsay + (site_packages / "pip-23.0.1.dist-info").mkdir() + (site_packages / "cowsay-5.0.dist-info").mkdir() + + # First update + sbom.update_sbom(tmp_path) + + # Verify initial state + sbom_path = tmp_path / "relenv-sbom.spdx.json" + with open(sbom_path) as f: + sbom_data = json.load(f) + assert len(sbom_data["packages"]) == 2 + pkg_names = {p["name"] for p in sbom_data["packages"]} + assert pkg_names == {"pip", "cowsay"} + + # Now "uninstall" cowsay and "install" relenv + (site_packages / "cowsay-5.0.dist-info").rmdir() + (site_packages / "relenv-0.21.2.dist-info").mkdir() + + # Second update + sbom.update_sbom(tmp_path) + + # Verify updated state + with open(sbom_path) as f: + sbom_data = json.load(f) + assert len(sbom_data["packages"]) == 2 + pkg_names = {p["name"] for p in sbom_data["packages"]} + assert pkg_names == {"pip", "relenv"} + + +def test_update_sbom_python_version_too_old(tmp_path: pathlib.Path) -> None: + """Test that update_sbom fails gracefully for Python < 3.12.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + # Mock Python version to be 3.10 + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 10, 18)): + with pytest.raises( + RuntimeError, match="SBOM generation is only supported for Python 3.12+" + ): + sbom.update_sbom(tmp_path) + + +def test_main_success(tmp_path: pathlib.Path, capsys: pytest.CaptureFixture) -> None: + """Test main() with valid relenv environment.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + (site_packages / "pip-23.0.1.dist-info").mkdir() + + # Create args + import argparse + + args = argparse.Namespace(path=str(tmp_path)) + + # Mock Python version + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 12, 1)): + # Run main + result = sbom.main(args) + assert result == 0 + + # Check output + captured = capsys.readouterr() + assert "Found relenv environment at:" in captured.out + assert "Updated" in captured.out + assert "relenv-sbom.spdx.json" in captured.out + + +def test_main_not_relenv(tmp_path: pathlib.Path, capsys: pytest.CaptureFixture) -> None: + """Test main() with non-relenv directory.""" + # Empty directory + import argparse + + args = argparse.Namespace(path=str(tmp_path)) + + # Run main + result = sbom.main(args) + assert result == 1 + + # Check error output + captured = capsys.readouterr() + assert "Error:" in captured.err + assert "Not a relenv environment" in captured.err diff --git a/tests/test_verify_build.py b/tests/test_verify_build.py index 6e26093b..bfa6d399 100644 --- a/tests/test_verify_build.py +++ b/tests/test_verify_build.py @@ -2323,3 +2323,197 @@ def test_xz_lzma_functionality(pyexec): f"{proc.stderr.decode()}" ) assert proc.stdout.decode().strip() == "OK" + + +@pytest.mark.skip_on_windows +def test_sbom_files_exist(build, minor_version): + """Test that SBOM file exists for all Python versions. + + We generate a single comprehensive relenv-sbom.spdx.json that documents + all build dependencies and runtime packages. Python's native SBOM files + (only present in 3.12+) are not copied as they contain inaccurate/incomplete + information for our builds. + """ + # All Python versions should have relenv-sbom.spdx.json + relenv_sbom = build / "relenv-sbom.spdx.json" + assert ( + relenv_sbom.exists() + ), "relenv-sbom.spdx.json should exist for all Python versions" + + # Python's native SBOMs should NOT be present (we don't copy them anymore) + python_sbom = build / "sbom.spdx.json" + externals_sbom = build / "externals.spdx.json" + assert ( + not python_sbom.exists() + ), "sbom.spdx.json should not be copied (relenv-sbom.spdx.json is authoritative)" + assert ( + not externals_sbom.exists() + ), "externals.spdx.json should not be copied (relenv-sbom.spdx.json is authoritative)" + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_structure(build, minor_version): + """Test that relenv-sbom.spdx.json has valid SPDX structure.""" + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + with open(relenv_sbom) as f: + sbom = json.load(f) + + # Validate SPDX structure + assert sbom.get("SPDXID") == "SPDXRef-DOCUMENT" + assert sbom.get("spdxVersion") == "SPDX-2.3" + assert sbom.get("dataLicense") == "CC0-1.0" + assert "name" in sbom + assert "creationInfo" in sbom + assert "created" in sbom["creationInfo"] + assert "creators" in sbom["creationInfo"] + assert len(sbom["creationInfo"]["creators"]) > 0 + assert sbom["creationInfo"]["creators"][0].startswith("Tool: relenv-") + + # Should have packages + assert "packages" in sbom + assert isinstance(sbom["packages"], list) + assert len(sbom["packages"]) > 0 + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_has_pip_packages(build, minor_version): + """Test that relenv-sbom.spdx.json includes pip-installed packages.""" + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + with open(relenv_sbom) as f: + sbom = json.load(f) + + packages = sbom.get("packages", []) + package_names = {pkg.get("name") for pkg in packages} + + # Should include core pip packages + expected_packages = {"pip", "setuptools", "wheel", "relenv"} + found_packages = expected_packages & package_names + + assert len(found_packages) >= 3, ( + f"SBOM should contain at least 3 of {expected_packages}, " + f"but only found: {found_packages}" + ) + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_includes_python(build, minor_version): + """Test that relenv-sbom.spdx.json includes Python itself as a package.""" + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + with open(relenv_sbom) as f: + sbom = json.load(f) + + packages = sbom.get("packages", []) + package_names = {pkg.get("name") for pkg in packages} + + # Python itself should be documented + assert ( + "Python" in package_names + ), "SBOM should include Python (CPython interpreter) as a package" + + # Find the Python package and verify its structure + python_pkg = next((pkg for pkg in packages if pkg.get("name") == "Python"), None) + assert python_pkg is not None, "Python package should exist" + + # Verify Python package has correct fields + assert "versionInfo" in python_pkg, "Python package should have version" + assert ( + python_pkg["versionInfo"] == minor_version + ), f"Python version should be {minor_version}, got {python_pkg.get('versionInfo')}" + assert ( + "downloadLocation" in python_pkg + ), "Python package should have download location" + assert ( + "python.org" in python_pkg["downloadLocation"].lower() + ), "Python download location should reference python.org" + assert ( + python_pkg.get("primaryPackagePurpose") == "APPLICATION" + ), "Python should be marked as APPLICATION" + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_validates_with_spdx_tools(build, minor_version): + """Test that relenv-sbom.spdx.json passes official SPDX validation.""" + pytest.importorskip("spdx_tools", reason="spdx-tools not installed") + + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + # Use official SPDX tools to validate the SBOM + from spdx_tools.spdx.parser.parse_anything import parse_file + from spdx_tools.spdx.validation.document_validator import ( + validate_full_spdx_document, + ) + + # Parse the SBOM file + try: + document = parse_file(str(relenv_sbom)) + except Exception as e: + pytest.fail(f"Failed to parse SBOM with spdx-tools: {e}") + + # Validate against SPDX 2.3 specification + validation_messages = validate_full_spdx_document(document) + + if validation_messages: + error_details = "\n".join(str(msg) for msg in validation_messages) + pytest.fail( + f"SBOM failed official SPDX validation with {len(validation_messages)} error(s):\n{error_details}" + ) + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_includes_bundled_dependencies(build, minor_version): + """Test that relenv-sbom.spdx.json includes Python's bundled dependencies.""" + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + with open(relenv_sbom) as f: + sbom = json.load(f) + + packages = sbom.get("packages", []) + package_names = {pkg.get("name") for pkg in packages} + + # expat is bundled in Python's source tree and updated by relenv + # It's a critical security component (XML parser) + assert ( + "expat" in package_names + ), "SBOM should include expat (bundled XML parser updated by relenv)" + + # Verify expat package has required fields + expat_pkg = next((pkg for pkg in packages if pkg.get("name") == "expat"), None) + assert expat_pkg is not None, "expat package should exist" + assert "versionInfo" in expat_pkg, "expat package should have version" + assert ( + "downloadLocation" in expat_pkg + ), "expat package should have download location" + assert ( + "libexpat" in expat_pkg["downloadLocation"].lower() + ), "expat download location should reference libexpat GitHub" + assert expat_pkg.get("checksums"), "expat should have SHA-256 checksum" + + # For Python 3.12+, check for additional bundled dependencies + # These are extracted from Python's own SBOM + major, minor = map(int, minor_version.split(".")) + if major > 3 or (major == 3 and minor >= 12): + expected_bundled = ["mpdecimal", "hacl-star", "libb2"] + found_bundled = [name for name in expected_bundled if name in package_names] + + assert len(found_bundled) >= 2, ( + f"For Python 3.12+, SBOM should include bundled dependencies from Python source. " + f"Expected at least {expected_bundled[:2]}, found: {found_bundled}" + ) + + # Verify at least one bundled dep has proper structure + if "mpdecimal" in package_names: + mpdec_pkg = next( + (pkg for pkg in packages if pkg.get("name") == "mpdecimal"), None + ) + assert mpdec_pkg is not None + assert ( + mpdec_pkg.get("versionInfo") != "NOASSERTION" + ), "mpdecimal should have specific version from Python's SBOM" From 8c3697b596d1c8f7b179af3557ab356a43131b53 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 5 Nov 2025 15:18:11 -0700 Subject: [PATCH 7/7] Fix sbom verification test --- relenv/build/common/install.py | 11 +++++++++++ tests/test_verify_build.py | 1 + 2 files changed, 12 insertions(+) diff --git a/relenv/build/common/install.py b/relenv/build/common/install.py index a938b1a6..60f81026 100644 --- a/relenv/build/common/install.py +++ b/relenv/build/common/install.py @@ -642,6 +642,16 @@ def generate_relenv_sbom(env: MutableMapping[str, str], dirs: Dirs) -> None: timestamp = time.strftime("%Y%m%d%H%M%S", time.gmtime()) doc_name = f"relenv-{env.get('RELENV_PY_VERSION', 'unknown')}-{env.get('RELENV_HOST', 'unknown')}" + # Create relationships - SPDX requires DESCRIBES relationship + # The document DESCRIBES the Python package (the primary component) + relationships = [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": "SPDXRef-PACKAGE-Python", + "relationshipType": "DESCRIBES", + } + ] + sbom = { "SPDXID": "SPDXRef-DOCUMENT", "spdxVersion": "SPDX-2.3", @@ -658,6 +668,7 @@ def generate_relenv_sbom(env: MutableMapping[str, str], dirs: Dirs) -> None: "vulnerability scanning and compliance.", }, "packages": packages, + "relationships": relationships, } # Write the SBOM file diff --git a/tests/test_verify_build.py b/tests/test_verify_build.py index bfa6d399..fc283cb8 100644 --- a/tests/test_verify_build.py +++ b/tests/test_verify_build.py @@ -2422,6 +2422,7 @@ def test_relenv_sbom_includes_python(build, minor_version): # Verify Python package has correct fields assert "versionInfo" in python_pkg, "Python package should have version" + # The versionInfo should match the full Python version (e.g., "3.11.14") assert ( python_pkg["versionInfo"] == minor_version ), f"Python version should be {minor_version}, got {python_pkg.get('versionInfo')}"