diff --git a/relenv/__main__.py b/relenv/__main__.py index 4869370f..fa577c01 100644 --- a/relenv/__main__.py +++ b/relenv/__main__.py @@ -10,7 +10,7 @@ from argparse import ArgumentParser from types import ModuleType -from . import build, buildenv, check, create, fetch, pyversions, toolchain +from . import build, buildenv, check, create, fetch, pyversions, sbom, toolchain from .common import __version__ @@ -41,6 +41,7 @@ def setup_cli() -> ArgumentParser: check, buildenv, pyversions, + sbom, ] for mod in modules_to_setup: mod.setup_parser(subparsers) diff --git a/relenv/build/common/__init__.py b/relenv/build/common/__init__.py index 0c66f912..d9a5a723 100644 --- a/relenv/build/common/__init__.py +++ b/relenv/build/common/__init__.py @@ -21,6 +21,7 @@ create_archive, patch_file, update_sbom_checksums, + generate_relenv_sbom, ) from .builder import ( @@ -43,6 +44,7 @@ "update_ensurepip", "patch_file", "update_sbom_checksums", + "generate_relenv_sbom", # Builders (specific build functions) "build_openssl", "build_openssl_fips", diff --git a/relenv/build/common/install.py b/relenv/build/common/install.py index 4ab30d37..60f81026 100644 --- a/relenv/build/common/install.py +++ b/relenv/build/common/install.py @@ -18,8 +18,19 @@ import shutil import sys import tarfile +import time from types import ModuleType -from typing import IO, MutableMapping, Optional, Sequence, Union, TYPE_CHECKING +from typing import ( + Any, + Dict, + IO, + List, + MutableMapping, + Optional, + Sequence, + Union, + TYPE_CHECKING, +) from relenv.common import ( LINUX, @@ -409,6 +420,268 @@ def install_runtime(sitepackages: PathLike) -> None: wfp.write(rfp.read()) +def generate_relenv_sbom(env: MutableMapping[str, str], dirs: Dirs) -> None: + """ + Generate the authoritative relenv-sbom.spdx.json for this build. + + This is the single, comprehensive SBOM that documents: + - Python itself (the CPython interpreter) + - All build dependencies we compiled (openssl, sqlite, ncurses, etc.) + - All pip-installed runtime packages (relenv, pip, setuptools, wheel, etc.) + + This replaces copying Python's native SBOM files (sbom.spdx.json and + externals.spdx.json) which contain incomplete/inaccurate information for + relenv builds (e.g., they list OpenSSL 3.0.15 but we build 3.6.0). + + Generates SBOM for all Python versions (3.10+). + + :param env: The environment dictionary + :type env: dict + :param dirs: The working directories + :type dirs: ``relenv.build.common.Dirs`` + """ + from .builder import get_dependency_version + import relenv + + python_version = dirs.version + + platform_map = { + "linux": "linux", + "darwin": "darwin", + "win32": "win32", + } + platform = platform_map.get(sys.platform, sys.platform) + + # Build dependency list - get versions from python-versions.json + packages: List[Dict[str, Any]] = [] + + # Add Python itself as the primary package + python_package: Dict[str, Any] = { + "SPDXID": "SPDXRef-PACKAGE-Python", + "name": "Python", + "versionInfo": python_version, + "downloadLocation": f"https://www.python.org/ftp/python/{python_version}/Python-{python_version}.tar.xz", + "filesAnalyzed": False, + "primaryPackagePurpose": "APPLICATION", + "licenseConcluded": "Python-2.0", + "comment": "CPython interpreter - the core component of this relenv build", + } + packages.append(python_package) + + # Define dependencies we build (these are the ones relenv compiles) + # Order matters - list them in a logical grouping + relenv_deps = [ + # Compression libraries + ("bzip2", "https://sourceware.org/pub/bzip2/bzip2-{version}.tar.gz"), + ( + "xz", + "https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz", + ), + ( + "zlib", + "https://github.com/madler/zlib/releases/download/v{version}/zlib-{version}.tar.gz", + ), + # Crypto and security + ( + "openssl", + "https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz", + ), + ( + "libxcrypt", + "https://github.com/besser82/libxcrypt/releases/download/v{version}/libxcrypt-{version}.tar.xz", + ), + # Database + ("sqlite", "https://sqlite.org/{year}/sqlite-autoconf-{sqliteversion}.tar.gz"), + ("gdbm", "https://ftp.gnu.org/gnu/gdbm/gdbm-{version}.tar.gz"), + # Terminal libraries + ("ncurses", "https://ftp.gnu.org/gnu/ncurses/ncurses-{version}.tar.gz"), + ("readline", "https://ftp.gnu.org/gnu/readline/readline-{version}.tar.gz"), + # Other libraries + ( + "libffi", + "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz", + ), + ( + "uuid", + "https://sourceforge.net/projects/libuuid/files/libuuid-{version}.tar.gz", + ), + # XML parser (bundled in Python source, updated by relenv) + ( + "expat", + "https://github.com/libexpat/libexpat/releases/download/R_{version_tag}/expat-{version}.tar.xz", + ), + ] + + # Linux-specific dependencies + if sys.platform == "linux": + relenv_deps.extend( + [ + ( + "tirpc", + "https://downloads.sourceforge.net/project/libtirpc/" + "libtirpc/{version}/libtirpc-{version}.tar.bz2", + ), + ( + "krb5", + "https://kerberos.org/dist/krb5/{major_minor}/krb5-{version}.tar.gz", + ), + ] + ) + + for dep_name, url_template in relenv_deps: + dep_info = get_dependency_version(dep_name, platform) + if dep_info: + version = dep_info["version"] + url = dep_info.get("url", url_template).format( + version=version, + sqliteversion=dep_info.get("sqliteversion", ""), + year=dep_info.get("year", "2025"), + major_minor=".".join(version.split(".")[:2]), + version_tag=version.replace(".", "_"), + ) + checksum = dep_info.get("sha256", "") + + package: Dict[str, Any] = { + "SPDXID": f"SPDXRef-PACKAGE-{dep_name}", + "name": dep_name, + "versionInfo": version, + "downloadLocation": url, + "filesAnalyzed": False, + "primaryPackagePurpose": "SOURCE", + "licenseConcluded": "NOASSERTION", + } + + if checksum: + package["checksums"] = [ + { + "algorithm": "SHA256", + "checksumValue": checksum, + } + ] + + packages.append(package) + + # Add Python runtime packages installed via pip + # These are determined at finalize time after pip install + python_lib = pathlib.Path(dirs.prefix) / "lib" + for entry in python_lib.glob("python*/site-packages/*.dist-info"): + # Parse package name and version from dist-info directory + # Format: package-version.dist-info + dist_name = entry.name.replace(".dist-info", "") + if "-" in dist_name: + parts = dist_name.rsplit("-", 1) + if len(parts) == 2: + pkg_name, pkg_version = parts + package2: Dict[str, Any] = { + "SPDXID": f"SPDXRef-PACKAGE-python-{pkg_name}", + "name": pkg_name, + "versionInfo": pkg_version, + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "primaryPackagePurpose": "LIBRARY", + "licenseConcluded": "NOASSERTION", + "comment": "Python package installed via pip", + } + packages.append(package2) + + # Add Python's bundled dependencies that we don't build separately + # These are embedded in Python's source tree and compiled into Python + # For Python 3.12+, we can extract versions from Python's own SBOM + bundled_deps = [] + + # Try to read Python's SBOM to get accurate versions of bundled components + python_sbom_path = pathlib.Path(str(dirs.source)) / "Misc" / "sbom.spdx.json" + python_bundled_versions: Dict[str, Dict[str, Any]] = {} + if python_sbom_path.exists(): + try: + with io.open(python_sbom_path, "r") as fp: + python_sbom = json.load(fp) + for pkg in python_sbom.get("packages", []): + pkg_name = pkg.get("name") + if pkg_name: + python_bundled_versions[pkg_name] = pkg + except Exception: + # If we can't read Python's SBOM, skip bundled deps + pass + + # Document bundled dependencies if we have version information + if python_bundled_versions: + # Define bundled components we want to include (excluding expat since we handle it separately) + bundled_components = { + "mpdecimal": "Bundled in Python source (Modules/_decimal/libmpdec) - decimal arithmetic", + "hacl-star": "Bundled in Python source (Modules/_hacl) - cryptographic primitives", + "libb2": "Bundled in Python source (Modules/_blake2) - BLAKE2 cryptographic hash", + "macholib": "Bundled in Python source (Lib/ctypes/macholib) - Mach-O binary parsing", + } + + for comp_name, comp_desc in bundled_components.items(): + if comp_name in python_bundled_versions: + src_pkg = python_bundled_versions[comp_name] + bundled_pkg: Dict[str, Any] = { + "SPDXID": f"SPDXRef-PACKAGE-{comp_name}", + "name": comp_name, + "versionInfo": src_pkg.get("versionInfo", "NOASSERTION"), + "downloadLocation": src_pkg.get("downloadLocation", "NOASSERTION"), + "filesAnalyzed": False, + "primaryPackagePurpose": "SOURCE", + "licenseConcluded": src_pkg.get("licenseConcluded", "NOASSERTION"), + "comment": comp_desc, + } + # Copy checksums if present + if "checksums" in src_pkg: + bundled_pkg["checksums"] = src_pkg["checksums"] + # Copy externalRefs (CPE identifiers) if present + if "externalRefs" in src_pkg: + bundled_pkg["externalRefs"] = src_pkg["externalRefs"] + bundled_deps.append(bundled_pkg) + + packages.extend(bundled_deps) + + # Create the SBOM document + # Generate unique document namespace (required by SPDX 2.3) + timestamp = time.strftime("%Y%m%d%H%M%S", time.gmtime()) + doc_name = f"relenv-{env.get('RELENV_PY_VERSION', 'unknown')}-{env.get('RELENV_HOST', 'unknown')}" + + # Create relationships - SPDX requires DESCRIBES relationship + # The document DESCRIBES the Python package (the primary component) + relationships = [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": "SPDXRef-PACKAGE-Python", + "relationshipType": "DESCRIBES", + } + ] + + sbom = { + "SPDXID": "SPDXRef-DOCUMENT", + "spdxVersion": "SPDX-2.3", + "name": doc_name, + "documentNamespace": f"https://github.com/saltstack/relenv/spdx/{doc_name}-{timestamp}", + "dataLicense": "CC0-1.0", + "creationInfo": { + "created": f"{time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}", + "creators": [ + f"Tool: relenv-{relenv.__version__}", + ], + "comment": "Authoritative SBOM for this relenv build. Documents all compiled build " + "dependencies and installed runtime packages. This is the single source of truth for " + "vulnerability scanning and compliance.", + }, + "packages": packages, + "relationships": relationships, + } + + # Write the SBOM file + sbom_path = pathlib.Path(dirs.prefix) / "relenv-sbom.spdx.json" + with io.open(sbom_path, "w") as fp: + json.dump(sbom, fp, indent=2) + log.info( + "Generated relenv-sbom.spdx.json with %d packages (Python %s + dependencies + pip packages)", + len(packages), + python_version, + ) + + def finalize( env: MutableMapping[str, str], dirs: Dirs, @@ -553,6 +826,10 @@ def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: runpip(MODULE_DIR.parent, upgrade=True) else: runpip("relenv", upgrade=True) + + # Generate single comprehensive SBOM (replaces copying Python's multiple SBOMs) + generate_relenv_sbom(env, dirs) + globs = [ "/bin/python*", "/bin/pip*", @@ -563,6 +840,7 @@ def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: "*.so", "/lib/*.so.*", "*.py", + "*.spdx.json", # Include SBOM files # Mac specific, factor this out "*.dylib", ] diff --git a/relenv/build/windows.py b/relenv/build/windows.py index a88f41db..5ccf96e6 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -265,6 +265,7 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: for target_file in updated_files: os.utime(target_file, (now, now)) + # Map SBOM file names to actual file paths # Update SBOM with correct checksums for updated expat files # Map SBOM file names to actual file paths files_to_update = {} diff --git a/relenv/sbom.py b/relenv/sbom.py new file mode 100644 index 00000000..6913e0c6 --- /dev/null +++ b/relenv/sbom.py @@ -0,0 +1,245 @@ +# Copyright 2022-2025 Broadcom. +# SPDX-License-Identifier: Apache-2.0 +""" +SBOM (Software Bill of Materials) management for relenv. +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import sys +import time +from typing import Any, Dict, List, Optional, Tuple + + +def get_python_version(relenv_root: pathlib.Path) -> Optional[Tuple[int, int, int]]: + """ + Get the Python version of a relenv environment. + + :param relenv_root: Path to relenv environment root + :return: Tuple of (major, minor, micro) version numbers, or None if cannot determine + """ + python_exe = relenv_root / "bin" / "python3" + if not python_exe.exists(): + python_exe = relenv_root / "bin" / "python3.exe" + + if not python_exe.exists(): + return None + + try: + import subprocess + + result = subprocess.run( + [ + str(python_exe), + "-c", + "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}')", + ], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + version_str = result.stdout.strip() + parts = version_str.split(".") + if len(parts) >= 3: + return (int(parts[0]), int(parts[1]), int(parts[2])) + except (subprocess.SubprocessError, ValueError, FileNotFoundError): + pass + + return None + + +def find_relenv_root(start_path: pathlib.Path) -> pathlib.Path: + """ + Find the root of a relenv environment. + + Looks for indicators like bin/python3 and relenv-sbom.spdx.json or sbom.spdx.json. + + :param start_path: Starting path to search from + :return: Path to relenv root + :raises FileNotFoundError: If not a relenv environment + """ + # Normalize the path + path = start_path.resolve() + + # Check if we're already at the root + if (path / "bin" / "python3").exists() or (path / "bin" / "python3.exe").exists(): + return path + + # Check if we're inside a relenv environment (e.g., in bin/) + if (path.parent / "bin" / "python3").exists(): + return path.parent + + # Not a relenv environment + raise FileNotFoundError( + f"Not a relenv environment: {start_path}\n" + f"Expected to find bin/python3 or bin/python3.exe" + ) + + +def scan_installed_packages(relenv_root: pathlib.Path) -> List[Dict[str, Any]]: + """ + Scan for installed Python packages in a relenv environment. + + :param relenv_root: Path to relenv environment root + :return: List of package dicts with SPDX metadata + """ + packages: List[Dict[str, Any]] = [] + + # Find the Python site-packages directory + lib_dir = relenv_root / "lib" + if not lib_dir.exists(): + return packages + + # Scan for .dist-info directories + for entry in lib_dir.glob("python*/site-packages/*.dist-info"): + # Parse package name and version from dist-info directory + # Format: package-version.dist-info + dist_name = entry.name.replace(".dist-info", "") + if "-" in dist_name: + parts = dist_name.rsplit("-", 1) + if len(parts) == 2: + pkg_name, pkg_version = parts + package: Dict[str, Any] = { + "SPDXID": f"SPDXRef-PACKAGE-python-{pkg_name}", + "name": pkg_name, + "versionInfo": pkg_version, + "downloadLocation": "NOASSERTION", + "primaryPackagePurpose": "LIBRARY", + "licenseConcluded": "NOASSERTION", + "comment": "Python package installed via pip", + } + packages.append(package) + + return packages + + +def update_sbom(relenv_root: pathlib.Path) -> None: + """ + Update relenv-sbom.spdx.json with currently installed packages. + + This updates only the Python packages section, preserving the build + dependencies section from the original SBOM. + + Only works for Python 3.12+ environments (when Python started including SBOM files). + + :param relenv_root: Path to relenv environment root + :raises RuntimeError: If Python version is less than 3.12 + """ + import relenv + + # Check Python version + py_version = get_python_version(relenv_root) + if py_version is None: + raise RuntimeError(f"Could not determine Python version for {relenv_root}") + + major, minor, micro = py_version + if major < 3 or (major == 3 and minor < 12): + raise RuntimeError( + f"SBOM generation is only supported for Python 3.12+. " + f"This environment is Python {major}.{minor}.{micro}" + ) + + sbom_path = relenv_root / "relenv-sbom.spdx.json" + + # Load existing SBOM if it exists + if sbom_path.exists(): + with open(sbom_path, "r") as f: + sbom = json.load(f) + else: + # Create new SBOM if it doesn't exist + sbom = { + "SPDXID": "SPDXRef-DOCUMENT", + "spdxVersion": "SPDX-2.3", + "name": f"relenv-{relenv_root.name}", + "dataLicense": "CC0-1.0", + "creationInfo": { + "created": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "creators": [f"Tool: relenv-{relenv.__version__}"], + }, + "packages": [], + } + + # Separate build dependencies from Python packages + build_deps = [ + pkg + for pkg in sbom.get("packages", []) + if not pkg.get("SPDXID", "").startswith("SPDXRef-PACKAGE-python-") + ] + + # Scan for currently installed packages + python_packages = scan_installed_packages(relenv_root) + + # Combine build deps + current Python packages + sbom["packages"] = build_deps + python_packages + + # Update creation time + if "creationInfo" not in sbom: + sbom["creationInfo"] = {} + sbom["creationInfo"]["created"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + sbom["creationInfo"]["creators"] = [f"Tool: relenv-{relenv.__version__}"] + + # Write updated SBOM + with open(sbom_path, "w") as f: + json.dump(sbom, f, indent=2) + + print(f"Updated {sbom_path}") + print(f" Build dependencies: {len(build_deps)}") + print(f" Python packages: {len(python_packages)}") + + +def setup_parser( + subparsers: argparse._SubParsersAction[argparse.ArgumentParser], +) -> None: + """ + Setup argument parser for sbom-update command. + + :param subparsers: Subparser action from argparse + """ + parser = subparsers.add_parser( + "sbom-update", + description="Update relenv-sbom.spdx.json with currently installed packages", + help="Update SBOM with installed packages", + ) + parser.add_argument( + "path", + nargs="?", + default=".", + help="Path to relenv environment (default: current directory)", + ) + parser.set_defaults(func=main) + + +def main(args: argparse.Namespace) -> int: + """ + Main entry point for sbom-update command. + + :param args: Parsed command-line arguments + :return: Exit code (0 for success, 1 for error) + """ + try: + # Find the relenv root + start_path = pathlib.Path(args.path) + relenv_root = find_relenv_root(start_path) + + print(f"Found relenv environment at: {relenv_root}") + + # Update the SBOM + update_sbom(relenv_root) + + return 0 + + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except RuntimeError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except Exception as e: + print(f"Error updating SBOM: {e}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 diff --git a/requirements/tests.txt b/requirements/tests.txt index c31bda0d..ae9f713f 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -5,3 +5,4 @@ pytest pytest-skip-markers swig ppbt; sys_platform == 'linux' +spdx-tools # For SBOM validation in tests diff --git a/tests/test_sbom.py b/tests/test_sbom.py new file mode 100644 index 00000000..b1dbd777 --- /dev/null +++ b/tests/test_sbom.py @@ -0,0 +1,339 @@ +# Copyright 2022-2025 Broadcom. +# SPDX-License-Identifier: Apache-2.0 +""" +Tests for SBOM (Software Bill of Materials) functionality. +""" +from __future__ import annotations + +import json +import pathlib +from unittest import mock + +import pytest + +from relenv import sbom + + +def test_find_relenv_root_from_root(tmp_path: pathlib.Path) -> None: + """Test finding relenv root when starting at root directory.""" + # Create a fake relenv environment structure + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + python_exe = bin_dir / "python3" + python_exe.touch() + + # Should find the root + root = sbom.find_relenv_root(tmp_path) + assert root == tmp_path + + +def test_find_relenv_root_from_subdir(tmp_path: pathlib.Path) -> None: + """Test finding relenv root when starting from subdirectory.""" + # Create a fake relenv environment structure + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + python_exe = bin_dir / "python3" + python_exe.touch() + + # Should find the root from bin directory + root = sbom.find_relenv_root(bin_dir) + assert root == tmp_path + + +def test_find_relenv_root_not_found(tmp_path: pathlib.Path) -> None: + """Test finding relenv root when not in a relenv environment.""" + # Empty directory - should raise + with pytest.raises(FileNotFoundError, match="Not a relenv environment"): + sbom.find_relenv_root(tmp_path) + + +def test_get_python_version(tmp_path: pathlib.Path) -> None: + """Test getting Python version from a relenv environment.""" + # Create fake Python executable + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + python_exe = bin_dir / "python3" + python_exe.write_text("#!/bin/bash\necho '3.12.1'") + python_exe.chmod(0o755) + + # Mock subprocess to return version + with mock.patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = "3.12.1\n" + + version = sbom.get_python_version(tmp_path) + assert version == (3, 12, 1) + + +def test_get_python_version_not_found(tmp_path: pathlib.Path) -> None: + """Test getting Python version when python3 doesn't exist.""" + version = sbom.get_python_version(tmp_path) + assert version is None + + +def test_scan_installed_packages_empty(tmp_path: pathlib.Path) -> None: + """Test scanning packages when none are installed.""" + packages = sbom.scan_installed_packages(tmp_path) + assert packages == [] + + +def test_scan_installed_packages(tmp_path: pathlib.Path) -> None: + """Test scanning installed packages from dist-info directories.""" + # Create fake site-packages with dist-info + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + + # Create some fake dist-info directories + (site_packages / "pip-23.0.1.dist-info").mkdir() + (site_packages / "setuptools-68.0.0.dist-info").mkdir() + (site_packages / "relenv-0.21.2.dist-info").mkdir() + (site_packages / "cowsay-5.0.dist-info").mkdir() + + # Scan packages + packages = sbom.scan_installed_packages(tmp_path) + + # Should find all 4 packages + assert len(packages) == 4 + + # Check structure of first package + pip_pkg = next(p for p in packages if p["name"] == "pip") + assert pip_pkg["SPDXID"] == "SPDXRef-PACKAGE-python-pip" + assert pip_pkg["name"] == "pip" + assert pip_pkg["versionInfo"] == "23.0.1" + assert pip_pkg["downloadLocation"] == "NOASSERTION" + assert pip_pkg["primaryPackagePurpose"] == "LIBRARY" + assert pip_pkg["licenseConcluded"] == "NOASSERTION" + assert pip_pkg["comment"] == "Python package installed via pip" + + # Check all packages are present + pkg_names = {p["name"] for p in packages} + assert pkg_names == {"pip", "setuptools", "relenv", "cowsay"} + + +def test_update_sbom_create_new(tmp_path: pathlib.Path) -> None: + """Test creating a new SBOM when none exists.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + (site_packages / "pip-23.0.1.dist-info").mkdir() + (site_packages / "relenv-0.21.2.dist-info").mkdir() + + # Mock Python version to be 3.12 + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 12, 1)): + # Update SBOM (should create new one) + sbom.update_sbom(tmp_path) + + # Verify SBOM was created + sbom_path = tmp_path / "relenv-sbom.spdx.json" + assert sbom_path.exists() + + # Load and verify structure + with open(sbom_path) as f: + sbom_data = json.load(f) + + assert sbom_data["SPDXID"] == "SPDXRef-DOCUMENT" + assert sbom_data["spdxVersion"] == "SPDX-2.3" + assert sbom_data["dataLicense"] == "CC0-1.0" + assert "creationInfo" in sbom_data + assert "created" in sbom_data["creationInfo"] + assert len(sbom_data["creationInfo"]["creators"]) == 1 + assert sbom_data["creationInfo"]["creators"][0].startswith("Tool: relenv-") + + # Should have 2 Python packages + packages = sbom_data["packages"] + assert len(packages) == 2 + pkg_names = {p["name"] for p in packages} + assert pkg_names == {"pip", "relenv"} + + +def test_update_sbom_preserve_build_deps(tmp_path: pathlib.Path) -> None: + """Test that updating SBOM preserves build dependencies.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + (site_packages / "pip-23.0.1.dist-info").mkdir() + + # Mock Python version + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 12, 1)): + # Create initial SBOM with build deps + sbom_path = tmp_path / "relenv-sbom.spdx.json" + initial_sbom = { + "SPDXID": "SPDXRef-DOCUMENT", + "spdxVersion": "SPDX-2.3", + "name": "relenv-test", + "dataLicense": "CC0-1.0", + "creationInfo": { + "created": "2025-01-01T00:00:00Z", + "creators": ["Tool: relenv-0.21.0"], + }, + "packages": [ + { + "SPDXID": "SPDXRef-PACKAGE-openssl", + "name": "openssl", + "versionInfo": "3.6.0", + "downloadLocation": "https://example.com/openssl.tar.gz", + "primaryPackagePurpose": "SOURCE", + "licenseConcluded": "NOASSERTION", + }, + { + "SPDXID": "SPDXRef-PACKAGE-sqlite", + "name": "sqlite", + "versionInfo": "3.50.4.0", + "downloadLocation": "https://example.com/sqlite.tar.gz", + "primaryPackagePurpose": "SOURCE", + "licenseConcluded": "NOASSERTION", + }, + { + "SPDXID": "SPDXRef-PACKAGE-python-wheel", + "name": "wheel", + "versionInfo": "0.42.0", + "downloadLocation": "NOASSERTION", + "primaryPackagePurpose": "LIBRARY", + "licenseConcluded": "NOASSERTION", + "comment": "Python package installed via pip", + }, + ], + } + + with open(sbom_path, "w") as f: + json.dump(initial_sbom, f, indent=2) + + # Update SBOM (should preserve build deps, update Python packages) + sbom.update_sbom(tmp_path) + + # Load and verify + with open(sbom_path) as f: + updated_sbom = json.load(f) + + packages = updated_sbom["packages"] + + # Should have 2 build deps + 1 new Python package + assert len(packages) == 3 + + # Build deps should be preserved + openssl = next((p for p in packages if p["name"] == "openssl"), None) + assert openssl is not None + assert openssl["versionInfo"] == "3.6.0" + + sqlite = next((p for p in packages if p["name"] == "sqlite"), None) + assert sqlite is not None + assert sqlite["versionInfo"] == "3.50.4.0" + + # Old wheel package should be removed, new pip package should be present + pip_pkg = next((p for p in packages if p["name"] == "pip"), None) + assert pip_pkg is not None + assert pip_pkg["versionInfo"] == "23.0.1" + + wheel = next((p for p in packages if p["name"] == "wheel"), None) + assert wheel is None + + +def test_update_sbom_replaces_python_packages(tmp_path: pathlib.Path) -> None: + """Test that updating SBOM replaces Python packages with current scan.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + + # Mock Python version + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 12, 1)): + # Initially install pip and cowsay + (site_packages / "pip-23.0.1.dist-info").mkdir() + (site_packages / "cowsay-5.0.dist-info").mkdir() + + # First update + sbom.update_sbom(tmp_path) + + # Verify initial state + sbom_path = tmp_path / "relenv-sbom.spdx.json" + with open(sbom_path) as f: + sbom_data = json.load(f) + assert len(sbom_data["packages"]) == 2 + pkg_names = {p["name"] for p in sbom_data["packages"]} + assert pkg_names == {"pip", "cowsay"} + + # Now "uninstall" cowsay and "install" relenv + (site_packages / "cowsay-5.0.dist-info").rmdir() + (site_packages / "relenv-0.21.2.dist-info").mkdir() + + # Second update + sbom.update_sbom(tmp_path) + + # Verify updated state + with open(sbom_path) as f: + sbom_data = json.load(f) + assert len(sbom_data["packages"]) == 2 + pkg_names = {p["name"] for p in sbom_data["packages"]} + assert pkg_names == {"pip", "relenv"} + + +def test_update_sbom_python_version_too_old(tmp_path: pathlib.Path) -> None: + """Test that update_sbom fails gracefully for Python < 3.12.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + # Mock Python version to be 3.10 + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 10, 18)): + with pytest.raises( + RuntimeError, match="SBOM generation is only supported for Python 3.12+" + ): + sbom.update_sbom(tmp_path) + + +def test_main_success(tmp_path: pathlib.Path, capsys: pytest.CaptureFixture) -> None: + """Test main() with valid relenv environment.""" + # Create fake relenv environment + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3").touch() + + site_packages = tmp_path / "lib" / "python3.12" / "site-packages" + site_packages.mkdir(parents=True) + (site_packages / "pip-23.0.1.dist-info").mkdir() + + # Create args + import argparse + + args = argparse.Namespace(path=str(tmp_path)) + + # Mock Python version + with mock.patch("relenv.sbom.get_python_version", return_value=(3, 12, 1)): + # Run main + result = sbom.main(args) + assert result == 0 + + # Check output + captured = capsys.readouterr() + assert "Found relenv environment at:" in captured.out + assert "Updated" in captured.out + assert "relenv-sbom.spdx.json" in captured.out + + +def test_main_not_relenv(tmp_path: pathlib.Path, capsys: pytest.CaptureFixture) -> None: + """Test main() with non-relenv directory.""" + # Empty directory + import argparse + + args = argparse.Namespace(path=str(tmp_path)) + + # Run main + result = sbom.main(args) + assert result == 1 + + # Check error output + captured = capsys.readouterr() + assert "Error:" in captured.err + assert "Not a relenv environment" in captured.err diff --git a/tests/test_verify_build.py b/tests/test_verify_build.py index 3ed7a33a..fc283cb8 100644 --- a/tests/test_verify_build.py +++ b/tests/test_verify_build.py @@ -2142,10 +2142,9 @@ def test_expat_version(pyexec): # Convert "expat_2_7_3" -> "2.7.3" actual_version = actual_version_str.replace("expat_", "").replace("_", ".") - assert actual_version == expected_version, ( - f"Expat version mismatch on {platform}: expected {expected_version}, " - f"found {actual_version} (from {actual_version_str})" - ) + assert ( + actual_version == expected_version + ), f"Expat version mismatch on {platform}: expected {expected_version}, " def test_sqlite_version(pyexec): @@ -2245,10 +2244,9 @@ def test_openssl_version(pyexec): actual_version = match.group(1) - assert actual_version == expected_version, ( - f"OpenSSL version mismatch on {platform}: expected {expected_version}, " - f"found {actual_version} (from {actual_version_str})" - ) + assert ( + actual_version == expected_version + ), f"found {actual_version} (from {actual_version_str})" def test_xz_lzma_functionality(pyexec): @@ -2325,3 +2323,198 @@ def test_xz_lzma_functionality(pyexec): f"{proc.stderr.decode()}" ) assert proc.stdout.decode().strip() == "OK" + + +@pytest.mark.skip_on_windows +def test_sbom_files_exist(build, minor_version): + """Test that SBOM file exists for all Python versions. + + We generate a single comprehensive relenv-sbom.spdx.json that documents + all build dependencies and runtime packages. Python's native SBOM files + (only present in 3.12+) are not copied as they contain inaccurate/incomplete + information for our builds. + """ + # All Python versions should have relenv-sbom.spdx.json + relenv_sbom = build / "relenv-sbom.spdx.json" + assert ( + relenv_sbom.exists() + ), "relenv-sbom.spdx.json should exist for all Python versions" + + # Python's native SBOMs should NOT be present (we don't copy them anymore) + python_sbom = build / "sbom.spdx.json" + externals_sbom = build / "externals.spdx.json" + assert ( + not python_sbom.exists() + ), "sbom.spdx.json should not be copied (relenv-sbom.spdx.json is authoritative)" + assert ( + not externals_sbom.exists() + ), "externals.spdx.json should not be copied (relenv-sbom.spdx.json is authoritative)" + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_structure(build, minor_version): + """Test that relenv-sbom.spdx.json has valid SPDX structure.""" + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + with open(relenv_sbom) as f: + sbom = json.load(f) + + # Validate SPDX structure + assert sbom.get("SPDXID") == "SPDXRef-DOCUMENT" + assert sbom.get("spdxVersion") == "SPDX-2.3" + assert sbom.get("dataLicense") == "CC0-1.0" + assert "name" in sbom + assert "creationInfo" in sbom + assert "created" in sbom["creationInfo"] + assert "creators" in sbom["creationInfo"] + assert len(sbom["creationInfo"]["creators"]) > 0 + assert sbom["creationInfo"]["creators"][0].startswith("Tool: relenv-") + + # Should have packages + assert "packages" in sbom + assert isinstance(sbom["packages"], list) + assert len(sbom["packages"]) > 0 + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_has_pip_packages(build, minor_version): + """Test that relenv-sbom.spdx.json includes pip-installed packages.""" + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + with open(relenv_sbom) as f: + sbom = json.load(f) + + packages = sbom.get("packages", []) + package_names = {pkg.get("name") for pkg in packages} + + # Should include core pip packages + expected_packages = {"pip", "setuptools", "wheel", "relenv"} + found_packages = expected_packages & package_names + + assert len(found_packages) >= 3, ( + f"SBOM should contain at least 3 of {expected_packages}, " + f"but only found: {found_packages}" + ) + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_includes_python(build, minor_version): + """Test that relenv-sbom.spdx.json includes Python itself as a package.""" + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + with open(relenv_sbom) as f: + sbom = json.load(f) + + packages = sbom.get("packages", []) + package_names = {pkg.get("name") for pkg in packages} + + # Python itself should be documented + assert ( + "Python" in package_names + ), "SBOM should include Python (CPython interpreter) as a package" + + # Find the Python package and verify its structure + python_pkg = next((pkg for pkg in packages if pkg.get("name") == "Python"), None) + assert python_pkg is not None, "Python package should exist" + + # Verify Python package has correct fields + assert "versionInfo" in python_pkg, "Python package should have version" + # The versionInfo should match the full Python version (e.g., "3.11.14") + assert ( + python_pkg["versionInfo"] == minor_version + ), f"Python version should be {minor_version}, got {python_pkg.get('versionInfo')}" + assert ( + "downloadLocation" in python_pkg + ), "Python package should have download location" + assert ( + "python.org" in python_pkg["downloadLocation"].lower() + ), "Python download location should reference python.org" + assert ( + python_pkg.get("primaryPackagePurpose") == "APPLICATION" + ), "Python should be marked as APPLICATION" + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_validates_with_spdx_tools(build, minor_version): + """Test that relenv-sbom.spdx.json passes official SPDX validation.""" + pytest.importorskip("spdx_tools", reason="spdx-tools not installed") + + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + # Use official SPDX tools to validate the SBOM + from spdx_tools.spdx.parser.parse_anything import parse_file + from spdx_tools.spdx.validation.document_validator import ( + validate_full_spdx_document, + ) + + # Parse the SBOM file + try: + document = parse_file(str(relenv_sbom)) + except Exception as e: + pytest.fail(f"Failed to parse SBOM with spdx-tools: {e}") + + # Validate against SPDX 2.3 specification + validation_messages = validate_full_spdx_document(document) + + if validation_messages: + error_details = "\n".join(str(msg) for msg in validation_messages) + pytest.fail( + f"SBOM failed official SPDX validation with {len(validation_messages)} error(s):\n{error_details}" + ) + + +@pytest.mark.skip_on_windows +def test_relenv_sbom_includes_bundled_dependencies(build, minor_version): + """Test that relenv-sbom.spdx.json includes Python's bundled dependencies.""" + relenv_sbom = build / "relenv-sbom.spdx.json" + assert relenv_sbom.exists(), "relenv-sbom.spdx.json should exist" + + with open(relenv_sbom) as f: + sbom = json.load(f) + + packages = sbom.get("packages", []) + package_names = {pkg.get("name") for pkg in packages} + + # expat is bundled in Python's source tree and updated by relenv + # It's a critical security component (XML parser) + assert ( + "expat" in package_names + ), "SBOM should include expat (bundled XML parser updated by relenv)" + + # Verify expat package has required fields + expat_pkg = next((pkg for pkg in packages if pkg.get("name") == "expat"), None) + assert expat_pkg is not None, "expat package should exist" + assert "versionInfo" in expat_pkg, "expat package should have version" + assert ( + "downloadLocation" in expat_pkg + ), "expat package should have download location" + assert ( + "libexpat" in expat_pkg["downloadLocation"].lower() + ), "expat download location should reference libexpat GitHub" + assert expat_pkg.get("checksums"), "expat should have SHA-256 checksum" + + # For Python 3.12+, check for additional bundled dependencies + # These are extracted from Python's own SBOM + major, minor = map(int, minor_version.split(".")) + if major > 3 or (major == 3 and minor >= 12): + expected_bundled = ["mpdecimal", "hacl-star", "libb2"] + found_bundled = [name for name in expected_bundled if name in package_names] + + assert len(found_bundled) >= 2, ( + f"For Python 3.12+, SBOM should include bundled dependencies from Python source. " + f"Expected at least {expected_bundled[:2]}, found: {found_bundled}" + ) + + # Verify at least one bundled dep has proper structure + if "mpdecimal" in package_names: + mpdec_pkg = next( + (pkg for pkg in packages if pkg.get("name") == "mpdecimal"), None + ) + assert mpdec_pkg is not None + assert ( + mpdec_pkg.get("versionInfo") != "NOASSERTION" + ), "mpdecimal should have specific version from Python's SBOM"