diff --git a/relenv/build/common/__init__.py b/relenv/build/common/__init__.py index 4bfc0d1c..0c66f912 100644 --- a/relenv/build/common/__init__.py +++ b/relenv/build/common/__init__.py @@ -20,6 +20,7 @@ finalize, create_archive, patch_file, + update_sbom_checksums, ) from .builder import ( @@ -41,6 +42,7 @@ "create_archive", "update_ensurepip", "patch_file", + "update_sbom_checksums", # Builders (specific build functions) "build_openssl", "build_openssl_fips", diff --git a/relenv/build/common/install.py b/relenv/build/common/install.py index 04ab379b..4ab30d37 100644 --- a/relenv/build/common/install.py +++ b/relenv/build/common/install.py @@ -6,7 +6,9 @@ from __future__ import annotations import fnmatch +import hashlib import io +import json import logging import os import os.path @@ -75,6 +77,82 @@ def patch_file(path: PathLike, old: str, new: str) -> None: fp.write(new_content) +def update_sbom_checksums( + source_dir: PathLike, files_to_update: MutableMapping[str, PathLike] +) -> None: + """ + Update checksums in sbom.spdx.json for modified files. + + Python 3.12+ includes an SBOM (Software Bill of Materials) that tracks + file checksums. When we update files (e.g., expat sources), we need to + recalculate their checksums. + + :param source_dir: Path to the Python source directory + :type source_dir: PathLike + :param files_to_update: Mapping of SBOM relative paths to actual file paths + :type files_to_update: MutableMapping[str, PathLike] + """ + source_path = pathlib.Path(source_dir) + spdx_json = source_path / "Misc" / "sbom.spdx.json" + + # SBOM only exists in Python 3.12+ + if not spdx_json.exists(): + log.debug("SBOM file not found, skipping checksum updates") + return + + # Read the SBOM JSON + with open(spdx_json, "r") as f: + data = json.load(f) + + # Compute checksums for each file + checksums = {} + for relative_path, file_path in files_to_update.items(): + file_path = pathlib.Path(file_path) + if not file_path.exists(): + log.warning("File not found for checksum: %s", file_path) + continue + + # Compute SHA1 and SHA256 + sha1 = hashlib.sha1() + sha256 = hashlib.sha256() + with open(file_path, "rb") as f: + content = f.read() + sha1.update(content) + sha256.update(content) + + checksums[relative_path] = [ + { + "algorithm": "SHA1", + "checksumValue": sha1.hexdigest(), + }, + { + "algorithm": "SHA256", + "checksumValue": sha256.hexdigest(), + }, + ] + log.debug( + "Computed checksums for %s: SHA1=%s, SHA256=%s", + relative_path, + sha1.hexdigest(), + sha256.hexdigest(), + ) + + # Update the SBOM with new checksums + updated_count = 0 + for file_entry in data.get("files", []): + file_name = file_entry.get("fileName") + if file_name in checksums: + file_entry["checksums"] = checksums[file_name] + updated_count += 1 + log.info("Updated SBOM checksums for %s", file_name) + + # Write back the updated SBOM + with open(spdx_json, "w") as f: + json.dump(data, f, indent=2) + + log.info("Updated %d file checksums in SBOM", updated_count) + + def patch_shebang(path: PathLike, old: str, new: str) -> bool: """ Replace a file's shebang. diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index f9979c85..2bf0c019 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -6,7 +6,14 @@ """ from __future__ import annotations +import glob import io +import os +import pathlib +import shutil +import tarfile +import time +import urllib.request from typing import IO, MutableMapping from ..common import DARWIN, MACOS_DEVELOPMENT_TARGET, arches, runcmd @@ -17,6 +24,7 @@ builds, finalize, get_dependency_version, + update_sbom_checksums, ) ARCHES = arches[DARWIN] @@ -52,12 +60,6 @@ def update_expat(dirs: Dirs, env: MutableMapping[str, str]) -> None: Python ships with an older bundled expat. This function updates it to the latest version for security and bug fixes. """ - import pathlib - import shutil - import glob - import urllib.request - import tarfile - # Get version from JSON expat_info = get_dependency_version("expat", "darwin") if not expat_info: @@ -84,6 +86,7 @@ def update_expat(dirs: Dirs, env: MutableMapping[str, str]) -> None: # Copy source files to Modules/expat/ expat_source_dir = tmpbuild / f"expat-{version}" / "lib" + updated_files = [] for source_file in ["*.h", "*.c"]: for file_path in glob.glob(str(expat_source_dir / source_file)): target_file = expat_dir / pathlib.Path(file_path).name @@ -91,6 +94,22 @@ def update_expat(dirs: Dirs, env: MutableMapping[str, str]) -> None: if target_file.exists(): target_file.unlink() shutil.copy2(file_path, str(expat_dir)) + updated_files.append(target_file) + + # Touch all updated files to ensure make rebuilds them + # (The tarball may contain files with newer timestamps) + now = time.time() + for target_file in updated_files: + os.utime(target_file, (now, now)) + + # Update SBOM with correct checksums for updated expat files + files_to_update = {} + for target_file in updated_files: + # SBOM uses relative paths from Python source root + relative_path = f"Modules/expat/{target_file.name}" + files_to_update[relative_path] = target_file + + update_sbom_checksums(dirs.source, files_to_update) def build_python(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> None: diff --git a/relenv/build/linux.py b/relenv/build/linux.py index 85939101..5a0024c5 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -6,11 +6,15 @@ """ from __future__ import annotations +import glob import io import os import pathlib import shutil +import tarfile import tempfile +import time +import urllib.request from typing import IO, MutableMapping from .common import ( @@ -21,6 +25,7 @@ builds, finalize, get_dependency_version, + update_sbom_checksums, ) from ..common import LINUX, Version, arches, runcmd @@ -367,13 +372,6 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: Python ships with an older bundled expat. This function updates it to the latest version for security and bug fixes. """ - from .common import get_dependency_version - import urllib.request - import tarfile - import glob - import pathlib - import shutil - # Get version from JSON expat_info = get_dependency_version("expat", "linux") if not expat_info: @@ -400,6 +398,7 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: # Copy source files to Modules/expat/ expat_source_dir = tmpbuild / f"expat-{version}" / "lib" + updated_files = [] for source_file in ["*.h", "*.c"]: for file_path in glob.glob(str(expat_source_dir / source_file)): target_file = expat_dir / pathlib.Path(file_path).name @@ -407,6 +406,22 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: if target_file.exists(): target_file.unlink() shutil.copy2(file_path, str(expat_dir)) + updated_files.append(target_file) + + # Touch all updated files to ensure make rebuilds them + # (The tarball may contain files with newer timestamps) + now = time.time() + for target_file in updated_files: + os.utime(target_file, (now, now)) + + # Update SBOM with correct checksums for updated expat files + files_to_update = {} + for target_file in updated_files: + # SBOM uses relative paths from Python source root + relative_path = f"Modules/expat/{target_file.name}" + files_to_update[relative_path] = target_file + + update_sbom_checksums(dirs.source, files_to_update) def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: diff --git a/relenv/build/windows.py b/relenv/build/windows.py index 1a4c80af..a88f41db 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -14,6 +14,7 @@ import shutil import sys import tarfile +import time from typing import IO, MutableMapping, Union from .common import ( @@ -24,10 +25,10 @@ install_runtime, patch_file, update_ensurepip, + update_sbom_checksums, ) from ..common import ( WIN32, - Version, arches, MODULE_DIR, download_url, @@ -149,6 +150,25 @@ def update_sqlite(dirs: Dirs, env: EnvMapping) -> None: def update_xz(dirs: Dirs, env: EnvMapping) -> None: """ Update the XZ library. + + COMPATIBILITY NOTE: We use config.h from XZ 5.4.7 for all XZ versions. + Starting with XZ 5.5.0, the project removed Visual Studio .vcxproj files + and switched to CMake. Python's build system (PCbuild/liblzma.vcxproj) + still expects MSBuild-compatible builds, so we maintain a compatibility + shim at relenv/_resources/xz/config.h. + + When updating XZ versions, verify compatibility by checking: + 1. Build completes without compiler errors + 2. test_xz_lzma_functionality passes + 3. No new HAVE_* defines required in src/liblzma source files + 4. No removed HAVE_* defines that config.h references + + If compatibility breaks, you have two options: + - Use CMake to generate new config.h for Windows (see discussion at + https://discuss.python.org/t/building-python-from-source-on-windows-using-a-custom-version-of-xz/74717) + - Update relenv/_resources/xz/config.h manually from newer XZ source + + See also: relenv/_resources/xz/readme.md """ # Try to get version from JSON # Note: Windows may use a different XZ version than Linux/Darwin due to MSBuild compatibility @@ -225,134 +245,40 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None: # Copy *.h and *.c to expat directory expat_lib_dir = dirs.source / "Modules" / "expat" / f"expat-{version}" / "lib" expat_dir = dirs.source / "Modules" / "expat" + updated_files = [] for file in glob.glob(str(expat_lib_dir / "*.h")): - if expat_dir / os.path.basename(file): - (expat_dir / os.path.basename(file)).unlink() + target = expat_dir / os.path.basename(file) + if target.exists(): + target.unlink() shutil.move(file, str(expat_dir)) + updated_files.append(target) for file in glob.glob(str(expat_lib_dir / "*.c")): - if expat_dir / os.path.basename(file): - (expat_dir / os.path.basename(file)).unlink() + target = expat_dir / os.path.basename(file) + if target.exists(): + target.unlink() shutil.move(file, str(expat_dir)) - # Update sbom.spdx.json with the correct hashes. This became a thing in 3.12 - # python Tools/build/generate_sbom.py doesn't work because it requires a git - # repository, so we have to do it manually. - if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: - checksums = { - "Modules/expat/expat.h": [ - { - "algorithm": "SHA1", - "checksumValue": "a4395dd0589a97aab0904f7a5f5dc5781a086aa2", - }, - { - "algorithm": "SHA256", - "checksumValue": "610b844bbfa3ec955772cc825db4d4db470827d57adcb214ad372d0eaf00e591", - }, - ], - "Modules/expat/expat_external.h": [ - { - "algorithm": "SHA1", - "checksumValue": "8fdf2e79a7ab46a3c76c74ed7e5fe641cbef308d", - }, - { - "algorithm": "SHA256", - "checksumValue": "ffb960af48b80935f3856a16e87023524b104f7fc1e58104f01db88ba7bfbcc9", - }, - ], - "Modules/expat/internal.h": [ - { - "algorithm": "SHA1", - "checksumValue": "7dce7d98943c5db33ae05e54801dcafb4547b9dd", - }, - { - "algorithm": "SHA256", - "checksumValue": "6bfe307d52e7e4c71dbc30d3bd902a4905cdd83bbe4226a7e8dfa8e4c462a157", - }, - ], - "Modules/expat/refresh.sh": [ - { - "algorithm": "SHA1", - "checksumValue": "71812ca27328697a8dcae1949cd638717538321a", - }, - { - "algorithm": "SHA256", - "checksumValue": "64fd1368de41e4ebc14593c65f5a676558aed44bd7d71c43ae05d06f9086d3b0", - }, - ], - "Modules/expat/xmlparse.c": [ - { - "algorithm": "SHA1", - "checksumValue": "4c81a1f04fc653877c63c834145c18f93cd95f3e", - }, - { - "algorithm": "SHA256", - "checksumValue": "04a379615f476d55f95ca1853107e20627b48ca4afe8d0fd5981ac77188bf0a6", - }, - ], - "Modules/expat/xmlrole.h": [ - { - "algorithm": "SHA1", - "checksumValue": "ac2964cca107f62dd133bfd4736a9a17defbc401", - }, - { - "algorithm": "SHA256", - "checksumValue": "92e41f373b67f6e0dcd7735faef3c3f1e2c17fe59e007e6b74beef6a2e70fa88", - }, - ], - "Modules/expat/xmltok.c": [ - { - "algorithm": "SHA1", - "checksumValue": "1e2d35d90a1c269217f83d3bdf3c71cc22cb4c3f", - }, - { - "algorithm": "SHA256", - "checksumValue": "98d0fc735041956cc2e7bbbe2fb8f03130859410e0aee5e8015f406a37c02a3c", - }, - ], - "Modules/expat/xmltok.h": [ - { - "algorithm": "SHA1", - "checksumValue": "d126831eaa5158cff187a8c93f4bc1c8118f3b17", - }, - { - "algorithm": "SHA256", - "checksumValue": "91bf003a725a675761ea8d92cebc299a76fd28c3a950572f41bc7ce5327ee7b5", - }, - ], - } - spdx_json = dirs.source / "Misc" / "sbom.spdx.json" - with open(str(spdx_json), "r") as f: - data = json.load(f) - for file in data["files"]: - if file["fileName"] in checksums.keys(): - print(file["fileName"]) - file["checksums"] = checksums[file["fileName"]] - with open(str(spdx_json), "w") as f: - json.dump(data, f, indent=2) + updated_files.append(target) + + # Touch all updated files to ensure MSBuild rebuilds them + # (The original files may have newer timestamps) + now = time.time() + for target_file in updated_files: + os.utime(target_file, (now, now)) + + # Update SBOM with correct checksums for updated expat files + # Map SBOM file names to actual file paths + files_to_update = {} + for target_file in updated_files: + # SBOM uses relative paths from Python source root + relative_path = f"Modules/expat/{target_file.name}" + files_to_update[relative_path] = target_file + + # Also include refresh.sh which was patched + bash_refresh = dirs.source / "Modules" / "expat" / "refresh.sh" + if bash_refresh.exists(): + files_to_update["Modules/expat/refresh.sh"] = bash_refresh - -def update_expat_check(env: EnvMapping) -> bool: - """ - Check if the given python version should get an updated libexpat. - - Patch libexpat on these versions and below: - - 3.9.23 - - 3.10.18 - - 3.11.13 - - 3.12.11 - - 3.13.7 - """ - relenv_version = Version(env["RELENV_PY_VERSION"]) - if relenv_version.minor == 9 and relenv_version.micro <= 23: - return True - elif relenv_version.minor == 10 and relenv_version.micro <= 18: - return True - elif relenv_version.minor == 11 and relenv_version.micro <= 13: - return True - elif relenv_version.minor == 12 and relenv_version.micro <= 11: - return True - elif relenv_version.minor == 13 and relenv_version.micro <= 7: - return True - return False + update_sbom_checksums(dirs.source, files_to_update) def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: @@ -372,27 +298,11 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: externals_dir = dirs.source / "externals" externals_dir.mkdir(parents=True, exist_ok=True) - # SQLITE - # TODO: Python 3.12 started creating an SBOM. We're doing something wrong - # TODO: updating sqlite so SBOM creation is failing. Gating here until we - # TODO: fix this. Here's the original gate: - # if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12"]: - if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11"]: - update_sqlite(dirs=dirs, env=env) - - # XZ-Utils - # TODO: Python 3.12 started creating an SBOM. We're doing something wrong - # TODO: updating XZ so SBOM creation is failing. Gating here until we fix - # TODO: this. Here's the original gate: - # if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11", "3.12", "3.13", "3.14"]: - if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11"]: - update_xz(dirs=dirs, env=env) - - # TODO: This was my attempt to fix the expat error during build... it failed - # TODO: so we're just skipping for now. - if env["RELENV_PY_MAJOR_VERSION"] in ["3.10", "3.11"]: - if update_expat_check(env=env): - update_expat(dirs=dirs, env=env) + update_sqlite(dirs=dirs, env=env) + + update_xz(dirs=dirs, env=env) + + update_expat(dirs=dirs, env=env) arch_to_plat = { "amd64": "x64", diff --git a/tests/test_verify_build.py b/tests/test_verify_build.py index 22066ad2..3ed7a33a 100644 --- a/tests/test_verify_build.py +++ b/tests/test_verify_build.py @@ -2096,3 +2096,232 @@ def test_install_setuptools_25_2_to_25_3(pipexec, build, minor_version, pip_vers ], check=True, ) + + +def test_expat_version(pyexec): + """ + Verify that the build contains the correct expat version. + + This validates that update_expat() successfully updated the bundled + expat library to match the version in python-versions.json. + + Works on all platforms: Linux, Darwin (macOS), and Windows. + """ + from relenv.build.common import get_dependency_version + + # Map sys.platform to relenv platform names + platform_map = { + "linux": "linux", + "darwin": "darwin", + "win32": "win32", + } + platform = platform_map.get(sys.platform) + if not platform: + pytest.skip(f"Unknown platform: {sys.platform}") + + # Get expected version from python-versions.json + expat_info = get_dependency_version("expat", platform) + if not expat_info: + pytest.skip(f"No expat version defined in python-versions.json for {platform}") + + expected_version = expat_info["version"] + + # Get actual version from the build + proc = subprocess.run( + [str(pyexec), "-c", "import pyexpat; print(pyexpat.EXPAT_VERSION)"], + capture_output=True, + check=True, + ) + + actual_version_str = proc.stdout.decode().strip() + # Format is "expat_X_Y_Z", extract version + assert actual_version_str.startswith( + "expat_" + ), f"Unexpected EXPAT_VERSION format: {actual_version_str}" + + # Convert "expat_2_7_3" -> "2.7.3" + actual_version = actual_version_str.replace("expat_", "").replace("_", ".") + + assert actual_version == expected_version, ( + f"Expat version mismatch on {platform}: expected {expected_version}, " + f"found {actual_version} (from {actual_version_str})" + ) + + +def test_sqlite_version(pyexec): + """ + Verify that the build contains the correct SQLite version. + + This validates that SQLite was built with the version specified + in python-versions.json. + + Works on all platforms: Linux, Darwin (macOS), and Windows. + """ + from relenv.build.common import get_dependency_version + + # Map sys.platform to relenv platform names + platform_map = { + "linux": "linux", + "darwin": "darwin", + "win32": "win32", + } + platform = platform_map.get(sys.platform) + if not platform: + pytest.skip(f"Unknown platform: {sys.platform}") + + # Get expected version from python-versions.json + sqlite_info = get_dependency_version("sqlite", platform) + if not sqlite_info: + pytest.skip(f"No sqlite version defined in python-versions.json for {platform}") + + expected_version = sqlite_info["version"] + + # Get actual version from the build + proc = subprocess.run( + [str(pyexec), "-c", "import sqlite3; print(sqlite3.sqlite_version)"], + capture_output=True, + check=True, + ) + + actual_version = proc.stdout.decode().strip() + + # SQLite version in JSON is like "3.50.4.0" but runtime shows "3.50.4" + # So we need to handle both formats + if expected_version.count(".") == 3: + # Remove trailing .0 for comparison + expected_version = ".".join(expected_version.split(".")[:3]) + + assert actual_version == expected_version, ( + f"SQLite version mismatch on {platform}: expected {expected_version}, " + f"found {actual_version}" + ) + + +def test_openssl_version(pyexec): + """ + Verify that the build contains the correct OpenSSL version. + + This validates that OpenSSL was built with the version specified + in python-versions.json. + + Works on all platforms: Linux, Darwin (macOS), and Windows. + """ + import re + + from relenv.build.common import get_dependency_version + + # Map sys.platform to relenv platform names + platform_map = { + "linux": "linux", + "darwin": "darwin", + "win32": "win32", + } + platform = platform_map.get(sys.platform) + if not platform: + pytest.skip(f"Unknown platform: {sys.platform}") + + # Get expected version from python-versions.json + openssl_info = get_dependency_version("openssl", platform) + if not openssl_info: + pytest.skip( + f"No openssl version defined in python-versions.json for {platform}" + ) + + expected_version = openssl_info["version"] + + # Get actual version from the build + proc = subprocess.run( + [str(pyexec), "-c", "import ssl; print(ssl.OPENSSL_VERSION)"], + capture_output=True, + check=True, + ) + + actual_version_str = proc.stdout.decode().strip() + # Format is "OpenSSL 3.5.4 30 Sep 2025" + # Extract version number + match = re.search(r"OpenSSL (\d+\.\d+\.\d+)", actual_version_str) + if not match: + pytest.fail(f"Could not parse OpenSSL version from: {actual_version_str}") + + actual_version = match.group(1) + + assert actual_version == expected_version, ( + f"OpenSSL version mismatch on {platform}: expected {expected_version}, " + f"found {actual_version} (from {actual_version_str})" + ) + + +def test_xz_lzma_functionality(pyexec): + """ + Verify that the lzma module works correctly. + + This is especially important for Windows builds which use a config.h + compatibility shim from XZ 5.4.7 to support newer XZ versions that + removed MSBuild support. + + If this test fails, it indicates that the config.h in + relenv/_resources/xz/config.h is no longer compatible with the + current XZ version being used. + + Works on all platforms: Linux, Darwin (macOS), and Windows. + """ + # Test that lzma module loads and basic compression works + test_code = """ +import lzma +import sys + +# Test basic compression/decompression +test_data = b"Hello, World! " * 100 +compressed = lzma.compress(test_data) +decompressed = lzma.decompress(compressed) + +if test_data != decompressed: + print("ERROR: Decompressed data does not match original", file=sys.stderr) + sys.exit(1) + +# Verify compression actually happened +if len(compressed) >= len(test_data): + print("ERROR: Compression did not reduce size", file=sys.stderr) + sys.exit(2) + +# Test different formats (skip FORMAT_RAW as it requires explicit filters) +for fmt in [lzma.FORMAT_XZ, lzma.FORMAT_ALONE]: + try: + data = lzma.compress(test_data, format=fmt) + result = lzma.decompress(data) + if result != test_data: + print(f"ERROR: Format {fmt} failed round-trip", file=sys.stderr) + sys.exit(3) + except Exception as e: + print(f"ERROR: Format {fmt} raised exception: {e}", file=sys.stderr) + sys.exit(4) + +# Test streaming compression/decompression +import io +output = io.BytesIO() +with lzma.LZMAFile(output, "w") as f: + f.write(test_data) + +compressed_stream = output.getvalue() +input_stream = io.BytesIO(compressed_stream) +with lzma.LZMAFile(input_stream, "r") as f: + decompressed_stream = f.read() + +if decompressed_stream != test_data: + print("ERROR: Streaming compression/decompression failed", file=sys.stderr) + sys.exit(5) + +print("OK") +""" + + proc = subprocess.run( + [str(pyexec), "-c", test_code], + capture_output=True, + check=False, + ) + + assert proc.returncode == 0, ( + f"LZMA functionality test failed (exit code {proc.returncode}): " + f"{proc.stderr.decode()}" + ) + assert proc.stdout.decode().strip() == "OK"