Skip to content

Commit 9e2a80b

Browse files
dwozdwoz
committed
Add tests to verify our version updates are working
1 parent add013c commit 9e2a80b

File tree

5 files changed

+265
-107
lines changed

5 files changed

+265
-107
lines changed

‎relenv/build/common.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,72 @@ def verify_checksum(file: PathLike, checksum: Optional[str]) -> bool:
236236
return True
237237

238238

239+
def compute_checksums(file: PathLike) -> Dict[str, str]:
240+
"""
241+
Compute SHA-1 and SHA-256 checksums for a file.
242+
243+
Returns a dictionary with 'sha1' and 'sha256' keys.
244+
245+
:param file: The path to the file to checksum
246+
:type file: str or PathLike
247+
248+
:return: Dictionary with 'sha1' and 'sha256' checksums
249+
:rtype: dict
250+
"""
251+
sha1 = hashlib.sha1()
252+
sha256 = hashlib.sha256()
253+
254+
with open(file, "rb") as fp:
255+
data = fp.read()
256+
sha1.update(data)
257+
sha256.update(data)
258+
259+
return {
260+
"sha1": sha1.hexdigest(),
261+
"sha256": sha256.hexdigest(),
262+
}
263+
264+
265+
def update_sbom_checksums(
266+
source_dir: PathLike, files_to_update: Dict[str, PathLike]
267+
) -> None:
268+
"""
269+
Update checksums in Python's SBOM file for modified source files.
270+
271+
This function updates Misc/sbom.spdx.json with correct checksums for
272+
files that have been replaced (e.g., updated expat library files).
273+
274+
:param source_dir: The Python source directory containing Misc/sbom.spdx.json
275+
:type source_dir: PathLike
276+
:param files_to_update: Dict mapping SBOM file names to actual file paths
277+
:type files_to_update: dict
278+
"""
279+
import json
280+
import pathlib
281+
282+
spdx_json = pathlib.Path(source_dir) / "Misc" / "sbom.spdx.json"
283+
if not spdx_json.exists():
284+
# Python < 3.12 doesn't have SBOM files
285+
return
286+
287+
with open(str(spdx_json), "r") as f:
288+
data = json.load(f)
289+
290+
# Update checksums for each file in the SBOM
291+
for file_entry in data.get("files", []):
292+
file_name = file_entry.get("fileName", "")
293+
if file_name in files_to_update:
294+
file_path = files_to_update[file_name]
295+
checksums = compute_checksums(file_path)
296+
file_entry["checksums"] = [
297+
{"algorithm": "SHA1", "checksumValue": checksums["sha1"]},
298+
{"algorithm": "SHA256", "checksumValue": checksums["sha256"]},
299+
]
300+
301+
with open(str(spdx_json), "w") as f:
302+
json.dump(data, f, indent=2)
303+
304+
239305
def all_dirs(root: PathLike, recurse: bool = True) -> List[str]:
240306
"""
241307
Get all directories under and including the given root.

‎relenv/build/darwin.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,13 +85,35 @@ def update_expat(dirs: Dirs, env: MutableMapping[str, str]) -> None:
8585

8686
# Copy source files to Modules/expat/
8787
expat_source_dir = tmpbuild / f"expat-{version}" / "lib"
88+
updated_files = []
8889
for source_file in ["*.h", "*.c"]:
8990
for file_path in glob.glob(str(expat_source_dir / source_file)):
9091
target_file = expat_dir / pathlib.Path(file_path).name
9192
# Remove old file if it exists
9293
if target_file.exists():
9394
target_file.unlink()
9495
shutil.copy2(file_path, str(expat_dir))
96+
updated_files.append(target_file)
97+
98+
# Touch all updated files to ensure make rebuilds them
99+
# (The tarball may contain files with newer timestamps)
100+
import time
101+
import os
102+
103+
now = time.time()
104+
for target_file in updated_files:
105+
os.utime(target_file, (now, now))
106+
107+
# Update SBOM with correct checksums for updated expat files
108+
from relenv.build.common import update_sbom_checksums
109+
110+
files_to_update = {}
111+
for target_file in updated_files:
112+
# SBOM uses relative paths from Python source root
113+
relative_path = f"Modules/expat/{target_file.name}"
114+
files_to_update[relative_path] = target_file
115+
116+
update_sbom_checksums(dirs.source, files_to_update)
95117

96118

97119
def build_python(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> None:

‎relenv/build/linux.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,13 +401,34 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None:
401401

402402
# Copy source files to Modules/expat/
403403
expat_source_dir = tmpbuild / f"expat-{version}" / "lib"
404+
updated_files = []
404405
for source_file in ["*.h", "*.c"]:
405406
for file_path in glob.glob(str(expat_source_dir / source_file)):
406407
target_file = expat_dir / pathlib.Path(file_path).name
407408
# Remove old file if it exists
408409
if target_file.exists():
409410
target_file.unlink()
410411
shutil.copy2(file_path, str(expat_dir))
412+
updated_files.append(target_file)
413+
414+
# Touch all updated files to ensure make rebuilds them
415+
# (The tarball may contain files with newer timestamps)
416+
import time
417+
418+
now = time.time()
419+
for target_file in updated_files:
420+
os.utime(target_file, (now, now))
421+
422+
# Update SBOM with correct checksums for updated expat files
423+
from relenv.build.common import update_sbom_checksums
424+
425+
files_to_update = {}
426+
for target_file in updated_files:
427+
# SBOM uses relative paths from Python source root
428+
relative_path = f"Modules/expat/{target_file.name}"
429+
files_to_update[relative_path] = target_file
430+
431+
update_sbom_checksums(dirs.source, files_to_update)
411432

412433

413434
def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None:

‎relenv/build/windows.py‎

Lines changed: 34 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -221,109 +221,44 @@ def update_expat(dirs: Dirs, env: EnvMapping) -> None:
221221
# Copy *.h and *.c to expat directory
222222
expat_lib_dir = dirs.source / "Modules" / "expat" / f"expat-{version}" / "lib"
223223
expat_dir = dirs.source / "Modules" / "expat"
224+
updated_files = []
224225
for file in glob.glob(str(expat_lib_dir / "*.h")):
225-
if expat_dir / os.path.basename(file):
226-
(expat_dir / os.path.basename(file)).unlink()
226+
target = expat_dir / os.path.basename(file)
227+
if target.exists():
228+
target.unlink()
227229
shutil.move(file, str(expat_dir))
230+
updated_files.append(target)
228231
for file in glob.glob(str(expat_lib_dir / "*.c")):
229-
if expat_dir / os.path.basename(file):
230-
(expat_dir / os.path.basename(file)).unlink()
232+
target = expat_dir / os.path.basename(file)
233+
if target.exists():
234+
target.unlink()
231235
shutil.move(file, str(expat_dir))
232-
# Update sbom.spdx.json with the correct hashes. This became a thing in 3.12
233-
# python Tools/build/generate_sbom.py doesn't work because it requires a git
234-
# repository, so we have to do it manually.
235-
if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]:
236-
checksums = {
237-
"Modules/expat/expat.h": [
238-
{
239-
"algorithm": "SHA1",
240-
"checksumValue": "a4395dd0589a97aab0904f7a5f5dc5781a086aa2",
241-
},
242-
{
243-
"algorithm": "SHA256",
244-
"checksumValue": "610b844bbfa3ec955772cc825db4d4db470827d57adcb214ad372d0eaf00e591",
245-
},
246-
],
247-
"Modules/expat/expat_external.h": [
248-
{
249-
"algorithm": "SHA1",
250-
"checksumValue": "8fdf2e79a7ab46a3c76c74ed7e5fe641cbef308d",
251-
},
252-
{
253-
"algorithm": "SHA256",
254-
"checksumValue": "ffb960af48b80935f3856a16e87023524b104f7fc1e58104f01db88ba7bfbcc9",
255-
},
256-
],
257-
"Modules/expat/internal.h": [
258-
{
259-
"algorithm": "SHA1",
260-
"checksumValue": "7dce7d98943c5db33ae05e54801dcafb4547b9dd",
261-
},
262-
{
263-
"algorithm": "SHA256",
264-
"checksumValue": "6bfe307d52e7e4c71dbc30d3bd902a4905cdd83bbe4226a7e8dfa8e4c462a157",
265-
},
266-
],
267-
"Modules/expat/refresh.sh": [
268-
{
269-
"algorithm": "SHA1",
270-
"checksumValue": "71812ca27328697a8dcae1949cd638717538321a",
271-
},
272-
{
273-
"algorithm": "SHA256",
274-
"checksumValue": "64fd1368de41e4ebc14593c65f5a676558aed44bd7d71c43ae05d06f9086d3b0",
275-
},
276-
],
277-
"Modules/expat/xmlparse.c": [
278-
{
279-
"algorithm": "SHA1",
280-
"checksumValue": "4c81a1f04fc653877c63c834145c18f93cd95f3e",
281-
},
282-
{
283-
"algorithm": "SHA256",
284-
"checksumValue": "04a379615f476d55f95ca1853107e20627b48ca4afe8d0fd5981ac77188bf0a6",
285-
},
286-
],
287-
"Modules/expat/xmlrole.h": [
288-
{
289-
"algorithm": "SHA1",
290-
"checksumValue": "ac2964cca107f62dd133bfd4736a9a17defbc401",
291-
},
292-
{
293-
"algorithm": "SHA256",
294-
"checksumValue": "92e41f373b67f6e0dcd7735faef3c3f1e2c17fe59e007e6b74beef6a2e70fa88",
295-
},
296-
],
297-
"Modules/expat/xmltok.c": [
298-
{
299-
"algorithm": "SHA1",
300-
"checksumValue": "1e2d35d90a1c269217f83d3bdf3c71cc22cb4c3f",
301-
},
302-
{
303-
"algorithm": "SHA256",
304-
"checksumValue": "98d0fc735041956cc2e7bbbe2fb8f03130859410e0aee5e8015f406a37c02a3c",
305-
},
306-
],
307-
"Modules/expat/xmltok.h": [
308-
{
309-
"algorithm": "SHA1",
310-
"checksumValue": "d126831eaa5158cff187a8c93f4bc1c8118f3b17",
311-
},
312-
{
313-
"algorithm": "SHA256",
314-
"checksumValue": "91bf003a725a675761ea8d92cebc299a76fd28c3a950572f41bc7ce5327ee7b5",
315-
},
316-
],
317-
}
318-
spdx_json = dirs.source / "Misc" / "sbom.spdx.json"
319-
with open(str(spdx_json), "r") as f:
320-
data = json.load(f)
321-
for file in data["files"]:
322-
if file["fileName"] in checksums.keys():
323-
print(file["fileName"])
324-
file["checksums"] = checksums[file["fileName"]]
325-
with open(str(spdx_json), "w") as f:
326-
json.dump(data, f, indent=2)
236+
updated_files.append(target)
237+
238+
# Touch all updated files to ensure MSBuild rebuilds them
239+
# (The original files may have newer timestamps)
240+
import time
241+
242+
now = time.time()
243+
for target_file in updated_files:
244+
os.utime(target_file, (now, now))
245+
246+
# Update SBOM with correct checksums for updated expat files
247+
# Map SBOM file names to actual file paths
248+
from relenv.build.common import update_sbom_checksums
249+
250+
files_to_update = {}
251+
for target_file in updated_files:
252+
# SBOM uses relative paths from Python source root
253+
relative_path = f"Modules/expat/{target_file.name}"
254+
files_to_update[relative_path] = target_file
255+
256+
# Also include refresh.sh which was patched
257+
bash_refresh = dirs.source / "Modules" / "expat" / "refresh.sh"
258+
if bash_refresh.exists():
259+
files_to_update["Modules/expat/refresh.sh"] = bash_refresh
260+
261+
update_sbom_checksums(dirs.source, files_to_update)
327262

328263

329264
def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None:

0 commit comments

Comments
 (0)