Skip to content

Commit

Permalink
Stop compressing applehv and hyperv by default
Browse files Browse the repository at this point in the history
Apple Hypervisor doesn't inherently require images to be compressed with
gzip. It's just that when we _do_ compress it, it's the most convenient
format to use because gzip is guaranteed to be available on macOS.

Similarly for Windows Hyper-V and ZIP.

Notably, this is different from e.g. GCP, where the platform itself
dictates a `tar.gz` file.

And so for consistency we should have the output from the build
step for `applehv` and `hyperv` just return the disk image in the
format it's intended to be used in, and then `cosa compress` just
compresses them using e.g. `gzip` or `zip`. This requires adding a new
`platform-compressor` key in the `image.yaml` file to allow overridding
the default `compressor` setting for certain platforms.

This allows folks using the same code to build the disk images for those
platforms and compress them with the compressor of their choice.
  • Loading branch information
jlebon committed Dec 3, 2024
1 parent 1353d16 commit 3ca025b
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 15 deletions.
26 changes: 17 additions & 9 deletions src/cmd-compress
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ else:
print(f"Targeting build: {build}")

# common extensions for known compressors
ext_dict = {'xz': '.xz', 'gzip': '.gz', 'zstd': '.zst'}
ext_dict = {'xz': '.xz', 'gzip': '.gz', 'zstd': '.zst', 'zip': '.zip'}


def get_cpu_param(param):
Expand All @@ -65,7 +65,7 @@ def strip_ext(path):
return path.rsplit(".", 1)[0]


def compress_one_builddir(builddir):
def compress_one_builddir(builddir, platform_compressors):
print(f"Compressing: {builddir}")
buildmeta_path = os.path.join(builddir, 'meta.json')
# In the case where we are doing builds for different architectures
Expand All @@ -77,9 +77,6 @@ def compress_one_builddir(builddir):
with open(buildmeta_path) as f:
buildmeta = json.load(f)

# Find what extension to use based on the selected compressor
ext = ext_dict[args.compressor]

tmpdir = 'tmp/compress'
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
Expand All @@ -106,6 +103,10 @@ def compress_one_builddir(builddir):
if only_artifacts is not None and img_format not in only_artifacts:
continue

compressor = platform_compressors.get(img_format) or args.compressor
# Find what extension to use based on the selected compressor
ext = ext_dict[compressor]

file = img['path']
filepath = os.path.join(builddir, file)
if img.get('uncompressed-sha256') is None:
Expand All @@ -116,12 +117,16 @@ def compress_one_builddir(builddir):
img['uncompressed-size'] = img['size']
with open(tmpfile, 'wb') as f:
t = ncpu()
if args.compressor == 'xz':
if compressor == 'xz':
runcmd(['xz', '-c9', f'-T{t}', filepath], stdout=f)
elif args.compressor == 'zstd':
elif compressor == 'zstd':
runcmd(['zstd', '-10', '-c', f'-T{t}', filepath], stdout=f)
else:
elif compressor == 'gzip':
runcmd(['gzip', f'-{gzip_level}', '-c', filepath], stdout=f) # pylint: disable=E0606
elif compressor == 'zip':
runcmd(['zip', '-9j', '-', filepath], stdout=f) # pylint: disable=E0606
else:
raise Exception(f"Unknown compressor: {compressor}")
file_with_ext = file + ext
filepath_with_ext = filepath + ext
compressed_size = os.path.getsize(tmpfile)
Expand Down Expand Up @@ -202,6 +207,8 @@ def uncompress_one_builddir(builddir):
runcmd(['zstd', '-dc', filepath], stdout=f)
elif file.endswith('gz'):
runcmd(['gzip', '-dc', filepath], stdout=f)
elif file.endswith('zip'):
runcmd(['unzip', '-p', filepath], stdout=f)
else:
print(f"Unknown sufix of file {file}")
file_without_ext = strip_ext(file)
Expand Down Expand Up @@ -275,7 +282,8 @@ if args.mode == "compress":
args.compressor = image_json.get('compressor', DEFAULT_COMPRESSOR)
for arch in builds.get_build_arches(build):
builddir = builds.get_build_dir(build, arch)
changed.append(compress_one_builddir(builddir))
changed.append(compress_one_builddir(builddir,
image_json.get('platform-compressor', {})))
if not any(changed):
print("All builds already compressed")
elif args.mode == "uncompress":
Expand Down
8 changes: 2 additions & 6 deletions src/cosalib/qemuvariants.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,7 @@
},
"applehv": {
"image_format": "raw",
"image_suffix": "raw.gz",
"platform": "applehv",
"compression": "gzip"
"platform": "applehv"
},
"azure": {
"image_format": "vpc",
Expand Down Expand Up @@ -99,9 +97,7 @@
},
"hyperv": {
"image_format": "vhdx",
"image_suffix": "vhdx.zip",
"platform": "hyperv",
"compression": "zip"
"platform": "hyperv"
},
"kubevirt": {
"image_format": "qcow2",
Expand Down

0 comments on commit 3ca025b

Please sign in to comment.