Skip to content

Commit 279a0b7

Browse files
committed
experiment with using artifacts as cache
1 parent c29d803 commit 279a0b7

File tree

9 files changed

+278
-335
lines changed

9 files changed

+278
-335
lines changed

.github/workflows/build-images.yml

Lines changed: 145 additions & 244 deletions
Large diffs are not rendered by default.

docs/workflow-dag.txt

Whitespace-only changes.

images/README.md

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,27 +17,46 @@ images. It also has some helper functions. It should be sourced before running
1717

1818
Then run `build.sh`, providing it an image directory.
1919

20-
When building locally for testing, you need podman installed. Then do the
21-
following:
20+
Building locally has the following requirements:
21+
22+
- podman installed
23+
- docker installed
24+
- docker registry running on localhost:5000
25+
- i.e. with `docker run -p 5000:5000 --rm --name registry registry`
26+
- bioconda-utils installed, along with test dependencies
27+
- i.e. with `conda create -p ./env --file bioconda_utils/bioconda_utils-requirements.txt --file test-requirements.txt -y`
28+
- followed by `conda activate ./env && pip install -e .`
2229

2330
```bash
31+
32+
cd images
2433
source versions.sh
2534

26-
# When running on GitHub Actions, this would be ghcr.io or quay.io
2735
export BUILD_ENV_REGISTRY="localhost"
2836

29-
# Similarly, when running on GitHub Actions, this would normally pull the
30-
# manifest (which does not have the -amd64 suffix) from ghcr.io or quay.io. There
31-
# does not seem to be a way to get podman-created manifests over to docker, or
32-
# even to make local docker manifests. So we need to reference the image
33-
# directly including the arch suffix.
34-
export BUILD_ENV_IMAGE="localhost/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}-amd64"
35-
36-
# Each takes 3-10 min (build-env takes the longest)
37-
bash build.sh base-glibc-busybox-bash
38-
bash build.sh base-glibc-debian-bash
39-
bash build.sh build-env
40-
bash build.sh create-env
37+
time bash build.sh base-glibc-busybox-bash
38+
time bash build.sh base-glibc-debian-bash
39+
time bash build.sh build-env
40+
time bash build.sh create-env
41+
42+
build_and_push_manifest ${BASE_DEBIAN_IMAGE_NAME} ${BASE_TAG} docker://localhost:5000
43+
build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME} ${BASE_TAG} docker://localhost:5000
44+
build_and_push_manifest ${CREATE_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} docker://localhost:5000
45+
build_and_push_manifest ${BUILD_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} docker://localhost:5000
46+
47+
# Run bioconda-utils tests
48+
cd ../
49+
export DEFAULT_BASE_IMAGE="localhost:5000/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}"
50+
export DEFAULT_EXTENDED_BASE_IMAGE="localhost:5000/${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG}"
51+
export BUILD_ENV_IMAGE="localhost:5000/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}"
52+
export CREATE_ENV_IMAGE="localhost:5000/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}"
53+
54+
docker pull $DEFAULT_BASE_IMAGE
55+
docker pull $DEFAULT_EXTENDED_BASE_IMAGE
56+
docker pull $BUILD_ENV_IMAGE
57+
docker pull $CREATE_ENV_IMAGE
58+
59+
py.test --durations=0 test/ -v --log-level=DEBUG -k "docker" --tb=native
4160
```
4261

4362
# Details

images/base-glibc-busybox-bash/prepare.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ TAG="$BASE_TAG"
1010
BUILD_ARGS=()
1111
BUILD_ARGS+=("--build-arg=debian_version=${DEBIAN_VERSION}")
1212
BUILD_ARGS+=("--build-arg=busybox_version=${BUSYBOX_VERSION}")
13-
iidfile="$( mktemp )"
13+
iidfile="$(mktemp)"
1414
buildah bud \
1515
--iidfile="${iidfile}" \
1616
--file=Dockerfile.busybox \
1717
${BUILD_ARGS[@]}
18-
busybox_image="$( cat "${iidfile}" )"
18+
busybox_image="$(cat "${iidfile}")"
1919
rm "${iidfile}"
2020

2121
# Override build args for what's needed in main Dockerfile

images/build-env/prepare.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ TAG="${BIOCONDA_UTILS_VERSION}_base$BASE_TAG"
88
cp -r ../locale/C.utf8 .
99

1010
# We are aiming to get the entire repo into the container so that versioneer
11-
# can correctly determine the version. But the direc
11+
# can correctly determine the version.
1212
if [ -e "./bioconda-utils" ]; then
1313
rm -rf "./bioconda-utils"
1414
fi

images/build.sh

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,23 @@
1919
# ARCHS: space-separated string of archs to build
2020
# IMAGE_NAME: name of image; created manifest will be IMAGE_NAME:tag
2121
# BUILD_ARGS: array of arguments like ("--build-arg=argument1=the-value", "--build-arg=arg2=a")
22+
#
23+
# Output is:
24+
# - images for each arch
25+
# - manifest containing those images
26+
# - tar files of images ready to upload as artifacts
27+
# - tar files built by podman loaded into docker
2228

2329
set -xeu
2430

2531
IMAGE_DIR=$1
2632

33+
echo === BUILDING $IMAGE_DIR ===
34+
2735
cd $IMAGE_DIR
2836

2937
[ -e prepare.sh ] && source prepare.sh
3038

31-
3239
# Clean up any manifests before we start.
3340
# IMAGE_NAME and TAG should be created by prepare.sh
3441
buildah manifest rm "${IMAGE_NAME}:${TAG}" || true
@@ -50,29 +57,33 @@ for arch in $ARCHS; do
5057
fi
5158
fi
5259

60+
# Make arch available as an environment variable and source prepare.sh again
61+
export CURRENT_ARCH="$arch"
62+
[ -e prepare.sh ] && source prepare.sh
63+
5364
source ../env_var_inventory.sh
5465

5566
# Actual building happens here. We will keep track of the built image in
5667
# $image_id.
57-
iidfile="$( mktemp )"
68+
iidfile="$(mktemp)"
5869
buildah bud \
5970
--arch="${arch}" \
6071
--iidfile="${iidfile}" \
6172
--file=Dockerfile \
6273
${BUILD_ARGS[@]} \
6374
$BASE_IMAGE_BUILD_ARG
64-
image_id="$( cat "${iidfile}" )"
75+
image_id="$(cat "${iidfile}")"
6576
rm "${iidfile}"
6677

6778
# In order for GitHub Actions to inherit container permissions from the repo
6879
# permissions, we need to add a special label. However `buildah config
6980
# --label` operates on a container, not an image. So we add the label to
7081
# a temporary container and then save the resulting image.
71-
container="$( buildah from "${image_id}" )"
82+
container="$(buildah from "${image_id}")"
7283
buildah config \
7384
--label="org.opencontainers.image.source=https://github.com/bioconda/bioconda-utils" \
7485
"${container}"
75-
image_id="$( buildah commit "${container}" )"
86+
image_id="$(buildah commit "${container}")"
7687
buildah rm "${container}"
7788

7889
# Add -$arch suffix to image's tag
@@ -81,9 +92,9 @@ for arch in $ARCHS; do
8192
# Add it to the manifest, which has no -$arch suffix
8293
buildah manifest add "${IMAGE_NAME}:${TAG}" "${image_id}"
8394

84-
# copy image over to local docker registry, when running locally. Otherwise,
85-
# the CI will use ghcr.io as an intermediate registry.
86-
if [ "${CI:-false}" == "false" ]; then
87-
podman save "${IMAGE_NAME}:${TAG}-${arch}" | docker load
88-
fi
95+
# Save image for storing as artifact or to load into docker
96+
mkdir -p ../../image-artifacts
97+
podman save "${IMAGE_NAME}:${TAG}-${arch}" >"../../image-artifacts/${IMAGE_NAME}-${arch}.tar"
98+
ls ../../image-artifacts
99+
89100
done

images/create-env/prepare.sh

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,35 +3,21 @@ IMAGE_NAME="${CREATE_ENV_IMAGE_NAME}"
33
TAG=$BIOCONDA_IMAGE_TAG
44
BUILD_ARGS=()
55

6-
7-
86
# Get the exact versions of mamba and conda that were installed in build-env.
97
#
10-
# If this tag exists on quay.io (that is, this create-env is being built in
11-
# a subsequent run), then use that. Otherwise, we assume this tag has already
12-
# been built and pushed to GitHub Container Registry (and the GitHub Actions job
13-
# dependency should reflect this)
14-
BUILD_ENV_REGISTRY=${BUILD_ENV_REGISTRY:="ghcr.io/bioconda"}
15-
if [ $(tag_exists $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then
16-
BUILD_ENV_REGISTRY=quay.io/bioconda
17-
fi
18-
8+
# TODO: here we're hard-coding the amd64 on the reasonable assumption that it
9+
# matches the arm64
1910
CONDA_VERSION=$(
20-
podman run -t $BUILD_ENV_REGISTRY/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} \
21-
bash -c "/opt/conda/bin/conda list --export '^conda$'| sed -n 's/=[^=]*$//p'"
11+
podman run -t ${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}-amd64 \
12+
bash -c "/opt/conda/bin/conda list --export '^conda$'| sed -n 's/=[^=]*$//p'"
2213
)
2314
# Remove trailing \r with parameter expansion
2415
export CONDA_VERSION=${CONDA_VERSION%$'\r'}
2516

2617
BUILD_ARGS+=("--build-arg=CONDA_VERSION=$CONDA_VERSION")
2718

28-
# Needs busybox image to copy some items over
29-
BASE_BUSYBOX_REGISTRY=${BASE_BUSYBOX_REGISTRY:="ghcr.io/bioconda"}
30-
if [ $(tag_exists $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG) ]; then
31-
BASE_BUSYBOX_REGISTRY=quay.io/bioconda
32-
fi
33-
34-
BUILD_ARGS+=("--build-arg=BUSYBOX_IMAGE=${BASE_BUSYBOX_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}")
19+
# Arguments to buildah bud are architecture-dependent
20+
BUILD_ARGS+=("--build-arg=BUSYBOX_IMAGE=localhost/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}-${CURRENT_ARCH}")
3521

3622
# TEST_BUILD_ARGS=()
3723
# TEST_BUILD_ARGS+=("--build-arg=BUSYBOX_IMAGE=$BUSYBOX_IMAGE")

images/env_var_inventory.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# There are a lot of environment variables used here. This script aims to
44
# document them as well as provide a mechanism (e.g., in CI workflows) to show
55
# their current values.
6-
#
6+
#
77
# Typical usage:
88
#
99
# source versions.sh
@@ -14,7 +14,7 @@ while read -r name description; do
1414
description="${description//\"/}"
1515
value="${!name:-}"
1616
echo -e "${name}\t${value}\t$description"
17-
done << 'EOF' | column -t -s $'\t'
17+
done <<'EOF' | column -t -s $'\t'
1818
ARCHS "architectures to build images for"
1919
DEBIAN_VERSION "version of debian for extended base image"
2020
BUSYBOX_VERSION "version of busybox for base image"
@@ -23,6 +23,7 @@ BASE_DEBIAN_IMAGE_NAME "name for debian image"
2323
BASE_BUSYBOX_IMAGE_NAME "name for busybox image"
2424
BUILD_ENV_IMAGE_NAME "name for build image"
2525
CREATE_ENV_IMAGE_NAME "name for create image"
26+
CURRENT_ARCH "Arch for current iteration of loop"
2627
BASE_TAG "the base version tag to add to image tags"
2728
BIOCONDA_IMAGE_TAG "full bioconda + image version"
2829
BUILD_ENV_REGISTRY "where the build image should come from (used in CI)"

images/versions.sh

Lines changed: 66 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,38 @@
11
#!/bin/bash
22

3-
# Configures various versions to be used throughout infrastructure
4-
ARCHS="amd64 arm64"
5-
DEBIAN_VERSION=12.5
6-
BUSYBOX_VERSION=1.36.1
7-
BASE_DEBIAN_IMAGE_NAME="tmp-base-debian"
8-
BASE_BUSYBOX_IMAGE_NAME="tmp-base-busybox"
9-
BUILD_ENV_IMAGE_NAME="tmp-build-env"
10-
CREATE_ENV_IMAGE_NAME="tmp-create-env"
11-
BASE_TAG="0.2"
12-
BASE_IMAGE_CONDAFORGE_AMD64="quay.io/condaforge/linux-anvil-x86_64:cos7"
13-
BASE_IMAGE_CONDAFORGE_ARM64="quay.io/condaforge/linux-anvil-aarch64:cos7"
14-
3+
# Configures various versions to be used throughout infrastructure.
4+
#
5+
# Anything with GITHUB_ENV at the end of the line will be exported to $GITHUB_ENV during GitHub Actions jobs.
6+
ARCHS="amd64 arm64" # GITHUB_ENV
7+
DEBIAN_VERSION=12.5 # GITHUB_ENV
8+
BUSYBOX_VERSION=1.36.1 # GITHUB_ENV
9+
BASE_DEBIAN_IMAGE_NAME="tmp-base-debian" # GITHUB_ENV
10+
BASE_BUSYBOX_IMAGE_NAME="tmp-base-busybox" # GITHUB_ENV
11+
BUILD_ENV_IMAGE_NAME="tmp-build-env" # GITHUB_ENV
12+
CREATE_ENV_IMAGE_NAME="tmp-create-env" # GITHUB_ENV
13+
BASE_TAG="0.2" # GITHUB_ENV
14+
BASE_IMAGE_CONDAFORGE_AMD64="quay.io/condaforge/linux-anvil-x86_64:cos7" # GITHUB_ENV
15+
BASE_IMAGE_CONDAFORGE_ARM64="quay.io/condaforge/linux-anvil-aarch64:cos7" # GITHUB_ENV
16+
CURRENT_ARCH=${CURRENT_ARCH:-""} # GITHUB_ENV
17+
function export_github_env() {
18+
HERE=$1
19+
GITHUB_ENV=${GITHUB_ENV:-/dev/null}
20+
for var in $(grep "# GITHUB_ENV$" $HERE | cut -f1 -d "="); do
21+
echo "$var=\"${!var}\"" >>$GITHUB_ENV
22+
done
23+
}
1524

1625
# Inspect this repo to get the currently-checked-out version, but if
1726
# BIOCONDA_UTILS_VERSION was set outside this script, use that instead.
18-
BIOCONDA_UTILS_VERSION=${BIOCONDA_UTILS_VERSION:-$(git describe --tags --dirty --always)}
27+
BIOCONDA_UTILS_VERSION=${BIOCONDA_UTILS_VERSION:-$(git describe --tags --dirty --always)} # GITHUB_ENV
1928

2029
# This will be used as the tag for create-env and build-env images, which
2130
# depend on bioconda-utils
22-
BIOCONDA_IMAGE_TAG=${BIOCONDA_UTILS_VERSION}_base${BASE_TAG}
31+
BIOCONDA_IMAGE_TAG=${BIOCONDA_UTILS_VERSION}_base${BASE_TAG} # GITHUB_ENV
2332

2433
# FUNCTIONS --------------------------------------------------------------------
2534

26-
function tag_exists () {
35+
function tag_exists() {
2736
# Returns 0 if the tag for the image exists on quay.io, otherwise returns 1.
2837
# Skips "latest" tags (likely they will always be present)
2938
# $1: image name
@@ -35,49 +44,65 @@ function tag_exists () {
3544

3645
# Images can be set to expire; the jq query selects only non-expired images.
3746
existing_tags="$(
38-
printf %s "${response}" \
39-
| jq -r '.tags[]|select(.end_ts == null or .end_ts >= now)|.name'
40-
)" \
41-
|| {
47+
printf %s "${response}" |
48+
jq -r '.tags[]|select(.end_ts == null or .end_ts >= now)|.name'
49+
)" ||
50+
{
4251
printf %s\\n \
4352
'Could not get list of image tags.' \
4453
'Does the repository exist on Quay.io?' \
4554
'Quay.io REST API response was:' \
4655
"${response}" >&2
4756
return 1
4857
}
49-
for tag in $TAGS ; do
58+
for tag in $TAGS; do
5059
case "${tag}" in
51-
"latest" ) ;;
52-
* )
53-
if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then
54-
printf 'Tag %s already exists for %s on quay.io!\n' "${tag}" "${IMAGE_NAME}" >&2
55-
echo "exists"
56-
fi
60+
"latest") ;;
61+
*)
62+
if printf %s "${existing_tags}" | grep -qxF "${tag}"; then
63+
printf 'Tag %s already exists for %s on quay.io!\n' "${tag}" "${IMAGE_NAME}" >&2
64+
echo "exists"
65+
fi
66+
;;
5767
esac
5868
done
5969
}
6070

61-
# Helper function to push a just-built image to GitHub Container
62-
# Respository, which is used as a temporary storage mechanism.
63-
function push_to_ghcr () {
64-
podman manifest push --all localhost/${1}:${2} ghcr.io/bioconda/${1}:${2}
65-
}
71+
function build_and_push_manifest() {
72+
# Builds a local manifest containing multiple archs for an image/tag, and
73+
# pushes to a registry. E.g.,
74+
#
75+
# build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} docker://localhost:5000
76+
#
77+
# or
78+
#
79+
# build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} quay.io/bioconda
80+
#
81+
local image=$1
82+
local registry=$2
6683

67-
# Helper function to move an image from gchr to quay.io for public use.
68-
function move_from_ghcr_to_quay () {
69-
local image_name=$1
70-
local tag=$2
84+
buildah manifest rm "local_${image}" || true
7185

7286
# Locally-named manifest to which we'll add the different archs.
73-
buildah manifest create "local_${image_name}:${tag}"
87+
buildah manifest create "local_${image}"
7488

75-
# Expects images for archs to be built already; add them to local manifest.
89+
# Expects images for archs to be built already; here we add them to local
90+
# manifest.
7691
for arch in $ARCHS; do
77-
imgid=$(buildah pull --arch=$arch "ghcr.io/bioconda/${image_name}:${tag}")
78-
buildah manifest add "local_${image_name}:${tag}" "${imgid}"
92+
imgid=$(buildah pull --arch=$arch "${image}-${arch}")
93+
buildah manifest add "local_${image}" "${imgid}"
7994
done
8095

81-
# Publish
82-
podman manifest push "local_${image_name}:${tag}" "quay.io/bioconda/${image_name}:${tag}"
96+
if [ "$registry" == "docker://localhost:5000" ]; then
97+
if ! curl -X GET http://localhost:5000/v2/_catalog; then
98+
echo "Local docker registry does not appear to be running on localhost:5000!"
99+
return 1
100+
fi
101+
# To avoid setting up TLS certs for local registry which seems like overkill
102+
additional_args="--tls-verify=false"
103+
else
104+
additional_args=""
105+
fi
106+
107+
podman manifest push --all $additional_args "local_${image}" "$registry/${image}"
83108
}

0 commit comments

Comments
 (0)