Skip to content

Commit 7ed76a8

Browse files
BtbNBtbN
committed
Overhaul cache system
1 parent 7590268 commit 7ed76a8

File tree

116 files changed

+179
-358
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

116 files changed

+179
-358
lines changed

‎.github/workflows/build.yml

+24-30
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,8 @@ jobs:
6363
id: imagename
6464
run: |
6565
IMG="${GITHUB_REPOSITORY,,}/base"
66-
DLIMG="${GITHUB_REPOSITORY,,}/dl_cache"
6766
echo "name=ghcr.io/${IMG/ /-}" >> $GITHUB_OUTPUT
6867
echo "rawname=${IMG/ /-}" >> $GITHUB_OUTPUT
69-
echo "dlname=ghcr.io/${DLIMG/ /-}" >> $GITHUB_OUTPUT
70-
echo "dlrawname=${DLIMG/ /-}" >> $GITHUB_OUTPUT
7168
- name: Stop Commands
7269
run: T="$(echo -n ${{ github.token }} | sha256sum | head -c 64)" && echo -e "::add-mask::${T}\n::stop-commands::${T}"
7370
- name: Build target base image
@@ -80,23 +77,22 @@ jobs:
8077
tags: ${{ steps.imagename.outputs.name }}:latest
8178
cache-to: type=registry,mode=max,ref=${{ steps.imagename.outputs.name }}:cache
8279
cache-from: type=registry,ref=${{ steps.imagename.outputs.name }}:cache
83-
- name: Generate download cache Dockerfile
80+
- name: Get download cache key
8481
id: dl_cache
8582
run: |
86-
./generate.sh dl only
8783
echo "dltagname=$(./util/get_dl_cache_tag.sh)" >> $GITHUB_OUTPUT
88-
cat Dockerfile.dl
89-
- name: Build download cache image
90-
uses: docker/build-push-action@v5
84+
- name: Cache
85+
uses: actions/cache@v3
9186
with:
92-
context: .
93-
file: Dockerfile.dl
94-
pull: true
95-
push: true
96-
provenance: false
97-
tags: ${{ steps.imagename.outputs.dlname }}:${{ steps.dl_cache.outputs.dltagname }}
98-
cache-to: type=registry,mode=max,ref=${{ steps.imagename.outputs.dlname }}:cache
99-
cache-from: type=registry,ref=${{ steps.imagename.outputs.dlname }}:cache
87+
path: .cache/downloads
88+
key: download-cache-${{ steps.dl_cache.outputs.dltagname }}
89+
restore-keys: |
90+
download-cache-
91+
- name: Update Cache
92+
run: |
93+
set -e
94+
./download.sh
95+
./util/clean_cache.sh
10096
- name: Cleanup
10197
if: ${{ env.HAVE_CLEANUP_PAT == 'true' }}
10298
continue-on-error: true
@@ -108,18 +104,6 @@ jobs:
108104
repository: ${{ github.repository }}
109105
owner_type: user
110106
untagged_only: true
111-
- name: Cleanup Download Cache
112-
if: ${{ env.HAVE_CLEANUP_PAT == 'true' }}
113-
continue-on-error: true
114-
uses: BtbN/delete-untagged-ghcr-action@main
115-
with:
116-
token: ${{ secrets.CLEANUP_PAT }}
117-
package_name: ${{ steps.imagename.outputs.dlrawname }}
118-
repository_owner: ${{ github.repository_owner }}
119-
repository: ${{ github.repository }}
120-
owner_type: user
121-
untagged_only: false
122-
keep_latest: 20
123107
build_target_bases:
124108
name: Build target base image
125109
if: ${{ github.event.inputs.buildOnly != 'true' }}
@@ -201,16 +185,26 @@ jobs:
201185
registry: ghcr.io
202186
username: ${{ github.actor }}
203187
password: ${{ github.token }}
204-
- name: Generate Dockerfile
205-
run: ./generate.sh ${{ matrix.target }} ${{ matrix.variant }}
206188
- name: Image Name
207189
id: imagename
208190
run: |
209191
IMG="${GITHUB_REPOSITORY,,}/${{ matrix.target }}-${{ matrix.variant }}"
210192
echo "name=ghcr.io/${IMG/ /-}" >> $GITHUB_OUTPUT
211193
echo "rawname=${IMG/ /-}" >> $GITHUB_OUTPUT
194+
- name: Get download cache key
195+
id: dl_cache
196+
run: |
197+
echo "dltagname=$(./util/get_dl_cache_tag.sh)" >> $GITHUB_OUTPUT
212198
- name: Stop Commands
213199
run: T="$(echo -n ${{ github.token }} | sha256sum | head -c 64)" && echo -e "::add-mask::${T}\n::stop-commands::${T}"
200+
- name: Generate Dockerfile
201+
run: ./generate.sh ${{ matrix.target }} ${{ matrix.variant }}
202+
- name: Cache
203+
uses: actions/cache/restore@v3
204+
with:
205+
path: .cache/downloads
206+
key: download-cache-${{ steps.dl_cache.outputs.dltagname }}
207+
fail-on-cache-miss: true
214208
- name: Build target base image
215209
uses: docker/build-push-action@v5
216210
with:

‎.github/workflows/pr.yml

+11
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,17 @@ jobs:
4040
with:
4141
config: .github/buildkit.toml
4242
driver: docker
43+
- name: Get download cache key
44+
id: dl_cache
45+
run: |
46+
echo "dltagname=$(./util/get_dl_cache_tag.sh)" >> $GITHUB_OUTPUT
47+
- name: Cache
48+
uses: actions/cache/restore@v3
49+
with:
50+
path: .cache/downloads
51+
key: download-cache-${{ steps.dl_cache.outputs.dltagname }}
52+
restore-keys: |
53+
download-cache-
4354
- name: Build Image
4455
run: ./makeimage.sh ${{ matrix.target }} ${{ matrix.variant }}
4556
env:

‎build.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ docker run --rm -v "$PWD:/uidtestdir" "$IMAGE" touch "/uidtestdir/$TESTFILE"
4646
DOCKERUID="$(stat -c "%u" "$TESTFILE")"
4747
rm -f "$TESTFILE"
4848
[[ "$DOCKERUID" != "$(id -u)" ]] && UIDARGS=( -u "$(id -u):$(id -g)" ) || UIDARGS=()
49+
unset TESTFILE
4950

5051
rm -rf ffbuild
5152
mkdir ffbuild
@@ -76,7 +77,7 @@ EOF
7677

7778
[[ -t 1 ]] && TTY_ARG="-t" || TTY_ARG=""
7879

79-
docker run --rm -i $TTY_ARG "${UIDARGS[@]}" -v $PWD/ffbuild:/ffbuild -v "$BUILD_SCRIPT":/build.sh "$IMAGE" bash /build.sh
80+
docker run --rm -i $TTY_ARG "${UIDARGS[@]}" -v "$PWD/ffbuild":/ffbuild -v "$BUILD_SCRIPT":/build.sh "$IMAGE" bash /build.sh
8081

8182
mkdir -p artifacts
8283
ARTIFACTS_PATH="$PWD/artifacts"

‎download.sh

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/bin/bash
2+
set -xe
3+
cd "$(dirname "$0")"
4+
source util/vars.sh dl only
5+
6+
TESTFILE="uidtestfile"
7+
rm -f "$TESTFILE"
8+
docker run --rm -v "$PWD:/uidtestdir" "${REGISTRY}/${REPO}/base:latest" touch "/uidtestdir/$TESTFILE"
9+
DOCKERUID="$(stat -c "%u" "$TESTFILE")"
10+
rm -f "$TESTFILE"
11+
[[ "$DOCKERUID" != "$(id -u)" ]] && UIDARGS=( -u "$(id -u):$(id -g)" ) || UIDARGS=()
12+
unset TESTFILE
13+
14+
[[ -t 1 ]] && TTY_ARG="-t" || TTY_ARG=""
15+
16+
DL_SCRIPT_DIR="$(mktemp -d)"
17+
trap "rm -rf -- '$DL_SCRIPT_DIR'" EXIT
18+
19+
mkdir -p "${PWD}"/.cache/downloads
20+
21+
for STAGE in scripts.d/*.sh scripts.d/*/*.sh; do
22+
STAGENAME="$(basename "$STAGE" | sed 's/.sh$//')"
23+
24+
cat <<-EOF >"${DL_SCRIPT_DIR}/${STAGENAME}.sh"
25+
set -xe -o pipefail
26+
shopt -s dotglob
27+
28+
source /dl_functions.sh
29+
source "/$STAGE"
30+
STG="\$(ffbuild_dockerdl)"
31+
32+
if [[ -z "\$STG" ]]; then
33+
exit 0
34+
fi
35+
36+
DLHASH="\$(sha256sum <<<"\$STG" | cut -d" " -f1)"
37+
DLNAME="$STAGENAME"
38+
39+
if [[ "$1" == "hashonly" ]]; then
40+
echo "\$DLHASH"
41+
exit 0
42+
fi
43+
44+
TGT="/dldir/\${DLNAME}_\${DLHASH}.tar.xz"
45+
if [[ -f "\$TGT" ]]; then
46+
rm -f "/dldir/\${DLNAME}.tar.xz"
47+
ln -s "\${DLNAME}_\${DLHASH}.tar.xz" "/dldir/\${DLNAME}.tar.xz"
48+
exit 0
49+
fi
50+
51+
WORKDIR="\$(mktemp -d)"
52+
trap "rm -rf -- '\$WORKDIR'" EXIT
53+
cd "\$WORKDIR"
54+
55+
eval "\$STG"
56+
57+
tar cpJf "\$TGT.tmp" .
58+
mv "\$TGT.tmp" "\$TGT"
59+
rm -f "/dldir/\${DLNAME}.tar.xz"
60+
ln -s "\${DLNAME}_\${DLHASH}.tar.xz" "/dldir/\${DLNAME}.tar.xz"
61+
EOF
62+
done
63+
64+
docker run -i $TTY_ARG --rm "${UIDARGS[@]}" -v "${DL_SCRIPT_DIR}":/stages -v "${PWD}/.cache/downloads":/dldir -v "${PWD}/scripts.d":/scripts.d -v "${PWD}/util/dl_functions.sh":/dl_functions.sh "${REGISTRY}/${REPO}/base:latest" \
65+
bash -c 'set -xe && for STAGE in /stages/*.sh; do bash $STAGE; done'

‎generate.sh

+14-44
Original file line numberDiff line numberDiff line change
@@ -18,49 +18,6 @@ to_df() {
1818
echo >> "$_of"
1919
}
2020

21-
default_dl() {
22-
to_df "RUN git-mini-clone \"$SCRIPT_REPO\" \"$SCRIPT_COMMIT\" \"$1\""
23-
}
24-
25-
###
26-
### Generate download Dockerfile
27-
###
28-
29-
exec_dockerstage_dl() {
30-
SCRIPT="$1"
31-
(
32-
SELF="$SCRIPT"
33-
SELFLAYER="$(layername "$STAGE")"
34-
source "$SCRIPT"
35-
ffbuild_dockerdl || exit $?
36-
TODF="Dockerfile.dl.final" ffbuild_dockerlayer_dl || exit $?
37-
)
38-
}
39-
40-
export TODF="Dockerfile.dl"
41-
42-
to_df "FROM ${REGISTRY}/${REPO}/base:latest AS base"
43-
to_df "ENV TARGET=$TARGET VARIANT=$VARIANT REPO=$REPO ADDINS_STR=$ADDINS_STR"
44-
to_df "WORKDIR \$FFBUILD_DLDIR"
45-
46-
for STAGE in scripts.d/*.sh scripts.d/*/*.sh; do
47-
to_df "FROM base AS $(layername "$STAGE")"
48-
exec_dockerstage_dl "$STAGE"
49-
done
50-
51-
to_df "FROM base AS intermediate"
52-
cat Dockerfile.dl.final >> "$TODF"
53-
rm Dockerfile.dl.final
54-
55-
to_df "FROM scratch"
56-
to_df "COPY --from=intermediate /opt/ffdl/. /"
57-
58-
if [[ "$TARGET" == "dl" && "$VARIANT" == "only" ]]; then
59-
exit 0
60-
fi
61-
62-
DL_IMAGE="${DL_IMAGE_RAW}:$(./util/get_dl_cache_tag.sh)"
63-
6421
###
6522
### Generate main Dockerfile
6623
###
@@ -69,11 +26,23 @@ exec_dockerstage() {
6926
SCRIPT="$1"
7027
(
7128
SELF="$SCRIPT"
29+
STAGENAME="$(basename "$SCRIPT" | sed 's/.sh$//')"
30+
source util/dl_functions.sh
7231
source "$SCRIPT"
7332

7433
ffbuild_enabled || exit 0
7534

76-
to_df "ENV SELF=\"$SELF\""
35+
to_df "ENV SELF=\"$SELF\" STAGENAME=\"$STAGENAME\""
36+
37+
set -x
38+
39+
STG="$(ffbuild_dockerdl)"
40+
if [[ -n "$STG" ]]; then
41+
HASH="$(sha256sum <<<"$STG" | cut -d" " -f1)"
42+
to_df "ADD .cache/downloads/${STAGENAME}_${HASH}.tar.xz /${STAGENAME}"
43+
to_df "WORKDIR /${STAGENAME}"
44+
fi
45+
7746
ffbuild_dockerstage || exit $?
7847
)
7948
}
@@ -82,6 +51,7 @@ export TODF="Dockerfile"
8251

8352
to_df "FROM ${REGISTRY}/${REPO}/base-${TARGET}:latest AS base"
8453
to_df "ENV TARGET=$TARGET VARIANT=$VARIANT REPO=$REPO ADDINS_STR=$ADDINS_STR"
54+
to_df "COPY util/run_stage.sh /usr/bin/run_stage"
8555

8656
for addin in "${ADDINS[@]}"; do
8757
(

‎images/base/Dockerfile

-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
FROM ubuntu:23.04
22

3-
ENV FFBUILD_DLDIR="/opt/ffdl"
4-
53
ENV DEBIAN_FRONTEND noninteractive
64
RUN \
7-
mkdir -p "$FFBUILD_DLDIR" && \
85
apt-get -y update && \
96
apt-get -y dist-upgrade && \
107
apt-get -y install build-essential yasm nasm \

‎images/base/retry-tool.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
#!/bin/bash
2-
set -xe
2+
set -xe -o pipefail
33

44
RETRY_COUNTER=0
55
MAX_RETRY=10
66
CUR_TIMEOUT=120
77
while [[ $RETRY_COUNTER -lt $MAX_RETRY ]]; do
8-
timeout $CUR_TIMEOUT "$@" && break || sleep $(shuf -i 5-90 -n 1)
8+
(timeout $CUR_TIMEOUT "$@") && break || sleep $(shuf -i 5-90 -n 1)
99
RETRY_COUNTER=$(( $RETRY_COUNTER + 1 ))
1010
CUR_TIMEOUT=$(( $CUR_TIMEOUT + 60 ))
1111
echo "Retry $RETRY_COUNTER..."

‎makeimage.sh

+2-17
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@ docker container inspect ffbuildreg &>/dev/null || \
2323
LOCAL_REG_PORT="$(docker container inspect --format='{{range $p, $conf := .NetworkSettings.Ports}}{{(index $conf 0).HostPort}}{{end}}' ffbuildreg)"
2424
LOCAL_ROOT="127.0.0.1:${LOCAL_REG_PORT}/local"
2525

26-
export REGISTRY_OVERRIDE_DL="127.0.0.1:${LOCAL_REG_PORT}" GITHUB_REPOSITORY_DL="local"
27-
2826
if [[ -z "$QUICKBUILD" ]]; then
2927
if grep "FROM.*base.*" "images/base-${TARGET}/Dockerfile" >/dev/null 2>&1; then
3028
docker buildx --builder ffbuilder build \
@@ -39,24 +37,11 @@ if [[ -z "$QUICKBUILD" ]]; then
3937
--push --tag "${LOCAL_ROOT}/base-${TARGET}:latest" \
4038
--build-arg GH_REPO="$LOCAL_ROOT" "images/base-${TARGET}"
4139

42-
export REGISTRY_OVERRIDE="$REGISTRY_OVERRIDE_DL" GITHUB_REPOSITORY="$GITHUB_REPOSITORY_DL"
40+
export REGISTRY_OVERRIDE="127.0.0.1:${LOCAL_REG_PORT}" GITHUB_REPOSITORY="local"
4341
fi
4442

43+
./download.sh
4544
./generate.sh "$TARGET" "$VARIANT" "${ADDINS[@]}"
46-
DL_CACHE_TAG="$(./util/get_dl_cache_tag.sh)"
47-
DL_IMAGE="${DL_IMAGE_RAW}:${DL_CACHE_TAG}"
48-
49-
if docker pull "${DL_IMAGE}"; then
50-
export REGISTRY_OVERRIDE_DL="$REGISTRY" GITHUB_REPOSITORY_DL="$REPO"
51-
./generate.sh "$TARGET" "$VARIANT" "${ADDINS[@]}"
52-
else
53-
DL_IMAGE="${LOCAL_ROOT}/dl_cache:${DL_CACHE_TAG}"
54-
docker manifest inspect --insecure "${DL_IMAGE}" >/dev/null ||
55-
docker buildx --builder ffbuilder build -f Dockerfile.dl \
56-
--cache-from=type=local,src=.cache/dl_image_cache \
57-
--cache-to=type=local,mode=max,dest=.cache/dl_image_cache \
58-
--push --tag "${DL_IMAGE}" .
59-
fi
6045

6146
docker buildx --builder ffbuilder build \
6247
--cache-from=type=local,src=.cache/"${IMAGE/:/_}" \

‎scripts.d/10-mingw-std-threads.sh

-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ ffbuild_enabled() {
99
}
1010

1111
ffbuild_dockerbuild() {
12-
cd "$FFBUILD_DLDIR/$SELF"
13-
1412
mkdir -p "$FFBUILD_PREFIX"/include
1513
cp *.h "$FFBUILD_PREFIX"/include
1614
}

‎scripts.d/10-mingw.sh

+1-3
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,10 @@ ffbuild_dockerfinal() {
1818
}
1919

2020
ffbuild_dockerdl() {
21-
to_df "RUN retry-tool sh -c \"rm -rf mingw && git clone '$SCRIPT_REPO' mingw\" && cd mingw && git checkout \"$SCRIPT_COMMIT\""
21+
echo "retry-tool sh -c \"rm -rf mingw && git clone '$SCRIPT_REPO' mingw\" && cd mingw && git checkout \"$SCRIPT_COMMIT\""
2222
}
2323

2424
ffbuild_dockerbuild() {
25-
cd "$FFBUILD_DLDIR/mingw"
26-
2725
cd mingw-w64-headers
2826

2927
unset CFLAGS

‎scripts.d/10-xorg-macros.sh

-2
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,6 @@ ffbuild_dockerlayer() {
1414
}
1515

1616
ffbuild_dockerbuild() {
17-
cd "$FFBUILD_DLDIR/$SELF"
18-
1917
autoreconf -i
2018
./configure --prefix="$FFBUILD_PREFIX"
2119
make -j"$(nproc)"

‎scripts.d/20-libiconv.sh

+2-4
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,11 @@ ffbuild_enabled() {
88
}
99

1010
ffbuild_dockerdl() {
11-
to_df "RUN retry-tool sh -c \"rm -rf $SELF && git clone '$SCRIPT_REPO' $SELF\" && git -C $SELF checkout \"$SCRIPT_COMMIT\""
12-
to_df "RUN cd $SELF && retry-tool ./autopull.sh --one-time"
11+
echo "retry-tool sh -c \"rm -rf iconv && git clone '$SCRIPT_REPO' iconv\" && git -C iconv checkout \"$SCRIPT_COMMIT\""
12+
echo "cd iconv && retry-tool ./autopull.sh --one-time"
1313
}
1414

1515
ffbuild_dockerbuild() {
16-
cd "$FFBUILD_DLDIR/$SELF"
17-
1816
(unset CC CFLAGS GMAKE && ./autogen.sh)
1917

2018
local myconf=(

‎scripts.d/20-libxml2.sh

-2
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ ffbuild_enabled() {
88
}
99

1010
ffbuild_dockerbuild() {
11-
cd "$FFBUILD_DLDIR/$SELF"
12-
1311
local myconf=(
1412
--prefix="$FFBUILD_PREFIX"
1513
--without-python

0 commit comments

Comments
 (0)