Help automating the update process #267
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build filesystem and export to OSTree | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| jobs: | |
| build: | |
| # https://runs-on.com/configuration/job-labels/ | |
| # https://aws.amazon.com/ec2/instance-types/ | |
| runs-on: | |
| # x64 | |
| - runs-on=${{github.run_id}}/family=m7+c7/cpu=8+16/ram=32+64/image=ubuntu24-full-x64/disk=large/extras=s3-cache | |
| timeout-minutes: 600 # 10 hours | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Fetch BuildStream cache credentials from Vault | |
| uses: hashicorp/vault-action@v3 | |
| with: | |
| url: https://vault.endlessos.org | |
| method: jwt | |
| path: ghactions | |
| role: endlessm-eos-build-meta | |
| pki: | | |
| pki/issue/endless-client {"common_name": "[email protected]", "ttl": "12h"} ; | |
| - name: Fetch signing credentials from Vault | |
| uses: hashicorp/vault-action@v3 | |
| with: | |
| url: https://vault.endlessos.org | |
| method: jwt | |
| path: ghactions | |
| role: endlessm-eos-build-meta | |
| secrets: | | |
| secret/ostree-builder/gpg/eoask1 private | OSTREE_BUILDER_GPG_PRIVATE ; | |
| secret/ostree-builder/ssh private | OSTREE_BUILDER_SSH_PRIVATE ; | |
| secret/secure-boot-signer/api-users/ostree-builder password | SBSIGNER_PASSWORD ; | |
| if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| - name: Fetch Github read credentials from Vault | |
| uses: hashicorp/vault-action@v3 | |
| with: | |
| url: https://vault.endlessos.org | |
| method: jwt | |
| path: ghactions | |
| role: endlessm-eos-build-meta | |
| secrets: | | |
| secret/github/users/eos-backup token | EOS_BACKUP_TOKEN ; | |
| - name: Configure BuildStream | |
| run: | | |
| # Certificate for BuildStream cache | |
| echo "${ENDLESSCLIENT_CERT}" > client.crt | |
| echo "${ENDLESSCLIENT_CA_CHAIN}" >> client.crt | |
| echo "${ENDLESSCLIENT_KEY}" > client.key | |
| chmod 600 client.key | |
| mkdir -p ~/.config | |
| cat > ~/.config/buildstream.conf << EOF | |
| # BuildStream user configuration | |
| scheduler: | |
| fetchers: 4 | |
| builders: 2 | |
| pushers: 2 | |
| network-retries: 2 | |
| build: | |
| # Allow up to 4 parallel processes to execute within the scope of one build | |
| max-jobs: 4 | |
| logging: | |
| key-length: 0 | |
| verbose: true | |
| error-lines: 20 | |
| message-lines: 20 | |
| debug: false | |
| element-format: | | |
| %{state: >12} %{full-key} %{name} %{workspace-dirs} | |
| message-format: | | |
| [%{elapsed}][%{key}][%{element}] %{action} %{message} | |
| # Artifact cache configuration | |
| artifacts: | |
| servers: | |
| - url: https://bstcache.endlessos.org | |
| push: true | |
| auth: | |
| client-cert: $(pwd)/client.crt | |
| client-key: $(pwd)/client.key | |
| EOF | |
| - name: Setup image version | |
| run: | | |
| build_num="${GITHUB_RUN_ID}" | |
| if [ "${GITHUB_REF_NAME-}" = main ]; then | |
| IMAGE_VERSION="nightly.$build_num" | |
| else | |
| # Assume this will always be a stable branch string like "gnome-44" | |
| IMAGE_VERSION=$(echo "${GITHUB_REF_NAME:-unknown}.$build_num" | sed "s|/|_|g" | sed "s|-|_|g") | |
| fi | |
| echo "image-version: ${IMAGE_VERSION}" > include/image-version.yml | |
| export IMAGE_VERSION | |
| commit_time=$(git log -1 --format=format:%ct) | |
| echo "filesystem-time: ${commit_time}" >> include/image-version.yml | |
| cat include/image-version.yml | |
| - name: Install BuildStream and dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| python3-pip python3-venv bubblewrap \ | |
| lzip xz-utils bzip2 gzip \ | |
| git wget curl \ | |
| ostree | |
| python3 -m venv venv | |
| source venv/bin/activate | |
| pip install -r ./utils/requirements.txt | |
| - name: Enable unprivileged userns | |
| run: | | |
| source venv/bin/activate | |
| sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: mozilla-actions/[email protected] | |
| - name: Build root filesystem (signed_boot=endless) | |
| run: | | |
| echo "Set up ~/.netrc" | |
| cat >> ~/.netrc << EOF | |
| machine sb-signer.endlessm-sf.com | |
| login ostree-builder | |
| password ${SBSIGNER_PASSWORD} | |
| machine github.com | |
| login eos-backup | |
| password ${EOS_BACKUP_TOKEN} | |
| EOF | |
| chmod 600 ~/.netrc | |
| echo "Set up private key for eos-sb-signer" | |
| echo "${OSTREE_BUILDER_GPG_PRIVATE}" > files/apitrustedkey.gpg | |
| chmod 600 files/apitrustedkey.gpg | |
| echo "Set up eos_sb_signer element plugin" | |
| cat > include/eos_sb_signer.yml << EOF | |
| elements: | |
| eos_sb_signer: | |
| config: | |
| endpoint: https://sb-signer.endlessm-sf.com | |
| private-key-file: files/apitrustedkey.gpg | |
| timeout: 30 | |
| EOF | |
| source venv/bin/activate | |
| bst -o payg true -o signed_boot endless build --retry-failed eos/repo.bst | |
| if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| - name: Build root filesystem (signed_boot=snakeoil) | |
| run: | | |
| echo "Set up ~/.netrc" | |
| cat >> ~/.netrc << EOF | |
| machine github.com | |
| login eos-backup | |
| password ${EOS_BACKUP_TOKEN} | |
| EOF | |
| chmod 600 ~/.netrc | |
| source venv/bin/activate | |
| bst -o payg true -o signed_boot snakeoil build --retry-failed eos/repo.bst | |
| if: ${{ github.event_name == 'pull_request' || github.ref != 'refs/heads/main' }} | |
| - name: Export OSTree commit and push it | |
| if: github.ref == 'refs/heads/main' | |
| env: | |
| product: eos | |
| platform: amd64 | |
| branch: master | |
| build_element: eos/repo.bst | |
| build_ref: "os/eos/amd64/master" | |
| ostree_gpg_key_id: "00EA12D9A37DD2A7BD810643DFB958DC725AE7CA" | |
| push_host: ostree.endlessm-sf.com | |
| push_user: uploader | |
| push_repo: "eos" | |
| run: | | |
| push_ref="os/${product}/${platform}/${branch}" | |
| echo "Signing key for OSTree commits" | |
| mkdir ./ostree-gpg | |
| chmod 700 ostree-gpg | |
| echo "${OSTREE_BUILDER_GPG_PRIVATE}" | gpg --homedir ./ostree-gpg --import | |
| echo "SSH key for OSTree pushes" | |
| echo "${OSTREE_BUILDER_SSH_PRIVATE}" > $HOME/.ssh/ostree_ssh | |
| chmod 600 $HOME/.ssh/ostree_ssh | |
| echo "Install ostree-push" | |
| sudo apt-get install -y gir1.2-ostree-1.0 openssh-client | |
| python3 -m venv --system-site-packages ./ostree-push.venv | |
| # Pip will try to build pygobject and pycairo from source; | |
| # passing --no-deps to avoid that. | |
| ./ostree-push.venv/bin/pip install --no-deps ostree-push | |
| echo "Check out repo to a temporary directory" | |
| checkout="$(mktemp --suffix="-update-repo" -d -p "$(pwd)")" | |
| ./venv/bin/bst -o signed_boot endless artifact checkout --hardlinks "${build_element}" --directory "${checkout}" | |
| commit="$(ostree --repo="${checkout}/ostree/repo" rev-parse "${build_ref}")" | |
| echo "Create a new OSTree repo and import commit ${commit}" | |
| ostree init --repo="./ostree-repo" --mode=archive | |
| ostree pull-local --repo ./ostree-repo "${checkout}/ostree/repo" "${commit}" | |
| echo "Sign and recommit the tree as ${push_ref}" | |
| ostree commit --repo ./ostree-repo \ | |
| --gpg-homedir=ostree-gpg --gpg-sign="$ostree_gpg_key_id" \ | |
| --branch="${push_ref}" --tree=ref="${commit}" | |
| # FIXME: Drop --command when ostree-receive is updated on the | |
| # server. | |
| target="${push_user}@${push_host}:${push_repo}" | |
| echo "Push to $target" | |
| ./ostree-push.venv/bin/ostree-push \ | |
| --command ostree-receive \ | |
| -i $HOME/.ssh/ostree_ssh \ | |
| -o StrictHostKeyChecking=no \ | |
| --repo ./ostree-repo "${target}" "${push_ref}" |