From 4e176488b5081928a56f45907da40827efb3e798 Mon Sep 17 00:00:00 2001 From: James Healy Date: Sun, 10 Nov 2024 18:26:52 +1100 Subject: [PATCH 1/2] Update rails-ci pipeline sto run on Buildkite hosted agents An experiment in changing the rails CI pipeline from "self-hosted" agents to "hosted" agents, a recently release Buildkite feature [1]. The hosted agents linux environment is superficially quite similar to the Elastic Stack for AWS, so the required changes are fairly minimal. Roughly half the changes are to take advantage of some performance optimisations available on hosted agents (like cache volumes, and remote buildkit builders with cache that last across builds). The essential changes: * Read the OCI registry from the environment rather than hard code an ECR registry. The current self-hosted agents run in AWS and can access ECR, but the hosted agent environment has access to its own registry specifically for use cases like this - building an image at the start of the build and then reusing it in later jobs * Changing the queue from `default` or `builder`, to `hosted` Optimisations: * There's no need to use the docker-compose plugins cache_from and image_name shenanigans. The images built at the start of each build use a remote buildkit builder with cache that is s hared between builds. The cache is typically warm, and when it is the image build time drops from ~2 mins to ~18sec * Use plain buildkit to build the images, without the docker compose plugin. This avoids the image being exported from buildkit to docker, and when the buildkit cache is warm the jobs complete in as little as 18s. This bypasses the docker-compse built in support for separating building and running, but the docker-compose.yml already kinda bypasses that by hard coding the image used in the run jobs (using the IMAGE_NAME env var) * Create a cache volume for ruby gems that are installed in docker during the initial step. This shaves ~30s off the build time [1] https://buildkite.com/docs/pipelines/hosted-agents/overview --- lib/buildkite/config/build_context.rb | 2 +- lib/buildkite/config/docker_build.rb | 17 ++--------------- lib/buildkite/config/rake_command.rb | 3 +-- pipelines/rails-ci/initial.yml | 17 +++++++++++++++-- pipelines/rails-ci/pipeline.rb | 2 +- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/lib/buildkite/config/build_context.rb b/lib/buildkite/config/build_context.rb index c5f77603..9f6874cf 100644 --- a/lib/buildkite/config/build_context.rb +++ b/lib/buildkite/config/build_context.rb @@ -190,7 +190,7 @@ def min_ruby end def remote_image_base - "973266071021.dkr.ecr.us-east-1.amazonaws.com/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds" + ENV.fetch("REGISTRY") + "/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds" end end end diff --git a/lib/buildkite/config/docker_build.rb b/lib/buildkite/config/docker_build.rb index b43f6301..ff64c83d 100644 --- a/lib/buildkite/config/docker_build.rb +++ b/lib/buildkite/config/docker_build.rb @@ -30,11 +30,7 @@ def cache_from(build_context) end def build_push(build_context) - [ - build_context.local_branch =~ /:/ ? - build_context.image_name_for("pr-#{build_context.pull_request}") : - build_context.image_name_for("br-#{build_context.local_branch}"), - ] + build_context.image_name_for(build_context.build_id, prefix: nil) end end @@ -66,20 +62,11 @@ def builder(ruby) compressed: ".buildkite.tgz" } - plugin :docker_compose, { - build: "base", - config: ".buildkite/docker-compose.yml", - env: %w[PRE_STEPS RACK], - "image-name" => build_context.ruby.image_name_for(build_context.build_id), - "cache-from" => cache_from(build_context), - push: build_push(build_context), - "image-repository" => build_context.image_base, - } + command "docker build --push --build-arg RUBY_IMAGE=#{build_context.ruby.ruby_image} --tag #{build_push(build_context)} --file .buildkite/Dockerfile ." env({ BUNDLER: build_context.bundler, RUBYGEMS: build_context.rubygems, - RUBY_IMAGE: build_context.ruby.ruby_image, encrypted_0fb9444d0374_key: nil, encrypted_0fb9444d0374_iv: nil }) diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb index 7c78a9f8..1b33996b 100644 --- a/lib/buildkite/config/rake_command.rb +++ b/lib/buildkite/config/rake_command.rb @@ -52,8 +52,7 @@ def install_plugins(service = "default", env = nil, dir = ".") plugin :docker_compose, { "env" => env, "run" => service, - "pull" => service, - "pull-retries" => 3, + "tty" => "true", "config" => ".buildkite/docker-compose.yml", "shell" => ["runner", *dir], }.compact diff --git a/pipelines/rails-ci/initial.yml b/pipelines/rails-ci/initial.yml index 299245db..92b3b9dc 100644 --- a/pipelines/rails-ci/initial.yml +++ b/pipelines/rails-ci/initial.yml @@ -1,9 +1,15 @@ # This file is never read -- it's just a copy of the pipeline's # configuration in the Buildkite UI. +env: + CONFIG_REPO: "https://github.com/yob/buildkite-config" + CONFIG_BRANCH: "hosted" steps: - name: ":pipeline: rails-initial-pipeline" command: | + echo "Fetching registry details" + export REGISTRY="$$(nsc workspace describe -o json -k registry_url)" + PATH=/bin:/usr/bin set -e @@ -30,17 +36,22 @@ steps: echo "Fetching pull-request metadata:" (docker run --rm \ -v "$$PWD":/app:ro -w /app \ + -v "$$PWD/cache/bundler":/usr/local/bundle \ -e GITHUB_PUBLIC_REPO_TOKEN \ -e BUILDKITE_REPO \ -e BUILDKITE_PULL_REQUEST \ ruby:latest \ .buildkite/bin/fetch-pr > .buildkite/tmp/.pr-meta.json) || true + echo "Generating pipeline:" sh -c "$$PIPELINE_COMMAND" ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true - + cache: + paths: + - "cache/bundler" + name: "rails-initial-bundler-cache" plugins: - artifacts#v1.9.3: upload: ".dockerignore" @@ -58,6 +69,7 @@ steps: PIPELINE_COMMAND: >- docker run --rm -v "$$PWD":/app:ro -w /app + -v "$$PWD/cache/bundler":/usr/local/bundle -e CI -e BUILDKITE -e BUILDKITE_AGENT_META_DATA_QUEUE @@ -72,9 +84,10 @@ steps: -e DOCKER_IMAGE -e RUN_QUEUE -e QUEUE + -e REGISTRY ruby:latest .buildkite/bin/pipeline-generate rails-ci | buildkite-agent pipeline upload timeout_in_minutes: 5 agents: - queue: "${QUEUE-builder}" + queue: hosted diff --git a/pipelines/rails-ci/pipeline.rb b/pipelines/rails-ci/pipeline.rb index f740dcbb..3e3dd56c 100644 --- a/pipelines/rails-ci/pipeline.rb +++ b/pipelines/rails-ci/pipeline.rb @@ -7,7 +7,7 @@ use Buildkite::Config::RakeCommand use Buildkite::Config::RubyGroup - plugin :docker_compose, "docker-compose#v4.16.0" + plugin :docker_compose, "docker-compose#v5.4.1" plugin :artifacts, "artifacts#v1.9.3" if build_context.nightly? From da0ac85c76d15dda314501d09e574542f0400b59 Mon Sep 17 00:00:00 2001 From: James Healy Date: Thu, 14 Nov 2024 10:45:44 +1100 Subject: [PATCH 2/2] inspect the docker image store at the start of each job --- lib/buildkite/config/rake_command.rb | 3 +++ pipelines/rails-ci/pipeline.rb | 1 + 2 files changed, 4 insertions(+) diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb index 1b33996b..babfbcad 100644 --- a/lib/buildkite/config/rake_command.rb +++ b/lib/buildkite/config/rake_command.rb @@ -48,6 +48,9 @@ def install_plugins(service = "default", env = nil, dir = ".") ], compressed: ".buildkite.tgz" } + plugin :metahook, { + "pre-command": "echo \"+++ inspect docker image store\"\ndocker image ls" + } plugin :docker_compose, { "env" => env, diff --git a/pipelines/rails-ci/pipeline.rb b/pipelines/rails-ci/pipeline.rb index 3e3dd56c..86eb3e3c 100644 --- a/pipelines/rails-ci/pipeline.rb +++ b/pipelines/rails-ci/pipeline.rb @@ -9,6 +9,7 @@ plugin :docker_compose, "docker-compose#v5.4.1" plugin :artifacts, "artifacts#v1.9.3" + plugin :metahook, "improbable-eng/metahook#v0.4.1" if build_context.nightly? build_context.rubies << Buildkite::Config::RubyConfig.master_ruby