apacheGH-40954: [CI] Fix use of obsolete docker-compose command on Github Actions (apache#40949)

### Rationale for this change

The `docker-compose` utility is progressively being removed from GHA-provided runners:
actions/runner-images#9557

### What changes are included in this PR?

Use `docker` client CLI directly instead of `docker-compose` where possible.

### Are these changes tested?

Yes, this should fix the sporadic CI failures because of the above migration.

### Are there any user-facing changes?

No, except additional optional env var `ARCHERY_DEBUG`.
* GitHub Issue: apache#40954

Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>

Author: pitrou

.github/workflows/archery.yml

@@ -32,7 +32,9 @@ on:
       - 'docker-compose.yml'
 
 env:
+  ARCHERY_DEBUG: 1
   ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+  ARCHERY_USE_DOCKER_CLI: 1
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}

.github/workflows/cpp.yml

@@ -53,6 +53,7 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 
@@ -94,6 +95,7 @@ jobs:
             cat <<JSON >> "$GITHUB_OUTPUT"
           {
             "arch": "arm64v8",
+            "archery-use-docker-cli": "0",
             "clang-tools": "10",
             "image": "ubuntu-cpp",
             "llvm": "10",
@@ -118,6 +120,9 @@ jobs:
         include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
+      # By default, use Docker CLI because docker-compose v1 is obsolete,
+      # except where the Docker client version is too old.
+      ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
       ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
       CLANG_TOOLS: ${{ matrix.clang-tools }}
       LLVM: ${{ matrix.llvm }}

.github/workflows/dev.yml

@@ -29,6 +29,10 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
+
 jobs:
 
   lint:

.github/workflows/docs.yml

@@ -24,6 +24,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 

.github/workflows/docs_light.yml

@@ -33,6 +33,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 

.github/workflows/go.yml

@@ -41,6 +41,10 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
+
 jobs:
 
   docker-targets:
@@ -75,12 +79,14 @@ jobs:
           {
             "arch-label": "ARM64",
             "arch": "arm64v8",
+            "archery-use-docker-cli": "0",
             "go": "1.21",
             "runs-on": ["self-hosted", "arm", "linux"]
           },
           {
             "arch-label": "ARM64",
             "arch": "arm64v8",
+            "archery-use-docker-cli": "0",
             "go": "1.22",
             "runs-on": ["self-hosted", "arm", "linux"]
           }
@@ -101,6 +107,9 @@ jobs:
         include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
+      # By default, use Docker CLI because docker-compose v1 is obsolete,
+      # except where the Docker client version is too old.
+      ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
       GO: ${{ matrix.go }}
     steps:
       - name: Checkout Arrow

.github/workflows/integration.yml

@@ -51,6 +51,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/java.yml

@@ -45,6 +45,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/java_jni.yml

@@ -45,6 +45,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/js.yml

@@ -38,6 +38,10 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
+
 jobs:
 
   docker:

.github/workflows/python.yml

@@ -41,6 +41,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/r.yml

@@ -51,6 +51,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/ruby.yml

@@ -53,6 +53,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/swift.yml

@@ -41,6 +41,8 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

dev/archery/archery/cli.py

@@ -44,6 +44,7 @@
 
 @click.group(context_settings={"help_option_names": ["-h", "--help"]})
 @click.option("--debug", type=BOOL, is_flag=True, default=False,
+              envvar='ARCHERY_DEBUG',
               help="Increase logging with debugging output.")
 @click.option("--pdb", type=BOOL, is_flag=True, default=False,
               help="Invoke pdb on uncaught exception.")

dev/archery/archery/docker/cli.py

@@ -46,10 +46,19 @@ def _execute(self, *args, **kwargs):
               callback=validate_arrow_sources,
               help="Specify Arrow source directory.")
 @click.option('--dry-run/--execute', default=False,
-              help="Display the docker-compose commands instead of executing "
-                   "them.")
+              help="Display the docker commands instead of executing them.")
+@click.option('--using-docker-cli', default=False, is_flag=True,
+              envvar='ARCHERY_USE_DOCKER_CLI',
+              help="Use docker CLI directly for building instead of calling "
+                   "docker-compose. This may help to reuse cached layers.")
+@click.option('--using-docker-buildx', default=False, is_flag=True,
+              envvar='ARCHERY_USE_DOCKER_BUILDX',
+              help="Use buildx with docker CLI directly for building instead "
+                   "of calling docker-compose or the plain docker build "
+                   "command. This option makes the build cache reusable "
+                   "across hosts.")
 @click.pass_context
-def docker(ctx, src, dry_run):
+def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx):
     """
     Interact with docker-compose based builds.
     """
@@ -64,7 +73,10 @@ def docker(ctx, src, dry_run):
 
     # take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the
     # environment variables to keep the usage similar to docker-compose
+    using_docker_cli |= using_docker_buildx
     compose = DockerCompose(config_path, params=os.environ,
+                            using_docker=using_docker_cli,
+                            using_buildx=using_docker_buildx,
                             debug=ctx.obj.get('debug', False))
     if dry_run:
         _mock_compose_calls(compose)
@@ -83,24 +95,19 @@ def check_config(obj):
 
 @docker.command('pull')
 @click.argument('image')
-@click.option('--using-docker-cli', default=False, is_flag=True,
-              envvar='ARCHERY_USE_DOCKER_CLI',
-              help="Use docker CLI directly for pulling instead of calling "
-                   "docker-compose. This may help to reuse cached layers.")
 @click.option('--pull-leaf/--no-leaf', default=True,
               help="Whether to pull leaf images too.")
 @click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True,
               help="Whether to ignore pull failures.")
 @click.pass_obj
-def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
-                ignore_pull_failures):
+def docker_pull(obj, image, *, pull_leaf, ignore_pull_failures):
     """
     Execute docker-compose pull.
     """
     compose = obj['compose']
 
     try:
-        compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli,
+        compose.pull(image, pull_leaf=pull_leaf,
                      ignore_pull_failures=ignore_pull_failures)
     except UndefinedImage as e:
         raise click.ClickException(
@@ -115,16 +122,6 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
 @click.argument('image')
 @click.option('--force-pull/--no-pull', default=True,
               help="Whether to force pull the image and its ancestor images")
-@click.option('--using-docker-cli', default=False, is_flag=True,
-              envvar='ARCHERY_USE_DOCKER_CLI',
-              help="Use docker CLI directly for building instead of calling "
-                   "docker-compose. This may help to reuse cached layers.")
-@click.option('--using-docker-buildx', default=False, is_flag=True,
-              envvar='ARCHERY_USE_DOCKER_BUILDX',
-              help="Use buildx with docker CLI directly for building instead "
-                   "of calling docker-compose or the plain docker build "
-                   "command. This option makes the build cache reusable "
-                   "across hosts.")
 @click.option('--use-cache/--no-cache', default=True,
               help="Whether to use cache when building the image and its "
                    "ancestor images")
@@ -133,22 +130,17 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
                    "passed as the argument. To disable caching for both the "
                    "image and its ancestors use --no-cache option.")
 @click.pass_obj
-def docker_build(obj, image, *, force_pull, using_docker_cli,
-                 using_docker_buildx, use_cache, use_leaf_cache):
+def docker_build(obj, image, *, force_pull, use_cache, use_leaf_cache):
     """
     Execute docker-compose builds.
     """
     compose = obj['compose']
 
-    using_docker_cli |= using_docker_buildx
     try:
         if force_pull:
-            compose.pull(image, pull_leaf=use_leaf_cache,
-                         using_docker=using_docker_cli)
+            compose.pull(image, pull_leaf=use_leaf_cache)
         compose.build(image, use_cache=use_cache,
                       use_leaf_cache=use_leaf_cache,
-                      using_docker=using_docker_cli,
-                      using_buildx=using_docker_buildx,
                       pull_parents=force_pull)
     except UndefinedImage as e:
         raise click.ClickException(
@@ -172,16 +164,6 @@ def docker_build(obj, image, *, force_pull, using_docker_cli,
               help="Whether to force build the image and its ancestor images")
 @click.option('--build-only', default=False, is_flag=True,
               help="Pull and/or build the image, but do not run it")
-@click.option('--using-docker-cli', default=False, is_flag=True,
-              envvar='ARCHERY_USE_DOCKER_CLI',
-              help="Use docker CLI directly for building instead of calling "
-                   "docker-compose. This may help to reuse cached layers.")
-@click.option('--using-docker-buildx', default=False, is_flag=True,
-              envvar='ARCHERY_USE_DOCKER_BUILDX',
-              help="Use buildx with docker CLI directly for building instead "
-                   "of calling docker-compose or the plain docker build "
-                   "command. This option makes the build cache reusable "
-                   "across hosts.")
 @click.option('--use-cache/--no-cache', default=True,
               help="Whether to use cache when building the image and its "
                    "ancestor images")
@@ -191,16 +173,16 @@ def docker_build(obj, image, *, force_pull, using_docker_cli,
                    "image and its ancestors use --no-cache option.")
 @click.option('--resource-limit', default=None,
               help="A CPU/memory limit preset to mimic CI environments like "
-                   "GitHub Actions. Implies --using-docker-cli. Note that "
+                   "GitHub Actions. Mandates --using-docker-cli. Note that "
                    "exporting ARCHERY_DOCKER_BIN=\"sudo docker\" is likely "
                    "required, unless Docker is configured with cgroups v2 "
                    "(else Docker will silently ignore the limits).")
 @click.option('--volume', '-v', multiple=True,
               help="Set volume within the container")
 @click.pass_obj
 def docker_run(obj, image, command, *, env, user, force_pull, force_build,
-               build_only, using_docker_cli, using_docker_buildx, use_cache,
-               use_leaf_cache, resource_limit, volume):
+               build_only, use_cache, use_leaf_cache, resource_limit,
+               volume):
     """
     Execute docker-compose builds.
 
@@ -234,26 +216,21 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
     archery docker run ubuntu-cpp bash
     """
     compose = obj['compose']
-    using_docker_cli |= using_docker_buildx
 
     env = dict(kv.split('=', 1) for kv in env)
     try:
         if force_pull:
-            compose.pull(image, pull_leaf=use_leaf_cache,
-                         using_docker=using_docker_cli)
+            compose.pull(image, pull_leaf=use_leaf_cache)
         if force_build:
             compose.build(image, use_cache=use_cache,
-                          use_leaf_cache=use_leaf_cache,
-                          using_docker=using_docker_cli,
-                          using_buildx=using_docker_buildx)
+                          use_leaf_cache=use_leaf_cache)
         if build_only:
             return
         compose.run(
             image,
             command=command,
             env=env,
             user=user,
-            using_docker=using_docker_cli,
             resource_limit=resource_limit,
             volumes=volume
         )
@@ -273,15 +250,11 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
 @click.option('--password', '-p', required=False,
               envvar='ARCHERY_DOCKER_PASSWORD',
               help='Docker repository password')
-@click.option('--using-docker-cli', default=False, is_flag=True,
-              help="Use docker CLI directly for building instead of calling "
-                   "docker-compose. This may help to reuse cached layers.")
 @click.pass_obj
-def docker_compose_push(obj, image, user, password, using_docker_cli):
+def docker_compose_push(obj, image, user, password):
     """Push the generated docker-compose image."""
     compose = obj['compose']
-    compose.push(image, user=user, password=password,
-                 using_docker=using_docker_cli)
+    compose.push(image, user=user, password=password)
 
 
 @docker.command('images')