Try fix

.github/workflows/cpp.yml

@@ -53,6 +53,7 @@ permissions:
  contents: read
 
 env:
+ ARCHERY_DEBUG: 1
  ARROW_ENABLE_TIMING_TESTS: OFF
  DOCKER_VOLUME_PREFIX: ".docker/"
 
@@ -93,6 +94,7 @@ jobs:
  echo "," >> "$GITHUB_OUTPUT"
  cat <<JSON >> "$GITHUB_OUTPUT"
  {
+ "archery-use-docker-cli": "0",
  "arch": "arm64v8",
  "clang-tools": "10",
  "image": "ubuntu-cpp",
@@ -118,6 +120,9 @@ jobs:
  include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
  env:
  ARCH: ${{ matrix.arch }}
+ # By default, use Docker CLI because docker-compose v1 is obsolete,
+ # except where the Docker client version is too old.
+ ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
  ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
  CLANG_TOOLS: ${{ matrix.clang-tools }}
  LLVM: ${{ matrix.llvm }}
@@ -140,8 +145,6 @@ jobs:
  sudo apt install -y --no-install-recommends python3 python3-dev python3-pip
  - name: Setup Archery
  run: python3 -m pip install -e dev/archery[docker]
- - name: Find docker-compose
- run: which -a docker-compose
  - name: Execute Docker Build
  env:
  ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -151,7 +154,7 @@ jobs:
  sudo sysctl -w vm.mmap_rnd_bits=28
  sudo sysctl -w kernel.core_pattern="core.%e.%p"
  ulimit -c unlimited
- archery docker run ${{ matrix.image }}
+ archery --debug docker run ${{ matrix.image }}
  - name: Docker Push
  if: >-
  success() &&

.github/workflows/java.yml

@@ -45,6 +45,8 @@ permissions:
  contents: read
 
 env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
  DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
@@ -80,8 +82,6 @@ jobs:
  python-version: 3.8
  - name: Setup Archery
  run: pip install -e dev/archery[docker]
- - name: Find docker-compose
- run: which -a docker-compose
  - name: Execute Docker Build
  env:
  ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}

.github/workflows/python.yml

@@ -41,6 +41,8 @@ permissions:
  contents: read
 
 env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
  DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
@@ -105,8 +107,6 @@ jobs:
  python-version: 3.8
  - name: Setup Archery
  run: pip install -e dev/archery[docker]
- - name: Find docker-compose
- run: which -a docker-compose
  - name: Execute Docker Build
  env:
  ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}

.github/workflows/ruby.yml

@@ -53,6 +53,8 @@ permissions:
  contents: read
 
 env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
  DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
@@ -87,8 +89,6 @@ jobs:
  python-version: 3.8
  - name: Setup Archery
  run: pip install -e dev/archery[docker]
- - name: Find docker-compose
- run: which -a docker-compose
  - name: Execute Docker Build
  env:
  ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}

dev/archery/archery/cli.py

@@ -44,6 +44,7 @@
 
 @click.group(context_settings={"help_option_names": ["-h", "--help"]})
 @click.option("--debug", type=BOOL, is_flag=True, default=False,
+ envvar='ARCHERY_DEBUG',
  help="Increase logging with debugging output.")
 @click.option("--pdb", type=BOOL, is_flag=True, default=False,
  help="Invoke pdb on uncaught exception.")

dev/archery/archery/docker/cli.py

@@ -46,10 +46,19 @@ def _execute(self, *args, **kwargs):
  callback=validate_arrow_sources,
  help="Specify Arrow source directory.")
 @click.option('--dry-run/--execute', default=False,
- help="Display the docker-compose commands instead of executing "
- "them.")
+ help="Display the docker commands instead of executing them.")
+@click.option('--using-docker-cli', default=False, is_flag=True,
+ envvar='ARCHERY_USE_DOCKER_CLI',
+ help="Use docker CLI directly for building instead of calling "
+ "docker-compose. This may help to reuse cached layers.")
+@click.option('--using-docker-buildx', default=False, is_flag=True,
+ envvar='ARCHERY_USE_DOCKER_BUILDX',
+ help="Use buildx with docker CLI directly for building instead "
+ "of calling docker-compose or the plain docker build "
+ "command. This option makes the build cache reusable "
+ "across hosts.")
 @click.pass_context
-def docker(ctx, src, dry_run):
+def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx):
  """
  Interact with docker-compose based builds.
  """
@@ -64,7 +73,10 @@ def docker(ctx, src, dry_run):
 
  # take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the
  # environment variables to keep the usage similar to docker-compose
+ using_docker_cli |= using_docker_buildx
  compose = DockerCompose(config_path, params=os.environ,
+ using_docker=using_docker_cli,
+ using_buildx=using_docker_buildx,
  debug=ctx.obj.get('debug', False))
  if dry_run:
  _mock_compose_calls(compose)
@@ -83,24 +95,19 @@ def check_config(obj):
 
 @docker.command('pull')
 @click.argument('image')
-@click.option('--using-docker-cli', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_CLI',
- help="Use docker CLI directly for pulling instead of calling "
- "docker-compose. This may help to reuse cached layers.")
 @click.option('--pull-leaf/--no-leaf', default=True,
  help="Whether to pull leaf images too.")
 @click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True,
  help="Whether to ignore pull failures.")
 @click.pass_obj
-def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
- ignore_pull_failures):
+def docker_pull(obj, image, *, pull_leaf, ignore_pull_failures):
  """
  Execute docker-compose pull.
  """
  compose = obj['compose']
 
  try:
- compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli,
+ compose.pull(image, pull_leaf=pull_leaf,
  ignore_pull_failures=ignore_pull_failures)
  except UndefinedImage as e:
  raise click.ClickException(
@@ -115,16 +122,6 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
 @click.argument('image')
 @click.option('--force-pull/--no-pull', default=True,
  help="Whether to force pull the image and its ancestor images")
-@click.option('--using-docker-cli', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_CLI',
- help="Use docker CLI directly for building instead of calling "
- "docker-compose. This may help to reuse cached layers.")
-@click.option('--using-docker-buildx', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_BUILDX',
- help="Use buildx with docker CLI directly for building instead "
- "of calling docker-compose or the plain docker build "
- "command. This option makes the build cache reusable "
- "across hosts.")
 @click.option('--use-cache/--no-cache', default=True,
  help="Whether to use cache when building the image and its "
  "ancestor images")
@@ -133,22 +130,17 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
  "passed as the argument. To disable caching for both the "
  "image and its ancestors use --no-cache option.")
 @click.pass_obj
-def docker_build(obj, image, *, force_pull, using_docker_cli,
- using_docker_buildx, use_cache, use_leaf_cache):
+def docker_build(obj, image, *, force_pull, use_cache, use_leaf_cache):
  """
  Execute docker-compose builds.
  """
  compose = obj['compose']
 
- using_docker_cli |= using_docker_buildx
  try:
  if force_pull:
- compose.pull(image, pull_leaf=use_leaf_cache,
- using_docker=using_docker_cli)
+ compose.pull(image, pull_leaf=use_leaf_cache)
  compose.build(image, use_cache=use_cache,
  use_leaf_cache=use_leaf_cache,
- using_docker=using_docker_cli,
- using_buildx=using_docker_buildx,
  pull_parents=force_pull)
  except UndefinedImage as e:
  raise click.ClickException(
@@ -172,16 +164,6 @@ def docker_build(obj, image, *, force_pull, using_docker_cli,
  help="Whether to force build the image and its ancestor images")
 @click.option('--build-only', default=False, is_flag=True,
  help="Pull and/or build the image, but do not run it")
-@click.option('--using-docker-cli', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_CLI',
- help="Use docker CLI directly for building instead of calling "
- "docker-compose. This may help to reuse cached layers.")
-@click.option('--using-docker-buildx', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_BUILDX',
- help="Use buildx with docker CLI directly for building instead "
- "of calling docker-compose or the plain docker build "
- "command. This option makes the build cache reusable "
- "across hosts.")
 @click.option('--use-cache/--no-cache', default=True,
  help="Whether to use cache when building the image and its "
  "ancestor images")
@@ -191,16 +173,16 @@ def docker_build(obj, image, *, force_pull, using_docker_cli,
  "image and its ancestors use --no-cache option.")
 @click.option('--resource-limit', default=None,
  help="A CPU/memory limit preset to mimic CI environments like "
- "GitHub Actions. Implies --using-docker-cli. Note that "
+ "GitHub Actions. Mandates --using-docker-cli. Note that "
  "exporting ARCHERY_DOCKER_BIN=\"sudo docker\" is likely "
  "required, unless Docker is configured with cgroups v2 "
  "(else Docker will silently ignore the limits).")
 @click.option('--volume', '-v', multiple=True,
  help="Set volume within the container")
 @click.pass_obj
 def docker_run(obj, image, command, *, env, user, force_pull, force_build,
- build_only, using_docker_cli, using_docker_buildx, use_cache,
- use_leaf_cache, resource_limit, volume):
+ build_only, use_cache, use_leaf_cache, resource_limit,
+ volume):
  """
  Execute docker-compose builds.
 
@@ -234,26 +216,21 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
  archery docker run ubuntu-cpp bash
  """
  compose = obj['compose']
- using_docker_cli |= using_docker_buildx
 
  env = dict(kv.split('=', 1) for kv in env)
  try:
  if force_pull:
- compose.pull(image, pull_leaf=use_leaf_cache,
- using_docker=using_docker_cli)
+ compose.pull(image, pull_leaf=use_leaf_cache)
  if force_build:
  compose.build(image, use_cache=use_cache,
- use_leaf_cache=use_leaf_cache,
- using_docker=using_docker_cli,
- using_buildx=using_docker_buildx)
+ use_leaf_cache=use_leaf_cache)
  if build_only:
  return
  compose.run(
  image,
  command=command,
  env=env,
  user=user,
- using_docker=using_docker_cli,
  resource_limit=resource_limit,
  volumes=volume
  )
@@ -273,15 +250,11 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
 @click.option('--password', '-p', required=False,
  envvar='ARCHERY_DOCKER_PASSWORD',
  help='Docker repository password')
-@click.option('--using-docker-cli', default=False, is_flag=True,
- help="Use docker CLI directly for building instead of calling "
- "docker-compose. This may help to reuse cached layers.")
 @click.pass_obj
-def docker_compose_push(obj, image, user, password, using_docker_cli):
+def docker_compose_push(obj, image, user, password):
  """Push the generated docker-compose image."""
  compose = obj['compose']
- compose.push(image, user=user, password=password,
- using_docker=using_docker_cli)
+ compose.push(image, user=user, password=password)
 
 
 @docker.command('images')