Skip to content

Commit

Permalink
GH-40954: [CI] Fix use of obsolete docker-compose command on Github A…
Browse files Browse the repository at this point in the history
…ctions (#40949)

### Rationale for this change

The `docker-compose` utility is progressively being removed from GHA-provided runners:
actions/runner-images#9557

### What changes are included in this PR?

Use `docker` client CLI directly instead of `docker-compose` where possible.

### Are these changes tested?

Yes, this should fix the sporadic CI failures because of the above migration.

### Are there any user-facing changes?

No, except additional optional env var `ARCHERY_DEBUG`.
* GitHub Issue: #40954

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Antoine Pitrou <antoine@python.org>
  • Loading branch information
pitrou authored Apr 3, 2024
1 parent 41a989c commit be3b789
Show file tree
Hide file tree
Showing 23 changed files with 117 additions and 87 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/archery.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ on:
- 'docker-compose.yml'

env:
ARCHERY_DEBUG: 1
ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
ARCHERY_USE_DOCKER_CLI: 1

concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/cpp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARROW_ENABLE_TIMING_TESTS: OFF
DOCKER_VOLUME_PREFIX: ".docker/"

Expand Down Expand Up @@ -94,6 +95,7 @@ jobs:
cat <<JSON >> "$GITHUB_OUTPUT"
{
"arch": "arm64v8",
"archery-use-docker-cli": "0",
"clang-tools": "10",
"image": "ubuntu-cpp",
"llvm": "10",
Expand All @@ -118,6 +120,9 @@ jobs:
include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
env:
ARCH: ${{ matrix.arch }}
# By default, use Docker CLI because docker-compose v1 is obsolete,
# except where the Docker client version is too old.
ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
CLANG_TOOLS: ${{ matrix.clang-tools }}
LLVM: ${{ matrix.llvm }}
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ concurrency:
permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1

jobs:

lint:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1
ARROW_ENABLE_TIMING_TESTS: OFF
DOCKER_VOLUME_PREFIX: ".docker/"

Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/docs_light.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1
ARROW_ENABLE_TIMING_TESTS: OFF
DOCKER_VOLUME_PREFIX: ".docker/"

Expand Down
9 changes: 9 additions & 0 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ concurrency:
permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1

jobs:

docker-targets:
Expand Down Expand Up @@ -75,12 +79,14 @@ jobs:
{
"arch-label": "ARM64",
"arch": "arm64v8",
"archery-use-docker-cli": "0",
"go": "1.21",
"runs-on": ["self-hosted", "arm", "linux"]
},
{
"arch-label": "ARM64",
"arch": "arm64v8",
"archery-use-docker-cli": "0",
"go": "1.22",
"runs-on": ["self-hosted", "arm", "linux"]
}
Expand All @@ -101,6 +107,9 @@ jobs:
include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
env:
ARCH: ${{ matrix.arch }}
# By default, use Docker CLI because docker-compose v1 is obsolete,
# except where the Docker client version is too old.
ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
GO: ${{ matrix.go }}
steps:
- name: Checkout Arrow
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"

jobs:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"

jobs:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/java_jni.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"

jobs:
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/js.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ concurrency:
permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1

jobs:

docker:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"

jobs:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/r.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"

jobs:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/ruby.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"

jobs:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/swift.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ permissions:
contents: read

env:
ARCHERY_DEBUG: 1
ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"

jobs:
Expand Down
1 change: 1 addition & 0 deletions dev/archery/archery/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@

@click.group(context_settings={"help_option_names": ["-h", "--help"]})
@click.option("--debug", type=BOOL, is_flag=True, default=False,
envvar='ARCHERY_DEBUG',
help="Increase logging with debugging output.")
@click.option("--pdb", type=BOOL, is_flag=True, default=False,
help="Invoke pdb on uncaught exception.")
Expand Down
79 changes: 26 additions & 53 deletions dev/archery/archery/docker/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,19 @@ def _execute(self, *args, **kwargs):
callback=validate_arrow_sources,
help="Specify Arrow source directory.")
@click.option('--dry-run/--execute', default=False,
help="Display the docker-compose commands instead of executing "
"them.")
help="Display the docker commands instead of executing them.")
@click.option('--using-docker-cli', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_CLI',
help="Use docker CLI directly for building instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.option('--using-docker-buildx', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_BUILDX',
help="Use buildx with docker CLI directly for building instead "
"of calling docker-compose or the plain docker build "
"command. This option makes the build cache reusable "
"across hosts.")
@click.pass_context
def docker(ctx, src, dry_run):
def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx):
"""
Interact with docker-compose based builds.
"""
Expand All @@ -64,7 +73,10 @@ def docker(ctx, src, dry_run):

# take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the
# environment variables to keep the usage similar to docker-compose
using_docker_cli |= using_docker_buildx
compose = DockerCompose(config_path, params=os.environ,
using_docker=using_docker_cli,
using_buildx=using_docker_buildx,
debug=ctx.obj.get('debug', False))
if dry_run:
_mock_compose_calls(compose)
Expand All @@ -83,24 +95,19 @@ def check_config(obj):

@docker.command('pull')
@click.argument('image')
@click.option('--using-docker-cli', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_CLI',
help="Use docker CLI directly for pulling instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.option('--pull-leaf/--no-leaf', default=True,
help="Whether to pull leaf images too.")
@click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True,
help="Whether to ignore pull failures.")
@click.pass_obj
def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
ignore_pull_failures):
def docker_pull(obj, image, *, pull_leaf, ignore_pull_failures):
"""
Execute docker-compose pull.
"""
compose = obj['compose']

try:
compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli,
compose.pull(image, pull_leaf=pull_leaf,
ignore_pull_failures=ignore_pull_failures)
except UndefinedImage as e:
raise click.ClickException(
Expand All @@ -115,16 +122,6 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
@click.argument('image')
@click.option('--force-pull/--no-pull', default=True,
help="Whether to force pull the image and its ancestor images")
@click.option('--using-docker-cli', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_CLI',
help="Use docker CLI directly for building instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.option('--using-docker-buildx', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_BUILDX',
help="Use buildx with docker CLI directly for building instead "
"of calling docker-compose or the plain docker build "
"command. This option makes the build cache reusable "
"across hosts.")
@click.option('--use-cache/--no-cache', default=True,
help="Whether to use cache when building the image and its "
"ancestor images")
Expand All @@ -133,22 +130,17 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
"passed as the argument. To disable caching for both the "
"image and its ancestors use --no-cache option.")
@click.pass_obj
def docker_build(obj, image, *, force_pull, using_docker_cli,
using_docker_buildx, use_cache, use_leaf_cache):
def docker_build(obj, image, *, force_pull, use_cache, use_leaf_cache):
"""
Execute docker-compose builds.
"""
compose = obj['compose']

using_docker_cli |= using_docker_buildx
try:
if force_pull:
compose.pull(image, pull_leaf=use_leaf_cache,
using_docker=using_docker_cli)
compose.pull(image, pull_leaf=use_leaf_cache)
compose.build(image, use_cache=use_cache,
use_leaf_cache=use_leaf_cache,
using_docker=using_docker_cli,
using_buildx=using_docker_buildx,
pull_parents=force_pull)
except UndefinedImage as e:
raise click.ClickException(
Expand All @@ -172,16 +164,6 @@ def docker_build(obj, image, *, force_pull, using_docker_cli,
help="Whether to force build the image and its ancestor images")
@click.option('--build-only', default=False, is_flag=True,
help="Pull and/or build the image, but do not run it")
@click.option('--using-docker-cli', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_CLI',
help="Use docker CLI directly for building instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.option('--using-docker-buildx', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_BUILDX',
help="Use buildx with docker CLI directly for building instead "
"of calling docker-compose or the plain docker build "
"command. This option makes the build cache reusable "
"across hosts.")
@click.option('--use-cache/--no-cache', default=True,
help="Whether to use cache when building the image and its "
"ancestor images")
Expand All @@ -191,16 +173,16 @@ def docker_build(obj, image, *, force_pull, using_docker_cli,
"image and its ancestors use --no-cache option.")
@click.option('--resource-limit', default=None,
help="A CPU/memory limit preset to mimic CI environments like "
"GitHub Actions. Implies --using-docker-cli. Note that "
"GitHub Actions. Mandates --using-docker-cli. Note that "
"exporting ARCHERY_DOCKER_BIN=\"sudo docker\" is likely "
"required, unless Docker is configured with cgroups v2 "
"(else Docker will silently ignore the limits).")
@click.option('--volume', '-v', multiple=True,
help="Set volume within the container")
@click.pass_obj
def docker_run(obj, image, command, *, env, user, force_pull, force_build,
build_only, using_docker_cli, using_docker_buildx, use_cache,
use_leaf_cache, resource_limit, volume):
build_only, use_cache, use_leaf_cache, resource_limit,
volume):
"""
Execute docker-compose builds.
Expand Down Expand Up @@ -234,26 +216,21 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
archery docker run ubuntu-cpp bash
"""
compose = obj['compose']
using_docker_cli |= using_docker_buildx

env = dict(kv.split('=', 1) for kv in env)
try:
if force_pull:
compose.pull(image, pull_leaf=use_leaf_cache,
using_docker=using_docker_cli)
compose.pull(image, pull_leaf=use_leaf_cache)
if force_build:
compose.build(image, use_cache=use_cache,
use_leaf_cache=use_leaf_cache,
using_docker=using_docker_cli,
using_buildx=using_docker_buildx)
use_leaf_cache=use_leaf_cache)
if build_only:
return
compose.run(
image,
command=command,
env=env,
user=user,
using_docker=using_docker_cli,
resource_limit=resource_limit,
volumes=volume
)
Expand All @@ -273,15 +250,11 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
@click.option('--password', '-p', required=False,
envvar='ARCHERY_DOCKER_PASSWORD',
help='Docker repository password')
@click.option('--using-docker-cli', default=False, is_flag=True,
help="Use docker CLI directly for building instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.pass_obj
def docker_compose_push(obj, image, user, password, using_docker_cli):
def docker_compose_push(obj, image, user, password):
"""Push the generated docker-compose image."""
compose = obj['compose']
compose.push(image, user=user, password=password,
using_docker=using_docker_cli)
compose.push(image, user=user, password=password)


@docker.command('images')
Expand Down
Loading

0 comments on commit be3b789

Please sign in to comment.