chore(ci): Add integration test CI docker image with all the arrows (#…

…557)

This PR adds an integration test run for every commit by caching a
recent docker image build (weekly) containing the built versions of
integration tests from the Arrow monorepo and arrow-rs. This should
result in a CI job with a reasonable run time that provides high-level
testing and type coverage.

This is based on the arrow-rs version of the integration test job (
https://github.com/apache/arrow-rs/blob/6d4e2f2ceaf423031b0bc72f54c547dd77a0ddbb/.github/workflows/integration.yml
).

---------

Co-authored-by: Jacob Wujciak-Jens <jacob@wujciak.de>

.github/workflows/docker-build.yaml

@@ -14,7 +14,7 @@
 # KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations
 # under the License.
-name: Build Docker Images
+name: docker-build
 
 on:
  workflow_dispatch:
@@ -30,24 +30,29 @@ permissions:
  contents: read
  packages: write
 
+concurrency:
+ group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+ cancel-in-progress: true
+
 jobs:
  build-docker:
- name: "docker-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
+ name: "${{ matrix.config.service }}-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
  runs-on: ${{ matrix.config.runs_on }}
  strategy:
  fail-fast: false
  matrix:
  config:
- - { runs_on: "ubuntu-latest", platform: "ubuntu", arch: "amd64" }
- - { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64" }
- - { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64" }
- - { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64" }
+ - { runs_on: "ubuntu-latest", platform: "ubuntu", arch: "amd64", service: "verify" }
+ - { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64", service: "verify" }
+ - { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64", service: "verify" }
+ - { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64", service: "verify" }
+ - { runs_on: "ubuntu-latest", arch: "amd64", platform: "integration", service: "integration" }
 
- - { runs_on: ["self-hosted", "arm"], platform: "ubuntu", arch: "arm64" }
- - { runs_on: ["self-hosted", "arm"], platform: "fedora", arch: "arm64" }
- - { runs_on: ["self-hosted", "arm"], platform: "alpine", arch: "arm64" }
+ - { runs_on: ["self-hosted", "arm"], platform: "ubuntu", arch: "arm64", service: "verify" }
+ - { runs_on: ["self-hosted", "arm"], platform: "fedora", arch: "arm64", service: "verify" }
+ - { runs_on: ["self-hosted", "arm"], platform: "alpine", arch: "arm64", service: "verify" }
 
- - { runs_on: "ubuntu-latest", platform: "alpine", arch: "s390x" }
+ - { runs_on: "ubuntu-latest", platform: "alpine", arch: "s390x", service: "verify" }
 
  env:
  DOCKER_CLIENT_TIMEOUT: 360
@@ -64,20 +69,20 @@ jobs:
  docker context create builders
 
  - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
+ uses: docker/setup-buildx-action@v3
  with:
  endpoint: builders
 
  - name: Login to GitHub Container Registry
- uses: docker/login-action@v2
+ uses: docker/login-action@v3
  with:
  registry: ghcr.io
  username: ${{ github.actor }}
  password: ${{ secrets.GITHUB_TOKEN }}
 
  - name: Set up QEMU
  if: matrix.config.arch == 's390x'
- uses: docker/setup-qemu-action@v2
+ uses: docker/setup-qemu-action@v3
 
  - name: Build
  env:
@@ -94,15 +99,15 @@ jobs:
  NANOARROW_PLATFORM: ${{ matrix.config.platform }}
  DOCKER_DEFAULT_PLATFORM: "linux/${{ matrix.config.arch }}"
  run: |
- docker-compose push verify
+ docker-compose push ${{ matrix.config.service }}
 
  # Build some arch-agnostic images for non-verify docker bits
  build-docker-manifest:
  runs-on: ubuntu-latest
  needs: build-docker
  steps:
  - name: Login to GitHub Container Registry
- uses: docker/login-action@v2
+ uses: docker/login-action@v3
  with:
  registry: ghcr.io
  username: ${{ github.actor }}

.github/workflows/integration.yaml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: integration
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ paths:
+ - .github/workflows/integration.yaml
+ - src/nanoarrow/**
+
+permissions:
+ contents: read
+
+jobs:
+ integration:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Run integration tests
+ run: |
+ echo "::group::Docker Pull"
+ docker compose run --rm -e GITHUB_ACTIONS integration

ci/docker/integration.dockerfile

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM apache/arrow-dev:amd64-conda-integration
+
+ENV ARROW_USE_CCACHE=OFF \
+ ARROW_CPP_EXE_PATH=/build/cpp/debug \
+ ARROW_NANOARROW_PATH=/build/nanoarrow \
+ ARROW_RUST_EXE_PATH=/build/rust/debug \
+ BUILD_DOCS_CPP=OFF \
+ ARROW_INTEGRATION_CPP=ON \
+ ARROW_INTEGRATION_CSHARP=ON \
+ ARROW_INTEGRATION_GO=ON \
+ ARROW_INTEGRATION_JAVA=ON \
+ ARROW_INTEGRATION_JS=ON \
+ ARCHERY_INTEGRATION_WITH_NANOARROW="1" \
+ ARCHERY_INTEGRATION_WITH_RUST="1"
+
+# These are necessary because the github runner overrides $HOME
+# https://github.com/actions/runner/issues/863
+ENV RUSTUP_HOME=/root/.rustup
+ENV CARGO_HOME=/root/.cargo
+
+ENV ARROW_USE_CCACHE=OFF
+ENV ARROW_CPP_EXE_PATH=/build/cpp/debug
+ENV ARROW_NANOARROW_PATH=/build/nanoarrow
+ENV ARROW_RUST_EXE_PATH=/build/rust/debug
+ENV BUILD_DOCS_CPP=OFF
+
+# Clone the arrow monorepo
+RUN git clone https://github.com/apache/arrow.git /arrow-integration --recurse-submodules
+
+# Clone the arrow-rs repo
+RUN git clone https://github.com/apache/arrow-rs /arrow-integration/rust
+
+# Workaround: stable rust is not compatible with glibc provided by the
+# provided arrow docker image https://github.com/apache/arrow/issues/41637
+RUN cd /arrow-integration/rust && rustup override set 1.77
+
+# Build all the integrations except nanoarrow (since we'll do that ourselves on each run)
+RUN ARCHERY_INTEGRATION_WITH_NANOARROW="0" \
+ conda run --no-capture-output \
+ /arrow-integration/ci/scripts/integration_arrow_build.sh \
+ /arrow-integration \
+ /build

docker-compose.yml

@@ -46,3 +46,21 @@ services:
  - ./ci/scripts/build-docs.sh:/build-docs.sh
  - ${NANOARROW_DOCKER_SOURCE_DIR}:/nanoarrow
  command: "/bin/bash /build-docs.sh /nanoarrow"
+
+ integration:
+ # Based on an arrow-dev repo image that is amd64 only
+ image: ${REPO}:integration-amd64
+ build:
+ context: .
+ cache_from:
+ - ${REPO}:integration-amd64
+ dockerfile: ci/docker/integration.dockerfile
+ volumes:
+ - ${NANOARROW_DOCKER_SOURCE_DIR}:/arrow-integration/nanoarrow
+ command:
+ ["echo '::group::Build nanoarrow' &&
+ conda run --no-capture-output /arrow-integration/ci/scripts/nanoarrow_build.sh /arrow-integration /build &&
+ echo '::endgroup::' &&
+ echo '::group::Run integration tests' &&
+ conda run --no-capture-output /arrow-integration/ci/scripts/integration_arrow.sh /arrow-integration /build &&
+ echo '::endgroup::'"]