ARROW-8743: [CI][C++] Add a test job for s390x

Closes #7131 from kou/cpp-ci-s390x

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

.travis.yml

@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+dist: bionic
+
+language: minimal
+
+addons:
+  apt:
+    packages:
+      - python3-pip
+
+services:
+  - docker
+
+jobs:
+  include:
+    - name: "C++ on s390x"
+      os: linux
+      arch: s390x
+      env:
+        ARCH: s390x
+        ARROW_CI_MODULES: "CPP"
+        ARROW_COMPUTE: "OFF"
+        ARROW_CSV: "OFF"
+        ARROW_DATASET: "OFF"
+        ARROW_FILESYSTEM: "OFF"
+        ARROW_FLIGHT: "OFF"
+        ARROW_GANDIVA: "OFF"
+        ARROW_HDFS: "OFF"
+        ARROW_IPC: "OFF"
+        ARROW_ORC: "OFF"
+        ARROW_PARQUET: "OFF"
+        ARROW_PLASMA: "OFF"
+        PARQUET_BUILD_EXAMPLES: "OFF"
+        PARQUET_BUILD_EXECUTABLES: "OFF"
+        DOCKER_IMAGE_ID: ubuntu-cpp
+        UBUNTU: "20.04"
+  allow_failures:
+    - arch: s390x
+
+env:
+  DOCKER_BUILDKIT: 0
+  COMPOSE_DOCKER_CLI_BUILD: 1
+  ARROW_ENABLE_TIMING_TESTS: "OFF"
+
+before_install:
+  - eval "$(python ci/detect-changes.py)"
+  - |
+    arrow_ci_affected=no
+    for arrow_ci_module in ${ARROW_CI_MODULES}; do
+      arrow_ci_affected_variable=ARROW_CI_${arrow_ci_module}_AFFECTED
+      if [ "$(eval 'echo ${arrow_ci_affected_variable}')" != "1" ]; then
+        arrow_ci_affected=yes
+      fi
+    done
+    if [ "${arrow_ci_affected}" = "no" ]; then
+      travis_terminate 0
+    fi
+
+install:
+  - pip3 install -e dev/archery[docker]
+
+script:
+  - sudo sysctl -w kernel.core_pattern="core.%e.%p"
+  # This isn't allowed on Travis CI:
+  #   /home/travis/.travis/functions: line 109: ulimit: core file size: cannot modify limit: Operation not permitted
+  - |
+    ulimit -c unlimited || :
+  - |
+    archery docker run \
+      -e ARROW_COMPUTE=${ARROW_COMPUTE:-ON} \
+      -e ARROW_CSV=${ARROW_CSV:-ON} \
+      -e ARROW_DATASET=${ARROW_DATASET:-ON} \
+      -e ARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \
+      -e ARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \
+      -e ARROW_GANDIVA=${ARROW_GANDIVA:-ON} \
+      -e ARROW_HDFS=${ARROW_HDFS:-ON} \
+      -e ARROW_IPC=${ARROW_IPC:-ON} \
+      -e ARROW_ORC=${ARROW_ORC:-ON} \
+      -e ARROW_PARQUET=${ARROW_PARQUET:-ON} \
+      -e ARROW_PLASMA=${ARROW_PLASMA:-ON} \
+      -e PARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-ON} \
+      -e PARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-ON} \
+      --volume ${PWD}/build:/build \
+      ${DOCKER_IMAGE_ID}
+
+after_success:
+  - |
+    archery docker push ${DOCKER_IMAGE_ID} || :

ci/docker/ubuntu-20.04-cpp.dockerfile

@@ -66,6 +66,7 @@ RUN apt-get update -y -q && \
         libssl-dev \
         libthrift-dev \
         libzstd-dev \
+        make \
         ninja-build \
         pkg-config \
         protobuf-compiler \
@@ -104,6 +105,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_WITH_ZSTD=ON \
     GTest_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
-    PARQUET_BUILD_EXECUTABLES=ON \
     PARQUET_BUILD_EXAMPLES=ON \
-    PATH=/usr/lib/ccache/:$PATH
+    PARQUET_BUILD_EXECUTABLES=ON \
+    PATH=/usr/lib/ccache/:$PATH \
+    PYTHON=python3

cpp/build-support/run-test.sh

@@ -97,7 +97,7 @@ function run_test() {
   rm -f $XMLFILE
 
   $TEST_EXECUTABLE "$@" 2>&1 \
-    | $ROOT/build-support/asan_symbolize.py \
+    | ${PYTHON:-python} $ROOT/build-support/asan_symbolize.py \
     | ${CXXFILT:-c++filt} \
     | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \
     | $pipe_cmd 2>&1 | tee $LOGFILE

cpp/cmake_modules/SetupCxxFlags.cmake

@@ -29,6 +29,8 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "armv7")
   set(ARROW_CPU_FLAG "armv7")
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc")
   set(ARROW_CPU_FLAG "ppc")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
+  set(ARROW_CPU_FLAG "s390x")
 endif()
 # Check architecture specific compiler flags
 if(ARROW_CPU_FLAG STREQUAL "x86")

dev/archery/archery/cli.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -696,9 +695,11 @@ def docker_compose(obj, src):
 @click.option('--dry-run/--execute', default=False,
               help="Display the docker-compose commands instead of executing "
                    "them.")
+@click.option('--volume', '-v', multiple=True,
+              help="Set volume within the container")
 @click.pass_obj
 def docker_compose_run(obj, image, command, env, force_pull, force_build,
-                       use_cache, use_leaf_cache, dry_run):
+                       use_cache, use_leaf_cache, dry_run, volume):
     """Execute docker-compose builds.
 
     To see the available builds run `archery docker list`.
@@ -723,6 +724,9 @@ def docker_compose_run(obj, image, command, env, force_pull, force_build,
     # pass runtime parameters via docker environment variables
     archery docker run -e CMAKE_BUILD_TYPE=release ubuntu-cpp
 
+    # set a volume
+    archery docker run -v $PWD/build:/build ubuntu-cpp
+
     # starting an interactive bash session for debugging
     archery docker run ubuntu-cpp bash
     """
@@ -749,7 +753,8 @@ def _print_command(self, *args, **kwargs):
             force_pull=force_pull,
             force_build=force_build,
             use_cache=use_cache,
-            use_leaf_cache=use_leaf_cache
+            use_leaf_cache=use_leaf_cache,
+            volumes=volume
         )
     except UndefinedImage as e:
         raise click.ClickException(

dev/archery/archery/docker.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -162,7 +161,8 @@ def build(self, image, use_cache=True, use_leaf_cache=True):
             self._execute('build', '--no-cache', image)
 
     def run(self, image, command=None, env=None, force_pull=False,
-            force_build=False, use_cache=True, use_leaf_cache=True):
+            force_build=False, use_cache=True, use_leaf_cache=True,
+            volumes=None):
         self._validate_image(image)
 
         if force_pull:
@@ -175,6 +175,9 @@ def run(self, image, command=None, env=None, force_pull=False,
         if env is not None:
             for k, v in env.items():
                 args.extend(['-e', '{}={}'.format(k, v)])
+        if volumes is not None:
+            for volume in volumes:
+                args.extend(['--volume', volume])
 
         args.append(image)
         if command is not None:

dev/archery/archery/tests/test_cli.py

@@ -34,7 +34,8 @@
             force_pull=True,
             force_build=True,
             use_cache=True,
-            use_leaf_cache=True
+            use_leaf_cache=True,
+            volumes=()
         )
     ),
     (
@@ -46,7 +47,8 @@
             force_pull=True,
             force_build=True,
             use_cache=True,
-            use_leaf_cache=True
+            use_leaf_cache=True,
+            volumes=()
         )
     ),
     (
@@ -58,7 +60,8 @@
             force_pull=False,
             force_build=False,
             use_cache=True,
-            use_leaf_cache=True
+            use_leaf_cache=True,
+            volumes=()
         )
     ),
     (
@@ -73,11 +76,12 @@
             force_pull=False,
             force_build=True,
             use_cache=False,
-            use_leaf_cache=False
+            use_leaf_cache=False,
+            volumes=()
         )
     ),
     (
-        ['-e', 'ARROW_GANDIVA=OFF', '-e',  'ARROW_FLIGHT=ON', 'ubuntu-cpp'],
+        ['-e', 'ARROW_GANDIVA=OFF', '-e', 'ARROW_FLIGHT=ON', 'ubuntu-cpp'],
         ['ubuntu-cpp'],
         dict(
             command=None,
@@ -88,7 +92,27 @@
             force_pull=True,
             force_build=True,
             use_cache=True,
-            use_leaf_cache=True
+            use_leaf_cache=True,
+            volumes=()
+        )
+    ),
+    (
+        [
+            '--volume', './build:/build', '-v', './ccache:/ccache:delegated',
+            'ubuntu-cpp'
+        ],
+        ['ubuntu-cpp'],
+        dict(
+            command=None,
+            env={},
+            force_pull=True,
+            force_build=True,
+            use_cache=True,
+            use_leaf_cache=True,
+            volumes=(
+                './build:/build',
+                './ccache:/ccache:delegated',
+            )
         )
     )
 ])

dev/archery/archery/tests/test_docker.py

@@ -384,6 +384,17 @@ def test_compose_run(arrow_compose_path):
         ])
         compose.run('conda-python', env=env)
 
+    expected_calls = [
+        (
+            "run --rm --volume /host/build:/build "
+            "--volume /host/ccache:/ccache:delegated conda-python"
+        )
+    ]
+    compose = DockerCompose(arrow_compose_path)
+    with assert_compose_calls(compose, expected_calls):
+        volumes = ("/host/build:/build", "/host/ccache:/ccache:delegated")
+        compose.run('conda-python', volumes=volumes)
+
 
 def test_compose_run_force_pull_and_build(arrow_compose_path):
     compose = DockerCompose(arrow_compose_path)

docker-compose.yml

@@ -22,8 +22,8 @@
 # defaults are set in .env file.
 #
 # Example:
-# $ ARCH=arm64v8 docker-compose build ubuntu-bionic-cpp
-# $ ARCH=arm64v8 docker-compose run ubuntu-bionic-cpp
+# $ ARCH=arm64v8 docker-compose build ubuntu-cpp
+# $ ARCH=arm64v8 docker-compose run ubuntu-cpp
 #
 #
 # Coredumps
@@ -257,7 +257,7 @@ services:
     #   docker-compose build ubuntu-cpp
     #   docker-compose run --rm ubuntu-cpp
     # Parameters:
-    #   ARCH: amd64, arm64v8, ...
+    #   ARCH: amd64, arm64v8, s390x, ...
     #   UBUNTU: 16.04, 18.04, 20.04
     image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
     build: