GH-34543: [CI] Self-hosted ARM workflows improvements (#34512)

### Rationale for this change Fixes some post-merge review comments ### What changes are included in this PR? Comments from original: #34482 ### Are these changes tested? On CI * Closes: #34543 Authored-by: Raúl Cumplido <raulcumplido@gmail.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
apache · Mar 14, 2023 · 0582122 · 0582122
1 parent 591042f
commit 0582122
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 124 deletions.
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -55,22 +55,39 @@ env:
 jobs:
   docker:
     name: ${{ matrix.title }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
     strategy:
       fail-fast: false
       matrix:
-        image:
-          - conda-cpp
-          - ubuntu-cpp-sanitizer
         include:
-          - image: conda-cpp
+          - arch: amd64
+            clang-tools: "14"
+            image: conda-cpp
+            llvm: "14"
+            runs-on: ubuntu-latest
             title: AMD64 Conda C++
-          - image: ubuntu-cpp-sanitizer
+            ubuntu: "22.04"
+          - arch: amd64
+            clang-tools: "14"
+            image: ubuntu-cpp-sanitizer
+            llvm: "14"
+            runs-on: ubuntu-latest
             title: AMD64 Ubuntu 22.04 C++ ASAN UBSAN
+            ubuntu: "22.04"
+          - arch: arm64v8
+            clang-tools: "10"
+            image: ubuntu-cpp
+            llvm: "10"
+            runs-on: ["self-hosted", "arm", "linux"]
+            title: ARM64 Ubuntu 20.04 C++
+            ubuntu: "20.04"
     env:
-      UBUNTU: "22.04"
+      ARCH: ${{ matrix.arch }}
+      CLANG_TOOLS: ${{ matrix.clang-tools }}
+      LLVM: ${{ matrix.llvm }}
+      UBUNTU: ${{ matrix.ubuntu }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v3
@@ -84,11 +101,10 @@ jobs:
           key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
           restore-keys: ${{ matrix.image }}-
       - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
+        run: |
+          sudo apt install -y --no-install-recommends python3 python3-pip
       - name: Setup Archery
-        run: pip install -e dev/archery[docker]
+        run: python3 -m pip install -e dev/archery[docker]
       - name: Execute Docker Build
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -400,54 +416,3 @@ jobs:
         run: |
           PATH="$(cygpath --unix ${PYTHON_BIN_DIR}):${PATH}"
           ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build"
-
-  linux-arm:
-    name: ARM64 Ubuntu 20.04 C++
-    runs-on: ["arm", "linux"]
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 75
-    env:
-      ARROW_ENABLE_TIMING_TESTS: "OFF"
-      ARCH: arm64v8
-      ARROW_CI_MODULES: "CPP"
-      DOCKER_IMAGE_ID: ubuntu-cpp
-      # ARROW_USE_GLOG=OFF is needed to avoid build error caused by
-      # glog and CMAKE_UNITY_BUILD=ON.
-      DOCKER_RUN_ARGS: >-
-        "
-        -e ARROW_BUILD_STATIC=OFF
-        -e ARROW_ORC=OFF
-        -e ARROW_USE_GLOG=OFF
-        -e CMAKE_UNITY_BUILD=ON
-        "
-      # The LLVM's APT repository doesn't provide arm64 binaries.
-      # We should use LLVM provided by Ubuntu.
-      CLANG_TOOLS: "10"
-      LLVM: "10"
-      UBUNTU: "20.04"
-    steps:
-      - name: Checkout Arrow
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          submodules: recursive
-      - name: Setup Python
-        run: |
-          sudo apt install -y --no-install-recommends python3 python3-pip
-      - name: Setup Archery
-        run: python3 -m pip install -e dev/archery[docker]
-      - name: Execute Docker Build
-        env:
-          ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
-          ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: |
-          sudo sysctl -w kernel.core_pattern="core.%e.%p"
-          ulimit -c unlimited
-          archery docker run ubuntu-cpp
-      - name: Docker Push
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
-        env:
-          ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
-          ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        continue-on-error: true
-        run: archery docker push ubuntu-cpp
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -42,20 +42,31 @@ permissions:
 jobs:
 
   docker:
-    name: AMD64 Debian 11 Go ${{ matrix.go }}
-    runs-on: ubuntu-latest
+    name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
+    runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
         include:
-          - go: 1.17
+          - arch-label: AMD64
+            arch: amd64
+            go: 1.17
+            runs-on: ubuntu-latest
             staticcheck: v0.2.2
-          - go: 1.18
+          - arch-label: AMD64
+            arch: amd64
+            go: 1.18
+            runs-on: ubuntu-latest
             staticcheck: v0.3.3
+          - arch-label: ARM64
+            arch: arm64v8
+            go: 1.17
+            staticcheck: v0.2.2
+            runs-on: ["self-hosted", "arm", "linux"]
     env:
+      ARCH: ${{ matrix.arch }}
       GO: ${{ matrix.go }}
       STATICCHECK: ${{ matrix.staticcheck }}
     steps:
@@ -65,44 +76,54 @@ jobs:
           fetch-depth: 0
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
+        run: |
+          sudo apt install -y --no-install-recommends python3 python3-pip
       - name: Setup Archery
-        run: pip install -e dev/archery[docker]
+        run: python3 -m pip install -e dev/archery[docker]
       - name: Execute Docker Build
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         run: archery docker run debian-go
       - name: Docker Push
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
+        if: >-
+          success() &&
+          github.event_name == 'push' &&
+          github.repository == 'apache/arrow'
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
         run: archery docker push debian-go
       - name: Install Go ${{ matrix.go }} for Benchmarks
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
+        if: >-
+          success() &&
+          matrix.arch == 'amd64' &&
+          github.event_name == 'push' &&
+          github.repository == 'apache/arrow'
         uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
           cache: true
           cache-dependency-path: go/go.sum
       - name: Run Benchmarks
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
+        if: >-
+          success() &&
+          matrix.arch == 'amd64' &&
+          github.event_name == 'push' &&
+          github.repository == 'apache/arrow'
         env:
           CONBENCH_URL: https://conbench.ursa.dev
           CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
           CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
           CONBENCH_REF: ${{ github.ref_name }}
-          CONBENCH_MACHINE_INFO_NAME: amd64-debian-11
+          CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
         run: |
-          pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
-          python ci/scripts/go_bench_adapt.py
+          python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
+          python3 ci/scripts/go_bench_adapt.py
 
   docker_cgo:
-    name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO
+    name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 15
@@ -145,7 +166,7 @@ jobs:
 
 
   docker_cgo_python:
-    name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO Python
+    name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 15
@@ -373,41 +394,3 @@ jobs:
       - name: Test
         shell: bash
         run: ci/scripts/go_test.sh $(pwd)
-
-  linux-arm:
-    name: ARM64 Debian 11 GO ${{ matrix.go }}
-    runs-on: ["self-hosted", "linux"]
-    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        go: [1.17]
-    env:
-      GO: ${{ matrix.go }}
-      ARROW_ENABLE_TIMING_TESTS: "OFF"
-      ARCH: arm64v8
-      ARROW_CI_MODULES: "GO"
-      DOCKER_IMAGE_ID: debian-go
-    steps:
-      - name: Checkout Arrow
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          submodules: recursive
-      - name: Setup Python
-        run: |
-          sudo apt install -y --no-install-recommends python3 python3-pip
-      - name: Setup Archery
-        run: python3 -m pip install -e dev/archery[docker]
-      - name: Execute Docker Build
-        env:
-          ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
-          ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: archery docker run debian-go
-      - name: Docker Push
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
-        env:
-          ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
-          ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        continue-on-error: true
-        run: archery docker push debian-go
diff --git a/ci/scripts/go_build.sh b/ci/scripts/go_build.sh
@@ -20,13 +20,6 @@
 set -ex
 
 source_dir=${1}/go
-ARCH=`uname -m`
-
-# Arm64 CI is triggered by travis and run in arm64v8/golang:1.17-bullseye
-if [ "aarch64" == "$ARCH" ]; then
-# Install `staticcheck`
-  GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck@v0.2.2
-fi
 
 pushd ${source_dir}/arrow