No commit message

.circleci/config.yml

@@ -1,3 +1,4 @@
+
 version: 2
 
 jobs:

.github/workflows/crossbow.yml

@@ -0,0 +1,241 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# NOTE: must set "Crossbow" as name to have the badge links working in the
+# github comment reports!
+name: Crossbow
+on:
+ push:
+ branches:
+ - "*-github-*"
+
+
+jobs:
+ package:
+ name: Package
+ runs-on: ["self-hosted", "Linux", "arm64"]
+ env:
+ ARCHITECTURE: arm64
+ steps:
+ - name: Checkout Arrow
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 1
+ path: arrow
+ repository: apache/arrow
+ ref: f5a9a59a10a0022aba13bb1b7caa4b8e5c164c1a
+ submodules: recursive
+
+ - name: Login to Dockerhub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USER }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Free up disk space
+ if: runner.os == 'Linux' && runner.arch == 'X64'
+ shell: bash
+ run: |
+ df -h
+ echo "::group::/usr/local/*"
+ du -hsc /usr/local/*
+ echo "::endgroup::"
+ echo "::group::/usr/local/bin/*"
+ du -hsc /usr/local/bin/*
+ echo "::endgroup::"
+ # ~1GB (From 1.2GB to 214MB)
+ sudo rm -rf \
+ /usr/local/bin/aliyun \
+ /usr/local/bin/azcopy \
+ /usr/local/bin/bicep \
+ /usr/local/bin/cmake-gui \
+ /usr/local/bin/cpack \
+ /usr/local/bin/helm \
+ /usr/local/bin/hub \
+ /usr/local/bin/kubectl \
+ /usr/local/bin/minikube \
+ /usr/local/bin/node \
+ /usr/local/bin/packer \
+ /usr/local/bin/pulumi* \
+ /usr/local/bin/stack \
+ /usr/local/bin/terraform || :
+ echo "::group::/usr/local/share/*"
+ du -hsc /usr/local/share/*
+ echo "::endgroup::"
+ # 1.3GB
+ sudo rm -rf /usr/local/share/powershell || :
+ echo "::group::/opt/*"
+ du -hsc /opt/*
+ echo "::endgroup::"
+ echo "::group::/opt/hostedtoolcache/*"
+ du -hsc /opt/hostedtoolcache/*
+ echo "::endgroup::"
+ # 5.3GB
+ sudo rm -rf /opt/hostedtoolcache/CodeQL || :
+ # 1.4GB
+ sudo rm -rf /opt/hostedtoolcache/go || :
+ # 489MB
+ sudo rm -rf /opt/hostedtoolcache/PyPy || :
+ # 376MB
+ sudo rm -rf /opt/hostedtoolcache/node || :
+ # Remove Web browser packages
+ sudo apt-get purge -y -f firefox \
+ google-chrome-stable \
+ microsoft-edge-stable
+ df -h
+
+
+ - name: Set up Ruby
+ run: |
+ sudo apt update
+ sudo apt install -y \
+ rake \
+ ruby \
+ ruby-dev
+ - name: Prepare apache-arrow-apt-source for arm64
+ if: |
+ env.ARCHITECTURE == 'arm64'
+ run: |
+ pushd arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt
+ for target in *-*; do
+ cp -a ${target} ${target}-arm64
+ done
+ popd
+ - name: Prepare apache-arrow-release for arm64
+ if: |
+ env.ARCHITECTURE == 'arm64'
+ run: |
+ pushd arrow/dev/tasks/linux-packages/apache-arrow-release/yum
+ for target in *-*; do
+ cp -a ${target} ${target}-aarch64
+ done
+ popd
+ - name: Build
+ run: |
+ set -e
+ pushd arrow/dev/tasks/linux-packages
+ rake version:update
+ rake docker:pull || :
+ rake --trace yum:build BUILD_DIR=build
+ popd
+ env:
+ APT_TARGETS: almalinux-8-aarch64
+ ARROW_VERSION: 13.0.0.dev498
+ REPO: ${{ secrets.REPO }}
+ YUM_TARGETS: almalinux-8-aarch64
+ - name: Docker Push
+ continue-on-error: true
+ shell: bash
+ run: |
+ pushd arrow/dev/tasks/linux-packages
+ rake docker:push
+ popd
+ env:
+ APT_TARGETS: almalinux-8-aarch64
+ REPO: ${{ secrets.REPO }}
+ YUM_TARGETS: almalinux-8-aarch64
+ - name: Set up test
+ run: |
+ sudo apt install -y \
+ apt-utils \
+ cpio \
+ createrepo-c \
+ devscripts \
+ gpg \
+ rpm \
+ rsync
+ gem install --user-install apt-dists-merge
+ (echo "Key-Type: RSA"; \
+ echo "Key-Length: 4096"; \
+ echo "Name-Real: Test"; \
+ echo "Name-Email: test@example.com"; \
+ echo "%no-protection") | \
+ gpg --full-generate-key --batch
+ GPG_KEY_ID=$(gpg --list-keys --with-colon test@example.com | grep fpr | cut -d: -f10)
+ echo "GPG_KEY_ID=${GPG_KEY_ID}" >> ${GITHUB_ENV}
+ case "almalinux-8-aarch64" in
+ almalinux-*|amazon-linux-*|centos-*)
+ repositories_dir=arrow/dev/tasks/linux-packages/apache-arrow-release/yum/repositories
+ rpm2cpio ${repositories_dir}/*/*/*/Packages/apache-arrow-release-*.rpm | \
+ cpio -id
+ mv etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow \
+ arrow/dev/tasks/linux-packages/KEYS
+ ;;
+ esac
+ gpg --export --armor test@example.com >> arrow/dev/tasks/linux-packages/KEYS
+ - name: Test
+ run: |
+ set -e
+ pushd arrow/dev/tasks/linux-packages
+ rake --trace yum:test
+ rm -rf yum/repositories
+ popd
+ env:
+ APT_TARGETS: almalinux-8-aarch64
+ ARROW_VERSION: 13.0.0.dev498
+ YUM_TARGETS: almalinux-8-aarch64
+
+ - name: Set up Python by actions/setup-python
+ if: runner.arch == 'X64'
+ uses: actions/setup-python@v4
+ with:
+ python-version: 3.8
+ - name: Set up Python by apt
+ if: runner.os == 'Linux' && runner.arch != 'X64'
+ run: |
+ sudo apt update
+ sudo apt install -y \
+ libgit2-dev \
+ libpython3-dev \
+ python3-pip
+ sudo python3 -m pip install --upgrade pip
+ - name: Checkout Crossbow
+ uses: actions/checkout@v3
+ with:
+ path: crossbow
+ ref: maint-13-nightly-packaging-10
+ - name: Setup Crossbow
+ shell: bash
+ run: |
+ python3 -m pip install -e arrow/dev/archery[crossbow]
+ echo "$HOME/.local/bin" >> $GITHUB_PATH
+ - name: Upload artifacts
+ shell: bash
+ run: |
+ archery crossbow \
+ --queue-path $(pwd)/crossbow \
+ --queue-remote https://github.com/ursacomputing/crossbow.git \
+ upload-artifacts \
+ --sha maint-13-nightly-packaging-10-github-almalinux-8-arm64 \
+ --tag maint-13-nightly-packaging-10-github-almalinux-8-arm64 \
+ "arrow/dev/tasks/linux-packages/*/*/repositories/**/*.rpm" 
+ env:
+ CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ - name: Verify uploaded artifacts
+ shell: bash
+ run: |
+ archery crossbow \
+ --queue-path $(pwd)/crossbow \
+ --queue-remote https://github.com/ursacomputing/crossbow.git \
+ status \
+ --task-filter 'almalinux-8-arm64' \
+ --no-fetch \
+ --validate \
+ maint-13-nightly-packaging-10
+ env:
+ CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}