Skip to content

Upload Java Nightly builds #700

Upload Java Nightly builds

Upload Java Nightly builds #700

Workflow file for this run

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Upload Java Nightly builds
on:
workflow_dispatch:
inputs:
prefix:
description: Job prefix to use.
required: false
default: ''
keep:
description: Number of versions to keep.
required: false
default: 14
schedule:
- cron: '0 14 * * *'
permissions:
contents: read
jobs:
upload:
if: github.repository == 'apache/arrow'
env:
PREFIX: ${{ github.event.inputs.prefix || ''}}
CROSSBOW_GITHUB_TOKEN: ${{ github.token }}
runs-on: ubuntu-latest
steps:
- name: Checkout Arrow
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
with:
fetch-depth: 1
path: arrow
repository: apache/arrow
ref: main
submodules: recursive
- name: Checkout Crossbow
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
with:
fetch-depth: 0
path: crossbow
repository: ursacomputing/crossbow
ref: main
- name: Set up Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
cache: 'pip'
python-version: 3.8
- name: Install Archery
shell: bash
run: pip install -e arrow/dev/archery[all]
- run: mkdir -p binaries
- name: Download Artifacts
run: |
if [ -z $PREFIX ]; then
PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0
fi
echo $PREFIX
archery crossbow download-artifacts -f java-jars -t binaries $PREFIX
- name: Sync from Remote
uses: ./arrow/.github/actions/sync-nightlies
with:
switches: -avzh --update --delete --progress
local_path: repo
remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}
- shell: bash
name: Show local repo sync from remote
run: |
for i in `ls -t repo/org/apache/arrow`; do
echo "- $i: $(find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d \
| wc -l \
| xargs) versions available"
done
- shell: bash
name: Build Repository
run: |
DATE=$(date +%Y-%m-%d)
if [ -z $PREFIX ]; then
PREFIX=nightly-packaging-${DATE}-0
fi
PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+-SNAPSHOT)'
mkdir -p repo/org/apache/arrow/
for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E '.jar|.json|.pom|.xml' | grep SNAPSHOT); do
[[ $LIBRARY =~ $PATTERN_TO_GET_LIB_AND_VERSION ]]
mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE}
# Copy twice to maintain a latest snapshot and some earlier versions
cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
touch repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE}
echo "Artifacts $LIBRARY configured"
done
- name: Prune Repository
shell: bash
env:
KEEP: ${{ github.event.inputs.keep || 14 }}
run: |
for i in `ls -t repo/org/apache/arrow`; do
find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d -print0 \
| xargs -0 ls -t -d \
| tail -n +$((KEEP + 1)) \
| xargs rm -rf
done
- name: Show repo contents
run: tree repo
- name: Sync to Remote
if: ${{ github.repository == 'apache/arrow' }}
uses: ./arrow/.github/actions/sync-nightlies
with:
upload: true
switches: -avzh --update --delete --progress
local_path: repo
remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}