Skip to content

Upload R Nightly builds #423

Upload R Nightly builds

Upload R Nightly builds #423

Workflow file for this run

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Upload R Nightly builds
# This workflow downloads the (nightly) binaries created in crossbow and uploads them
# to nightlies.apache.org. Due to authorization requirements, this upload can't be done
# from the crossbow repository.
on:
workflow_dispatch:
inputs:
prefix:
description: Job prefix to use.
required: false
default: ''
keep:
description: Number of versions to keep.
required: false
default: 14
schedule:
#Crossbow packaging runs at 0 8 * * *
- cron: '0 14 * * *'
permissions:
contents: read
jobs:
upload:
if: github.repository == 'apache/arrow'
runs-on: ubuntu-latest
steps:
- name: Checkout Arrow
uses: actions/checkout@v3
with:
fetch-depth: 1
path: arrow
repository: apache/arrow
ref: main
submodules: recursive
- name: Checkout Crossbow
uses: actions/checkout@v3
with:
fetch-depth: 0
path: crossbow
repository: ursacomputing/crossbow
ref: main
- name: Set up Python
uses: actions/setup-python@v4
with:
cache: 'pip'
python-version: 3.8
- name: Install Archery
shell: bash
run: pip install -e arrow/dev/archery[all]
- run: mkdir -p binaries
- name: Download Artifacts
env:
PREFIX: ${{ github.event.inputs.prefix || ''}}
run: |
if [ -z $PREFIX ]; then
PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0
fi
echo $PREFIX
archery crossbow download-artifacts -f r-binary-packages -t binaries $PREFIX
if [ -n "$(ls -A binaries/*/*/)" ]; then
echo "Found files!"
else
echo "No files found. Stopping upload."
exit 1
fi
- name: Cache Repo
uses: actions/cache@v3
with:
path: repo
key: r-nightly-${{ github.run_id }}
restore-keys: r-nightly-
- name: Sync from Remote
uses: ./arrow/.github/actions/sync-nightlies
with:
switches: -avzh --update --delete --progress
local_path: repo
remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}
- run: tree repo
- name: Build Repository
shell: Rscript {0}
run: |
# folder that we sync to nightlies.apache.org
repo_root <- "repo"
# The binaries are in a nested dir
# so we need to find the correct path.
art_path <- list.files("binaries",
recursive = TRUE,
include.dirs = TRUE,
pattern = "r-binary-packages$",
full.names = TRUE
)
current_path <- list.files(art_path, full.names = TRUE, recursive = TRUE)
files <- sub("r-(pkg|lib)", repo_root, current_path)
# decode contrib.url from artifact name:
# bin__windows__contrib__4.1 -> bin/windows/contrib/4.1
new_paths <- gsub("__", "/", files)
# strip superfluous nested dirs
new_paths <- sub(art_path, ".", new_paths)
dirs <- dirname(new_paths)
sapply(dirs, dir.create, recursive = TRUE, showWarnings = FALSE)
# overwrite allows us to "force push" a new version with the same name
copy_result <- file.copy(current_path, new_paths, overwrite = TRUE)
if (!all(copy_result)) {
stop("There was an issue while copying the files!")
}
- name: Prune Repository
shell: bash
env:
KEEP: ${{ github.event.inputs.keep || 14 }}
run: |
prune() {
# list files | retain $KEEP newest files | delete everything else
ls -t $1/arrow* | tail -n +$((KEEP + 1)) | xargs --no-run-if-empty rm
}
# find leaf sub dirs
repo_dirs=$(find repo -type d -links 2)
# We want to retain $keep (14) versions of each pkg/lib so we call
# prune on each leaf dir and not on repo/.
for dir in ${repo_dirs[@]}; do
prune $dir
done
- name: Update Repository Index
shell: Rscript {0}
run: |
# folder that we sync to nightlies.apache.org
repo_root <- "repo"
tools::write_PACKAGES(file.path(repo_root, "src/contrib"),
type = "source",
verbose = TRUE,
latestOnly = FALSE
)
repo_dirs <- list.dirs(repo_root)
# find dirs with binary R packages: e.g. */contrib/4.1
pkg_dirs <- grep(".+contrib\\/\\d.+", repo_dirs, value = TRUE)
for (dir in pkg_dirs) {
on_win <- grepl("windows", dir)
tools::write_PACKAGES(dir,
type = ifelse(on_win, "win.binary", "mac.binary"),
verbose = TRUE,
latestOnly = FALSE
)
}
- name: Show repo contents
run: tree repo
- name: Sync to Remote
uses: ./arrow/.github/actions/sync-nightlies
with:
upload: true
switches: -avzh --update --delete --progress
local_path: repo
remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}