bootstrap-testdata

#!/bin/sh -x
# Bootstrap minimum test data on Thredds.
# To run testsuite in https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests.
#
# Need to open temporary Thredds "testdata/secure/" on Birdhouse production to anonymous group.
#
# To be run locally on the host running Thredds.  Will populate test data files
# under DATASET_ROOT, customizable by env var.
#
# Need write-access to DATASET_ROOT (/data/datasets/).


THIS_FILE="$(readlink -f "$0" || realpath "$0")"
THIS_DIR="$(dirname "${THIS_FILE}")"
COMPOSE_DIR="${COMPOSE_DIR:-$(dirname "${THIS_DIR}")}"

if [ -f "${COMPOSE_DIR}/read-configs.include.sh" ]; then
    . "${COMPOSE_DIR}/read-configs.include.sh"

    # Get THREDDS_SERVICE_DATA_LOCATION_ON_HOST, BIRDHOUSE_DATA_PERSIST_ROOT
    read_configs
fi


if [ -z "${DATASET_ROOT}" ]; then
    # Default for when unable to source read-configs.include.sh (ie when
    # used standalone outside of the checkout).
    DATASET_ROOT="${DATASET_ROOT:-${THREDDS_SERVICE_DATA_LOCATION_ON_HOST:-${BIRDHOUSE_DATA_PERSIST_ROOT:-/data}/datasets}}"
fi

FROM_SERVER=${FROM_SERVER:-"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/fileServer/birdhouse"}

FILE_LIST="
nrcan/nrcan_canada_daily/tasmin/nrcan_canada_daily_tasmin_2013.nc
testdata/flyingpigeon/cmip3/pr.sresa2.miub_echo_g.run1.atm.da.nc
testdata/flyingpigeon/cmip3/tasmin.sresa2.miub_echo_g.run1.atm.da.nc
testdata/flyingpigeon/cmip5/tasmax_Amon_MPI-ESM-MR_rcp45_r1i1p1_200601-200612.nc
testdata/flyingpigeon/cmip5/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc
testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc
testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc
ets/Watersheds_5797_cfcompliant.nc
testdata/xclim/cmip6/sic_SImon_CCCma-CanESM5_ssp245_r13i1p2f1_2020.nc
cccma/CanESM2/historical/day/atmos/r1i1p1/tasmin/tasmin_day_CanESM2_historical_r1i1p1_18500101-20051231.nc
cccma/CanESM2/historical/fx/atmos/r0i0p0/sftlf/sftlf_fx_CanESM2_historical_r0i0p0.nc
testdata/xclim/NRCANdaily/nrcan_canada_daily_tasmin_1990.nc
"

cd "${DATASET_ROOT}" || exit 1

for afile in ${FILE_LIST}; do
    if [ ! -f "${afile}" ]; then
        PARENT_DIRS="$(dirname "${afile}")"
        if [ ! -d "${PARENT_DIRS}" ]; then
            mkdir -p "${PARENT_DIRS}"
        fi
        set -e  # Fail on error.
        curl "${FROM_SERVER}/${afile}" --output "${afile}"
        set +e
    fi
done