Skip to content

Commit

Permalink
Simple fix to remove trailing slash in GCS_SAVE_PATH to avoid double …
Browse files Browse the repository at this point in the history
…slashes in GCS_RESULTS_DIR (#4873)
  • Loading branch information
SHuang-Broad authored Jun 11, 2018
1 parent 24b4f7b commit 56ad6cc
Showing 1 changed file with 4 additions and 0 deletions.
4 changes: 4 additions & 0 deletions scripts/sv/copy_sv_results.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ COPY_FASTQ=${COPY_FASTQ:-"Y"}
shift $(($# < 6 ? $# : 6))
SV_ARGS=${*:-${SV_ARGS:-""}}

GCS_SAVE_PATH=${GCS_SAVE_PATH%/} # remove trailing slash to avoid double slashes

# get appropriate ZONE for cluster
echo "CLUSTER_INFO=\$(gcloud dataproc clusters list --project=${PROJECT_NAME} --filter='clusterName=${CLUSTER_NAME}')"
CLUSTER_INFO=$(gcloud dataproc clusters list --project=${PROJECT_NAME} --filter="clusterName=${CLUSTER_NAME}" --format="csv(NAME, WORKER_COUNT, PREEMPTIBLE_WORKER_COUNT, STATUS, ZONE)")
Expand All @@ -62,11 +64,13 @@ if [ -z "${RESULTS_DIR}" ]; then
echo "RESULTS_DIR=${RESULTS_DIR}" 2>&1 | tee -a ${LOCAL_LOG_FILE}
GCS_RESULTS_DIR="${GCS_SAVE_PATH}/${RESULTS_DIR}"
if [[ "${GCS_RESULTS_DIR}" != gs://* ]]; then GCS_RESULTS_DIR="gs://${GCS_RESULTS_DIR}"; fi
echo "Saving results to bucket ${GCS_RESULTS_DIR}"
else
# copy the latest results to google cloud
echo "RESULTS_DIR=${RESULTS_DIR}" 2>&1 | tee -a ${LOCAL_LOG_FILE}
GCS_RESULTS_DIR="${GCS_SAVE_PATH}/${RESULTS_DIR}"
if [[ "${GCS_RESULTS_DIR}" != gs://* ]]; then GCS_RESULTS_DIR="gs://${GCS_RESULTS_DIR}"; fi
echo "Saving results to bucket ${GCS_RESULTS_DIR}"
# chose semi-optimal parallel args for distcp
# 1) count number of files to copy
COUNT_FILES_CMD="hadoop fs -count /${RESULTS_DIR}/ | tr -s ' ' | cut -d ' ' -f 3"
Expand Down

0 comments on commit 56ad6cc

Please sign in to comment.