shellcheck: lint shell script files

.travis.yml

@@ -55,6 +55,7 @@ unittest: &unittest
   script:
     - export PYTHONPATH=$(pwd -P)/lib/
     - pycodestyle
+    - .travis/shellchecker
     - pytest --cov-append --cov=lib/cylc --cov=lib/parsec
   after_success:
     # Report metrics, such as coverage

.travis/after_failure.sh

@@ -22,5 +22,7 @@
 
 cylc scan -f --color=never
 
-find $HOME/cylc-run -name '*.err' -type f -exec echo '==== {} ====' \; -exec cat '{}' \;
-find /tmp/${USER}/cylctb-* -type f -exec echo '==== {} ====' \; -exec cat '{}' \;
+find "$HOME/cylc-run" -name '*.err' -type f \
+    -exec echo '==== {} ====' \; -exec cat '{}' \;
+find "/tmp/${USER}/cylctb-"* -type f \
+    -exec echo '==== {} ====' \; -exec cat '{}' \;

.travis/install.sh

@@ -33,11 +33,12 @@ if grep -E '(unit-tests|functional-tests)' <<< "${args[@]}"; then
 fi
 
 if grep 'unit-tests' <<< "${args[@]}"; then
+    sudo apt-get install shellcheck
     pip install pycodestyle pytest testfixtures empy
 fi
 
 # install dependencies required for building documentation
-if grep 'docs' <<< "${args[@]}$"; then
+if grep 'docs' <<< "${args[@]}"; then
     pip install sphinx
     # for PDF output via LaTeX builder
     sudo apt-get install texlive-latex-base

.travis/shellchecker

@@ -0,0 +1,94 @@
+#!/bin/bash
+#
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Wrapper for running `shellcheck` over projects.
+
+set -e
+cd "$(dirname "$0")/../"
+
+# find shell files under the specified directory
+find_files () {
+    SEARCH_PATH="$1"
+    grep --color=never -Rl '^#\!.*bash' "${SEARCH_PATH}" | sed 's/^\.\///'
+    find "${SEARCH_PATH}" -name '*.sh' | sed 's/^\.\///'
+}
+
+main () {
+    # parse CLI
+    includes=()
+    excludes=()
+    args=()
+    while [[ "$#" -gt 0 ]]; do
+        if [[ ${1} == '--' ]]; then
+            shift
+            break
+        elif [[ ${1} == "--exclude" ]]; then
+            shift
+            excludes+=("$1")
+        elif [[ ${1} == "-*" ]]; then
+            args+=("$1")
+        else
+            mapfile -t files < <(find_files "${1}")
+            includes=("${includes[@]}" "${files[@]}")
+        fi
+        shift
+    done
+
+    # extract files list
+    files=()
+    for include in "${includes[@]}"; do
+        skip=0
+        for exclude in "${excludes[@]}"; do
+            if [[ "${include}" == "${exclude}"* ]]; then
+                skip=1
+                break
+            fi
+        done
+        if [[ $skip == 0 ]]; then
+            files+=("${include}")
+        fi
+    done
+
+    # sort $files removing duplicates
+    readarray -t files < <(for x in "${files[@]}"; do echo "$x"; done | sort -u)
+
+    # run shellcheck
+    shellcheck "$@" "${files[@]}"
+}
+
+# default configuration for linting cylc
+default () {
+    # run a strict check on all "functional" scripts
+    main '.' \
+        --exclude 'tests/' \
+        --exclude 'lib/parsec/tests' \
+        --exclude 'lib/cylc/job.sh' \
+        --exclude 'etc/dev-bin/live-graph-movie.sh' \
+        -- -e SC1090
+    main 'lib/cylc/job.sh' -- -e SC1090 -s bash
+    main 'lib/cylc/job.sh' -- -e SC1090 -s ksh
+
+    # run a lenient check on all test scripts
+    main 'tests' 'lib/parsec/tests' -- -S error -e SC1090
+}
+
+if [[ $# -gt 0 ]]; then
+    main "$@"
+else
+    default
+fi

bin/cylc

@@ -31,7 +31,7 @@ print_version() {
     if [[ "$#" -eq 0 ]]; then
         echo "$CYLC_VERSION"
     fi
-    if [[ "$@" == 'long' || "$@" == '--long' ]]; then
+    if [[ "$*" == 'long' || "$*" == '--long' ]]; then
     echo "Cylc ${CYLC_VERSION} (${CYLC_DIR})"    
     fi
 }
@@ -52,11 +52,6 @@ init_cylc() {
 help_util() {
     # Deals with form 'cylc [COMMAND] --help'
     cd "${CYLC_HOME_BIN}"
-    # deal with graph which is a weird edge case...
-    if [[ "$@" == "graph" ]]; then
-        local COMMAND="${CYLC_HOME_BIN}/cylc-graph"
-        exec "${COMMAND}" "--help"
-    fi
     # For help command/option with no args
     if [[ $# == 0 && "${HELP_OPTS[*]} " == *"$UTIL"* ]]; then
         exec "${CYLC_HOME_BIN}/cylc-help"
@@ -100,14 +95,15 @@ __HELP__
         fi
     fi
     # If category name is used as arg, call the help func with the category
+    local COMMAND
     if [[ $# -gt 1  &&  ( "${CATEGORIES[*]} " == *" $1 "* 
             || " ${HELP_OPTS[*]} " == *" $1 "* ) ]]; then
-        local COMMAND="${CYLC_HOME_BIN}/cylc-help"
+        COMMAND="${CYLC_HOME_BIN}/cylc-help"
         exec "${COMMAND}" "$@"
     fi
     # Deal with cases like 'cylc --help [COMMAND/CATEGORY]'
     if [[ $# -gt 1 && -f "$(ls "cylc-$2"* 2>'/dev/null')" ]]; then
-        local COMMAND="${CYLC_HOME_BIN}/$(ls "cylc-$2"*)"
+        COMMAND="${CYLC_HOME_BIN}/$(ls "cylc-$2"*)"
         exec "${COMMAND}" "--help"
     fi
     # If not a category or not an actual command in the bin dir, exit
@@ -156,7 +152,8 @@ path_lead() {
 }
 
 get_command_from_abbr() {
-    local COMMAND="$(cd "${CYLC_HOME_BIN}" && ls "cylc-$1"* 2>'/dev/null')"
+    local COMMAND
+    COMMAND="$(cd "${CYLC_HOME_BIN}" && ls "cylc-$1"* 2>'/dev/null')"
     if [[ -z "${COMMAND}" ]]; then
         # Abbreviation has no match, bad
         echo "cylc $1: unknown utility. Abort." >&2

bin/cylc-conditions

@@ -38,5 +38,5 @@ fi
 echo
 echo "The GNU General Public License v3.0"
 echo
-cat $CYLC_DIR/COPYING
+cat "$CYLC_DIR/COPYING"
 echo

bin/cylc-graph-diff

@@ -19,7 +19,7 @@ set -eu
 
 _clean() {
     if [[ -n "${DIFF_FILES:-}" ]]; then
-        rm -f ${DIFF_FILES:-}
+        rm -f "${DIFF_FILES[@]:-}"
     fi
 }
 
@@ -55,7 +55,7 @@ for ARG in "$@"; do
     else
         # A suite - check it's registered.
         if ! cylc print --fail "$ARG$" >/dev/null 2>&1; then
-            echo "Suite not found: "$ARG >&2
+            echo "Suite not found: $ARG" >&2
             exit 1
         fi
         SUITES="$SUITES $ARG"
@@ -69,22 +69,22 @@ fi
 
 trap _clean EXIT
 trap _clean ERR
-DIFF_FILES=""
+DIFF_FILES=()
 for SUITE in $SUITES; do
     FILE=$(mktemp -t "$(tr '/' '.' <<<"${SUITE}").graph.ref.XXXX")
-    cylc graph --reference "${SUITE}" "$@" >$FILE
-    DIFF_FILES="$DIFF_FILES $FILE"
+    cylc graph --reference "${SUITE}" "$@" >"$FILE"
+    DIFF_FILES+=("$FILE")
 done
 
 if $GRAPHICAL_DIFF; then
     for DIFFTOOL in xxdiff diffuse kdiff3 kompare gvimdiff2 meld tkdiff; do
         if type -P $DIFFTOOL >/dev/null; then
-            "$DIFFTOOL" $DIFF_FILES
+            "$DIFFTOOL" "${DIFF_FILES[@]}"
             exit 0
         fi
     done
     echo "Error: no graphical diff tool found." >&2
     exit 1
 fi
 
-$DIFF_CMD $DIFF_FILES
+$DIFF_CMD "${DIFF_FILES[@]}"

bin/cylc-import-examples

@@ -55,16 +55,16 @@ if [[ -d $DESTINATION ]]; then
 fi
 
 echo " + Copying example suites"
-mkdir -p $DESTINATION
-cp -r $CYLC_DIR/etc/examples/* $DESTINATION
+mkdir -p "$DESTINATION"
+cp -r "$CYLC_DIR/etc/examples/*" "$DESTINATION"
 
 echo " + Registering example suites"
-cd $DESTINATION
+cd "$DESTINATION"
 SUITE_RCS=$(find . -name suite.rc | sed -e 's@./@@')
 for SUITE_RC in $SUITE_RCS; do
-    SUITE_DEF_DIR=$(dirname $SUITE_RC)
+    SUITE_DEF_DIR="$(dirname "$SUITE_RC")"
     SUITE_REG_NAME=$BASE_DIR/$SUITE_DEF_DIR
-    cylc register $SUITE_REG_NAME $SUITE_DEF_DIR
+    cylc register "$SUITE_REG_NAME" "$SUITE_DEF_DIR"
 done
 
 echo "DONE"

bin/cylc-jobscript

@@ -54,7 +54,7 @@ if sys.argv[1] in ['-g', '--gedit']:
     print glbl_cfg().get(['editors', 'gui'])
 elif sys.argv[1] in ['-e', '--edit']:
     print glbl_cfg().get(['editors', 'terminal'])
-" $1
+" "$1"
 }
 
 PLAIN=false
@@ -66,20 +66,21 @@ for arg in "${@}"; do
         exit 0
     elif [[ "${arg}" == '--edit' ]] || [[ "${arg}" == '--gedit' ]] || \
          [[ "${arg}" == '-e' ]] || [[ "${arg}" == '-g' ]]; then
-        EDITOR_CMD="$(editor ${arg})"
+        EDITOR_CMD="$(editor "${arg}")"
     elif [[ "${arg}" == '--plain' ]]; then
         PLAIN=true
     else
         SUBMIT_ARGS="${SUBMIT_ARGS} ${arg}"
     fi
 done
 
-if CYLC_SUBMIT_OUT="$(cylc submit --dry-run ${SUBMIT_ARGS})"; then
+if CYLC_SUBMIT_OUT="$(cylc submit --dry-run "${SUBMIT_ARGS}")"; then
     JOBSCRIPT="$(sed -n 's/^JOB SCRIPT=//p' <<<"${CYLC_SUBMIT_OUT}")"
     if ! "${PLAIN}"; then
         echo "Task Job Script Generated: ${JOBSCRIPT}" >&2
     fi
     if [[ -n ${EDITOR_CMD} ]]; then
+        # shellcheck disable=2086
         exec $EDITOR_CMD "${JOBSCRIPT}"
     else
         exec less "${JOBSCRIPT}"

bin/cylc-make-docs

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+set -eu
+
 usage() {
     echo "Usage: cylc [admin] make-docs [--help]"
     echo "Build the HTML User Guide, with Sphinx."
@@ -36,7 +38,7 @@ fi
 echo >&2
 echo "Building the HTML Cylc Documentation with Sphinx:"
 echo >&2
-cd "$CYLC_DIR"/doc/
+cd "$CYLC_DIR/doc/"
 echo "... Generating the command reference ..."
 ./src/custom/make-commands.sh
 echo >&2

bin/cylc-scp-transfer

@@ -69,18 +69,18 @@ for T in $SRCE; do
         RMACH=${D%:*}
         RPATH=${D#*:}
 
-        RDIR=$( dirname $RPATH )
+        RDIR="$( dirname "$RPATH" )"
 
         cylc task message "Creating remote destination directory, $RDIR"
-        ssh $RMACH mkdir -p $RDIR
+        ssh "$RMACH" mkdir -p "$RDIR"
     else
-        DIR=$( dirname $D )
+        DIR="$( dirname "$D" )"
         cylc task message "Creating destination directory, $DIR"
-        mkdir -p $DIR
+        mkdir -p "$DIR"
     fi
 
     COMMAND="scp -B -r $T $D"
-    echo $COMMAND
+    echo "$COMMAND"
     if $VERBOSE; then
         $COMMAND
     else

bin/cylc-test-battery

@@ -99,12 +99,12 @@ eof
 
 chunk () {
     # argument in the format chunk_no/no_chunks
-    IFS=$'/' read CHUNK_NO CHUNKS <<< "$1"
+    IFS=$'/' read -r CHUNK_NO CHUNKS <<< "$1"
     # create lists of tests in a temp file
     TEST_FILE="$(mktemp)"
     cylc test-battery --dry | sort > "$TEST_FILE"
     LINES_PER_FILE=$(( ( $(wc -l "$TEST_FILE" | cut -d ' ' -f 1) \
-        + $CHUNKS - 1 ) / CHUNKS ))
+        + CHUNKS - 1 ) / CHUNKS ))
     # chunk tests
     split -d -l "$LINES_PER_FILE" "$TEST_FILE" "$TEST_FILE"
     # select chunk
@@ -117,11 +117,12 @@ export CYLC_TEST_RUN_GENERIC=${CYLC_TEST_RUN_GENERIC:-true}
 export CYLC_TEST_RUN_PLATFORM=${CYLC_TEST_RUN_PLATFORM:-true}
 export CYLC_TEST_SKIP=${CYLC_TEST_SKIP:-}
 export CYLC_TEST_IS_GENERIC=true
-export CYLC_TEST_TIME_INIT="$(date -u +'%Y%m%dT%H%M%SZ')"
+CYLC_TEST_TIME_INIT="$(date -u +'%Y%m%dT%H%M%SZ')"
+export CYLC_TEST_TIME_INIT
 
 if [[ "$PWD" != "$CYLC_DIR" ]]; then
     echo "cd \"$CYLC_DIR\""
-    cd "$CYLC_DIR"
+    cd "$CYLC_DIR" || exit 1
 fi
 
 ARG_COUNT=1
@@ -134,7 +135,7 @@ for ARG in "$@"; do
         --chunk)
             # Replace "--chunk a/b" with the appropriate tests.
             set -- "${@:1:$(( ARG_COUNT - 1 ))}" \
-                $(chunk ${@:$(( ARG_COUNT + 1 )):1}) \
+                "$(chunk "${@:$(( ARG_COUNT + 1 )):1}")" \
                 "${@:$(( ARG_COUNT + 2 ))}"
             ;;
         *)