verify shellcheck

README.md

@@ -49,3 +49,15 @@ Cheat sheet:
 - `git subtree add --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once)
 - `git subtree pull --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes)
 - edit, `git commit`, `git subtree push --prefix=release-tools git@github.com:<user>/csi-release-tools.git <my-new-or-existing-branch>` - push to a new branch before submitting a PR
+
+verify-shellcheck.sh
+--------------------
+
+The [verify-shellcheck.sh](./verify-shellcheck.sh) script in this repo
+is a stripped down copy of the [corresponding
+script](https://github.com/kubernetes/kubernetes/blob/release-1.14/hack/verify-shellcheck.sh)
+in the Kubernetes repository. It can be used to check for certain
+errors shell scripts, like missing quotation marks. The default
+`test-shellcheck` target in [build.make](./build.make) only checks the
+scripts in this directory. Components can add more directories to
+`TEST_SHELLCHECK_DIRS` to check also other scripts.

build.make

@@ -98,7 +98,7 @@ test:
 test: test-go
 test-go:
 	@ echo; echo "### $@:"
-	go test `go list ./... | grep -v 'vendor' $(TEST_GO_FILTER_CMD)` $(TESTARGS)
+	go test `go list ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)
 
 .PHONY: test-vet
 test: test-vet
@@ -122,3 +122,18 @@ test: test-subtree
 test-subtree:
 	@ echo; echo "### $@:"
 	./release-tools/verify-subtree.sh release-tools
+
+# Components can extend the set of directories which must pass shellcheck.
+# The default is to check only the release-tools directory itself.
+TEST_SHELLCHECK_DIRS=release-tools
+.PHONY: test-shellcheck
+test: test-shellcheck
+test-shellcheck:
+	@ echo; echo "### $@:"
+	@ ret=0; \
+	for dir in $(abspath $(TEST_SHELLCHECK_DIRS)); do \
+		echo; \
+		echo "$$dir:"; \
+		./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \
+	done; \
+	return $$ret

util.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+# Copyright 2014 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function kube::util::sourced_variable {
+  # Call this function to tell shellcheck that a variable is supposed to
+  # be used from other calling context. This helps quiet an "unused
+  # variable" warning from shellcheck and also document your code.
+  true
+}
+
+kube::util::sortable_date() {
+  date "+%Y%m%d-%H%M%S"
+}
+
+# arguments: target, item1, item2, item3, ...
+# returns 0 if target is in the given items, 1 otherwise.
+kube::util::array_contains() {
+  local search="$1"
+  local element
+  shift
+  for element; do
+    if [[ "${element}" == "${search}" ]]; then
+      return 0
+     fi
+  done
+  return 1
+}
+
+# Example:  kube::util::trap_add 'echo "in trap DEBUG"' DEBUG
+# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal
+kube::util::trap_add() {
+  local trap_add_cmd
+  trap_add_cmd=$1
+  shift
+
+  for trap_add_name in "$@"; do
+    local existing_cmd
+    local new_cmd
+
+    # Grab the currently defined trap commands for this trap
+    existing_cmd=$(trap -p "${trap_add_name}" |  awk -F"'" '{print $2}')
+
+    if [[ -z "${existing_cmd}" ]]; then
+      new_cmd="${trap_add_cmd}"
+    else
+      new_cmd="${trap_add_cmd};${existing_cmd}"
+    fi
+
+    # Assign the test. Disable the shellcheck warning telling that trap
+    # commands should be single quoted to avoid evaluating them at this
+    # point instead evaluating them at run time. The logic of adding new
+    # commands to a single trap requires them to be evaluated right away.
+    # shellcheck disable=SC2064
+    trap "${new_cmd}" "${trap_add_name}"
+  done
+}
+
+kube::util::download_file() {
+  local -r url=$1
+  local -r destination_file=$2
+
+  rm "${destination_file}" 2&> /dev/null || true
+
+  for i in $(seq 5)
+  do
+    if ! curl -fsSL --retry 3 --keepalive-time 2 "${url}" -o "${destination_file}"; then
+      echo "Downloading ${url} failed. $((5-i)) retries left."
+      sleep 1
+    else
+      echo "Downloading ${url} succeed"
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Wait for background jobs to finish. Return with
+# an error status if any of the jobs failed.
+kube::util::wait-for-jobs() {
+  local fail=0
+  local job
+  for job in $(jobs -p); do
+    wait "${job}" || fail=$((fail + 1))
+  done
+  return ${fail}
+}
+
+# kube::util::join <delim> <list...>
+# Concatenates the list elements with the delimiter passed as first parameter
+#
+# Ex: kube::util::join , a b c
+#  -> a,b,c
+function kube::util::join {
+  local IFS="$1"
+  shift
+  echo "$*"
+}
+
+# kube::util::check-file-in-alphabetical-order <file>
+# Check that the file is in alphabetical order
+#
+function kube::util::check-file-in-alphabetical-order {
+  local failure_file="$1"
+  if ! diff -u "${failure_file}" <(LC_ALL=C sort "${failure_file}"); then
+    {
+      echo
+      echo "${failure_file} is not in alphabetical order. Please sort it:"
+      echo
+      echo "  LC_ALL=C sort -o ${failure_file} ${failure_file}"
+      echo
+    } >&2
+    false
+  fi
+}
+
+# Some useful colors.
+if [[ -z "${color_start-}" ]]; then
+  declare -r color_start="\033["
+  declare -r color_red="${color_start}0;31m"
+  declare -r color_yellow="${color_start}0;33m"
+  declare -r color_green="${color_start}0;32m"
+  declare -r color_blue="${color_start}1;34m"
+  declare -r color_cyan="${color_start}1;36m"
+  declare -r color_norm="${color_start}0m"
+
+  kube::util::sourced_variable "${color_start}"
+  kube::util::sourced_variable "${color_red}"
+  kube::util::sourced_variable "${color_yellow}"
+  kube::util::sourced_variable "${color_green}"
+  kube::util::sourced_variable "${color_blue}"
+  kube::util::sourced_variable "${color_cyan}"
+  kube::util::sourced_variable "${color_norm}"
+fi
+
+# ex: ts=2 sw=2 et filetype=sh

verify-shellcheck.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# The csi-release-tools directory.
+TOOLS="$(dirname "${BASH_SOURCE[0]}")"
+. "${TOOLS}/util.sh"
+
+# Directory to check. Default is the parent of the tools themselves.
+ROOT="${1:-${TOOLS}/..}"
+
+# required version for this script, if not installed on the host we will
+# use the official docker image instead. keep this in sync with SHELLCHECK_IMAGE
+SHELLCHECK_VERSION="0.6.0"
+# upstream shellcheck latest stable image as of January 10th, 2019
+SHELLCHECK_IMAGE="koalaman/shellcheck-alpine:v0.6.0@sha256:7d4d712a2686da99d37580b4e2f45eb658b74e4b01caf67c1099adc294b96b52"
+
+# fixed name for the shellcheck docker container so we can reliably clean it up
+SHELLCHECK_CONTAINER="k8s-shellcheck"
+
+# disabled lints
+disabled=(
+  # this lint disallows non-constant source, which we use extensively without
+  # any known bugs
+  1090
+  # this lint prefers command -v to which, they are not the same
+  2230
+)
+# comma separate for passing to shellcheck
+join_by() {
+  local IFS="$1";
+  shift;
+  echo "$*";
+}
+SHELLCHECK_DISABLED="$(join_by , "${disabled[@]}")"
+readonly SHELLCHECK_DISABLED
+
+# creates the shellcheck container for later use
+create_container () {
+  # TODO(bentheelder): this is a performance hack, we create the container with
+  # a sleep MAX_INT32 so that it is effectively paused.
+  # We then repeatedly exec to it to run each shellcheck, and later rm it when
+  # we're done.
+  # This is incredibly much faster than creating a container for each shellcheck
+  # call ...
+  docker run --name "${SHELLCHECK_CONTAINER}" -d --rm -v "${ROOT}:${ROOT}" -w "${ROOT}" --entrypoint="sleep" "${SHELLCHECK_IMAGE}" 2147483647
+}
+# removes the shellcheck container
+remove_container () {
+  docker rm -f "${SHELLCHECK_CONTAINER}" &> /dev/null || true
+}
+
+# ensure we're linting the source tree
+cd "${ROOT}"
+
+# find all shell scripts excluding ./_*, ./.git/*, ./vendor*,
+# and anything git-ignored
+all_shell_scripts=()
+while IFS=$'\n' read -r script;
+  do git check-ignore -q "$script" || all_shell_scripts+=("$script");
+done < <(find . -name "*.sh" \
+  -not \( \
+    -path ./_\*      -o \
+    -path ./.git\*   -o \
+    -path ./vendor\*    \
+  \))
+
+# detect if the host machine has the required shellcheck version installed
+# if so, we will use that instead.
+HAVE_SHELLCHECK=false
+if which shellcheck &>/dev/null; then
+  detected_version="$(shellcheck --version | grep 'version: .*')"
+  if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then
+    HAVE_SHELLCHECK=true
+  fi
+fi
+
+# tell the user which we've selected and possibly set up the container
+if ${HAVE_SHELLCHECK}; then
+  echo "Using host shellcheck ${SHELLCHECK_VERSION} binary."
+else
+  echo "Using shellcheck ${SHELLCHECK_VERSION} docker image."
+  # remove any previous container, ensure we will attempt to cleanup on exit,
+  # and create the container
+  remove_container
+  kube::util::trap_add 'remove_container' EXIT
+  if ! output="$(create_container 2>&1)"; then
+      {
+        echo "Failed to create shellcheck container with output: "
+        echo ""
+        echo "${output}"
+      } >&2
+      exit 1
+  fi
+fi
+
+# lint each script, tracking failures
+errors=()
+for f in "${all_shell_scripts[@]}"; do
+  set +o errexit
+  if ${HAVE_SHELLCHECK}; then
+    failedLint=$(shellcheck --exclude="${SHELLCHECK_DISABLED}" "${f}")
+  else
+    failedLint=$(docker exec -t ${SHELLCHECK_CONTAINER} \
+                 shellcheck --exclude="${SHELLCHECK_DISABLED}" "${f}")
+  fi
+  set -o errexit
+  if [[ -n "${failedLint}" ]]; then
+      errors+=( "${failedLint}" )
+  fi
+done
+
+# Check to be sure all the packages that should pass lint are.
+if [ ${#errors[@]} -eq 0 ]; then
+  echo 'Congratulations! All shell files are passing lint.'
+else
+  {
+    echo "Errors from shellcheck:"
+    for err in "${errors[@]}"; do
+      echo "$err"
+    done
+    echo
+    echo 'Please review the above warnings. You can test via "./hack/verify-shellcheck"'
+    echo 'If the above warnings do not make sense, you can exempt them from shellcheck'
+    echo 'checking by adding the "shellcheck disable" directive'
+    echo '(https://github.com/koalaman/shellcheck/wiki/Directive#disable).'
+    echo
+  } >&2
+  false
+fi