Merge pull request #22 from weaveworks/shell-lint

Shell lint

README.md

@@ -4,11 +4,15 @@ Included in this repo are tools shared by weave.git and scope.git.  They include
 
 - ```cover```: a tool which merges overlapping coverage reports generated by go
   test
+- ```files-with-type```: a tool to search directories for files of a given
+  MIME type
 - ```lint```: a script to lint Go project; runs various tools like golint, go
   vet, errcheck etc
 - ```rebuild-image```: a script to rebuild docker images when their input files
   change; useful when you using docker images to build your software, but you
   don't want to build the image every time.
+- ```shell-lint```: a script to lint multiple shell files with 
+  [shellcheck](http://www.shellcheck.net/)
 - ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto
   the Weave network
 - ```test```: a script to run all go unit tests in subdirectories, gather the

cover/gather_coverage.sh

@@ -3,19 +3,18 @@
 # merges them and produces a complete report.
 
 set -ex
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 DESTINATION=$1
 FROMDIR=$2
-mkdir -p $DESTINATION
+mkdir -p "$DESTINATION"
 
 if [ -n "$CIRCLECI" ]; then
-    for i in $(seq 1 $(($CIRCLE_NODE_TOTAL - 1))); do
-        scp node$i:$FROMDIR/* $DESTINATION || true
+    for i in $(seq 1 $((CIRCLE_NODE_TOTAL - 1))); do
+        scp "node$i:$FROMDIR"/* "$DESTINATION" || true
     done
 fi
 
 go get github.com/weaveworks/build-tools/cover
-cover $DESTINATION/* >profile.cov
+cover "$DESTINATION"/* >profile.cov
 go tool cover -html=profile.cov -o coverage.html
 go tool cover -func=profile.cov -o coverage.txt
-tar czf coverage.tar.gz $DESTINATION
+tar czf coverage.tar.gz "$DESTINATION"

files-with-type

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Find all files with a given MIME type.
+#
+# e.g.
+#     $ files-with-type text/x-shellscript k8s infra
+#
+# Assumes `find`, `xargs`, and `file` are all installed.
+
+mime_type=$1
+shift
+
+find  "$@" -print0 -type f |xargs -0 file --mime-type | grep "${mime_type}" | sed -e 's/:.*$//'

integration/assert.sh

@@ -24,14 +24,14 @@ export INVARIANT=${INVARIANT:-}
 export CONTINUE=${CONTINUE:-}
 
 args="$(getopt -n "$0" -l \
-    verbose,help,stop,discover,invariant,continue vhxdic $*)" \
+    verbose,help,stop,discover,invariant,continue vhxdic "$@")" \
 || exit -1
 for arg in $args; do
     case "$arg" in
         -h)
             echo "$0 [-vxidc]" \
                 "[--verbose] [--stop] [--invariant] [--discover] [--continue]"
-            echo "`sed 's/./ /g' <<< "$0"` [-h] [--help]"
+            echo "$(sed 's/./ /g' <<< "$0") [-h] [--help]"
             exit 0;;
         --help)
             cat <<EOF
@@ -83,17 +83,17 @@ assert_end() {
     # to get report_time split tests_time on 2 substrings:
     #   ${tests_time:0:${#tests_time}-9} - seconds
     #   ${tests_time:${#tests_time}-9:3} - milliseconds
-    [[ -z "$INVARIANT" ]] \
-        && report_time=" in ${tests_time:0:${#tests_time}-9}.${tests_time:${#tests_time}-9:3}s" \
-        || report_time=
-
+    if [[ -z "$INVARIANT" ]]; then
+        report_time=" in ${tests_time:0:${#tests_time}-9}.${tests_time:${#tests_time}-9:3}s"
+    else
+        report_time=
+    fi
     if [[ "$tests_failed" -eq 0 ]]; then
         echo "all $tests passed$report_time."
     else
         for error in "${tests_errors[@]}"; do echo "$error"; done
         echo "$tests_failed of $tests failed$report_time."
     fi
-    tests_failed_previous=$tests_failed
     [[ $tests_failed -gt 0 ]] && tests_suite_status=1
     _assert_reset
 }
@@ -103,7 +103,7 @@ assert() {
     (( tests_ran++ )) || :
     [[ -z "$DISCOVERONLY" ]] || return
     expected=$(echo -ne "${2:-}")
-    result="$(eval 2>/dev/null $1 <<< ${3:-})" || true
+    result="$(eval 2>/dev/null "$1" <<< "${3:-}")" || true
     if [[ "$result" == "$expected" ]]; then
         [[ -z "$DEBUG" ]] || echo -n .
         return
@@ -119,7 +119,7 @@ assert_raises() {
     (( tests_ran++ )) || :
     [[ -z "$DISCOVERONLY" ]] || return
     status=0
-    (eval $1 <<< ${3:-}) > /dev/null 2>&1 || status=$?
+    (eval "$1" <<< "${3:-}") > /dev/null 2>&1 || status=$?
     expected=${2:-0}
     if [[ "$status" -eq "$expected" ]]; then
         [[ -z "$DEBUG" ]] || echo -n .
@@ -143,7 +143,7 @@ _assert_fail() {
 
 skip_if() {
     # skip_if <command ..>
-    (eval $@) > /dev/null 2>&1 && status=0 || status=$?
+    (eval "$@") > /dev/null 2>&1 && status=0 || status=$?
     [[ "$status" -eq 0 ]] || return
     skip
 }

integration/gce.sh

@@ -7,15 +7,15 @@
 
 set -e
 
-: ${KEY_FILE:=/tmp/gce_private_key.json}
-: ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}
-: ${IMAGE:=ubuntu-14-04}
-: ${ZONE:=us-central1-a}
-: ${PROJECT:=}
-: ${TEMPLATE_NAME:=}
-: ${NUM_HOSTS:=}
-
-if [ -z "${PROJECT}" -o -z "${NUM_HOSTS}" -o -z "${TEMPLATE_NAME}" ]; then
+: "${KEY_FILE:=/tmp/gce_private_key.json}"
+: "${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}"
+: "${IMAGE:=ubuntu-14-04}"
+: "${ZONE:=us-central1-a}"
+: "${PROJECT:=}"
+: "${TEMPLATE_NAME:=}"
+: "${NUM_HOSTS:=}"
+
+if [ -z "${PROJECT}" ] || [ -z "${NUM_HOSTS}" ] || [ -z "${TEMPLATE_NAME}" ]; then
 	echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME"
 	exit 1
 fi
@@ -26,21 +26,21 @@ if [ -n "$CIRCLECI" ]; then
 fi
 
 # Setup authentication
-gcloud auth activate-service-account --key-file $KEY_FILE 1>/dev/null
-gcloud config set project $PROJECT
+gcloud auth activate-service-account --key-file "$KEY_FILE" 1>/dev/null
+gcloud config set project "$PROJECT"
 
 function vm_names {
 	local names=
-	for i in $(seq 1 $NUM_HOSTS); do
-		names="host$i$SUFFIX $names"
+	for i in $(seq 1 "$NUM_HOSTS"); do
+		names=( "host$i$SUFFIX" "${names[@]}" )
 	done
-	echo "$names"
+	echo "${names[@]}"
 }
 
 # Delete all vms in this account
 function destroy {
 	names="$(vm_names)"
-	if [ $(gcloud compute instances list --zone $ZONE -q $names | wc -l) -le 1 ] ; then
+	if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ] ; then
 		return 0
 	fi
 	for i in {0..10}; do
@@ -60,23 +60,23 @@ function destroy {
 }
 
 function internal_ip {
-	jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" $1
+	jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1"
 }
 
 function external_ip {
-	jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" $1
+	jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1"
 }
 
 function try_connect {
 	for i in {0..10}; do
-		ssh -t $1 true && return
+		ssh -t "$1" true && return
 		sleep 2
 	done
 }
 
 function install_docker_on {
 	name=$1
-	ssh -t $name sudo bash -x -s <<EOF
+	ssh -t "$name" sudo bash -x -s <<EOF
 curl -sSL https://get.docker.com/gpg | sudo apt-key add -
 curl -sSL https://get.docker.com/ | sh
 apt-get update -qq;
@@ -87,72 +87,72 @@ service docker restart
 EOF
 	# It seems we need a short delay for docker to start up, so I put this in
 	# a separate ssh connection.  This installs nsenter.
-	ssh -t $name sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
+	ssh -t "$name" sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
 }
 
 function copy_hosts {
 	hostname=$1
 	hosts=$2
-	cat $hosts | ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\""
+	ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" < "$hosts"
 }
 
 # Create new set of VMs
 function setup {
 	destroy
 
-	names="$(vm_names)"
-	gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE
-	gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
+	names=( $(vm_names) )
+	gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE"
+	gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
 	sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
 
 	# build an /etc/hosts file for these vms
 	hosts=$(mktemp hosts.XXXXXXXXXX)
 	json=$(mktemp json.XXXXXXXXXX)
-	gcloud compute instances list --format=json >$json
-	for name in $names; do
-		echo "$(internal_ip $json $name) $name.$ZONE.$PROJECT" >>$hosts
+	gcloud compute instances list --format=json > "$json"
+	for name in "${names[@]}"; do
+		echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >> "$hosts"
 	done
 
-	for name in $names; do
+	for name in "${names[@]}"; do
 		hostname="$name.$ZONE.$PROJECT"
 
 		# Add the remote ip to the local /etc/hosts
 		sudo sed -i "/$hostname/d" /etc/hosts
-		sudo sh -c "echo \"$(external_ip $json $name) $hostname\" >>/etc/hosts"
-		try_connect $hostname
+		sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts"
+		try_connect "$hostname"
 
-		copy_hosts $hostname $hosts &
+		copy_hosts "$hostname" "$hosts" &
 	done
 
 	wait
 
-	rm $hosts $json
+	rm "$hosts" "$json"
 }
 
 function make_template {
-	gcloud compute instances create $TEMPLATE_NAME --image $IMAGE --zone $ZONE
-	gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
+	gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE"
+	gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
 	name="$TEMPLATE_NAME.$ZONE.$PROJECT"
-	try_connect $name
-	install_docker_on $name
-	gcloud -q compute instances delete $TEMPLATE_NAME --keep-disks boot --zone $ZONE
-	gcloud compute images create $TEMPLATE_NAME --source-disk $TEMPLATE_NAME --source-disk-zone $ZONE
+	try_connect "$name"
+	install_docker_on "$name"
+	gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE"
+	gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE"
 }
 
 function hosts {
 	hosts=
 	args=
 	json=$(mktemp json.XXXXXXXXXX)
-	gcloud compute instances list --format=json >$json
+	gcloud compute instances list --format=json > "$json"
 	for name in $(vm_names); do
 		hostname="$name.$ZONE.$PROJECT"
-		hosts="$hostname $hosts"
-		args="--add-host=$hostname:$(internal_ip $json $name) $args"
+		hosts=( $hostname "${hosts[@]}" )
+		args=( "--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}" )
 	done
 	echo export SSH=\"ssh -l vagrant\"
-	echo export HOSTS=\"$hosts\"
-	echo export ADD_HOST_ARGS=\"$args\"
-	rm $json
+	echo "export HOSTS=\"${hosts[*]}\""
+	echo "export ADD_HOST_ARGS=\"${args[*]}\""
+	rm "$json"
 }
 
 case "$1" in
@@ -170,7 +170,7 @@ destroy)
 
 make_template)
 	# see if template exists
-	if ! gcloud compute images list | grep $PROJECT | grep $TEMPLATE_NAME; then
+	if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then
 		make_template
 	fi
 esac

integration/run_all.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 
+set -ex
+
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck disable=SC1090
 . "$DIR/config.sh"
 
 whitely echo Sanity checks
@@ -10,18 +13,19 @@ if ! bash "$DIR/sanity_check.sh"; then
 fi
 whitely echo ...ok
 
-TESTS="${@:-$(find . -name '*_test.sh')}"
-RUNNER_ARGS=""
+# shellcheck disable=SC2068
+TESTS=( ${@:-$(find . -name '*_test.sh')} )
+RUNNER_ARGS=( )
 
 # If running on circle, use the scheduler to work out what tests to run
-if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
-    RUNNER_ARGS="$RUNNER_ARGS -scheduler"
+if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ]; then
+    RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -scheduler )
 fi
 
 # If running on circle or PARALLEL is not empty, run tests in parallel
-if [ -n "$CIRCLECI" -o -n "$PARALLEL" ]; then
-    RUNNER_ARGS="$RUNNER_ARGS -parallel"
+if [ -n "$CIRCLECI" ] || [ -n "$PARALLEL" ]; then
+    RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -parallel )
 fi
 
-make -C ${DIR}/../runner
-HOSTS="$HOSTS" "${DIR}/../runner/runner" $RUNNER_ARGS $TESTS
+make -C "${DIR}/../runner"
+HOSTS="$HOSTS" "${DIR}/../runner/runner" "${RUNNER_ARGS[@]}" "${TESTS[@]}"

integration/sanity_check.sh

@@ -1,26 +1,26 @@
 #! /bin/bash
-
+# shellcheck disable=SC1091
 . ./config.sh
 
 set -e
 
 whitely echo Ping each host from the other
 for host in $HOSTS; do
     for other in $HOSTS; do
-        [ $host = $other ] || run_on $host $PING $other
+        [ "$host" = "$other" ] || run_on "$host" "$PING" "$other"
     done
 done
 
 whitely echo Check we can reach docker
 
 for host in $HOSTS; do
     echo
-    echo Host Version Info: $host
-    echo =====================================
+    echo "Host Version Info: $host"
+    echo "====================================="
     echo "# docker version"
-    docker_on $host version
+    docker_on "$host" version
     echo "# docker info"
-    docker_on $host info
+    docker_on "$host" info
     echo "# weave version"
-    weave_on $host version
+    weave_on "$host" version
 done