general: remove import paths and sort out CI (#36)

.docker_details

@@ -0,0 +1,5 @@
+source "$(git rev-parse --show-toplevel)/_scripts/commonFunctions.bash"
+
+autostash_or_export DOCKER_IMAGE="x_monorepo"
+autostash_or_export DOCKERFILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+autostash_or_export DOCKER_WORKING_DIR="$HOME/gopath/src/myitcv.io"

.docker_env_file

@@ -0,0 +1,6 @@
+GH_USER
+GH_TOKEN
+GO_VERSION
+CHROMEDRIVER_VERSION
+CHROME_CHANNEL
+TRAVIS

.gitignore

@@ -1 +1,2 @@
 /.bin
+_scripts/.vbash*

.travis.yml

@@ -1,4 +1,5 @@
-language: go
+language: generic
+sudo: required
 
 notifications:
   email:
@@ -9,20 +10,15 @@ notifications:
 
 matrix:
   include:
-    - env: CHROMEDRIVER_VERSION=2.40 CHROME_CHANNEL=beta
-      go: 1.10.x
-      addons:
-        chrome: beta
+    - env: CHROMEDRIVER_VERSION=2.40 CHROME_CHANNEL=beta GO_VERSION=go1.10.3
 
-go_import_path: myitcv.io
+services:
+  - docker
 
-before_script:
-  - ./_scripts/install_go.sh
-  - ./_scripts/setupCIEnv.sh
-
-install: true
+before_install:
+  - docker --version
+  - source .docker_details
+  - docker build -q --build-arg USER=$USER --build-arg UID=$UID --build-arg DOCKER_WORKING_DIR=$DOCKER_WORKING_DIR --build-arg CHROME_CHANNEL=$CHROME_CHANNEL -t $DOCKER_IMAGE $DOCKERFILE_DIR
 
 script:
-  - ./_scripts/run_tests.sh
-  - ./_scripts/check_code_is_formatted.sh
-  - ./_scripts/check_git_is_clean.sh
+  - docker run -u $USER:$USER --env-file .docker_env_file -v $PWD:$DOCKER_WORKING_DIR -w $DOCKER_WORKING_DIR --rm $DOCKER_IMAGE ./_scripts/docker_run.sh

Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:18.04
+
+RUN apt-get -qq update
+RUN apt-get -qq -y install sudo apt-utils git curl jq unzip gnupg2
+
+RUN export go_bootstrap="$(curl -s https://golang.org/dl/?mode=json | jq -r '.[0] .version')" && \
+  curl -sL https://dl.google.com/go/$go_bootstrap.linux-amd64.tar.gz | tar -C / -zx
+
+ENV PATH=/go/bin:$PATH
+
+RUN git clone -q https://github.com/myitcv/vbash /vbash/src/github.com/myitcv/vbash && \
+  export GOPATH=/vbash && \
+  go install github.com/myitcv/vbash
+
+ARG USER
+ARG UID
+ARG DOCKER_WORKING_DIR
+ARG CHROME_CHANNEL
+
+RUN echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \
+  curl -sL https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - && \
+  apt-get -qq update && \
+  apt-get -qq -y install google-chrome-${CHROME_CHANNEL}
+
+ENV PATH=/vbash/bin:$PATH
+ENV GOPATH=/home/$USER/gopath
+
+RUN groupadd -g $UID $USER && \
+    adduser --uid $UID --gid $UID --disabled-password --gecos "" $USER
+
+RUN sudo -u $USER mkdir -p $DOCKER_WORKING_DIR
+
+# enable sudo
+RUN usermod -aG sudo $USER
+RUN echo "$USER ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER
+
+USER $USER

_scripts/check_code_is_formatted.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env vbash
 
 # Copyright (c) 2016 Paul Jolly <paul@myitcv.org.uk>, all rights reserved.
 # Use of this document is governed by a license found in the LICENSE document.

_scripts/check_git_is_clean.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env vbash
 
 # Copyright (c) 2016 Paul Jolly <paul@myitcv.org.uk>, all rights reserved.
 # Use of this document is governed by a license found in the LICENSE document.

_scripts/common.bash

@@ -45,12 +45,7 @@ source "${BASH_SOURCE%/*}/commonEnv.bash"
 
 if [ $(running_on_ci_server) == "yes" ]
 then
-	# Travis bug with GOROOT
-	unset GOROOT
-	if [ "${USE_GO_TIP:-}" == "true" ]
-	then
-		export PATH="$HOME/gotip/bin:$PATH"
-	fi
+	export PATH="$HOME/go/bin:$PATH"
 fi
 
 # We assume that PATH will have been setup in such a way as to leave

_scripts/commonEnv.bash

@@ -20,15 +20,6 @@ then
 	export PROTOBUF_INSTALL_DIR=$CI_DEPENDENCIES_DIR/protobuf
 
 	export CHROMEDRIVER_INSTALL_DIR=$CI_DEPENDENCIES_DIR/chromedriver
-
-	# gross hack because Travis does not update the google-chrome alternative
-	# properly
-	if [ ! -d /tmp/google-chrome-bin ]
-	then
-		mkdir /tmp/google-chrome-bin
-		ln -s /usr/bin/google-chrome-$CHROME_CHANNEL /tmp/google-chrome-bin/google-chrome
-	fi
-	export PATH=/tmp/google-chrome-bin:$PATH
 fi
 
 if [ "${PROTOBUF_VERSION:-}" == "" ]

_scripts/commonFunctions.bash

@@ -37,4 +37,145 @@ only_run_on_ci_server()
 }
 export -f only_run_on_ci_server
 
+cwd_as_import_path()
+{
+	local i
+	for i in $(sed -e "s/:/\n/g" <<< "$GOPATH")
+	do
+		if [[ "$PWD" =~ $i* ]]
+		then
+			echo ${PWD#${i}/src/}
+			return
+		fi
+	done
+
+	echo "could not resolve $PWD to import path"
+	exit 1
+}
+export -f cwd_as_import_path
+
+subpackages()
+{
+	local ip=$(cwd_as_import_path)
+
+	go list ./... | ( grep -v -f <(sed -e 's+^\(.*\)$+^myitcv.io/\1/+' <<< "$(nested_test_dirs)") || true )
+}
+export -f subpackages
+
+nested_test_dirs()
+{
+	for i in $(find -path ./_scripts -prune -o -name run_tests.sh -printf '%P\n')
+	do
+		dirname $(dirname "$i")
+	done
+}
+export -f nested_test_dirs
+
+nested_test_dir_patterns()
+{
+	nested_test_dirs | sed -e 's+^\(.*\)$+^\1/$+'
+}
+export -f nested_test_dir_patterns
+
+sub_git_files()
+{
+	git ls-files | ( grep -v -f <(sed -e 's+^\(.*\)$+^\1/.*$+' <<< "$(nested_test_dirs)") || true )
+}
+export -f sub_git_files
+
+ensure_go_formatted()
+{
+	if [ "$#" == "0" ]
+	then
+		return
+	fi
+	local z=$(goimports -l "$@")
+	if [ ! -z "$z" ]
+	then
+		echo "The following files are not formatted:"
+		echo ""
+		echo "$z"
+		exit 1
+	fi
+}
+export -f ensure_go_formatted
+
+ensure_go_gen_formatted()
+{
+	if [ "$#" == "0" ]
+	then
+		return
+	fi
+	local z=$(gofmt -l "$@")
+	if [ ! -z "$z" ]
+	then
+		echo "The following generated files are not formatted:"
+		echo ""
+		echo "$z"
+		exit 1
+	fi
+}
+export -f ensure_go_gen_formatted
+
+gen_files()
+{
+	grep '\(^gen_\|/gen_\)[^/]\+$' || true
+}
+export -f gen_files
+
+non_gen_files()
+{
+	grep -v '\(^gen_\|/gen_\)[^/]\+$' || true
+}
+export -f non_gen_files
+
+go_files()
+{
+	grep '/\?[^/]\+.go$' || true
+}
+export -f go_files
+
+non_gen_go_files()
+{
+	go_files | non_gen_files
+}
+export -f non_gen_go_files
+
+gen_go_files()
+{
+	go_files | gen_files
+}
+export -f gen_go_files
+
+run_nested_tests()
+{
+	for i in $(nested_test_dirs)
+	do
+		cat <<EOD
+bash -c "set -e; echo '---- $i'; cd $i; ./_scripts/run_tests.sh; echo '----'; echo ''"
+EOD
+	done | concsh
+}
+export -f run_nested_tests
+
+install_main_go()
+{
+	for i in $(go list -f "{{if eq .Name \"main\"}}{{.Dir}}{{end}}" "$@")
+	do
+		pushd $i > /dev/null
+		go install
+		popd > /dev/null
+	done
+}
+export -f install_main_go
+
+# workaround for Go 1.10 pre export data in Go 1.11
+install_deps()
+{
+	go list -f "{{ range .Deps}}{{.}}
+	{{end}}" "$@" | xargs go install
+}
+export -f install_deps
+
+# **********************
 LOADED_COMMON_FUNCTIONS=true

_tmpls/_scripts/run_tests.sh → _scripts/docker_run.sh

@@ -1,10 +1,12 @@
-#!/usr/bin/env bash
+#!/usr/bin/env vbash
 
 # Copyright (c) 2016 Paul Jolly <paul@myitcv.org.uk>, all rights reserved.
 # Use of this document is governed by a license found in the LICENSE document.
 
 source "$(git rev-parse --show-toplevel)/_scripts/common.bash"
 
-# we just generate and test in this template module
-$gg ./...
-go test ./...
+./_scripts/install_go.sh
+./_scripts/setupCIEnv.sh
+./_scripts/run_tests.sh
+./_scripts/check_code_is_formatted.sh
+./_scripts/check_git_is_clean.sh

_scripts/install_go.sh

@@ -1,21 +1,33 @@
-#!/usr/bin/env bash
+#!/usr/bin/env vbash
 
-source "${BASH_SOURCE%/*}/common.bash"
+source "$(git rev-parse --show-toplevel)/_scripts/common.bash"
 
 trap "go version" EXIT
 
-if [ "${USE_GO_TIP:-}" != "true" ]
+only_run_on_ci_server
+
+if [ "$(uname -m)" != "x86_64" ]
 then
-	# nothing to do
-	exit
+	echo "Unkown architecture"
+	exit 1
 fi
 
-source="https://github.com/myitcv/gobuilds/raw/master/linux_amd64/$GO_TIP_VERSION.tar.gz"
-target=$HOME/gotip
+os=$(uname | tr '[:upper:]' '[:lower:]')
+arch="amd64"
+
+if [[ "$GO_VERSION" = go* ]]
+then
+	cd $HOME
+	curl -sL  https://dl.google.com/go/$GO_VERSION.${os}-${arch}.tar.gz | tar -zx
+else
+	# tip
+	source="https://github.com/myitcv/gobuilds/raw/master/${os}_${arch}/$GO_VERSION.tar.gz"
+	target=$HOME/go
 
-echo "Will install ${GO_TIP_VERSION:0:10} from $source to $target"
+	echo "Will install ${GO_VERSION:0:10} from $source to $target"
 
-mkdir -p $target
-cd $target
+	mkdir -p $target
+	cd $target
 
-curl -L -s $source | tar -xz
+	curl -L -s $source | tar -xz
+fi