Better testing (#1476)

Better testing

- Support multiple Vim versions (currently 7.4, 8.0, and Neovim).

- Make sure that we always run Vim from a temporary installation in
  `/tmp/vim-go-test` without loading `~/.vim`. This makes it a lot
  easier to run on people's computers.

- Also add a handy `./script/run-vim` script to run the installed Vim
  from the temp directory. Useful for testing, debugging, etc. without a
  (potentially large) ~/.vim/ dir.

- Previously the tests weren't actually being run correctly; see:
  https://travis-ci.org/fatih/vim-go/builds/279277579

- Format the output of the tests a wee bit nicer, roughly similar to the
  `go test` output.

- Expand docs on testing a bit.

I also attempted to integrate code coverage support with
https://github.com/Vimjas/covimerage (which was the reason I started
working on this), but couldn't really get that to work. Need to look in
to that later.

.dockerignore

@@ -0,0 +1 @@
+.git/

.github/CONTRIBUTING.md

@@ -1,9 +1,12 @@
 Thanks for improving vim-go! Before you dive in please read the following:
 
 1. Please read our
-   [Documentation](https://github.com/fatih/vim-go/blob/master/doc/vim-go.txt), it might
-   have answers for your problem
-2. If you add a new feature please don't forget to update the documentation:
+   [Documentation](https://github.com/fatih/vim-go/blob/master/doc/vim-go.txt),
+   it might have a solution to your problem.
+2. If you add a new feature then please don't forget to update the documentation:
    [doc/vim-go.txt](https://github.com/fatih/vim-go/blob/master/doc/vim-go.txt).
-3. If it's a breaking change or exceed +100 lines please open an issue first
-   and describe the changes you want to make.
+3. If it's a breaking change or exceeds 100 lines of code then please open an
+   issue first and describe the changes you want to make.
+4. See `:help go-development` for instructions on how to run and write tests. If
+   you add a new feature be sure you also include a test if feasible.
+

.gitignore

@@ -1,7 +1,2 @@
 doc/tags
 .DS_Store
-
-# Test specific files
-FAILED
-test.log
-scripts/vim-vimhelplint

.travis.yml

@@ -1,21 +1,11 @@
 language: go
-
-env:
-  global:
-    - DEPS=$HOME/deps
-    - PATH=$DEPS/bin:$PATH
-    - PATCH="v8.0.0134"
-
-install: |
-  git config --global user.email "you@example.com"
-  git config --global user.name "Your Name"
-
-  # check out if we can pre-compiled Vim releases somehow,
-  git clone --branch $PATCH --depth 1 https://github.com/vim/vim
-  cd vim
-  ./configure --prefix=$DEPS --with-features=huge --disable-gui
-  make
-  make install
-  cd -
-
-script: ./scripts/test.sh
+matrix:
+  include:
+    - env: SCRIPT=test VIM_VERSION=vim-7.4
+    - env: SCRIPT=test VIM_VERSION=vim-8.0
+    - env: SCRIPT=test VIM_VERSION=nvim
+    - env: SCRIPT=lint VIM_VERSION=vim-8.0
+install:
+  - ./scripts/install-vim $VIM_VERSION
+script:
+  - ./scripts/$SCRIPT $VIM_VERSION

Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.9.1
+
+RUN apt-get update -y && \
+  apt-get install -y build-essential curl git libncurses5-dev && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN useradd -ms /bin/bash -d /vim-go vim-go
+USER vim-go
+WORKDIR /vim-go
+COPY . /vim-go/
+
+ENTRYPOINT ["make"]

Makefile

@@ -1,7 +1,24 @@
-all: test
+all: install test lint
+
+install:
+	@echo "==> Installing Vims"
+	@./scripts/install-vim vim-7.4
+	@./scripts/install-vim vim-8.0
+	@./scripts/install-vim nvim
 
 test:
 	@echo "==> Running tests"
-	@./scripts/test.sh
+	@./scripts/test vim-7.4
+	@./scripts/test vim-8.0
+	@./scripts/test nvim
+
+lint:
+	@echo "==> Running linting tools"
+	@./scripts/lint vim-8.0
+
+clean:
+	@echo "==> Cleaning /tmp/vim-go-test"
+	@rm -rf /tmp/vim-go-test
+
 
-.PHONY: all test
+.PHONY: all test install clean lint

autoload/go/tags_test.vim

@@ -5,7 +5,7 @@ func Test_add_tags()
   let expected = join(readfile("test-fixtures/tags/add_all_golden.go"), "\n")
 
   " run for offset 40, which is inside the struct
-  call go#tags#run(0, 0, 40, "add", input_file, 1)
+  silent call go#tags#run(0, 0, 40, "add", input_file, 1)
 
   let actual = join(readfile(input_file), "\n")
 
@@ -20,7 +20,7 @@ func Test_remove_tags()
   let expected = join(readfile("test-fixtures/tags/remove_all_golden.go"), "\n")
 
   " run for offset 40, which is inside the struct
-  call go#tags#run(0, 0, 40, "remove", input_file, 1)
+  silent call go#tags#run(0, 0, 40, "remove", input_file, 1)
 
   let actual = join(readfile(input_file), "\n")
 

doc/vim-go.txt

@@ -125,7 +125,7 @@ Supported Go plugins~                                         *vim-go-plugins*
 The following plugins are supported for use with vim-go:
 
 * Real-time completion (Vim):
-  https://github.com/Shougo/neocomplete.vim 
+  https://github.com/Shougo/neocomplete.vim
 
 * Real-time completion (Neovim):
   https://github.com/Shougo/deoplete.nvim and
@@ -566,13 +566,13 @@ CTRL-t
                                                                  *:GoFreevars*
 :GoFreevars
 
-    Enumerates the free variables of the selection. “Free variables” is a
+    Enumerates the free variables of the selection. "Free variables" is a
     technical term meaning the set of variables that are referenced but not
     defined within the selection, or loosely speaking, its inputs.
 
-    This information is useful if you’re considering whether to refactor the
+    This information is useful when considering whether to refactor the
     selection into a function of its own, as the free variables would be the
-    necessary parameters of that function.  It’s also useful when you want to
+    necessary parameters of that function. It's also useful when you want to
     understand what the inputs are to a complex block of code even if you
     don’t plan to change it.
 
@@ -1181,11 +1181,11 @@ used >
 Use this option to add additional options to the |'g:go_fmt_command'|. It's
 value type can be either a string or a dictionary. This is due backwards
 compatibility. The string version will be removed in the future so please use
-the dictionary version. Default is empty. 
+the dictionary version. Default is empty.
 >
   let g:go_fmt_options = ''
 
-  or 
+  or
 
   let g:go_fmt_options = {}
 <
@@ -1394,7 +1394,7 @@ Specifies the type of list to use for command outputs (such as errors from
 builds, results from static analysis commands, etc...). The list type for
 specific commands can be overridden with |'g:go_list_type_commands'|. The
 default value (empty) will use the appropriate kind of list for the command
-that was called. Supported values are "", "quickfix", and "locationlist". 
+that was called. Supported values are "", "quickfix", and "locationlist".
 >
   let g:go_list_type = ""
 <
@@ -1422,7 +1422,7 @@ As an example, the following settings will change all list types to
 
 Specifies whether the quickfix/location list should be closed automatically
 in the absence of errors.  The default value is 1.
-If you prefer to keep a long running error window open, you can disable 
+If you prefer to keep a long running error window open, you can disable
 this by setting the value to 0.
 >
   let g:go_list_autoclose = 1
@@ -1815,7 +1815,7 @@ manually, usually from an |autocommand|:
 If you have a lot of packages with the same prefix (`github.com/user`) you can
 use a single autocommand:
 >
-  autocmd BufRead /home/martin/go/src/*.go 
+  autocmd BufRead /home/martin/go/src/*.go
         \  let s:tmp = matchlist(expand('%:p'),
             \ '/home/martin/go/src/\(github.com/user/[^/]\+\)')
         \| if len(s:tmp) > 1 |  exe 'silent :GoGuruScope ' . s:tmp[1] | endif
@@ -1916,23 +1916,32 @@ By default new terminals are opened in a vertical split. To change it
 ==============================================================================
 DEVELOPMENT                                               *go-development*
 
-vim-go supports test files written in VimL. Please check `autoload` folder for
-examples. If you add a new feature be sure you also include the `_test.vim`
-file next to the script. Test functions should be starting with `Test_`,
-example:
+vim-go supports test files written in VimScript; the way they're run is
+roughly similar to Go tests:
+
+- A `*.vim` file has a corresponding `*_test.vim`.
+- All functions starting with `Test_` are run as test.
+- A test is considered to be "failed" if |v:errors| has any entries. You can
+  use one of the |test-functions| to set this, or append to it directly.
+
+A simple example:
 >
  function Test_run_fmt()
    call assert_equal(expected, actual)
    ...
  endfunction
 <
-You can locally test it by running:
->
- make
-<
-This will run all tests and print either `PASS` or `FAIL` to indicate the
-final status of all tests. Additionally, each new pull request will trigger a
-new Travis-ci job.
+To run tests vim-go comes with three small helper scripts:
+
+  `scripts/install-vim`   Install a pristine Vim to `/tmp/vim-go-test/`.
+  `scripts/run-vim`       Run a Vim version from `/tmp/vim-go-test/`.
+  `scripts/test`          Run all tests with a Vim from `/tmp/vim-go-test/`.
+
+All scripts accept a Vim version as the first argument, which can be
+`vim-7.4`, `vim-8.0`, or `nvim`. You will need to install a Vim version with
+`install-vim` before you can use `run-vim` or `test`.
+
+You can install and test all Vim versions by running `make`.
 
 
 ==============================================================================
@@ -1960,4 +1969,4 @@ CREDITS                                                         *go-credits*
 * vim-go contributors: https://github.com/fatih/vim-go/graphs/contributors.
 
 
-vim:ft=help:et:ts=2:sw=2:sts=2:norl
+ vim: ft=help tw=78 et ts=2 sw=2 sts=2 norl

scripts/install-vim

@@ -0,0 +1,106 @@
+#!/bin/sh
+#
+# Install and setup a Vim or Neovim for running tests.
+# This should work on both Travis and people's desktop computers, and be 100%
+# independent from any system installed Vim.
+#
+# It will echo the full path to a Vim binary, e.g.:
+#   /some/path/src/vim
+
+set -euC
+
+vimgodir=$(cd -P "$(dirname "$0")/.." > /dev/null && pwd)
+cd "$vimgodir"
+
+vim=${1:-}
+
+case "$vim" in
+  "vim-7.4")
+    # This is what the most recent Ubuntu LTS (16.04) ships with.
+    tag="v7.4.1689"
+    giturl="https://github.com/vim/vim"
+    ;;
+
+  "vim-8.0")
+    # This follows the version in Arch Linux. Vim's master branch isn't always
+    # stable, and we don't want to have the build fail because Vim introduced a
+    # bug.
+    tag="v8.0.1176"
+    giturl="https://github.com/vim/vim"
+    ;;
+
+  "nvim")
+    # Use latest stable version.
+    tag="v0.2.0"
+    giturl="https://github.com/neovim/neovim"
+    ;;
+
+  *)
+    echo "unknown version: '${1:-}'"
+    echo "First argument must be 'vim-7.4', 'vim-8.0', or 'nvim'."
+    exit 1
+    ;;
+esac
+
+srcdir="/tmp/vim-go-test/$1-src"
+installdir="/tmp/vim-go-test/$1-install"
+
+# Use cached installdir.
+if [ -d "$installdir" ]; then
+  echo "$installdir exists; skipping build."
+
+  # The ./scripts/test script relies on this.
+  echo "installed to: $installdir"
+  exit 0
+fi
+
+mkdir -p "$srcdir"
+cd "$srcdir"
+
+# Neovim build requires more deps than Vim and is annoying, so we use the
+# binary.
+# 0.2.0 doesn't have a binary build for Linux, so we use 0.2.1-dev for now.
+if [ "$1" = "nvim" ]; then
+
+  # TODO: Use macOS binaries on macOS
+  curl -Ls https://github.com/neovim/neovim/releases/download/nightly/nvim-linux64.tar.gz |
+    tar xzf - -C /tmp/vim-go-test/
+  mv /tmp/vim-go-test/nvim-linux64 /tmp/vim-go-test/nvim-install
+  mkdir -p "$installdir/share/nvim/runtime/pack/vim-go/start"
+  ln -s "$vimgodir" "$installdir/share/nvim/runtime/pack/vim-go/start/vim-go"
+
+  # Consistent paths makes calling things easier.
+  mv "$installdir/bin/nvim" "$installdir/bin/vim"
+  mkdir -p "$installdir/share/vim/vimgo/pack"
+  ln -s "$installdir/share/nvim/runtime/pack/vim-go" "$installdir/share/vim/vimgo/pack/vim-go"
+
+# Build Vim from source.
+else
+  if [ -d "$srcdir/.git" ]; then
+    echo "Skipping clone as $srcdir/.git exists"
+  else
+    echo "Cloning $tag from $giturl"
+    git clone --branch "$tag" --depth 1 "$giturl" "$srcdir"
+  fi
+
+  ./configure --prefix="$installdir" --with-features=huge --disable-gui
+  make install
+  mkdir -p "$installdir/share/vim/vimgo/pack/vim-go/start"
+  ln -s "$vimgodir" "$installdir/share/vim/vimgo/pack/vim-go/start/vim-go"
+fi
+
+# Make sure all Go tools and other dependencies are installed.
+export GOPATH=$installdir
+export PATH=${GOPATH}/bin:$PATH
+"$vimgodir/scripts/run-vim" $vim +':silent :GoUpdateBinaries' +':qa'
+
+[ -d "$installdir/share/vim/vimgo/pack/vim-go/start/vim-vimhelplint" ] || \
+  git clone --depth 1 --quiet https://github.com/machakann/vim-vimhelplint \
+    "$installdir/share/vim/vimgo/pack/vim-go/start/vim-vimhelplint"
+
+# Don't really need source after successful install.
+rm -rf "$srcdir"
+
+echo "installed to: $installdir"
+
+# vim:ts=2:sts=2:sw=2:et

scripts/lint

@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Run all linting tools.
+#
+
+set -euC
+vimgodir=$(cd -P "$(dirname "$0")/.." > /dev/null && pwd)
+cd "$vimgodir"
+
+### Setup Vim and other dependencies.
+#####################################
+if [ -z "${1:-}" ]; then
+  echo "unknown version: '${1:-}'"
+  echo "First argument must be 'vim-7.4', 'vim-8.0', or 'nvim'."
+	exit 1
+fi
+
+vim=$1
+vimdir="/tmp/vim-go-test/$vim-install"
+export GOPATH=$vimdir
+export PATH=${GOPATH}/bin:$PATH
+
+if [ ! -f "$vimdir/bin/vim" ]; then
+	echo "$vimdir/bin/vim doesn't exist; did you install it with the install-vim script?"
+	exit 1
+fi
+
+### Run vimhelplint.
+####################
+printf "Running vimhelplint ... "
+
+# set modeline explicitly so that the modeline will be respected when run as root.
+lint=$($vimdir/bin/vim -esNR \
+  --cmd "set rtp+=$vimdir/share/vim/vimgo/pack/vim-go/start/vim-vimhelplint/" \
+  --cmd 'set modeline' \
+  +'filetype plugin on' \
+  +"e $vimgodir/doc/vim-go.txt" \
+  +'verbose VimhelpLintEcho' \
+  +q \
+  2>&1)
+if [ "$lint" ]; then
+  echo "FAILED"
+  echo "$lint"
+  exit 6
+else
+  echo "PASSED"
+  exit 0
+fi