From 7efc3fe06532f56d9340429f4b05522426e1d240 Mon Sep 17 00:00:00 2001 From: Martin Tournoij Date: Mon, 30 Oct 2017 05:33:44 +0000 Subject: [PATCH] Better testing (#1476) Better testing - Support multiple Vim versions (currently 7.4, 8.0, and Neovim). - Make sure that we always run Vim from a temporary installation in `/tmp/vim-go-test` without loading `~/.vim`. This makes it a lot easier to run on people's computers. - Also add a handy `./script/run-vim` script to run the installed Vim from the temp directory. Useful for testing, debugging, etc. without a (potentially large) ~/.vim/ dir. - Previously the tests weren't actually being run correctly; see: https://travis-ci.org/fatih/vim-go/builds/279277579 - Format the output of the tests a wee bit nicer, roughly similar to the `go test` output. - Expand docs on testing a bit. I also attempted to integrate code coverage support with https://github.com/Vimjas/covimerage (which was the reason I started working on this), but couldn't really get that to work. Need to look in to that later. --- .dockerignore | 1 + .github/CONTRIBUTING.md | 13 +++-- .gitignore | 5 -- .travis.yml | 30 ++++------- Dockerfile | 13 +++++ Makefile | 23 +++++++-- autoload/go/tags_test.vim | 4 +- doc/vim-go.txt | 51 ++++++++++-------- scripts/install-vim | 106 ++++++++++++++++++++++++++++++++++++++ scripts/lint | 48 +++++++++++++++++ scripts/run-vim | 33 ++++++++++++ scripts/runtest.vim | 43 +++++++--------- scripts/test | 51 ++++++++++++++++++ scripts/test.sh | 47 ----------------- 14 files changed, 340 insertions(+), 128 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 scripts/install-vim create mode 100755 scripts/lint create mode 100755 scripts/run-vim create mode 100755 scripts/test delete mode 100755 scripts/test.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..2d2ecd68da --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.git/ diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index f3fc905c61..edabf91507 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,9 +1,12 @@ Thanks for improving vim-go! Before you dive in please read the following: 1. Please read our - [Documentation](https://github.com/fatih/vim-go/blob/master/doc/vim-go.txt), it might - have answers for your problem -2. If you add a new feature please don't forget to update the documentation: + [Documentation](https://github.com/fatih/vim-go/blob/master/doc/vim-go.txt), + it might have a solution to your problem. +2. If you add a new feature then please don't forget to update the documentation: [doc/vim-go.txt](https://github.com/fatih/vim-go/blob/master/doc/vim-go.txt). -3. If it's a breaking change or exceed +100 lines please open an issue first - and describe the changes you want to make. +3. If it's a breaking change or exceeds 100 lines of code then please open an + issue first and describe the changes you want to make. +4. See `:help go-development` for instructions on how to run and write tests. If + you add a new feature be sure you also include a test if feasible. + diff --git a/.gitignore b/.gitignore index 1dfba202d3..bdeb2b8264 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,2 @@ doc/tags .DS_Store - -# Test specific files -FAILED -test.log -scripts/vim-vimhelplint diff --git a/.travis.yml b/.travis.yml index d05b23ea51..d0613d6d96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,11 @@ language: go - -env: - global: - - DEPS=$HOME/deps - - PATH=$DEPS/bin:$PATH - - PATCH="v8.0.0134" - -install: | - git config --global user.email "you@example.com" - git config --global user.name "Your Name" - - # check out if we can pre-compiled Vim releases somehow, - git clone --branch $PATCH --depth 1 https://github.com/vim/vim - cd vim - ./configure --prefix=$DEPS --with-features=huge --disable-gui - make - make install - cd - - -script: ./scripts/test.sh +matrix: + include: + - env: SCRIPT=test VIM_VERSION=vim-7.4 + - env: SCRIPT=test VIM_VERSION=vim-8.0 + - env: SCRIPT=test VIM_VERSION=nvim + - env: SCRIPT=lint VIM_VERSION=vim-8.0 +install: + - ./scripts/install-vim $VIM_VERSION +script: + - ./scripts/$SCRIPT $VIM_VERSION diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..e6340b27cb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.9.1 + +RUN apt-get update -y && \ + apt-get install -y build-essential curl git libncurses5-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN useradd -ms /bin/bash -d /vim-go vim-go +USER vim-go +WORKDIR /vim-go +COPY . /vim-go/ + +ENTRYPOINT ["make"] diff --git a/Makefile b/Makefile index 20b296d773..ab14997a4b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,24 @@ -all: test +all: install test lint + +install: + @echo "==> Installing Vims" + @./scripts/install-vim vim-7.4 + @./scripts/install-vim vim-8.0 + @./scripts/install-vim nvim test: @echo "==> Running tests" - @./scripts/test.sh + @./scripts/test vim-7.4 + @./scripts/test vim-8.0 + @./scripts/test nvim + +lint: + @echo "==> Running linting tools" + @./scripts/lint vim-8.0 + +clean: + @echo "==> Cleaning /tmp/vim-go-test" + @rm -rf /tmp/vim-go-test + -.PHONY: all test +.PHONY: all test install clean lint diff --git a/autoload/go/tags_test.vim b/autoload/go/tags_test.vim index 6702ad6fba..8f5b464e92 100644 --- a/autoload/go/tags_test.vim +++ b/autoload/go/tags_test.vim @@ -5,7 +5,7 @@ func Test_add_tags() let expected = join(readfile("test-fixtures/tags/add_all_golden.go"), "\n") " run for offset 40, which is inside the struct - call go#tags#run(0, 0, 40, "add", input_file, 1) + silent call go#tags#run(0, 0, 40, "add", input_file, 1) let actual = join(readfile(input_file), "\n") @@ -20,7 +20,7 @@ func Test_remove_tags() let expected = join(readfile("test-fixtures/tags/remove_all_golden.go"), "\n") " run for offset 40, which is inside the struct - call go#tags#run(0, 0, 40, "remove", input_file, 1) + silent call go#tags#run(0, 0, 40, "remove", input_file, 1) let actual = join(readfile(input_file), "\n") diff --git a/doc/vim-go.txt b/doc/vim-go.txt index 73616e41d9..216aed2a97 100644 --- a/doc/vim-go.txt +++ b/doc/vim-go.txt @@ -125,7 +125,7 @@ Supported Go plugins~ *vim-go-plugins* The following plugins are supported for use with vim-go: * Real-time completion (Vim): - https://github.com/Shougo/neocomplete.vim + https://github.com/Shougo/neocomplete.vim * Real-time completion (Neovim): https://github.com/Shougo/deoplete.nvim and @@ -566,13 +566,13 @@ CTRL-t *:GoFreevars* :GoFreevars - Enumerates the free variables of the selection. “Free variables” is a + Enumerates the free variables of the selection. "Free variables" is a technical term meaning the set of variables that are referenced but not defined within the selection, or loosely speaking, its inputs. - This information is useful if you’re considering whether to refactor the + This information is useful when considering whether to refactor the selection into a function of its own, as the free variables would be the - necessary parameters of that function. It’s also useful when you want to + necessary parameters of that function. It's also useful when you want to understand what the inputs are to a complex block of code even if you don’t plan to change it. @@ -1181,11 +1181,11 @@ used > Use this option to add additional options to the |'g:go_fmt_command'|. It's value type can be either a string or a dictionary. This is due backwards compatibility. The string version will be removed in the future so please use -the dictionary version. Default is empty. +the dictionary version. Default is empty. > let g:go_fmt_options = '' - or + or let g:go_fmt_options = {} < @@ -1394,7 +1394,7 @@ Specifies the type of list to use for command outputs (such as errors from builds, results from static analysis commands, etc...). The list type for specific commands can be overridden with |'g:go_list_type_commands'|. The default value (empty) will use the appropriate kind of list for the command -that was called. Supported values are "", "quickfix", and "locationlist". +that was called. Supported values are "", "quickfix", and "locationlist". > let g:go_list_type = "" < @@ -1422,7 +1422,7 @@ As an example, the following settings will change all list types to Specifies whether the quickfix/location list should be closed automatically in the absence of errors. The default value is 1. -If you prefer to keep a long running error window open, you can disable +If you prefer to keep a long running error window open, you can disable this by setting the value to 0. > let g:go_list_autoclose = 1 @@ -1815,7 +1815,7 @@ manually, usually from an |autocommand|: If you have a lot of packages with the same prefix (`github.com/user`) you can use a single autocommand: > - autocmd BufRead /home/martin/go/src/*.go + autocmd BufRead /home/martin/go/src/*.go \ let s:tmp = matchlist(expand('%:p'), \ '/home/martin/go/src/\(github.com/user/[^/]\+\)') \| if len(s:tmp) > 1 | exe 'silent :GoGuruScope ' . s:tmp[1] | endif @@ -1916,23 +1916,32 @@ By default new terminals are opened in a vertical split. To change it ============================================================================== DEVELOPMENT *go-development* -vim-go supports test files written in VimL. Please check `autoload` folder for -examples. If you add a new feature be sure you also include the `_test.vim` -file next to the script. Test functions should be starting with `Test_`, -example: +vim-go supports test files written in VimScript; the way they're run is +roughly similar to Go tests: + +- A `*.vim` file has a corresponding `*_test.vim`. +- All functions starting with `Test_` are run as test. +- A test is considered to be "failed" if |v:errors| has any entries. You can + use one of the |test-functions| to set this, or append to it directly. + +A simple example: > function Test_run_fmt() call assert_equal(expected, actual) ... endfunction < -You can locally test it by running: -> - make -< -This will run all tests and print either `PASS` or `FAIL` to indicate the -final status of all tests. Additionally, each new pull request will trigger a -new Travis-ci job. +To run tests vim-go comes with three small helper scripts: + + `scripts/install-vim` Install a pristine Vim to `/tmp/vim-go-test/`. + `scripts/run-vim` Run a Vim version from `/tmp/vim-go-test/`. + `scripts/test` Run all tests with a Vim from `/tmp/vim-go-test/`. + +All scripts accept a Vim version as the first argument, which can be +`vim-7.4`, `vim-8.0`, or `nvim`. You will need to install a Vim version with +`install-vim` before you can use `run-vim` or `test`. + +You can install and test all Vim versions by running `make`. ============================================================================== @@ -1960,4 +1969,4 @@ CREDITS *go-credits* * vim-go contributors: https://github.com/fatih/vim-go/graphs/contributors. -vim:ft=help:et:ts=2:sw=2:sts=2:norl + vim: ft=help tw=78 et ts=2 sw=2 sts=2 norl diff --git a/scripts/install-vim b/scripts/install-vim new file mode 100755 index 0000000000..3dc796f298 --- /dev/null +++ b/scripts/install-vim @@ -0,0 +1,106 @@ +#!/bin/sh +# +# Install and setup a Vim or Neovim for running tests. +# This should work on both Travis and people's desktop computers, and be 100% +# independent from any system installed Vim. +# +# It will echo the full path to a Vim binary, e.g.: +# /some/path/src/vim + +set -euC + +vimgodir=$(cd -P "$(dirname "$0")/.." > /dev/null && pwd) +cd "$vimgodir" + +vim=${1:-} + +case "$vim" in + "vim-7.4") + # This is what the most recent Ubuntu LTS (16.04) ships with. + tag="v7.4.1689" + giturl="https://github.com/vim/vim" + ;; + + "vim-8.0") + # This follows the version in Arch Linux. Vim's master branch isn't always + # stable, and we don't want to have the build fail because Vim introduced a + # bug. + tag="v8.0.1176" + giturl="https://github.com/vim/vim" + ;; + + "nvim") + # Use latest stable version. + tag="v0.2.0" + giturl="https://github.com/neovim/neovim" + ;; + + *) + echo "unknown version: '${1:-}'" + echo "First argument must be 'vim-7.4', 'vim-8.0', or 'nvim'." + exit 1 + ;; +esac + +srcdir="/tmp/vim-go-test/$1-src" +installdir="/tmp/vim-go-test/$1-install" + +# Use cached installdir. +if [ -d "$installdir" ]; then + echo "$installdir exists; skipping build." + + # The ./scripts/test script relies on this. + echo "installed to: $installdir" + exit 0 +fi + +mkdir -p "$srcdir" +cd "$srcdir" + +# Neovim build requires more deps than Vim and is annoying, so we use the +# binary. +# 0.2.0 doesn't have a binary build for Linux, so we use 0.2.1-dev for now. +if [ "$1" = "nvim" ]; then + + # TODO: Use macOS binaries on macOS + curl -Ls https://github.com/neovim/neovim/releases/download/nightly/nvim-linux64.tar.gz | + tar xzf - -C /tmp/vim-go-test/ + mv /tmp/vim-go-test/nvim-linux64 /tmp/vim-go-test/nvim-install + mkdir -p "$installdir/share/nvim/runtime/pack/vim-go/start" + ln -s "$vimgodir" "$installdir/share/nvim/runtime/pack/vim-go/start/vim-go" + + # Consistent paths makes calling things easier. + mv "$installdir/bin/nvim" "$installdir/bin/vim" + mkdir -p "$installdir/share/vim/vimgo/pack" + ln -s "$installdir/share/nvim/runtime/pack/vim-go" "$installdir/share/vim/vimgo/pack/vim-go" + +# Build Vim from source. +else + if [ -d "$srcdir/.git" ]; then + echo "Skipping clone as $srcdir/.git exists" + else + echo "Cloning $tag from $giturl" + git clone --branch "$tag" --depth 1 "$giturl" "$srcdir" + fi + + ./configure --prefix="$installdir" --with-features=huge --disable-gui + make install + mkdir -p "$installdir/share/vim/vimgo/pack/vim-go/start" + ln -s "$vimgodir" "$installdir/share/vim/vimgo/pack/vim-go/start/vim-go" +fi + +# Make sure all Go tools and other dependencies are installed. +export GOPATH=$installdir +export PATH=${GOPATH}/bin:$PATH +"$vimgodir/scripts/run-vim" $vim +':silent :GoUpdateBinaries' +':qa' + +[ -d "$installdir/share/vim/vimgo/pack/vim-go/start/vim-vimhelplint" ] || \ + git clone --depth 1 --quiet https://github.com/machakann/vim-vimhelplint \ + "$installdir/share/vim/vimgo/pack/vim-go/start/vim-vimhelplint" + +# Don't really need source after successful install. +rm -rf "$srcdir" + +echo "installed to: $installdir" + +# vim:ts=2:sts=2:sw=2:et diff --git a/scripts/lint b/scripts/lint new file mode 100755 index 0000000000..dfafdeed84 --- /dev/null +++ b/scripts/lint @@ -0,0 +1,48 @@ +#!/bin/sh +# +# Run all linting tools. +# + +set -euC +vimgodir=$(cd -P "$(dirname "$0")/.." > /dev/null && pwd) +cd "$vimgodir" + +### Setup Vim and other dependencies. +##################################### +if [ -z "${1:-}" ]; then + echo "unknown version: '${1:-}'" + echo "First argument must be 'vim-7.4', 'vim-8.0', or 'nvim'." + exit 1 +fi + +vim=$1 +vimdir="/tmp/vim-go-test/$vim-install" +export GOPATH=$vimdir +export PATH=${GOPATH}/bin:$PATH + +if [ ! -f "$vimdir/bin/vim" ]; then + echo "$vimdir/bin/vim doesn't exist; did you install it with the install-vim script?" + exit 1 +fi + +### Run vimhelplint. +#################### +printf "Running vimhelplint ... " + +# set modeline explicitly so that the modeline will be respected when run as root. +lint=$($vimdir/bin/vim -esNR \ + --cmd "set rtp+=$vimdir/share/vim/vimgo/pack/vim-go/start/vim-vimhelplint/" \ + --cmd 'set modeline' \ + +'filetype plugin on' \ + +"e $vimgodir/doc/vim-go.txt" \ + +'verbose VimhelpLintEcho' \ + +q \ + 2>&1) +if [ "$lint" ]; then + echo "FAILED" + echo "$lint" + exit 6 +else + echo "PASSED" + exit 0 +fi diff --git a/scripts/run-vim b/scripts/run-vim new file mode 100755 index 0000000000..baf0be3e69 --- /dev/null +++ b/scripts/run-vim @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Run a "bare" Vim with just vim-go and ignoring ~/.vim +# + +set -euC +vimgodir=$(cd -P "$(dirname "$0")/.." > /dev/null && pwd) +cd "$vimgodir" + +if [ -z "${1:-}" ]; then + echo "unknown version: '${1:-}'" + echo "First argument must be 'vim-7.4', 'vim-8.0', or 'nvim'." + exit 1 +fi + +dir="/tmp/vim-go-test/$1-install" +export GOPATH=$dir +export PATH=${GOPATH}/bin:$PATH +shift + +if [ ! -f "$dir/bin/vim" ]; then + echo "$dir/bin/vim doesn't exist; did you install it with the install-vim script?" + exit 1 +fi + +$dir/bin/vim --noplugin -u NONE -N \ + +"set shm+=WAFI rtp=$dir/share/vim/vimgo packpath=$dir/share/vim/vimgo,$vimgodir" \ + +'filetype plugin indent on' \ + +'packloadall!' \ + "$@" + + +# vim:ts=2:sts=2:sw=2:et diff --git a/scripts/runtest.vim b/scripts/runtest.vim index 3026ac6c61..98d83db272 100644 --- a/scripts/runtest.vim +++ b/scripts/runtest.vim @@ -1,19 +1,15 @@ let total_started = reltime() -" add vim-go the only plugin inside the runtimepath -let git_root_path = system("git rev-parse --show-toplevel | tr -d '\\n'") -exe 'set rtp=' . git_root_path - " source the passed test file source % " cd into the folder of the test file let cd = exists('*haslocaldir') && haslocaldir() ? 'lcd ' : 'cd ' let dir = getcwd() +let testfile = expand('%:t') execute cd . expand('%:p:h') " initialize variables -let g:testname = expand('%') let s:fail = 0 let s:done = 0 let s:logs = [] @@ -21,7 +17,7 @@ let s:logs = [] " get a list of all Test_ functions for the given file set nomore redir @q -silent function /^Test_ + silent function /^Test_ redir END let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g')) @@ -29,7 +25,7 @@ let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g')) for s:test in sort(s:tests) let started = reltime() - call add(s:logs, printf("=== RUN %s", s:test[:-3])) + call add(s:logs, printf("=== RUN %s", s:test[:-3])) exe 'call ' . s:test let elapsed_time = reltimestr(reltime(started)) @@ -39,13 +35,13 @@ for s:test in sort(s:tests) if len(v:errors) > 0 let s:fail += 1 - call add(s:logs, printf("--- FAIL: %s (%ss)", s:test[:-3], elapsed_time)) + call add(s:logs, printf("--- FAIL %s (%ss)", s:test[:-3], elapsed_time)) call extend(s:logs, map(v:errors, '" ". v:val')) " reset so we can capture failures of next test let v:errors = [] else - call add(s:logs, printf("--- PASS: %s (%ss)", s:test[:-3], elapsed_time)) + call add(s:logs, printf("--- PASS %s (%ss)", s:test[:-3], elapsed_time)) endif endfor @@ -54,30 +50,27 @@ execute cd . fnameescape(dir) " create an empty fail to indicate that the test failed if s:fail > 0 - split FAILED - write + split /tmp/vim-go-test/FAILED + silent write endif let total_elapsed_time = reltimestr(reltime(total_started)) let total_elapsed_time = substitute(total_elapsed_time, '^\s*\(.\{-}\)\s*$', '\1', '') -let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test') . '. Total test time: '. total_elapsed_time .'s' -call add(s:logs, "") -call add(s:logs, message) - -" store all error messages from within vim into test.log -redir > test.log -silent messages +" Add all messages (usually errors). +redir => s:mess + silent messages redir END +let s:logs = s:logs + filter(split(s:mess, "\n"), 'v:val !~ "^Messages maintainer"') -" also store all internal messages from s:logs: as well -split test.log -call append(line('$'), '') -call append(line('$'), 'From ' . g:testname . ':') +" Also store all internal messages from s:logs as well. +silent! split /tmp/vim-go-test/test.tmp call append(line('$'), s:logs) -write +call append(line('$'), printf("%s%s %s / %s tests", + \ (s:fail > 0 ? 'FAIL ' : 'ok '), + \ testfile, total_elapsed_time, s:done)) +silent! write -" bye, bye! qall! -" vim: sw=2 ts=2 et +" vim:ts=2:sts=2:sw=2:et diff --git a/scripts/test b/scripts/test new file mode 100755 index 0000000000..8676505fb2 --- /dev/null +++ b/scripts/test @@ -0,0 +1,51 @@ +#!/bin/sh +# +# Run all tests. +# + +set -euC +vimgodir=$(cd -P "$(dirname "$0")/.." > /dev/null && pwd) +cd "$vimgodir" + +### Setup Vim and other dependencies. +##################################### +if [ -z "${1:-}" ]; then + echo "unknown version: '${1:-}'" + echo "First argument must be 'vim-7.4', 'vim-8.0', or 'nvim'." + exit 1 +fi + +vim=$1 +vimdir="/tmp/vim-go-test/$vim-install" +export GOPATH=$vimdir +export PATH=${GOPATH}/bin:$PATH + +if [ ! -f "$vimdir/bin/vim" ]; then + echo "$vimdir/bin/vim doesn't exist; did you install it with the install-vim script?" + exit 1 +fi + +### Run tests. +############## +# Clean stale log file. +[ -f '/tmp/vim-go-test/test.log' ] && rm '/tmp/vim-go-test/test.log' +[ -f '/tmp/vim-go-test/FAILED' ] && rm '/tmp/vim-go-test/FAILED' + +# Run the actual tests. +fail=0 +for test_file in "$vimgodir"/autoload/go/*_test.vim; do + "$vimgodir/scripts/run-vim" $vim -e +"silent e $test_file" -S ./scripts/runtest.vim + + # Append logs + cat '/tmp/vim-go-test/test.tmp' | tee '/tmp/vim-go-test/test.log' + rm '/tmp/vim-go-test/test.tmp' +done + +echo +if [ -f "/tmp/vim-go-test/FAILED" ]; then + echo 2>&1 "Some tests FAILED" + exit 1 +fi +echo 2>&1 "All tests PASSED" + +# vim:ts=2:sts=2:sw=2:et diff --git a/scripts/test.sh b/scripts/test.sh deleted file mode 100755 index 332043df2e..0000000000 --- a/scripts/test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# vim:ts=2:sts=2:sw=2:et - -set -e - -cd $(dirname $0) - -# install dependencies -go get github.com/fatih/gomodifytags -go get golang.org/x/tools/cmd/goimports - -# cleanup test.log -if [ -f "test.log" ]; then - rm test.log -fi - -if [ -f "FAILED" ]; then - rm FAILED -fi - -for test_file in ../autoload/go/*_test.vim -do - vim -u NONE -S runtest.vim $test_file -done - -if [ -f "test.log" ]; then - cat test.log -fi - -# if Failed exists, test failed -if [ -f "FAILED" ]; then - echo 2>&1 "FAIL" - exit 1 -fi -echo 2>&1 "PASS" - -# Run vimhelplint -[ -d vim-vimhelplint ] || git clone https://github.com/machakann/vim-vimhelplint -echo "Running vimhelplint" -lint=$(vim -esN --cmd 'set rtp+=./vim-vimhelplint' -c 'filetype plugin on' \ - -c 'e ../doc/vim-go.txt' -c 'verb VimhelpLintEcho' -c q 2>&1) -if [ -n "$lint" ]; then - echo $lint - exit 1 -else - exit 0 -fi