Add -list-files flag, test/files-toml-$version

Include an easier to access list of files for specific TOML versions,
making it easier to implement your own test runner.

Fixes #134

README.md

@@ -226,10 +226,37 @@ added.) Obviously, this advantage does not apply to testing TOML encoders since
 there must exist a TOML decoder that conforms to the specification in order to
 read the output of a TOML encoder.
 
+Usage without `toml-test` binary
+--------------------------------
+While the `toml-test` tool is a convenient way to run the tests, you can also
+re-implement its behaviour in your own language's test-suite, which may be an
+easier way to run the tests.
+
+Because multiple TOML versions are supported, not all tests are valid for every
+version of TOML; for example the 1.0.0 tests contain a test that trailing commas
+in tables are invalid, but in 1.1.0 this should be considered valid.
+
+In short: you can't "just" copy all .toml and .json files from the tests/
+directory. The [test/files-toml-1.0.0] and [test/files-toml-1.1.0] files contain
+a list of files you should run for that TOML version.
+
+You can use those lists to determine which tests to run, or include only those
+tests in your library by copying them with something like:
+
+    <files-toml-1.0.0 while read l; do
+        mkdir -p ~/my-test/"$(dirname "$l")"
+        cp -r "$l" ~/my-test/"$l"
+    done
+
+These files are generated with `toml-test -list-files`:
+
+    % toml-test -list-files -toml=1.0.0
+    % toml-test -list-files -toml=1.1.0
+
 Adding tests
 ------------
 `toml-test` was designed so that tests can be easily added and removed. As
-mentioned above, tests are split into two groups: invalid and valid tests. 
+mentioned above, tests are split into two groups: invalid and valid tests.
 
 Invalid tests **only check if a decoder rejects invalid TOML data**. Or, in the
 case of testing encoders, invalid tests **only check if an encoder rejects an

cmd/toml-test/main.go

@@ -5,6 +5,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	tomltest "github.com/BurntSushi/toml-test"
@@ -13,7 +14,7 @@ import (
 
 var hlErr = zli.Color256(224).Bg() | zli.Color256(0) | zli.Bold
 
-func parseFlags() (tomltest.Runner, []string, int, string) {
+func parseFlags() (tomltest.Runner, []string, int, string, bool) {
 	f := zli.NewFlags(os.Args)
 	var (
 		help        = f.Bool(false, "help", "h")
@@ -25,6 +26,7 @@ func parseFlags() (tomltest.Runner, []string, int, string) {
 		color       = f.String("always", "color")
 		skip        = f.StringList(nil, "skip")
 		run         = f.StringList(nil, "run")
+		listFiles   = f.Bool(false, "list-files")
 	)
 	zli.F(f.Parse())
 	if help.Bool() {
@@ -43,7 +45,7 @@ func parseFlags() (tomltest.Runner, []string, int, string) {
 		Version:   tomlVersion.String(),
 	}
 
-	if len(f.Args) == 0 {
+	if len(f.Args) == 0 && !listFiles.Bool() {
 		zli.Fatalf("no parser command")
 	}
 	for _, r := range r.RunTests {
@@ -104,11 +106,25 @@ func parseFlags() (tomltest.Runner, []string, int, string) {
 		}
 	}
 
-	return r, f.Args, showAll.Int(), testDir.String()
+	return r, f.Args, showAll.Int(), testDir.String(), listFiles.Bool()
 }
 
 func main() {
-	runner, cmd, showAll, testDir := parseFlags()
+	runner, cmd, showAll, testDir, listFiles := parseFlags()
+
+	if listFiles {
+		l, err := runner.List()
+		zli.F(err)
+
+		sort.Strings(l)
+		for _, ll := range l {
+			if strings.HasPrefix(ll, "valid/") {
+				fmt.Println(ll + ".json")
+			}
+			fmt.Println(ll + ".toml")
+		}
+		return
+	}
 
 	tests, err := runner.Run()
 	zli.F(err)

cmd/toml-test/usage.go

@@ -43,8 +43,12 @@ Flags:
                   the correctness of my TOML parser!
 
     -toml         Select TOML version to run tests for. Supported versions are
-                  "1.0.0" and "next" (which includes changes which aren't in a
-                  "tagged" TOML version yet). Defaults to 1.0.0.
+                  "1.0.0" and "1.1.0" (which isn't released yet and may change).
+                  Defaults to 1.0.0.
+
+    -list-files   List all test files, one file per line, and exit without
+                  running anything. This takes the -toml flag in to account, but
+                  none of the other flags.
 
     -v            List all tests, even passing ones. Add twice to show detailed
                   output for passing tests.

gen.py

@@ -19,6 +19,12 @@ def gen_multi():
             with open(path, 'wb+') as fp:
                 fp.write(line.encode())
 
+def gen_list():
+    with open('tests/files-toml-1.0.0', 'w+') as fp:
+        subprocess.run(['go', 'run', './cmd/toml-test', '-list-files', '-toml=1.0.0'], stdout=fp)
+    with open('tests/files-toml-1.1.0', 'w+') as fp:
+        subprocess.run(['go', 'run', './cmd/toml-test', '-list-files', '-toml=1.1.0'], stdout=fp)
+
 def gen_spec(tmp):
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -156,4 +162,5 @@ def has_active_invalid(block):
 if __name__ == "__main__":
     with tempfile.TemporaryDirectory() as tmp:
         gen_spec(tmp)
-        gen_multi()
+    gen_multi()
+    gen_list()

runner.go

@@ -1,4 +1,4 @@
-//go:generate ./gen-multi.py
+//go:generate ./gen.py
 
 package tomltest