separate conformance suite on CI.

filecoin-project · Aug 16, 2020 · 58c340b · 58c340b
1 parent b9d6729
commit 58c340b
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 5 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -146,6 +146,10 @@ jobs:
         description: |
           Upload coverage report to https://codecov.io/. Requires the codecov API token to be
           set as an environment variable for private projects.
+      trailing-flags:
+        type: string
+        default: ""
+        description: Flags to follow after packages.
     executor: << parameters.executor >>
     steps:
       - install-deps
@@ -171,7 +175,8 @@ jobs:
               -- \
               << parameters.coverage >> \
               << parameters.go-test-flags >> \
-              << parameters.packages >>
+              << parameters.packages >> \
+              << parameters.trailing-flags >>
           no_output_timeout: 30m
       - store_test_results:
           path: /tmp/test-reports
@@ -191,6 +196,8 @@ jobs:
     <<: *test
   test-window-post:
     <<: *test
+  test-conformance:
+    <<: *test
 
   build-macos:
     description: build darwin lotus binary
@@ -326,17 +333,24 @@ workflows:
       - test:
           codecov-upload: true
           test-suite-name: full
+          trailing-flags: "-conformance.skip=false"
       - test-window-post:
           go-test-flags: "-run=TestWindowedPost"
+          trailing-flags: "-conformance.skip=true"
           winpost-test: "1"
           test-suite-name: window-post
       - test-short:
           go-test-flags: "--timeout 10m --short"
           test-suite-name: short
+          trailing-flags: "-conformance.skip=true"
           filters:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
+      - test-conformance:
+          test-suite-name: conformance
+          packages: "./conformance"
+          trailing-flags: "-conformance.skip=false"
       - build-debug
       - build-all:
           requires:

diff --git a/conformance/runner_test.go b/conformance/runner_test.go
@@ -24,24 +24,26 @@ const (
 	// It is mounted on the Lotus repo as a git submodule.
 	//
 	// When running this test, the corpus root can be overridden through the
-	// -corpus.root CLI flag to run an alternate corpus.
+	// -conformance.corpus CLI flag to run an alternate corpus.
 	defaultCorpusRoot = "../extern/test-vectors/corpus"
 )
 
 var (
-	// corpusRoot is the effective corpus root path, taken from the `-corpus.root` CLI flag,
+	// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
 	// falling back to defaultCorpusRoot if not provided.
 	corpusRoot string
 	// ignore is a set of paths relative to root to skip.
 	ignore = map[string]struct{}{
 		".git":        {},
 		"schema.json": {},
 	}
+	skip bool
 )
 
 func init() {
-	// read the alternative root from the -corpus.root CLI flag.
-	flag.StringVar(&corpusRoot, "corpus.root", defaultCorpusRoot, "test vector corpus directory")
+	// read the alternative root from the -conformance.corpus CLI flag.
+	flag.StringVar(&corpusRoot, "conformance.corpus", defaultCorpusRoot, "test vector corpus directory")
+	flag.BoolVar(&skip, "conformance.skip", false, "whether to skip these tests; for shorthanding and CI")
 }
 
 // TestConformance is the entrypoint test that runs all test vectors found
@@ -51,6 +53,9 @@ func init() {
 // as well as files beginning with _. It parses each file as a test vector, and
 // runs it via the Driver.
 func TestConformance(t *testing.T) {
+	if skip {
+		t.SkipNow()
+	}
 	var vectors []string
 	err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
 		if err != nil {