driver, runner and CI for interoperable test-vector based testing.

.circleci/config.yml

@@ -113,7 +113,7 @@ jobs:
   test: &test
     description: |
       Run tests with gotestsum.
-    parameters:
+    parameters: &test-params
       executor:
         type: executor
         default: golang
@@ -161,6 +161,7 @@ jobs:
           name: go test
           environment:
             LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >>
+            SKIP_CONFORMANCE: "1"
           command: |
             mkdir -p /tmp/test-reports/<< parameters.test-suite-name >>
             mkdir -p /tmp/test-artifacts
@@ -191,6 +192,53 @@ jobs:
     <<: *test
   test-window-post:
     <<: *test
+  test-conformance:
+    description: |
+      Run tests using a corpus of interoperable test vectors for Filecoin 
+      implementations to test their correctness and compliance with the Filecoin
+      specifications.
+    parameters:
+      <<: *test-params
+      vectors-branch:
+        type: string
+        default: ""
+        description: |
+          Branch on github.com/filecoin-project/test-vectors to checkout and
+          test with. If empty (the default) the commit defined by the git
+          submodule is used.
+    executor: << parameters.executor >>
+    steps:
+      - install-deps
+      - prepare
+      - run:
+          command: make deps lotus
+          no_output_timeout: 30m
+      - download-params
+      - when:
+          condition:
+            not:
+              equal: [ "", << parameters.vectors-branch >> ]
+          steps:
+            - run:
+                name: checkout vectors branch
+                command: |
+                  cd extern/test-vectors
+                  git fetch
+                  git checkout origin/<< parameters.vectors-branch >>
+            - run:
+                name: go get vectors branch
+                command: go get github.com/filecoin-project/test-vectors@<< parameters.vectors-branch >>
+      - run:
+          name: go test
+          environment:
+            SKIP_CONFORMANCE: "0"
+          command: |
+            mkdir -p /tmp/test-artifacts
+            go test -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
+            go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
+          no_output_timeout: 30m
+      - store_artifacts:
+          path: /tmp/test-artifacts/conformance-coverage.html
 
   build-macos:
     description: build darwin lotus binary
@@ -356,6 +404,14 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
+      - test-conformance:
+          test-suite-name: conformance
+          packages: "./conformance"
+      - test-conformance:
+          name: test-conformance-bleeding-edge
+          test-suite-name: conformance-bleeding-edge
+          packages: "./conformance"
+          vectors-branch: master
       - build-debug
       - build-all:
           requires:

.gitmodules

@@ -5,3 +5,6 @@
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
 	url = https://github.com/filecoin-project/serialization-vectors
+[submodule "extern/test-vectors"]
+	path = extern/test-vectors
+	url = https://github.com/filecoin-project/test-vectors.git

conformance/driver.go

@@ -0,0 +1,72 @@
+package conformance
+
+import (
+	"context"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/puppet"
+	"github.com/filecoin-project/test-vectors/schema"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/sector-storage/ffiwrapper"
+
+	"github.com/filecoin-project/test-vectors/chaos"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+)
+
+var (
+	// BaseFee to use in the VM.
+	// TODO make parametrisable through vector.
+	BaseFee = abi.NewTokenAmount(100)
+)
+
+type Driver struct {
+	ctx    context.Context
+	vector *schema.TestVector
+}
+
+func NewDriver(ctx context.Context, vector *schema.TestVector) *Driver {
+	return &Driver{ctx: ctx, vector: vector}
+}
+
+// ExecuteMessage executes a conformance test vector message in a temporary VM.
+func (d *Driver) ExecuteMessage(msg *types.Message, preroot cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) (*vm.ApplyRet, cid.Cid, error) {
+	vmOpts := &vm.VMOpts{
+		StateBase:      preroot,
+		Epoch:          epoch,
+		Rand:           &testRand{}, // TODO always succeeds; need more flexibility.
+		Bstore:         bs,
+		Syscalls:       mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
+		CircSupplyCalc: nil,
+		BaseFee:        BaseFee,
+	}
+
+	lvm, err := vm.NewVM(vmOpts)
+	if err != nil {
+		return nil, cid.Undef, err
+	}
+
+	invoker := vm.NewInvoker()
+
+	// add support for the puppet and chaos actors.
+	if puppetOn, ok := d.vector.Selector["puppet_actor"]; ok && puppetOn == "true" {
+		invoker.Register(puppet.PuppetActorCodeID, puppet.Actor{}, puppet.State{})
+	}
+	if chaosOn, ok := d.vector.Selector["chaos_actor"]; ok && chaosOn == "true" {
+		invoker.Register(chaos.ChaosActorCodeCID, chaos.Actor{}, chaos.State{})
+	}
+
+	lvm.SetInvoker(invoker)
+
+	ret, err := lvm.ApplyMessage(d.ctx, msg)
+	if err != nil {
+		return nil, cid.Undef, err
+	}
+
+	root, err := lvm.Flush(d.ctx)
+	return ret, root, err
+}

conformance/runner_test.go

@@ -0,0 +1,221 @@
+package conformance
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+
+	"github.com/filecoin-project/statediff"
+	"github.com/filecoin-project/test-vectors/schema"
+
+	"github.com/ipld/go-car"
+	"github.com/logrusorgru/aurora"
+)
+
+const (
+	// EnvSkipConformance, if 1, skips the conformance test suite.
+	EnvSkipConformance = "SKIP_CONFORMANCE"
+
+	// EnvCorpusRootDir is the name of the environment variable where the path
+	// to an alternative corpus location can be provided.
+	//
+	// The default is defaultCorpusRoot.
+	EnvCorpusRootDir = "CORPUS_DIR"
+
+	// defaultCorpusRoot is the directory where the test vector corpus is hosted.
+	// It is mounted on the Lotus repo as a git submodule.
+	//
+	// When running this test, the corpus root can be overridden through the
+	// -conformance.corpus CLI flag to run an alternate corpus.
+	defaultCorpusRoot = "../extern/test-vectors/corpus"
+)
+
+// ignore is a set of paths relative to root to skip.
+var ignore = map[string]struct{}{
+	".git":        {},
+	"schema.json": {},
+}
+
+// TestConformance is the entrypoint test that runs all test vectors found
+// in the corpus root directory.
+//
+// It locates all json files via a recursive walk, skipping over the ignore set,
+// as well as files beginning with _. It parses each file as a test vector, and
+// runs it via the Driver.
+func TestConformance(t *testing.T) {
+	if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" {
+		t.SkipNow()
+	}
+	// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
+	// falling back to defaultCorpusRoot if not provided.
+	corpusRoot := defaultCorpusRoot
+	if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" {
+		corpusRoot = dir
+	}
+
+	var vectors []string
+	err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		filename := filepath.Base(path)
+		rel, err := filepath.Rel(corpusRoot, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if _, ok := ignore[rel]; ok {
+			// skip over using the right error.
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if info.IsDir() {
+			// dive into directories.
+			return nil
+		}
+		if filepath.Ext(path) != ".json" {
+			// skip if not .json.
+			return nil
+		}
+		if ignored := strings.HasPrefix(filename, "_"); ignored {
+			// ignore files starting with _.
+			t.Logf("ignoring: %s", rel)
+			return nil
+		}
+		vectors = append(vectors, rel)
+		return nil
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(vectors) == 0 {
+		t.Fatalf("no test vectors found")
+	}
+
+	// Run a test for each vector.
+	for _, v := range vectors {
+		path := filepath.Join(corpusRoot, v)
+		raw, err := ioutil.ReadFile(path)
+		if err != nil {
+			t.Fatalf("failed to read test raw file: %s", path)
+		}
+
+		var vector schema.TestVector
+		err = json.Unmarshal(raw, &vector)
+		if err != nil {
+			t.Errorf("failed to parse test vector %s: %s; skipping", path, err)
+			continue
+		}
+
+		t.Run(v, func(t *testing.T) {
+			for _, h := range vector.Hints {
+				if h == schema.HintIncorrect {
+					t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID)
+					t.SkipNow()
+				}
+			}
+
+			// dispatch the execution depending on the vector class.
+			switch vector.Class {
+			case "message":
+				executeMessageVector(t, &vector)
+			default:
+				t.Fatalf("test vector class not supported: %s", vector.Class)
+			}
+		})
+	}
+}
+
+// executeMessageVector executes a message-class test vector.
+func executeMessageVector(t *testing.T, vector *schema.TestVector) {
+	var (
+		ctx   = context.Background()
+		epoch = vector.Pre.Epoch
+		root  = vector.Pre.StateTree.RootCID
+	)
+
+	bs := blockstore.NewTemporary()
+
+	// Read the base64-encoded CAR from the vector, and inflate the gzip.
+	buf := bytes.NewReader(vector.CAR)
+	r, err := gzip.NewReader(buf)
+	if err != nil {
+		t.Fatalf("failed to inflate gzipped CAR: %s", err)
+	}
+	defer r.Close() // nolint
+
+	// Load the CAR embedded in the test vector into the Blockstore.
+	_, err = car.LoadCar(bs, r)
+	if err != nil {
+		t.Fatalf("failed to load state tree car from test vector: %s", err)
+	}
+
+	// Create a new Driver.
+	driver := NewDriver(ctx, vector)
+
+	// Apply every message.
+	for i, m := range vector.ApplyMessages {
+		msg, err := types.DecodeMessage(m.Bytes)
+		if err != nil {
+			t.Fatalf("failed to deserialize message: %s", err)
+		}
+
+		// add an epoch if one's set.
+		if m.Epoch != nil {
+			epoch = *m.Epoch
+		}
+
+		// Execute the message.
+		var ret *vm.ApplyRet
+		ret, root, err = driver.ExecuteMessage(msg, root, bs, epoch)
+		if err != nil {
+			t.Fatalf("fatal failure when executing message: %s", err)
+		}
+
+		// Assert that the receipt matches what the test vector expects.
+		receipt := vector.Post.Receipts[i]
+		if expected, actual := receipt.ExitCode, ret.ExitCode; expected != actual {
+			t.Errorf("exit code of msg %d did not match; expected: %s, got: %s", i, expected, actual)
+		}
+		if expected, actual := receipt.GasUsed, ret.GasUsed; expected != actual {
+			t.Errorf("gas used of msg %d did not match; expected: %d, got: %d", i, expected, actual)
+		}
+	}
+
+	// Once all messages are applied, assert that the final state root matches
+	// the expected postcondition root.
+	if root != vector.Post.StateTree.RootCID {
+		t.Errorf("wrong post root cid; expected %v, but got %v", vector.Post.StateTree.RootCID, root)
+
+		a := aurora.Bold(aurora.BrightRed("(A) expected final state"))
+		b := aurora.Bold(aurora.BrightYellow("(B) actual final state"))
+		c := aurora.Bold(aurora.BrightBlue("(C) initial state"))
+
+		// run state diffs.
+		t.Log(aurora.Sprintf(aurora.Bold("=== dumping 3-way diffs between %s, %s, %s ===\n"), a, b, c))
+
+		t.Log(aurora.Sprintf(aurora.Bold("--- %s left: %s; right: %s ---\n"), aurora.Bold(aurora.Green("[Δ1]")), a, b))
+		t.Log(statediff.Diff(context.Background(), bs, vector.Post.StateTree.RootCID, root))
+
+		t.Log(aurora.Sprintf(aurora.Bold("--- %s left: %s; right: %s ---\n"), aurora.Bold(aurora.Green("[Δ1]")), c, b))
+		t.Log(statediff.Diff(context.Background(), bs, vector.Pre.StateTree.RootCID, root))
+
+		t.Log(aurora.Sprintf(aurora.Bold("--- %s left: %s; right: %s ---\n"), aurora.Bold(aurora.Green("[Δ1]")), c, a))
+		t.Log(statediff.Diff(context.Background(), bs, vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID))
+	}
+}