test: e2e framework

.github/workflows/build.yml

@@ -34,5 +34,12 @@ jobs:
         run: make build
       - name: Run unit tests
         run: make test
+      - name: Run e2e tests
+        run:  | 
+          NOTATION_CWD=`pwd`
+          NOTATION_BIN=$NOTATION_CWD/bin/notation
+          cd ./test/e2e
+          ./run.sh $NOTATION_BIN
+          cd $NOTATION_CWD
       - name: Upload coverage to codecov.io
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v3

.gitignore

@@ -6,3 +6,4 @@ bin/
 vendor/
 coverage.txt
 dist/
+cache.db

Makefile

@@ -21,7 +21,7 @@ GO_BUILD_FLAGS = --ldflags="$(LDFLAGS)"
 
 .PHONY: help
 help:
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: all
 all: build
@@ -41,7 +41,13 @@ build: $(addprefix bin/,$(COMMANDS)) ## builds binaries
 
 .PHONY: test
 test: vendor check-line-endings ## run unit tests
-	go test -race -v -coverprofile=coverage.txt -covermode=atomic ./...
+	go test -race -v -coverprofile=coverage.txt -covermode=atomic $(shell go list ./... | grep -v test/e2e/)
+
+NOTATION_BIN_PATH = $(shell echo `pwd`/bin/$(COMMANDS))
+
+.PHONY: e2e
+e2e: build ## build notation cli and run e2e test
+	cd ./test/e2e; ./run.sh $(NOTATION_BIN_PATH)
 
 .PHONY: clean
 clean:

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	github.com/veraison/go-cose v1.0.0-rc.2
 	oras.land/oras-go/v2 v2.0.0-rc.6
 )
 
@@ -22,6 +21,7 @@ require (
 	github.com/go-ldap/ldap/v3 v3.4.4 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/veraison/go-cose v1.0.0-rc.2 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	golang.org/x/crypto v0.3.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect

test/e2e/Readme.md

@@ -0,0 +1,31 @@
+# A Quick Introduction on how Notation end-to-end test works
+
+## Framework
+Using [Ginkgo](https://onsi.github.io/ginkgo/) as the e2e framework, which is based on the Golang standard testing library.
+
+Using [Gomega](https://onsi.github.io/gomega/) as the matching library.
+
+## Introduction
+1. ### Data：testdata contains data needed by e2e tests, including:
+    - *config*: notation test key and cert files.
+    - *registry*: OCI layout files and registry config files.
+
+2. ### For developer
+    - *Test registry*: a test registry started before running tests.
+    - *Config isolation*: notation needs a few configuration files in user level directory, which can be isolated by modify `XDG_CONFIG_HOME` environment variable. In Notation E2E test framework, a VirtualHost abstraction is designed for isolating user level configuration.
+    - *Parallelization*: In order to speed up testing, Ginkgo will launch several processes to run e2e test cases. 
+    - *Randomization*: By default, Ginkgo will run specs in a suite in random order. Please make sure the test cases can be runned independently. 
+        If the test cases depend on the execution order, consider using [Ordered Containers](https://onsi.github.io/ginkgo/#ordered-containers).
+
+
+## Setting up
+1. Github Actions
+
+    * Please check `Run e2e tests` steps in **workflows/build.yml** for detail.
+2. Local environment
+
+    * Install Golang.
+    * Install Docker.
+    * Clone the repository.
+    * Run `cd ./test/e2e` 
+    * Run `./run.sh <absolute_path_to_notation_binary>` 

test/e2e/go.mod

@@ -0,0 +1,20 @@
+module github.com/notaryproject/notation/test/e2e
+
+go 1.19
+
+require (
+	github.com/onsi/ginkgo/v2 v2.3.0
+	github.com/onsi/gomega v1.22.1
+	oras.land/oras-go/v2 v2.0.0-rc.5
+)
+
+require (
+	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
+	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

test/e2e/go.sum

@@ -0,0 +1,29 @@
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
+github.com/onsi/ginkgo/v2 v2.3.0 h1:kUMoxMoQG3ogk/QWyKh3zibV7BKZ+xBpWil1cTylVqc=
+github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
+github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI=
+github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
+github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
+golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+oras.land/oras-go/v2 v2.0.0-rc.5 h1:enT2ZMNo383bH3INm1/+mw4d09AaMbqx0BMhsgEDUSg=
+oras.land/oras-go/v2 v2.0.0-rc.5/go.mod h1:YGHvWBGuqRlZgUyXUIoKsR3lcuCOb3DAtG0SEsEw1iY=

test/e2e/internal/notation/file.go

@@ -0,0 +1,85 @@
+package notation
+
+import (
+	"encoding/json"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+// copyDir copies the source directory to the destination directory
+func copyDir(src, dst string) error {
+	if err := filepath.WalkDir(src, func(srcPath string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		// generate the dst path
+		relPath, err := filepath.Rel(src, srcPath)
+		if err != nil {
+			return err
+		}
+		dstPath := filepath.Join(dst, relPath)
+
+		if d.IsDir() {
+			if err := os.MkdirAll(dstPath, os.ModePerm); err != nil {
+				return err
+			}
+		} else {
+			if err := copyFile(srcPath, dstPath); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// copyFile copies the source file to the destination file
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	if _, err := io.Copy(out, in); err != nil {
+		return err
+	}
+
+	if err := out.Sync(); err != nil {
+		return err
+	}
+
+	si, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	return os.Chmod(dst, si.Mode())
+}
+
+// saveJson marshals the data and save to the given path.
+func saveJson(data any, path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	b, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Write(b)
+	return err
+}

test/e2e/internal/notation/host.go

@@ -0,0 +1,114 @@
+package notation
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/notaryproject/notation/test/e2e/internal/utils"
+)
+
+// Host creates a virtualized notation testing host by modify
+// the "XDG_CONFIG_HOME" environment variable of the Executor.
+//
+// options is the required testing environment options
+// fn is the callback function containing the testing logic.
+func Host(options []utils.HostOption, fn func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost)) {
+	opts := []utils.HostOption{CreateNotationDirOption()}
+	opts = append(opts, options...)
+
+	// create a vhost
+	vhost, err := utils.NewVirtualHost(NotationBinPath, opts...)
+	if err != nil {
+		panic(err)
+	}
+
+	// generate a repository with an artifact
+	artifact := GenArtifact()
+
+	// run the main logic
+	fn(vhost.Executor, artifact, vhost)
+
+	// clean temporary user directory
+	if err := vhost.CleanUserDir(); err != nil {
+		panic(err)
+	}
+
+	// remove the generated repository and artifact
+	if err := artifact.Remove(); err != nil {
+		panic(err)
+	}
+}
+
+// Opts is a grammar sugar to generate a list of HostOption
+func Opts(options ...utils.HostOption) []utils.HostOption {
+	return options
+}
+
+// BaseOptions returns a list of base Options for a valid notation
+// testing environment.
+func BaseOptions() []utils.HostOption {
+	return Opts(
+		AuthOption("", ""),
+		AddTestKeyOption(),
+		AddTestTrustStoreOption(),
+		AddTestTrustPolicyOption(),
+	)
+}
+
+// CreateNotationDirOption creates the notation directory in temp user dir.
+func CreateNotationDirOption() utils.HostOption {
+	return func(vhost *utils.VirtualHost) error {
+		return os.MkdirAll(vhost.UserPath(notationDirName), os.ModePerm)
+	}
+}
+
+// AuthOption sets the auth environment variables for notation.
+func AuthOption(username, password string) utils.HostOption {
+	if username == "" {
+		username = TestRegistry.Username
+	}
+	if password == "" {
+		password = TestRegistry.Password
+	}
+	return func(vhost *utils.VirtualHost) error {
+		vhost.UpdateEnv(authEnv(username, password))
+		return nil
+	}
+}
+
+// AddTestKeyOption adds the test signingkeys.json, key and cert files to
+// the notation directory.
+func AddTestKeyOption() utils.HostOption {
+	return func(vhost *utils.VirtualHost) error {
+		return AddTestKeyPairs(vhost.UserPath(notationDirName))
+	}
+}
+
+// AddTestTrustStoreOption added the test cert to the trust store.
+func AddTestTrustStoreOption() utils.HostOption {
+	return func(vhost *utils.VirtualHost) error {
+		vhost.Executor.
+			MatchKeyWords("Successfully added following certificates").
+			Exec("cert", "add", "--type", "ca", "--store", "e2e", NotationE2ECertPath)
+		return nil
+	}
+}
+
+// AddTestTrustPolicyOption added a valid trust policy for testing
+func AddTestTrustPolicyOption() utils.HostOption {
+	return func(vhost *utils.VirtualHost) error {
+		return saveJson(
+			genTestTrustPolicy(),
+			filepath.Join(vhost.UserPath(notationDirName), trustPolicyName),
+		)
+	}
+}
+
+// authEnv creates an auth info
+// (By setting $NOTATION_USERNAME and $NOTATION_PASSWORD)
+func authEnv(username, password string) map[string]string {
+	env := make(map[string]string)
+	env["NOTATION_USERNAME"] = username
+	env["NOTATION_PASSWORD"] = password
+	return env
+}