test: e2e framework (#493)

- Added Notaton E2E integration test framework in ./test/e2e directory
- Added Github Action E2E test
- Added multi-registry support: zot, dockerhub

Signed-off-by: Junjie Gao <junjiegao@microsoft.com>
Co-authored-by: zaihaoyin <zaihaoyin@microsoft.com>

.github/workflows/build.yml

@@ -34,5 +34,7 @@ jobs:
         run: make build
       - name: Run unit tests
         run: make test
+      - name: Run e2e tests
+        run:  make e2e
       - name: Upload coverage to codecov.io
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v3

Makefile

@@ -21,7 +21,7 @@ GO_BUILD_FLAGS = --ldflags="$(LDFLAGS)"
 
 .PHONY: help
 help:
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: all
 all: build
@@ -43,6 +43,13 @@ build: $(addprefix bin/,$(COMMANDS)) ## builds binaries
 test: vendor check-line-endings ## run unit tests
 	go test -race -v -coverprofile=coverage.txt -covermode=atomic ./...
 
+
+.PHONY: e2e
+e2e: build ## build notation cli and run e2e test
+	NOTATION_BIN_PATH=`pwd`/bin/$(COMMANDS); \
+	cd ./test/e2e; \
+	./run.sh zot $$NOTATION_BIN_PATH
+
 .PHONY: clean
 clean:
 	git status --ignored --short | grep '^!! ' | sed 's/!! //' | xargs rm -rf

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	github.com/veraison/go-cose v1.0.0-rc.2
 	oras.land/oras-go/v2 v2.0.0-rc.6
 )
 
@@ -22,6 +21,7 @@ require (
 	github.com/go-ldap/ldap/v3 v3.4.4 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/veraison/go-cose v1.0.0-rc.2 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	golang.org/x/crypto v0.3.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect

test/e2e/README.md

@@ -0,0 +1,28 @@
+# A Quick Introduction on how Notation end-to-end test works
+
+## Framework
+Using [Ginkgo](https://onsi.github.io/ginkgo/) as the e2e framework, which is based on the Golang standard testing library.
+
+Using [Gomega](https://onsi.github.io/gomega/) as the matching library.
+
+## Introduction
+
+### Data：testdata contains data needed by e2e tests, including:
+- *config*: notation test key and cert files.
+- *registry*: OCI layout files and registry config files.
+### For developer
+- *Test registry*: a test registry started before running tests.
+- *Config isolation*: notation needs a few configuration files in user level directory, which can be isolated by modify `XDG_CONFIG_HOME` environment variable. In Notation E2E test framework, a VirtualHost abstraction is designed for isolating user level configuration.
+- *Parallelization*: In order to speed up testing, Ginkgo will launch several processes to run e2e test cases. 
+- *Randomization*: By default, Ginkgo will run specs in a suite in random order. Please make sure the test cases can be runned independently. If the test cases depend on the execution order, consider using [Ordered Containers](https://onsi.github.io/ginkgo/#ordered-containers).
+
+
+## Setting up
+### Github Actions
+- Please check `Run e2e tests` steps in **workflows/build.yml** for detail.
+### Local environment
+- Install Golang.
+- Install Docker.
+- Clone the repository.
+- Run `cd ./test/e2e` 
+- Run `./run.sh <absolute_path_to_notation_binary>` 

test/e2e/go.mod

@@ -0,0 +1,20 @@
+module github.com/notaryproject/notation/test/e2e
+
+go 1.19
+
+require (
+	github.com/onsi/ginkgo/v2 v2.3.0
+	github.com/onsi/gomega v1.22.1
+	oras.land/oras-go/v2 v2.0.0-rc.6
+)
+
+require (
+	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
+	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

test/e2e/go.sum

@@ -0,0 +1,26 @@
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/onsi/ginkgo/v2 v2.3.0 h1:kUMoxMoQG3ogk/QWyKh3zibV7BKZ+xBpWil1cTylVqc=
+github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
+github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI=
+github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
+github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
+golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+oras.land/oras-go/v2 v2.0.0-rc.6 h1:jGWysqm8flq+X0Vj8bZ6rkASAqTab5k18Mx9hEjFc8g=
+oras.land/oras-go/v2 v2.0.0-rc.6/go.mod h1:iVExH1NxrccIxjsiq17L91WCZ4KIw6jVQyCLsZsu1gc=

test/e2e/internal/notation/file.go

@@ -0,0 +1,45 @@
+package notation
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+)
+
+// copyFile copies the source file to the destination file
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	si, err := in.Stat()
+	if err != nil {
+		return err
+	}
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	if _, err := io.Copy(out, in); err != nil {
+		return err
+	}
+
+	if err := out.Sync(); err != nil {
+		return err
+	}
+	return out.Chmod(si.Mode())
+}
+
+// saveJSON marshals the data and save to the given path.
+func saveJSON(data any, path string) error {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		return err
+	}
+	return json.NewEncoder(f).Encode(data)
+}

test/e2e/internal/notation/host.go

@@ -0,0 +1,134 @@
+package notation
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/notaryproject/notation/test/e2e/internal/utils"
+)
+
+// CoreTestFunc is the test function running in a VirtualHost.
+//
+// notation is an Executor isolated by $XDG_CONFIG_HOME.
+// artifact is a generated artifact in a new repository.
+// vhost is the VirtualHost instance.
+type CoreTestFunc func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost)
+
+// Host creates a virtualized notation testing host by modify
+// the "XDG_CONFIG_HOME" environment variable of the Executor.
+//
+// options is the required testing environment options
+// fn is the callback function containing the testing logic.
+func Host(options []utils.HostOption, fn CoreTestFunc) {
+	// create a notation vhost
+	vhost, err := createNotationHost(NotationBinPath, options...)
+	if err != nil {
+		panic(err)
+	}
+
+	// generate a repository with an artifact
+	artifact := GenerateArtifact("", "")
+
+	// run the main logic
+	fn(vhost.Executor, artifact, vhost)
+}
+
+// OldNotation create an old version notation ExecOpts in a VirtualHost
+// for testing forward compatibility.
+func OldNotation(options ...utils.HostOption) *utils.ExecOpts {
+	if len(options) == 0 {
+		options = BaseOptions()
+	}
+
+	vhost, err := createNotationHost(NotationOldBinPath, options...)
+	if err != nil {
+		panic(err)
+	}
+
+	return vhost.Executor
+}
+
+func createNotationHost(path string, options ...utils.HostOption) (*utils.VirtualHost, error) {
+	vhost, err := utils.NewVirtualHost(path, CreateNotationDirOption())
+	if err != nil {
+		return nil, err
+	}
+
+	// set additional options
+	vhost.SetOption(options...)
+	return vhost, nil
+}
+
+// Opts is a grammar sugar to generate a list of HostOption.
+func Opts(options ...utils.HostOption) []utils.HostOption {
+	return options
+}
+
+// BaseOptions returns a list of base Options for a valid notation.
+// testing environment.
+func BaseOptions() []utils.HostOption {
+	return Opts(
+		AuthOption("", ""),
+		AddKeyOption("e2e.key", "e2e.crt"),
+		AddTrustStoreOption("e2e", filepath.Join(NotationE2ELocalKeysDir, "e2e.crt")),
+		AddTrustPolicyOption("trustpolicy.json"),
+	)
+}
+
+// CreateNotationDirOption creates the notation directory in temp user dir.
+func CreateNotationDirOption() utils.HostOption {
+	return func(vhost *utils.VirtualHost) error {
+		return os.MkdirAll(vhost.AbsolutePath(NotationDirName), os.ModePerm)
+	}
+}
+
+// AuthOption sets the auth environment variables for notation.
+func AuthOption(username, password string) utils.HostOption {
+	if username == "" {
+		username = TestRegistry.Username
+	}
+	if password == "" {
+		password = TestRegistry.Password
+	}
+	return func(vhost *utils.VirtualHost) error {
+		vhost.UpdateEnv(authEnv(username, password))
+		return nil
+	}
+}
+
+// AddKeyOption adds the test signingkeys.json, key and cert files to
+// the notation directory.
+func AddKeyOption(keyName, certName string) utils.HostOption {
+	return func(vhost *utils.VirtualHost) error {
+		return AddTestKeyPairs(vhost.AbsolutePath(NotationDirName), keyName, certName)
+	}
+}
+
+// AddTrustStoreOption added the test cert to the trust store.
+func AddTrustStoreOption(namedstore string, srcCertPath string) utils.HostOption {
+	return func(vhost *utils.VirtualHost) error {
+		vhost.Executor.
+			Exec("cert", "add", "--type", "ca", "--store", namedstore, srcCertPath).
+			MatchKeyWords("Successfully added following certificates")
+		return nil
+	}
+}
+
+// AddTrustPolicyOption added a valid trust policy for testing
+func AddTrustPolicyOption(trustpolicyName string) utils.HostOption {
+	return func(vhost *utils.VirtualHost) error {
+		return copyFile(
+			filepath.Join(NotationE2ETrustPolicyDir, trustpolicyName),
+			vhost.AbsolutePath(NotationDirName, TrustPolicyName),
+		)
+	}
+}
+
+// authEnv creates an auth info
+// (By setting $NOTATION_USERNAME and $NOTATION_PASSWORD)
+func authEnv(username, password string) map[string]string {
+	return map[string]string{
+		"NOTATION_USERNAME": username,
+		"NOTATION_PASSWORD": password,
+	}
+}

test/e2e/internal/notation/init.go

@@ -0,0 +1,87 @@
+package notation
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	NotationDirName   = "notation"
+	TrustPolicyName   = "trustpolicy.json"
+	TrustStoreDirName = "truststore"
+	TrustStoreTypeCA  = "ca"
+)
+
+const (
+	envKeyRegistryHost       = "NOTATION_E2E_REGISTRY_HOST"
+	envKeyRegistryUsername   = "NOTATION_E2E_REGISTRY_USERNAME"
+	envKeyRegistryPassword   = "NOTATION_E2E_REGISTRY_PASSWORD"
+	envKeyNotationBinPath    = "NOTATION_E2E_BINARY_PATH"
+	envKeyNotationOldBinPath = "NOTATION_E2E_OLD_BINARY_PATH"
+	envKeyNotationConfigPath = "NOTATION_E2E_CONFIG_PATH"
+	envKeyOCILayoutPath      = "NOTATION_E2E_OCI_LAYOUT_PATH"
+	envKeyTestRepo           = "NOTATION_E2E_TEST_REPO"
+	envKeyTestTag            = "NOTATION_E2E_TEST_TAG"
+)
+
+var (
+	// NotationBinPath is the notation binary path.
+	NotationBinPath string
+	// NotationOldBinPath is the path of an old version notation binary for
+	// testing forward compatibility.
+	NotationOldBinPath        string
+	NotationE2EConfigPath     string
+	NotationE2ELocalKeysDir   string
+	NotationE2ETrustPolicyDir string
+)
+
+var (
+	OCILayoutPath string
+	TestRepoUri   string
+	TestTag       string
+)
+
+func init() {
+	RegisterFailHandler(Fail)
+	setUpRegistry()
+	setUpNotationValues()
+}
+
+func setUpRegistry() {
+	setValue(envKeyRegistryHost, &TestRegistry.Host)
+	setValue(envKeyRegistryUsername, &TestRegistry.Username)
+	setValue(envKeyRegistryPassword, &TestRegistry.Password)
+
+	setPathValue(envKeyOCILayoutPath, &OCILayoutPath)
+	setValue(envKeyTestRepo, &TestRepoUri)
+	setValue(envKeyTestTag, &TestTag)
+}
+
+func setUpNotationValues() {
+	// set Notation binary path
+	setPathValue(envKeyNotationBinPath, &NotationBinPath)
+	setPathValue(envKeyNotationOldBinPath, &NotationOldBinPath)
+
+	// set Notation configuration paths
+	setPathValue(envKeyNotationConfigPath, &NotationE2EConfigPath)
+	NotationE2ETrustPolicyDir = filepath.Join(NotationE2EConfigPath, "trustpolicys")
+	NotationE2ELocalKeysDir = filepath.Join(NotationE2EConfigPath, LocalKeysDirName)
+}
+
+func setPathValue(envKey string, value *string) {
+	setValue(envKey, value)
+	if !filepath.IsAbs(*value) {
+		panic(fmt.Sprintf("env %s=%q is not a absolute path", envKey, *value))
+	}
+}
+
+func setValue(envKey string, value *string) {
+	if *value = os.Getenv(envKey); *value == "" {
+		panic(fmt.Sprintf("env %s is empty", envKey))
+	}
+	fmt.Printf("set test value $%s=%s\n", envKey, *value)
+}