test: generate coverage profile during e2e tests (#903)

Signed-off-by: Billy Zha <jinzha1@microsoft.com>

.github/workflows/build.yml

@@ -43,9 +43,10 @@ jobs:
         run: make test
       - name: Run E2E Tests
         run: |
-          sh $GITHUB_WORKSPACE/test/e2e/scripts/e2e.sh $GITHUB_WORKSPACE --clean
+          bash $GITHUB_WORKSPACE/test/e2e/scripts/e2e.sh $GITHUB_WORKSPACE --clean
         env:
           ORAS_PATH: bin/linux/amd64/oras
+          COVERAGE_DUMP_ROOT: .cover
       - name: Check Version
         run: bin/linux/amd64/oras version
       - name: Upload coverage to codecov.io

.gitignore

@@ -36,6 +36,9 @@ debug
 
 # Custom
 coverage.txt
+test/e2e/coverage.txt
+**/covcounters.*
+**/covmeta.*
 bin/
 dist/
 *.tar.gz

Makefile

@@ -124,3 +124,7 @@ sign:
 	for f in $$(ls _dist/*.{gz,txt} 2>/dev/null) ; do \
 		gpg --armor --detach-sign $${f} ; \
 	done
+
+.PHONY: e2e-covdata
+e2e-covdata:
+	$(GO_EXE) tool covdata textfmt -i="test/e2e/${COVERAGE_DUMP_ROOT}" -o test/e2e/coverage.txt

test/e2e/internal/utils/exec.go

@@ -189,6 +189,7 @@ func (opts *ExecOption) Exec() *gexec.Session {
 		opts.binary = ORASPath
 	}
 	cmd = exec.Command(opts.binary, opts.args...)
+	cmd.Env = append(os.Environ(), fmt.Sprintf("GOCOVERDIR=%s", CovDumpPath))
 	cmd.Stdin = opts.stdin
 	if opts.workDir != "" {
 		// switch working directory

test/e2e/internal/utils/init.go

@@ -36,6 +36,9 @@ var Host string
 // FallbackHost points to the registry service where fallback E2E specs will be run against.
 var FallbackHost string
 
+// Path to generate the coverage report.
+var CovDumpPath string
+
 func init() {
 	Host = os.Getenv(RegHostKey)
 	if Host == "" {
@@ -72,6 +75,25 @@ func init() {
 	}
 	BeforeSuite(func() {
 		ORASPath = os.Getenv("ORAS_PATH")
+		if covDumpRoot := os.Getenv("COVERAGE_DUMP_ROOT"); covDumpRoot != "" {
+			if ORASPath != "" {
+				fmt.Printf("Pre-built oras ignored: %s\n", ORASPath)
+				ORASPath = ""
+			}
+			if filepath.IsAbs(covDumpRoot) {
+				CovDumpPath = covDumpRoot
+			} else if workspacePath := os.Getenv("GITHUB_WORKSPACE"); workspacePath != "" {
+				CovDumpPath = filepath.Join(workspacePath, "test/e2e", covDumpRoot)
+			} else {
+				// local debugging
+				CovDumpPath = filepath.Join(pwd, "..", "..", covDumpRoot)
+			}
+
+			// confirm the existence of dump folder
+			err := os.MkdirAll(CovDumpPath, 0700)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+			fmt.Printf("Coverage file dump path: %q\n", CovDumpPath)
+		}
 		if filepath.IsAbs(ORASPath) {
 			fmt.Printf("Testing based on pre-built binary locates in %q\n", ORASPath)
 		} else if workspacePath := os.Getenv("GITHUB_WORKSPACE"); ORASPath != "" && workspacePath != "" {
@@ -82,7 +104,11 @@ func init() {
 			fmt.Printf("Testing based on pre-built binary locates in %q\n", ORASPath)
 		} else {
 			// fallback to native build to facilitate local debugging
-			ORASPath, err = gexec.Build("oras.land/oras/cmd/oras")
+			buildArgs := []string{}
+			if CovDumpPath != "" {
+				buildArgs = append(buildArgs, "-coverpkg", "oras.land/oras/cmd/oras/...,oras.land/oras/internal/...")
+			}
+			ORASPath, err = gexec.Build("oras.land/oras/cmd/oras", buildArgs...)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 			DeferCleanup(gexec.CleanupBuildArtifacts)
 			fmt.Printf("Testing based on temp binary locates in %q\n", ORASPath)

test/e2e/scripts/e2e.sh

@@ -43,22 +43,41 @@ fi
 
 oras_container_name="oras-e2e"
 upstream_container_name="oras-e2e-fallback"
+e2e_root="${repo_root}/test/e2e"
 echo " === preparing oras distribution === "
 run_registry \
-  ${repo_root}/test/e2e/testdata/distribution/mount \
+  ${e2e_root}/testdata/distribution/mount \
   ghcr.io/oras-project/registry:v1.0.0-rc.4 \
   $oras_container_name \
   $ORAS_REGISTRY_PORT
 
 echo " === preparing upstream distribution === "
 run_registry \
-  ${repo_root}/test/e2e/testdata/distribution/mount_fallback \
+  ${e2e_root}/testdata/distribution/mount_fallback \
   registry:2.8.1 \
   $upstream_container_name \
   $ORAS_REGISTRY_FALLBACK_PORT
 
+echo " === setup coverage instrumenting == "
+if [[ $GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag ]]; then
+    echo "coverage instrumentation skipped"
+    unset COVERAGE_DUMP_ROOT
+fi
+
+if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then
+  rm ${e2e_root}/${COVERAGE_DUMP_ROOT} -rf
+fi
+
 echo " === run tests === "
-if ! ginkgo -r -p --succinct suite; then 
+ginkgo -r -p --succinct suite || fail=true
+
+if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then
+  echo " === generating code cov report === "
+  make -C ${repo_root} e2e-covdata || true
+fi
+
+if [ "${fail}" = 'true' ]; then
+  echo " === retriving registry error logs === "
   echo '-------- oras distribution trace -------------'
   docker logs -t --tail 200 $oras_container_name
   echo '-------- upstream distribution trace -------------'

test/e2e/suite/command/blob.go

@@ -41,14 +41,14 @@ var _ = Describe("ORAS beginners:", func() {
 				repo := fmt.Sprintf(repoFmt, "push", "password-stdin")
 				ORAS("blob", "push", RegistryRef(Host, repo, ""), "--password-stdin", "-").
 					ExpectFailure().
-					MatchTrimmedContent("Error: `-` read file from input and `--password-stdin` read password from input cannot be both used").Exec()
+					MatchErrKeyWords("Error: `-` read file from input and `--password-stdin` read password from input cannot be both used").Exec()
 			})
 			It("should fail to push a blob from stdin but no blob size provided", func() {
 				repo := fmt.Sprintf(repoFmt, "push", "no-size")
 				ORAS("blob", "push", RegistryRef(Host, repo, pushDigest), "-").
 					WithInput(strings.NewReader(pushContent)).
 					ExpectFailure().
-					MatchTrimmedContent("Error: `--size` must be provided if the blob is read from stdin").Exec()
+					MatchErrKeyWords("Error: `--size` must be provided if the blob is read from stdin").Exec()
 			})
 
 			It("should fail to push a blob from stdin if invalid blob size provided", func() {