Add inline-comparison as acceptance test (#130)

* add inline-compare as acceptance test

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add additional RPM metadata

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add comments and doc strings to the compare-* make targets

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

.github/workflows/acceptance-test.yaml

@@ -1,14 +1,17 @@
 name: 'Acceptance'
 on:
+  workflow_dispatch:
   push:
     # ... only act on pushes to main
     branches:
       - main
     # ... do not act on release tags
     tags-ignore:
       - v*
+
 env:
   GO_VERSION: "1.14.x"
+
 jobs:
   Build-Snapshot-Artifacts:
     runs-on: ubuntu-latest
@@ -98,3 +101,29 @@ jobs:
 
       - name: Run Acceptance Tests (Mac)
         run: make acceptance-mac
+
+  # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
+  Inline-Compare:
+    needs: [ Build-Snapshot-Artifacts ]
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+
+      - name: Fingerprint inline-compare sources
+        run: make compare-fingerprint
+
+      - name: Restore inline reports cache
+        id: cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ github.workspace }}/test/inline-compare/inline-reports
+          key: inline-reports-${{ hashFiles('**/inline-compare.fingerprint') }}
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifacts
+          path: snapshot
+
+      - name: Compare Anchore inline-scan results against snapshot build output
+        run: make compare-snapshot

.github/workflows/release.yaml

@@ -7,8 +7,10 @@ on:
     # ... only act on release tags
     tags:
       - 'v*'
+
 env:
   GO_VERSION: "1.14.x"
+
 jobs:
   wait-for-checks:
     runs-on: ubuntu-latest
@@ -50,12 +52,22 @@ jobs:
           checkName: "Acceptance-Mac"
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
+      - name: Check inline comparison test results
+        uses: fountainhead/action-wait-for-check@v1.0.0
+        id: inline-compare
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml)
+          checkName: "Inline-Compare"
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
       - name: Quality gate
-        if: steps.sa-unit-int.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success'
+        if: steps.sa-unit-int.outputs.conclusion != 'success' || steps.inline-compare.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success'
         run: |
           echo "Static/Unit/Integration Status: ${{ steps.sa-unit-int.outputs.conclusion }}"
           echo "Acceptance Test (Linux) Status: ${{ steps.acceptance-linux.outputs.conclusion }}"
           echo "Acceptance Test (Mac) Status: ${{ steps.acceptance-mac.outputs.conclusion }}"
+          echo "Inline Compare Status: ${{ steps.inline-compare.outputs.conclusion }}"
           false
 
   release:

.gitignore

@@ -2,6 +2,7 @@
 /snapshot
 .server/
 .vscode/
+*.fingerprint
 *.tar
 *.jar
 *.war

Makefile

@@ -21,6 +21,7 @@ COVERAGE_THRESHOLD := 72
 DISTDIR=./dist
 SNAPSHOTDIR=./snapshot
 GITTREESTATE=$(if $(shell git status --porcelain),dirty,clean)
+SNAPSHOT_CMD=$(shell realpath $(shell pwd)/$(SNAPSHOTDIR)/syft_linux_amd64/syft)
 
 ifeq "$(strip $(VERSION))" ""
  override VERSION = $(shell git describe --always --tags --dirty)
@@ -58,10 +59,6 @@ endef
 all: clean static-analysis test ## Run all linux-based checks (linting, license check, unit, integration, and linux acceptance tests)
 	@printf '$(SUCCESS)All checks pass!$(RESET)\n'
 
-.PHONY: compare
-compare:
-	@cd test/inline-compare && make
-
 .PHONY: test
 test: unit integration acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests)
 
@@ -127,7 +124,8 @@ integration: ## Run integration tests
 	$(call title,Running integration tests)
 	go test -v -tags=integration ./test/integration
 
-test/integration/test-fixtures/tar-cache.key, integration-fingerprint:
+# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
+integration-fingerprint:
 	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/tar-cache.fingerprint
 
 .PHONY: java-packages-fingerprint
@@ -192,6 +190,20 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie
 .PHONY: acceptance-linux
 acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux)
 
+# note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests
+.PHONY: compare-fingerprint
+compare-fingerprint:
+	find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint
+
+.PHONY: compare-snapshot
+compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan
+	chmod 755 $(SNAPSHOT_CMD)
+	@cd test/inline-compare && SYFT_CMD=$(SNAPSHOT_CMD) make
+
+.PHONY: compare
+compare:  ## Compare the reports of a run of a main-branch build of syft against inline-scan
+	@cd test/inline-compare && make
+
 .PHONY: acceptance-test-deb-package-install
 acceptance-test-deb-package-install: $(SNAPSHOTDIR)
 	$(call title,Running acceptance test: DEB install)

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/spf13/viper v1.7.0
 	github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d
 	github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240
-	github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b
+	github.com/wagoodman/go-rpmdb v0.0.0-20200810111121-8136676cb95c
 	github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2
 	github.com/xeipuuv/gojsonschema v1.2.0

go.sum

@@ -832,8 +832,8 @@ github.com/wagoodman/go-progress v0.0.0-20200621122631-1a2120f0695a h1:lV3ioFpbq
 github.com/wagoodman/go-progress v0.0.0-20200621122631-1a2120f0695a/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA=
 github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240 h1:r6BlIP7CVZtMlxUQhT40h1IE1TzEgKVqwmsVGuscvdk=
 github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA=
-github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b h1:elYGLFZPymeTWJ6qA3tIzFet3LQ9D/Jl6HLWNyFjdQc=
-github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b/go.mod h1:MjoIZzKmbYfcpbC6ARWMcHijAjtLBViDaHcayXKWQWI=
+github.com/wagoodman/go-rpmdb v0.0.0-20200810111121-8136676cb95c h1:eEWc4HjIq0gSno1apdb5MjRn2995xNrNmRTiJyjUJd8=
+github.com/wagoodman/go-rpmdb v0.0.0-20200810111121-8136676cb95c/go.mod h1:MjoIZzKmbYfcpbC6ARWMcHijAjtLBViDaHcayXKWQWI=
 github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163 h1:qoZwR+bHbFFNirY4Yt7lqbOXnFAMnlFfR89w0TXwjrc=
 github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163/go.mod h1:DzXZ1wfRedNhC3KQTick8Gf3CEPMFHsP5k4R/ldjKtw=
 github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=

syft/cataloger/rpmdb/parse_rpmdb.go

@@ -45,12 +45,18 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
 	for _, entry := range pkgList {
 		p := pkg.Package{
 			Name:    entry.Name,
-			Version: entry.Version,
-			Type:    pkg.RpmPkg,
+			Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does
+			//Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch),
+			Type: pkg.RpmPkg,
 			Metadata: pkg.RpmMetadata{
-				Epoch:   entry.Epoch,
-				Arch:    entry.Arch,
-				Release: entry.Release,
+				Version:   entry.Version,
+				Epoch:     entry.Epoch,
+				Arch:      entry.Arch,
+				Release:   entry.Release,
+				SourceRpm: entry.SourceRpm,
+				Vendor:    entry.Vendor,
+				License:   entry.License,
+				Size:      entry.Size,
 			},
 		}
 

syft/cataloger/rpmdb/parse_rpmdb_test.go

@@ -11,12 +11,17 @@ func TestParseRpmDB(t *testing.T) {
 	expected := map[string]pkg.Package{
 		"dive": {
 			Name:    "dive",
-			Version: "0.9.2",
+			Version: "0.9.2-1",
 			Type:    pkg.RpmPkg,
 			Metadata: pkg.RpmMetadata{
-				Epoch:   0,
-				Arch:    "x86_64",
-				Release: "1",
+				Epoch:     0,
+				Arch:      "x86_64",
+				Release:   "1",
+				Version:   "0.9.2",
+				SourceRpm: "dive-0.9.2-1.src.rpm",
+				Size:      12406784,
+				License:   "MIT",
+				Vendor:    "",
 			},
 		},
 	}
@@ -31,11 +36,11 @@ func TestParseRpmDB(t *testing.T) {
 		t.Fatalf("failed to parse rpmdb: %+v", err)
 	}
 
-	if len(actual) != 1 {
+	if len(actual) != len(expected) {
 		for _, a := range actual {
 			t.Log("   ", a)
 		}
-		t.Fatalf("unexpected package count: %d!=%d", len(actual), 1)
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
 	}
 
 	for _, a := range actual {

syft/cataloger/rpmdb/test-fixtures/generate-fixture.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -eux
 
-docker create --name generate-rpmdb-fixture centos:latest sh -c 'tail -f /dev/null'
+docker create --name generate-rpmdb-fixture centos:8 sh -c 'tail -f /dev/null'
 
 function cleanup {
   docker kill generate-rpmdb-fixture

syft/pkg/metadata.go

@@ -10,10 +10,14 @@ type DpkgMetadata struct {
 }
 
 type RpmMetadata struct {
-	Epoch   int    `mapstructure:"Epoch" json:"epoch"`
-	Arch    string `mapstructure:"Arch" json:"architecture"`
-	Release string `mapstructure:"Release" json:"release"`
-	// TODO: consider keeping the remaining values as an embedded map
+	Version   string `mapstructure:"Version" json:"version"`
+	Epoch     int    `mapstructure:"Epoch" json:"epoch"`
+	Arch      string `mapstructure:"Arch" json:"architecture"`
+	Release   string `mapstructure:"Release" json:"release"`
+	SourceRpm string `mapstructure:"SourceRpm" json:"source-rpm"`
+	Size      int    `mapstructure:"Size" json:"size"`
+	License   string `mapstructure:"License" json:"license"`
+	Vendor    string `mapstructure:"Vendor" json:"vendor"`
 }
 
 type JavaManifest struct {

test/acceptance/compare.py

@@ -5,10 +5,9 @@
 
 Metadata = collections.namedtuple("Metadata", "metadata sources")
 Package = collections.namedtuple("Package", "name type version")
-Vulnerability = collections.namedtuple("Vulnerability", "cve package")
 
 
-class syft:
+class Syft:
     def __init__(self, report_path):
         self.report_path = report_path
 
@@ -35,10 +34,10 @@ def packages(self):
 
 
 def main(baseline_report, new_report):
-    report1_obj = syft(report_path=baseline_report)
+    report1_obj = Syft(report_path=baseline_report)
     report1_packages, report1_metadata = report1_obj.packages()
 
-    report2_obj = syft(report_path=new_report)
+    report2_obj = Syft(report_path=new_report)
     report2_packages, report2_metadata = report2_obj.packages()
 
     if len(report2_packages) == 0 and len(report1_packages) == 0:
@@ -102,9 +101,9 @@ def main(baseline_report, new_report):
 
 
 if __name__ == "__main__":
-    print("\nComparing two syft reports...\n")
+    print("\nComparing two Syft reports...\n")
     if len(sys.argv) != 3:
-        sys.exit("please provide two syft json files")
+        sys.exit("please provide two Syft json files")
 
     rc = main(sys.argv[1], sys.argv[2])
     sys.exit(rc)

test/inline-compare/Makefile

@@ -1,39 +1,49 @@
-IMAGE = "centos:8"
-IMAGE_CLEAN = $(shell echo $(IMAGE) | tr ":" "_")
-syft_DIR = syft-reports
-syft_REPORT = $(syft_DIR)/$(IMAGE_CLEAN).json
+ifndef SYFT_CMD
+	SYFT_CMD = go run ../../main.go
+endif
+
+IMAGE_CLEAN = $(shell echo $(COMPARE_IMAGE) | tr ":" "_")
+SYFT_DIR = syft-reports
+SYFT_REPORT = $(SYFT_DIR)/$(IMAGE_CLEAN).json
 INLINE_DIR = inline-reports
 INLINE_REPORT = $(INLINE_DIR)/$(IMAGE_CLEAN)-content-os.json
 
-ifndef syft_DIR
-	$(error syft_DIR is not set)
+ifndef SYFT_DIR
+	$(error SYFT_DIR is not set)
 endif
 
 ifndef INLINE_DIR
 	$(error INLINE_DIR is not set)
 endif
 
 .PHONY: all
-all: compare
+.DEFAULT_GOAL :=
+all: clean-syft
+	./compare-all.sh
 
-.PHONY: compare
-compare: $(INLINE_REPORT) $(syft_REPORT)
-	docker build -t compare-syft:latest .
-	docker run compare-syft:latest $(IMAGE)
+.PHONY: compare-image
+compare-image: $(SYFT_REPORT) $(INLINE_REPORT)
+	./compare.py $(COMPARE_IMAGE)
+
+.PHONY: gather-iamge
+gather-image: $(SYFT_REPORT) $(INLINE_REPORT)
 
 $(INLINE_REPORT):
 	echo "Creating $(INLINE_REPORT)..."
 	mkdir -p $(INLINE_DIR)
-	curl -s https://ci-tools.anchore.io/inline_scan-v0.7.0 | bash -s -- -p -r $(IMAGE)
+	curl -s https://ci-tools.anchore.io/inline_scan-v0.7.0 | bash -s -- -p -r $(COMPARE_IMAGE)
 	mv anchore-reports/* $(INLINE_DIR)/
 	rmdir anchore-reports
 
-$(syft_REPORT):
-	echo "Creating $(syft_REPORT)..."
-	mkdir -p $(syft_DIR)
-	docker pull $(IMAGE)
-	go run ../../main.go $(IMAGE) -o json > $(syft_REPORT)
+$(SYFT_REPORT):
+	echo "Creating $(SYFT_REPORT)..."
+	mkdir -p $(SYFT_DIR)
+	$(SYFT_CMD) $(COMPARE_IMAGE) -o json > $(SYFT_REPORT)
 
 .PHONY: clean
-clean:
-	rm -f $(INLINE_DIR)/*  $(syft_DIR)/*
+clean: clean-syft
+	rm -f $(INLINE_DIR)/*
+
+.PHONY: clean-syft
+clean-syft:
+	rm -f $(SYFT_DIR)/*