From 2d452bf59e0f44de6051ac3f316c8afafe0a929a Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 10 Aug 2020 10:33:44 -0400 Subject: [PATCH] Add inline-comparison as acceptance test (#130) * add inline-compare as acceptance test Signed-off-by: Alex Goodman * add additional RPM metadata Signed-off-by: Alex Goodman * add comments and doc strings to the compare-* make targets Signed-off-by: Alex Goodman --- .github/workflows/acceptance-test.yaml | 29 ++++ .github/workflows/release.yaml | 14 +- .gitignore | 1 + Makefile | 22 ++- go.mod | 2 +- go.sum | 4 +- syft/cataloger/rpmdb/parse_rpmdb.go | 16 +- syft/cataloger/rpmdb/parse_rpmdb_test.go | 17 +- .../rpmdb/test-fixtures/generate-fixture.sh | 2 +- syft/pkg/metadata.go | 12 +- test/acceptance/compare.py | 11 +- test/acceptance/compare.sh | 27 --- test/inline-compare/Dockerfile | 7 - test/inline-compare/Makefile | 48 +++--- test/inline-compare/compare-all.sh | 17 ++ test/inline-compare/compare.py | 157 ++++++++++++++---- test/integration/pkg_cases.go | 2 +- 17 files changed, 275 insertions(+), 113 deletions(-) delete mode 100755 test/acceptance/compare.sh delete mode 100644 test/inline-compare/Dockerfile create mode 100755 test/inline-compare/compare-all.sh diff --git a/.github/workflows/acceptance-test.yaml b/.github/workflows/acceptance-test.yaml index 148310b9237..344877b5a7f 100644 --- a/.github/workflows/acceptance-test.yaml +++ b/.github/workflows/acceptance-test.yaml @@ -1,5 +1,6 @@ name: 'Acceptance' on: + workflow_dispatch: push: # ... only act on pushes to main branches: @@ -7,8 +8,10 @@ on: # ... do not act on release tags tags-ignore: - v* + env: GO_VERSION: "1.14.x" + jobs: Build-Snapshot-Artifacts: runs-on: ubuntu-latest @@ -98,3 +101,29 @@ jobs: - name: Run Acceptance Tests (Mac) run: make acceptance-mac + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Inline-Compare: + needs: [ Build-Snapshot-Artifacts ] + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v2 + + - name: Fingerprint inline-compare sources + run: make compare-fingerprint + + - name: Restore inline reports cache + id: cache + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/test/inline-compare/inline-reports + key: inline-reports-${{ hashFiles('**/inline-compare.fingerprint') }} + + - uses: actions/download-artifact@v2 + with: + name: artifacts + path: snapshot + + - name: Compare Anchore inline-scan results against snapshot build output + run: make compare-snapshot \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3d6b66cf561..0f9cbc28302 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,8 +7,10 @@ on: # ... only act on release tags tags: - 'v*' + env: GO_VERSION: "1.14.x" + jobs: wait-for-checks: runs-on: ubuntu-latest @@ -50,12 +52,22 @@ jobs: checkName: "Acceptance-Mac" ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Check inline comparison test results + uses: fountainhead/action-wait-for-check@v1.0.0 + id: inline-compare + with: + token: ${{ secrets.GITHUB_TOKEN }} + # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml) + checkName: "Inline-Compare" + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Quality gate - if: steps.sa-unit-int.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success' + if: steps.sa-unit-int.outputs.conclusion != 'success' || steps.inline-compare.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success' run: | echo "Static/Unit/Integration Status: ${{ steps.sa-unit-int.outputs.conclusion }}" echo "Acceptance Test (Linux) Status: ${{ steps.acceptance-linux.outputs.conclusion }}" echo "Acceptance Test (Mac) Status: ${{ steps.acceptance-mac.outputs.conclusion }}" + echo "Inline Compare Status: ${{ steps.inline-compare.outputs.conclusion }}" false release: diff --git a/.gitignore b/.gitignore index 11203f62593..cfa6b4053de 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /snapshot .server/ .vscode/ +*.fingerprint *.tar *.jar *.war diff --git a/Makefile b/Makefile index e77a8783a27..e1828e10ed5 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,7 @@ COVERAGE_THRESHOLD := 72 DISTDIR=./dist SNAPSHOTDIR=./snapshot GITTREESTATE=$(if $(shell git status --porcelain),dirty,clean) +SNAPSHOT_CMD=$(shell realpath $(shell pwd)/$(SNAPSHOTDIR)/syft_linux_amd64/syft) ifeq "$(strip $(VERSION))" "" override VERSION = $(shell git describe --always --tags --dirty) @@ -58,10 +59,6 @@ endef all: clean static-analysis test ## Run all linux-based checks (linting, license check, unit, integration, and linux acceptance tests) @printf '$(SUCCESS)All checks pass!$(RESET)\n' -.PHONY: compare -compare: - @cd test/inline-compare && make - .PHONY: test test: unit integration acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests) @@ -127,7 +124,8 @@ integration: ## Run integration tests $(call title,Running integration tests) go test -v -tags=integration ./test/integration -test/integration/test-fixtures/tar-cache.key, integration-fingerprint: +# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted +integration-fingerprint: find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/tar-cache.fingerprint .PHONY: java-packages-fingerprint @@ -192,6 +190,20 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie .PHONY: acceptance-linux acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux) +# note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests +.PHONY: compare-fingerprint +compare-fingerprint: + find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint + +.PHONY: compare-snapshot +compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan + chmod 755 $(SNAPSHOT_CMD) + @cd test/inline-compare && SYFT_CMD=$(SNAPSHOT_CMD) make + +.PHONY: compare +compare: ## Compare the reports of a run of a main-branch build of syft against inline-scan + @cd test/inline-compare && make + .PHONY: acceptance-test-deb-package-install acceptance-test-deb-package-install: $(SNAPSHOTDIR) $(call title,Running acceptance test: DEB install) diff --git a/go.mod b/go.mod index c9357033a0d..e88830ad108 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/spf13/viper v1.7.0 github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240 - github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b + github.com/wagoodman/go-rpmdb v0.0.0-20200810111121-8136676cb95c github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163 github.com/x-cray/logrus-prefixed-formatter v0.5.2 github.com/xeipuuv/gojsonschema v1.2.0 diff --git a/go.sum b/go.sum index 6041ad86099..dfb9865a331 100644 --- a/go.sum +++ b/go.sum @@ -832,8 +832,8 @@ github.com/wagoodman/go-progress v0.0.0-20200621122631-1a2120f0695a h1:lV3ioFpbq github.com/wagoodman/go-progress v0.0.0-20200621122631-1a2120f0695a/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA= github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240 h1:r6BlIP7CVZtMlxUQhT40h1IE1TzEgKVqwmsVGuscvdk= github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA= -github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b h1:elYGLFZPymeTWJ6qA3tIzFet3LQ9D/Jl6HLWNyFjdQc= -github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b/go.mod h1:MjoIZzKmbYfcpbC6ARWMcHijAjtLBViDaHcayXKWQWI= +github.com/wagoodman/go-rpmdb v0.0.0-20200810111121-8136676cb95c h1:eEWc4HjIq0gSno1apdb5MjRn2995xNrNmRTiJyjUJd8= +github.com/wagoodman/go-rpmdb v0.0.0-20200810111121-8136676cb95c/go.mod h1:MjoIZzKmbYfcpbC6ARWMcHijAjtLBViDaHcayXKWQWI= github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163 h1:qoZwR+bHbFFNirY4Yt7lqbOXnFAMnlFfR89w0TXwjrc= github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163/go.mod h1:DzXZ1wfRedNhC3KQTick8Gf3CEPMFHsP5k4R/ldjKtw= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index 8c91898567e..0123dc5117a 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -45,12 +45,18 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) { for _, entry := range pkgList { p := pkg.Package{ Name: entry.Name, - Version: entry.Version, - Type: pkg.RpmPkg, + Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does + //Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch), + Type: pkg.RpmPkg, Metadata: pkg.RpmMetadata{ - Epoch: entry.Epoch, - Arch: entry.Arch, - Release: entry.Release, + Version: entry.Version, + Epoch: entry.Epoch, + Arch: entry.Arch, + Release: entry.Release, + SourceRpm: entry.SourceRpm, + Vendor: entry.Vendor, + License: entry.License, + Size: entry.Size, }, } diff --git a/syft/cataloger/rpmdb/parse_rpmdb_test.go b/syft/cataloger/rpmdb/parse_rpmdb_test.go index 2ff8e527358..bbcb25b8d6f 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb_test.go +++ b/syft/cataloger/rpmdb/parse_rpmdb_test.go @@ -11,12 +11,17 @@ func TestParseRpmDB(t *testing.T) { expected := map[string]pkg.Package{ "dive": { Name: "dive", - Version: "0.9.2", + Version: "0.9.2-1", Type: pkg.RpmPkg, Metadata: pkg.RpmMetadata{ - Epoch: 0, - Arch: "x86_64", - Release: "1", + Epoch: 0, + Arch: "x86_64", + Release: "1", + Version: "0.9.2", + SourceRpm: "dive-0.9.2-1.src.rpm", + Size: 12406784, + License: "MIT", + Vendor: "", }, }, } @@ -31,11 +36,11 @@ func TestParseRpmDB(t *testing.T) { t.Fatalf("failed to parse rpmdb: %+v", err) } - if len(actual) != 1 { + if len(actual) != len(expected) { for _, a := range actual { t.Log(" ", a) } - t.Fatalf("unexpected package count: %d!=%d", len(actual), 1) + t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected)) } for _, a := range actual { diff --git a/syft/cataloger/rpmdb/test-fixtures/generate-fixture.sh b/syft/cataloger/rpmdb/test-fixtures/generate-fixture.sh index 80bf0996c79..4356173e5a7 100755 --- a/syft/cataloger/rpmdb/test-fixtures/generate-fixture.sh +++ b/syft/cataloger/rpmdb/test-fixtures/generate-fixture.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eux -docker create --name generate-rpmdb-fixture centos:latest sh -c 'tail -f /dev/null' +docker create --name generate-rpmdb-fixture centos:8 sh -c 'tail -f /dev/null' function cleanup { docker kill generate-rpmdb-fixture diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go index 150bbb4908e..c2bac29b101 100644 --- a/syft/pkg/metadata.go +++ b/syft/pkg/metadata.go @@ -10,10 +10,14 @@ type DpkgMetadata struct { } type RpmMetadata struct { - Epoch int `mapstructure:"Epoch" json:"epoch"` - Arch string `mapstructure:"Arch" json:"architecture"` - Release string `mapstructure:"Release" json:"release"` - // TODO: consider keeping the remaining values as an embedded map + Version string `mapstructure:"Version" json:"version"` + Epoch int `mapstructure:"Epoch" json:"epoch"` + Arch string `mapstructure:"Arch" json:"architecture"` + Release string `mapstructure:"Release" json:"release"` + SourceRpm string `mapstructure:"SourceRpm" json:"source-rpm"` + Size int `mapstructure:"Size" json:"size"` + License string `mapstructure:"License" json:"license"` + Vendor string `mapstructure:"Vendor" json:"vendor"` } type JavaManifest struct { diff --git a/test/acceptance/compare.py b/test/acceptance/compare.py index af7a85ba3c6..bd199e4c2b4 100755 --- a/test/acceptance/compare.py +++ b/test/acceptance/compare.py @@ -5,10 +5,9 @@ Metadata = collections.namedtuple("Metadata", "metadata sources") Package = collections.namedtuple("Package", "name type version") -Vulnerability = collections.namedtuple("Vulnerability", "cve package") -class syft: +class Syft: def __init__(self, report_path): self.report_path = report_path @@ -35,10 +34,10 @@ def packages(self): def main(baseline_report, new_report): - report1_obj = syft(report_path=baseline_report) + report1_obj = Syft(report_path=baseline_report) report1_packages, report1_metadata = report1_obj.packages() - report2_obj = syft(report_path=new_report) + report2_obj = Syft(report_path=new_report) report2_packages, report2_metadata = report2_obj.packages() if len(report2_packages) == 0 and len(report1_packages) == 0: @@ -102,9 +101,9 @@ def main(baseline_report, new_report): if __name__ == "__main__": - print("\nComparing two syft reports...\n") + print("\nComparing two Syft reports...\n") if len(sys.argv) != 3: - sys.exit("please provide two syft json files") + sys.exit("please provide two Syft json files") rc = main(sys.argv[1], sys.argv[2]) sys.exit(rc) diff --git a/test/acceptance/compare.sh b/test/acceptance/compare.sh deleted file mode 100755 index 28bef47d516..00000000000 --- a/test/acceptance/compare.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -eu - -BOLD="$(tput -T linux bold)" -RED="$(tput -T linux setaf 1)" -RESET="$(tput -T linux sgr0)" -FAIL="${BOLD}${RED}" -SUCCESS="${BOLD}" -JQ_ARGS="-S .artifacts" - -if ! command -v jq &> /dev/null ;then - JQ_IMAGE="imega/jq:latest" - JQ_CMD="docker run --rm -i ${JQ_IMAGE} ${JQ_ARGS}" - docker pull "${JQ_IMAGE}" -else - JQ_CMD="jq ${JQ_ARGS}" -fi - -if [[ $(cat $1 | ${JQ_CMD}) ]]; then - set -x - # compare the output of both results - diff <(cat $1 | ${JQ_CMD}) <(cat $2 | ${JQ_CMD}) - set +x - echo "${SUCCESS}Comparison passed!${RESET}" -else - exit "${FAIL}Failing since one of the test files is empty ($1)${RESET}" -fi \ No newline at end of file diff --git a/test/inline-compare/Dockerfile b/test/inline-compare/Dockerfile deleted file mode 100644 index 71e42622143..00000000000 --- a/test/inline-compare/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM python:3 - -WORKDIR / -COPY syft-reports /syft-reports -COPY inline-reports /inline-reports -COPY compare.py . -ENTRYPOINT ["/compare.py"] \ No newline at end of file diff --git a/test/inline-compare/Makefile b/test/inline-compare/Makefile index aa1639942e5..58065ebfd34 100644 --- a/test/inline-compare/Makefile +++ b/test/inline-compare/Makefile @@ -1,12 +1,15 @@ -IMAGE = "centos:8" -IMAGE_CLEAN = $(shell echo $(IMAGE) | tr ":" "_") -syft_DIR = syft-reports -syft_REPORT = $(syft_DIR)/$(IMAGE_CLEAN).json +ifndef SYFT_CMD + SYFT_CMD = go run ../../main.go +endif + +IMAGE_CLEAN = $(shell echo $(COMPARE_IMAGE) | tr ":" "_") +SYFT_DIR = syft-reports +SYFT_REPORT = $(SYFT_DIR)/$(IMAGE_CLEAN).json INLINE_DIR = inline-reports INLINE_REPORT = $(INLINE_DIR)/$(IMAGE_CLEAN)-content-os.json -ifndef syft_DIR - $(error syft_DIR is not set) +ifndef SYFT_DIR + $(error SYFT_DIR is not set) endif ifndef INLINE_DIR @@ -14,26 +17,33 @@ ifndef INLINE_DIR endif .PHONY: all -all: compare +.DEFAULT_GOAL := +all: clean-syft + ./compare-all.sh -.PHONY: compare -compare: $(INLINE_REPORT) $(syft_REPORT) - docker build -t compare-syft:latest . - docker run compare-syft:latest $(IMAGE) +.PHONY: compare-image +compare-image: $(SYFT_REPORT) $(INLINE_REPORT) + ./compare.py $(COMPARE_IMAGE) + +.PHONY: gather-iamge +gather-image: $(SYFT_REPORT) $(INLINE_REPORT) $(INLINE_REPORT): echo "Creating $(INLINE_REPORT)..." mkdir -p $(INLINE_DIR) - curl -s https://ci-tools.anchore.io/inline_scan-v0.7.0 | bash -s -- -p -r $(IMAGE) + curl -s https://ci-tools.anchore.io/inline_scan-v0.7.0 | bash -s -- -p -r $(COMPARE_IMAGE) mv anchore-reports/* $(INLINE_DIR)/ rmdir anchore-reports -$(syft_REPORT): - echo "Creating $(syft_REPORT)..." - mkdir -p $(syft_DIR) - docker pull $(IMAGE) - go run ../../main.go $(IMAGE) -o json > $(syft_REPORT) +$(SYFT_REPORT): + echo "Creating $(SYFT_REPORT)..." + mkdir -p $(SYFT_DIR) + $(SYFT_CMD) $(COMPARE_IMAGE) -o json > $(SYFT_REPORT) .PHONY: clean -clean: - rm -f $(INLINE_DIR)/* $(syft_DIR)/* \ No newline at end of file +clean: clean-syft + rm -f $(INLINE_DIR)/* + +.PHONY: clean-syft +clean-syft: + rm -f $(SYFT_DIR)/* \ No newline at end of file diff --git a/test/inline-compare/compare-all.sh b/test/inline-compare/compare-all.sh new file mode 100755 index 00000000000..4a47fb3ab76 --- /dev/null +++ b/test/inline-compare/compare-all.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -eu + +# TODO: add "alpine:3.12.0" back +images=("debian:10.5" "centos:8.2.2004" ) + +# gather all image analyses +for img in "${images[@]}"; do + echo "Gathering facts for $img" + COMPARE_IMAGE=${img} make gather-image +done + +# compare all results +for img in "${images[@]}"; do + echo "Comparing results for $img" + COMPARE_IMAGE=${img} make compare-image +done \ No newline at end of file diff --git a/test/inline-compare/compare.py b/test/inline-compare/compare.py index edd730caf7c..32590ba89e3 100755 --- a/test/inline-compare/compare.py +++ b/test/inline-compare/compare.py @@ -2,14 +2,25 @@ import os import sys import json -import functools import collections -QUALITY_GATE_THRESHOLD = 0.9 +QUALITY_GATE_THRESHOLD = 0.95 +INDENT = " " +IMAGE_QUALITY_GATE = collections.defaultdict(lambda: QUALITY_GATE_THRESHOLD, **{ +}) + +# We additionally fail if an image is above a particular threshold. Why? We expect the lower threshold to be 90%, +# however additional functionality in grype is still being implemented, so this threshold may not be able to be met. +# In these cases the IMAGE_QUALITY_GATE is set to a lower value to allow the test to pass for known issues. Once these +# issues/enhancements are done we want to ensure that the lower threshold is bumped up to catch regression. The only way +# to do this is to select an upper threshold for images with known threshold values, so we have a failure that +# loudly indicates the lower threshold should be bumped. +IMAGE_UPPER_THRESHOLD = collections.defaultdict(lambda: 1, **{ + +}) Metadata = collections.namedtuple("Metadata", "version") Package = collections.namedtuple("Package", "name type") -Vulnerability = collections.namedtuple("Vulnerability", "cve package") class InlineScan: @@ -33,12 +44,17 @@ def _report_path(self, report): def _enumerate_section(self, report, section): report_path = self._report_path(report=report) + os_report_path = self._report_path(report="content-os") + + if os.path.exists(os_report_path) and not os.path.exists(report_path): + # if the OS report is there but the target report is not, that is engine's way of saying "no findings" + return + with open(report_path) as json_file: data = json.load(json_file) for entry in data[section]: yield entry - @functools.lru_cache def _python_packages(self): packages = set() metadata = collections.defaultdict(dict) @@ -51,7 +67,6 @@ def _python_packages(self): return packages, metadata - @functools.lru_cache def _os_packages(self): packages = set() metadata = collections.defaultdict(dict) @@ -63,7 +78,7 @@ def _os_packages(self): return packages, metadata -class syft: +class Syft: report_tmpl = "{image}.json" @@ -78,31 +93,54 @@ def _enumerate_section(self, section): for entry in data[section]: yield entry - @functools.lru_cache def packages(self): packages = set() metadata = collections.defaultdict(dict) for entry in self._enumerate_section(section="artifacts"): # normalize to inline - pType = entry["type"].lower() - if pType in ("wheel", "egg"): - pType = "python" - - package = Package(name=entry["name"], type=pType,) + pkg_type = entry["type"].lower() + if pkg_type in ("wheel", "egg"): + pkg_type = "python" + elif pkg_type in ("deb",): + pkg_type = "dpkg" + elif pkg_type in ("java-archive",): + pkg_type = "java" + elif pkg_type in ("apk",): + pkg_type = "apkg" + + package = Package(name=entry["name"], type=pkg_type,) packages.add(package) metadata[package.type][package] = Metadata(version=entry["version"]) return packages, metadata +def print_rows(rows): + if not rows: + return + widths = [] + for col, _ in enumerate(rows[0]): + width = max(len(row[col]) for row in rows) + 2 # padding + widths.append(width) + for row in rows: + print("".join(word.ljust(widths[col_idx]) for col_idx, word in enumerate(row))) + + def main(image): + print(colors.bold+"Image:", image, colors.reset) + inline = InlineScan(image=image, report_dir="inline-reports") inline_packages, inline_metadata = inline.packages() - syft = syft(image=image, report_dir="syft-reports") + syft = Syft(image=image, report_dir="syft-reports") syft_packages, syft_metadata = syft.packages() + if len(inline_packages) == 0: + # we are purposefully selecting test images that are guaranteed to have packages, so this should never happen + print(colors.bold + colors.fg.red + "inline found no packages!", colors.reset) + return 1 + if len(syft_packages) == 0 and len(inline_packages) == 0: print("nobody found any packages") return 0 @@ -120,51 +158,114 @@ def main(image): metadata = inline_metadata[package.type][package] inline_metadata_set.add((package, metadata)) - syft_metadata_set = set() + syft_overlap_metadata_set = set() for package in syft_packages: metadata = syft_metadata[package.type][package] - syft_metadata_set.add((package, metadata)) + # we only want to really count mismatched metadata for packages that are at least found by inline + if package in inline_metadata[package.type]: + syft_overlap_metadata_set.add((package, metadata)) - same_metadata = syft_metadata_set & inline_metadata_set + same_metadata = syft_overlap_metadata_set & inline_metadata_set percent_overlap_metadata = ( float(len(same_metadata)) / float(len(inline_metadata_set)) ) * 100.0 + missing_metadata = inline_metadata_set - same_metadata if len(bonus_packages) > 0: - print("syft Bonus packages:") + rows = [] + print(colors.bold + "Syft found extra packages:", colors.reset) for package in sorted(list(bonus_packages)): - print(" " + repr(package)) + rows.append([INDENT, repr(package)]) + print_rows(rows) print() if len(missing_pacakges) > 0: - print("syft Missing packages:") + rows = [] + print(colors.bold + "Syft missed packages:", colors.reset) for package in sorted(list(missing_pacakges)): - print(" " + repr(package)) + rows.append([INDENT, repr(package)]) + print_rows(rows) + print() + + if len(missing_metadata) > 0: + rows = [] + print(colors.bold + "Syft mismatched metadata:", colors.reset) + for inline_metadata_pair in sorted(list(missing_metadata)): + pkg, metadata = inline_metadata_pair + if pkg in syft_metadata[pkg.type]: + syft_metadata_item = syft_metadata[pkg.type][pkg] + else: + syft_metadata_item = "--- MISSING ---" + rows.append([INDENT, "for:", repr(pkg), ":", repr(syft_metadata_item), "!=", repr(metadata)]) + print_rows(rows) print() - print("Inline Packages: %d" % len(inline_packages)) - print("syft Packages: %d" % len(syft_packages)) - print() + print(colors.bold+"Summary:", colors.reset) + print(" Image: %s" % image) + print(" Inline Packages: %d" % len(inline_packages)) + print(" Syft Packages: %d" % len(syft_packages)) print( - "Baseline Packages Matched: %2.3f %% (%d/%d packages)" + " Baseline Packages Matched: %2.3f %% (%d/%d packages)" % (percent_overlap_packages, len(same_packages), len(inline_packages)) ) print( - "Baseline Metadata Matched: %2.3f %% (%d/%d metadata)" + " Baseline Metadata Matched: %2.3f %% (%d/%d metadata)" % (percent_overlap_metadata, len(same_metadata), len(inline_metadata_set)) ) overall_score = (percent_overlap_packages + percent_overlap_metadata) / 2.0 - print("Overall Score: %2.3f %%" % overall_score) + print(colors.bold + " Overall Score: %2.1f %%" % overall_score, colors.reset) - if overall_score < (QUALITY_GATE_THRESHOLD * 100): - print("failed quality gate (>= %d %%)" % (QUALITY_GATE_THRESHOLD * 100)) + upper_gate_value = IMAGE_UPPER_THRESHOLD[image] * 100 + lower_gate_value = IMAGE_QUALITY_GATE[image] * 100 + if overall_score < lower_gate_value: + print(colors.bold + " Quality Gate: " + colors.fg.red + "FAILED (is not >= %d %%)\n" % lower_gate_value, colors.reset) return 1 + elif overall_score > upper_gate_value: + print(colors.bold + " Quality Gate: " + colors.fg.orange + "FAILED (lower threshold is artificially low and should be updated)\n", colors.reset) + return 1 + else: + print(colors.bold + " Quality Gate: " + colors.fg.green + "pass (>= %d %%)\n" % lower_gate_value, colors.reset) return 0 +class colors: + reset='\033[0m' + bold='\033[01m' + disable='\033[02m' + underline='\033[04m' + reverse='\033[07m' + strikethrough='\033[09m' + invisible='\033[08m' + class fg: + black='\033[30m' + red='\033[31m' + green='\033[32m' + orange='\033[33m' + blue='\033[34m' + purple='\033[35m' + cyan='\033[36m' + lightgrey='\033[37m' + darkgrey='\033[90m' + lightred='\033[91m' + lightgreen='\033[92m' + yellow='\033[93m' + lightblue='\033[94m' + pink='\033[95m' + lightcyan='\033[96m' + class bg: + black='\033[40m' + red='\033[41m' + green='\033[42m' + orange='\033[43m' + blue='\033[44m' + purple='\033[45m' + cyan='\033[46m' + lightgrey='\033[47m' + + if __name__ == "__main__": if len(sys.argv) != 2: sys.exit("provide an image") diff --git a/test/integration/pkg_cases.go b/test/integration/pkg_cases.go index e5a329606b8..eefc3f43fc9 100644 --- a/test/integration/pkg_cases.go +++ b/test/integration/pkg_cases.go @@ -14,7 +14,7 @@ var cases = []struct { name: "find rpmdb packages", pkgType: pkg.RpmPkg, pkgInfo: map[string]string{ - "dive": "0.9.2", + "dive": "0.9.2-1", }, }, {