implement v5 db schema to support improved matching between rpm appst…

…ream modules (#944)

Adds support for a `package_qualifiers` column to allow evaluating package matches to vulnerabilities based on more than just version constraints. Currently adds an rpm-modularity qualifier in order to support matching to correct app stream module in order to reduce false positives within rpm-based distro ecosystems. In order to prevent an increase in false positive matches for previous versions of grype using the v4 schema, this change (along with the vulnerability source driver parser updates) requires bumping the schema to v5.

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
Co-authored-by: Alex Goodman <alex.goodman@anchore.com>

cmd/root.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/anchore/grype/grype"
 	"github.com/anchore/grype/grype/db"
-	grypeDb "github.com/anchore/grype/grype/db/v4"
+	grypeDb "github.com/anchore/grype/grype/db/v5"
 	"github.com/anchore/grype/grype/event"
 	"github.com/anchore/grype/grype/grypeerr"
 	"github.com/anchore/grype/grype/match"

cmd/root_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/anchore/grype/grype/db"
-	grypeDB "github.com/anchore/grype/grype/db/v4"
+	grypeDB "github.com/anchore/grype/grype/db/v5"
 	"github.com/anchore/grype/grype/match"
 	"github.com/anchore/grype/grype/pkg"
 	"github.com/anchore/grype/grype/vulnerability"

go.mod

@@ -55,6 +55,7 @@ require (
 	github.com/anchore/sqlite v1.4.6-0.20220607210448-bcc6ee5c4963
 	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
 	github.com/in-toto/in-toto-golang v0.3.4-0.20220709202702-fa494aaa0add
+	github.com/mitchellh/mapstructure v1.5.0
 	github.com/secure-systems-lab/go-securesystemslib v0.4.0
 	github.com/sigstore/cosign v1.13.0
 	github.com/sigstore/sigstore v1.4.2
@@ -185,7 +186,6 @@ require (
 	github.com/miekg/pkcs11 v1.1.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

grype/db/curator.go

@@ -17,8 +17,8 @@ import (
 	partybus "github.com/wagoodman/go-partybus"
 	progress "github.com/wagoodman/go-progress"
 
-	grypeDB "github.com/anchore/grype/grype/db/v4"
-	"github.com/anchore/grype/grype/db/v4/store"
+	grypeDB "github.com/anchore/grype/grype/db/v5"
+	"github.com/anchore/grype/grype/db/v5/store"
 	"github.com/anchore/grype/grype/event"
 	"github.com/anchore/grype/grype/vulnerability"
 	"github.com/anchore/grype/internal/bus"

grype/db/db_closer.go

@@ -1,9 +1,9 @@
 package db
 
-import v4 "github.com/anchore/grype/grype/db/v4"
+import v5 "github.com/anchore/grype/grype/db/v5"
 
 // Closer lets receiver close the db connection and free any allocated db resources.
 // It's especially useful if vulnerability DB loaded repeatedly during some periodic SBOM scanning process.
 type Closer struct {
-	v4.DBCloser
+	v5.DBCloser
 }

grype/db/match_exclusion_provider.go

@@ -3,7 +3,7 @@ package db
 import (
 	"fmt"
 
-	grypeDB "github.com/anchore/grype/grype/db/v4"
+	grypeDB "github.com/anchore/grype/grype/db/v5"
 	"github.com/anchore/grype/grype/match"
 )
 

grype/db/v5/advisory.go

@@ -0,0 +1,7 @@
+package v5
+
+// Advisory represents published statements regarding a vulnerability (and potentially about it's resolution).
+type Advisory struct {
+	ID   string `json:"id"`
+	Link string `json:"link"`
+}

grype/db/v5/diff.go

@@ -0,0 +1,16 @@
+package v5
+
+type DiffReason = string
+
+const (
+	DiffAdded   DiffReason = "added"
+	DiffChanged DiffReason = "changed"
+	DiffRemoved DiffReason = "removed"
+)
+
+type Diff struct {
+	Reason    DiffReason `json:"reason"`
+	ID        string     `json:"id"`
+	Namespace string     `json:"namespace"`
+	Packages  []string   `json:"packages"`
+}

grype/db/v5/fix.go

@@ -0,0 +1,16 @@
+package v5
+
+type FixState string
+
+const (
+	UnknownFixState FixState = "unknown"
+	FixedState      FixState = "fixed"
+	NotFixedState   FixState = "not-fixed"
+	WontFixState    FixState = "wont-fix"
+)
+
+// Fix represents all information about known fixes for a stated vulnerability.
+type Fix struct {
+	Versions []string `json:"versions"` // The version(s) which this particular vulnerability was fixed in
+	State    FixState `json:"state"`
+}

grype/db/v5/id.go

@@ -0,0 +1,28 @@
+package v5
+
+import (
+	"time"
+)
+
+// ID represents identifying information for a DB and the data it contains.
+type ID struct {
+	// BuildTimestamp is the timestamp used to define the age of the DB, ideally including the age of the data
+	// contained in the DB, not just when the DB file was created.
+	BuildTimestamp time.Time `json:"build_timestamp"`
+	SchemaVersion  int       `json:"schema_version"`
+}
+
+type IDReader interface {
+	GetID() (*ID, error)
+}
+
+type IDWriter interface {
+	SetID(ID) error
+}
+
+func NewID(age time.Time) ID {
+	return ID{
+		BuildTimestamp: age.UTC(),
+		SchemaVersion:  SchemaVersion,
+	}
+}

grype/db/v5/namespace/cpe/namespace.go

@@ -0,0 +1,54 @@
+package cpe
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/anchore/grype/grype/db/v5/pkg/resolver"
+	"github.com/anchore/grype/grype/db/v5/pkg/resolver/stock"
+)
+
+const ID = "cpe"
+
+type Namespace struct {
+	provider string
+	resolver resolver.Resolver
+}
+
+func NewNamespace(provider string) *Namespace {
+	return &Namespace{
+		provider: provider,
+		resolver: &stock.Resolver{},
+	}
+}
+
+func FromString(namespaceStr string) (*Namespace, error) {
+	if namespaceStr == "" {
+		return nil, errors.New("unable to create CPE namespace from empty string")
+	}
+
+	components := strings.Split(namespaceStr, ":")
+
+	if len(components) != 2 {
+		return nil, fmt.Errorf("unable to create CPE namespace from %s: incorrect number of components", namespaceStr)
+	}
+
+	if components[1] != ID {
+		return nil, fmt.Errorf("unable to create CPE namespace from %s: type %s is incorrect", namespaceStr, components[1])
+	}
+
+	return NewNamespace(components[0]), nil
+}
+
+func (n *Namespace) Provider() string {
+	return n.provider
+}
+
+func (n *Namespace) Resolver() resolver.Resolver {
+	return n.resolver
+}
+
+func (n Namespace) String() string {
+	return fmt.Sprintf("%s:%s", n.provider, ID)
+}

grype/db/v5/namespace/cpe/namespace_test.go

@@ -0,0 +1,51 @@
+package cpe
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFromString(t *testing.T) {
+	successTests := []struct {
+		namespaceString string
+		result          *Namespace
+	}{
+		{
+			namespaceString: "abc.xyz:cpe",
+			result:          NewNamespace("abc.xyz"),
+		},
+	}
+
+	for _, test := range successTests {
+		result, _ := FromString(test.namespaceString)
+		assert.Equal(t, result, test.result)
+	}
+
+	errorTests := []struct {
+		namespaceString string
+		errorMessage    string
+	}{
+		{
+			namespaceString: "",
+			errorMessage:    "unable to create CPE namespace from empty string",
+		},
+		{
+			namespaceString: "single-component",
+			errorMessage:    "unable to create CPE namespace from single-component: incorrect number of components",
+		},
+		{
+			namespaceString: "too:many:components",
+			errorMessage:    "unable to create CPE namespace from too:many:components: incorrect number of components",
+		},
+		{
+			namespaceString: "wrong:namespace_type",
+			errorMessage:    "unable to create CPE namespace from wrong:namespace_type: type namespace_type is incorrect",
+		},
+	}
+
+	for _, test := range errorTests {
+		_, err := FromString(test.namespaceString)
+		assert.EqualError(t, err, test.errorMessage)
+	}
+}

grype/db/v5/namespace/distro/namespace.go

@@ -0,0 +1,67 @@
+package distro
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/anchore/grype/grype/db/v5/pkg/resolver"
+	"github.com/anchore/grype/grype/db/v5/pkg/resolver/stock"
+	"github.com/anchore/grype/grype/distro"
+)
+
+const ID = "distro"
+
+type Namespace struct {
+	provider   string
+	distroType distro.Type
+	version    string
+	resolver   resolver.Resolver
+}
+
+func NewNamespace(provider string, distroType distro.Type, version string) *Namespace {
+	return &Namespace{
+		provider:   provider,
+		distroType: distroType,
+		version:    version,
+		resolver:   &stock.Resolver{},
+	}
+}
+
+func FromString(namespaceStr string) (*Namespace, error) {
+	if namespaceStr == "" {
+		return nil, errors.New("unable to create distro namespace from empty string")
+	}
+
+	components := strings.Split(namespaceStr, ":")
+
+	if len(components) != 4 {
+		return nil, fmt.Errorf("unable to create distro namespace from %s: incorrect number of components", namespaceStr)
+	}
+
+	if components[1] != ID {
+		return nil, fmt.Errorf("unable to create distro namespace from %s: type %s is incorrect", namespaceStr, components[1])
+	}
+
+	return NewNamespace(components[0], distro.Type(components[2]), components[3]), nil
+}
+
+func (n *Namespace) Provider() string {
+	return n.provider
+}
+
+func (n *Namespace) DistroType() distro.Type {
+	return n.distroType
+}
+
+func (n *Namespace) Version() string {
+	return n.version
+}
+
+func (n *Namespace) Resolver() resolver.Resolver {
+	return n.resolver
+}
+
+func (n Namespace) String() string {
+	return fmt.Sprintf("%s:%s:%s:%s", n.provider, ID, n.distroType, n.version)
+}

grype/db/v5/namespace/distro/namespace_test.go

@@ -0,0 +1,81 @@
+package distro
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	grypeDistro "github.com/anchore/grype/grype/distro"
+)
+
+func TestFromString(t *testing.T) {
+	successTests := []struct {
+		namespaceString string
+		result          *Namespace
+	}{
+		{
+			namespaceString: "alpine:distro:alpine:3.15",
+			result:          NewNamespace("alpine", grypeDistro.Alpine, "3.15"),
+		},
+		{
+			namespaceString: "redhat:distro:redhat:8",
+			result:          NewNamespace("redhat", grypeDistro.RedHat, "8"),
+		},
+		{
+			namespaceString: "abc.xyz:distro:unknown:abcd~~~",
+			result:          NewNamespace("abc.xyz", grypeDistro.Type("unknown"), "abcd~~~"),
+		},
+		{
+			namespaceString: "msrc:distro:windows:10111",
+			result:          NewNamespace("msrc", grypeDistro.Type("windows"), "10111"),
+		},
+		{
+			namespaceString: "amazon:distro:amazonlinux:2022",
+			result:          NewNamespace("amazon", grypeDistro.AmazonLinux, "2022"),
+		},
+		{
+			namespaceString: "amazon:distro:amazonlinux:2",
+			result:          NewNamespace("amazon", grypeDistro.AmazonLinux, "2"),
+		},
+	}
+
+	for _, test := range successTests {
+		result, _ := FromString(test.namespaceString)
+		assert.Equal(t, result, test.result)
+	}
+
+	errorTests := []struct {
+		namespaceString string
+		errorMessage    string
+	}{
+		{
+			namespaceString: "",
+			errorMessage:    "unable to create distro namespace from empty string",
+		},
+		{
+			namespaceString: "single-component",
+			errorMessage:    "unable to create distro namespace from single-component: incorrect number of components",
+		},
+		{
+			namespaceString: "two:components",
+			errorMessage:    "unable to create distro namespace from two:components: incorrect number of components",
+		},
+		{
+			namespaceString: "still:not:enough",
+			errorMessage:    "unable to create distro namespace from still:not:enough: incorrect number of components",
+		},
+		{
+			namespaceString: "too:many:components:a:b",
+			errorMessage:    "unable to create distro namespace from too:many:components:a:b: incorrect number of components",
+		},
+		{
+			namespaceString: "wrong:namespace_type:a:b",
+			errorMessage:    "unable to create distro namespace from wrong:namespace_type:a:b: type namespace_type is incorrect",
+		},
+	}
+
+	for _, test := range errorTests {
+		_, err := FromString(test.namespaceString)
+		assert.EqualError(t, err, test.errorMessage)
+	}
+}