rhcc: add machinery for the Red Hat Container Catalog

This implements a container scanning for Red Hat containers using the
new `cvemap.xml` data.

See-Also: #513
Signed-off-by: Jason Shepherd <jason@jasonshepherd.net>
Signed-off-by: Hank Donnay <hdonnay@redhat.com>

rhel/rhcc/coalescer.go

@@ -0,0 +1,48 @@
+package rhcc
+
+import (
+	"context"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/internal/indexer"
+)
+
+// coalescer takes individual layer artifacts and coalesces them to form the final image's
+// package results
+type coalescer struct{}
+
+func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) {
+	if ctx.Err() != nil {
+		return nil, ctx.Err()
+	}
+	ir := &claircore.IndexReport{
+		Environments: map[string][]*claircore.Environment{},
+		Packages:     map[string]*claircore.Package{},
+		Repositories: map[string]*claircore.Repository{},
+	}
+
+	for _, l := range ls {
+		if len(l.Repos) == 0 {
+			continue
+		}
+		rs := make([]string, len(l.Repos))
+		for i, r := range l.Repos {
+			rs[i] = r.ID
+			ir.Repositories[r.ID] = r
+		}
+		for _, pkg := range l.Pkgs {
+			if pkg.RepositoryHint != `rhcc` {
+				continue
+			}
+			ir.Packages[pkg.ID] = pkg
+			ir.Environments[pkg.ID] = []*claircore.Environment{
+				{
+					PackageDB:     pkg.PackageDB,
+					IntroducedIn:  l.Hash,
+					RepositoryIDs: rs,
+				},
+			}
+		}
+	}
+	return ir, nil
+}

rhel/rhcc/coalescer_test.go

@@ -0,0 +1,74 @@
+package rhcc
+
+import (
+	"context"
+	"strconv"
+	"testing"
+
+	"github.com/quay/zlog"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/internal/indexer"
+	"github.com/quay/claircore/test"
+)
+
+func TestCoalescer(t *testing.T) {
+	ctx := zlog.Test(context.Background(), t)
+	coalescer := &coalescer{}
+	pkgs := test.GenUniquePackages(6)
+	for _, p := range pkgs {
+		// Mark them as if they came from this package's package scanner
+		p.RepositoryHint = `rhcc`
+	}
+	repo := []*claircore.Repository{&goldRepo}
+	layerArtifacts := []*indexer.LayerArtifacts{
+		{
+			Hash: test.RandomSHA256Digest(t),
+			Pkgs: pkgs[:1],
+		},
+		{
+			Hash: test.RandomSHA256Digest(t),
+			Pkgs: pkgs[:2],
+		},
+		{
+			Hash:  test.RandomSHA256Digest(t),
+			Pkgs:  pkgs[:3],
+			Repos: repo,
+		},
+		{
+			Hash: test.RandomSHA256Digest(t),
+			Pkgs: pkgs[:4],
+		},
+		{
+			Hash:  test.RandomSHA256Digest(t),
+			Pkgs:  pkgs[:5],
+			Repos: repo,
+		},
+		{
+			Hash: test.RandomSHA256Digest(t),
+			Pkgs: pkgs,
+		},
+	}
+	ir, err := coalescer.Coalesce(ctx, layerArtifacts)
+	if err != nil {
+		t.Fatalf("received error from coalesce method: %v", err)
+	}
+	// Expect 0-5 to have gotten associated with the repository.
+	for i := range pkgs {
+		es, ok := ir.Environments[strconv.Itoa(i)]
+		if !ok && i == 5 {
+			// Left out the last package.
+			continue
+		}
+		e := es[0]
+		if len(e.RepositoryIDs) == 0 {
+			t.Error("expected some repositories")
+		}
+		for _, id := range e.RepositoryIDs {
+			r := ir.Repositories[id]
+			if got, want := r.Name, goldRepo.Name; got != want {
+				t.Errorf("got: %q, want: %q", got, want)
+			}
+		}
+	}
+}

rhel/rhcc/ecosystem.go

@@ -0,0 +1,24 @@
+package rhcc
+
+import (
+	"context"
+
+	"github.com/quay/claircore/internal/indexer"
+)
+
+func NewEcosystem(_ context.Context) *indexer.Ecosystem {
+	return &indexer.Ecosystem{
+		PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) {
+			return []indexer.PackageScanner{&scanner{}}, nil
+		},
+		DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) {
+			return nil, nil
+		},
+		RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) {
+			return []indexer.RepositoryScanner{&reposcanner{}}, nil
+		},
+		Coalescer: func(_ context.Context) (indexer.Coalescer, error) {
+			return &coalescer{}, nil
+		},
+	}
+}

rhel/rhcc/fetcher_test.go

@@ -0,0 +1,50 @@
+package rhcc
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/quay/zlog"
+
+	"github.com/quay/claircore/libvuln/driver"
+)
+
+func TestFetcher(t *testing.T) {
+	const serveFile = "testdata/cve-2021-3762.xml"
+	ctx := zlog.Test(context.Background(), t)
+
+	fi, err := os.Stat(serveFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tag := fmt.Sprintf(`"%d"`, fi.ModTime().UnixNano())
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("if-none-match") == tag {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+		w.Header().Set("etag", tag)
+		http.ServeFile(w, r, serveFile)
+	}))
+	defer srv.Close()
+
+	u := &updater{
+		url:    srv.URL,
+		client: srv.Client(),
+	}
+	rd, hint, err := u.Fetch(ctx, "")
+	if err != nil {
+		t.Error(err)
+	}
+	if rd != nil {
+		rd.Close()
+	}
+	_, _, err = u.Fetch(ctx, driver.Fingerprint(hint))
+	if got, want := err, driver.Unchanged; got != want {
+		t.Errorf("got: %v, want: %v", got, want)
+	}
+}

rhel/rhcc/mapper.go

@@ -0,0 +1,137 @@
+package rhcc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/quay/zlog"
+	"golang.org/x/time/rate"
+)
+
+// MappingFile is a struct for mapping file between container NAME label and
+// container registry repository location.
+type mappingFile struct {
+	Data map[string][]string `json:"data"`
+}
+
+func (m *mappingFile) Get(ctx context.Context, _ *http.Client, name string) []string {
+	if repos, ok := m.Data[name]; ok {
+		zlog.Debug(ctx).Str("name", name).
+			Msg("name present in mapping file")
+		return repos
+	}
+	return []string{}
+}
+
+// UpdatingMapper provides local container name -> repos mapping via a
+// continually updated local mapping file.
+type updatingMapper struct {
+	URL string
+	// an atomic value holding the latest
+	// parsed MappingFile
+	mapping atomic.Value
+
+	// Machinery for updating the mapping file.
+	reqRate      *rate.Limiter
+	mu           sync.Mutex // protects lastModified
+	lastModified string
+}
+
+// NewUpdatingMapper returns an UpdatingMapper.
+//
+// The update period is unconfigurable. The first caller after the period loses
+// and must update the mapping file.
+func newUpdatingMapper(url string, init *mappingFile) *updatingMapper {
+	lu := &updatingMapper{
+		URL:     url,
+		reqRate: rate.NewLimiter(rate.Every(10*time.Minute), 1),
+	}
+	lu.mapping.Store(init)
+	// If we were provided an initial mapping, pull the first token.
+	if init != nil {
+		lu.reqRate.Allow()
+	}
+	return lu
+}
+
+// Get translates container names to repos using a mapping file.
+//
+// Get is safe for concurrent usage.
+func (u *updatingMapper) Get(ctx context.Context, c *http.Client, name string) []string {
+	ctx = zlog.ContextWithValues(ctx, "component", "rhel/rhcc/name2repos/UpdatingMapper.Get")
+	if name == "" {
+		return []string{}
+	}
+	if u.reqRate.Allow() {
+		zlog.Debug(ctx).Msg("got unlucky, updating mapping file")
+		if err := u.do(ctx, c); err != nil {
+			zlog.Error(ctx).
+				Err(err).
+				Msg("error updating mapping file")
+		}
+	}
+
+	// interface conversion guaranteed to pass, see
+	// constructor.
+	m := u.mapping.Load().(*mappingFile)
+	if m == nil {
+		return []string{}
+	}
+	return m.Get(ctx, nil, name)
+}
+
+func (u *updatingMapper) Fetch(ctx context.Context, c *http.Client) error {
+	return u.do(ctx, c)
+}
+
+// Do is an internal method called to perform an atomic update of the mapping
+// file.
+//
+// This method may be ran concurrently.
+func (u *updatingMapper) do(ctx context.Context, c *http.Client) error {
+	ctx = zlog.ContextWithValues(ctx, "component", "rhel/rhcc/UpdatingMapper.do", "url", u.URL)
+	zlog.Debug(ctx).Msg("attempting fetch of name2repos mapping file")
+
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.URL, nil)
+	if err != nil {
+		return err
+	}
+	if u.lastModified != "" {
+		req.Header.Set("if-modified-since", u.lastModified)
+	}
+
+	resp, err := c.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK:
+	case http.StatusNotModified:
+		zlog.Debug(ctx).
+			Str("since", u.lastModified).
+			Msg("response not modified; no update necessary")
+		return nil
+	default:
+		return fmt.Errorf("received status code %q querying mapping url", resp.StatusCode)
+	}
+
+	var mapping mappingFile
+	err = json.NewDecoder(resp.Body).Decode(&mapping)
+	if err != nil {
+		return fmt.Errorf("failed to decode mapping file: %w", err)
+	}
+	u.lastModified = resp.Header.Get("last-modified")
+	// atomic store of mapping file
+	u.mapping.Store(&mapping)
+	zlog.Debug(ctx).Msg("atomic update of local mapping file complete")
+	return nil
+}

rhel/rhcc/matcher.go

@@ -0,0 +1,47 @@
+package rhcc
+
+import (
+	"context"
+
+	rpmVersion "github.com/knqyf263/go-rpm-version"
+	"github.com/quay/zlog"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/libvuln/driver"
+)
+
+var Matcher driver.Matcher = &matcher{}
+
+type matcher struct{}
+
+var _ driver.Matcher = (*matcher)(nil)
+
+func (*matcher) Name() string {
+	return "rhel-container-matcher"
+}
+
+func (*matcher) Filter(r *claircore.IndexRecord) bool {
+	return r.Repository != nil &&
+		r.Repository.Name == goldRepo.Name
+}
+
+func (*matcher) Query() []driver.MatchConstraint {
+	return []driver.MatchConstraint{driver.RepositoryName}
+}
+
+func (*matcher) Vulnerable(ctx context.Context, record *claircore.IndexRecord, vuln *claircore.Vulnerability) (bool, error) {
+	pkgVer, fixedInVer := rpmVersion.NewVersion(record.Package.Version), rpmVersion.NewVersion(vuln.FixedInVersion)
+	zlog.Debug(ctx).
+		Str("record", record.Package.Version).
+		Str("vulnerability", vuln.FixedInVersion).
+		Msg("comparing versions")
+	return pkgVer.LessThan(fixedInVer), nil
+}
+
+// Implement version filtering to have the database only return results for the
+// same minor version. Marking the results as not authoritative means the
+// Vulnerable method is still called.
+
+func (*matcher) VersionFilter() {}
+
+func (*matcher) VersionAuthoritative() bool { return false }