tarfs: rewrite to support random access archives

This change adds support for some schemes of encoding a table of contents and allowing individual files to be accessed independently. It breaks the API by adding a "size" parameter (like the archive/zip package) but attempts to autodetect when given a nonsense value. Signed-off-by: Hank Donnay <hdonnay@redhat.com>
quay · Sep 7, 2023 · 4ed05e3 · 4ed05e3
1 parent 4a301d7
commit 4ed05e3
Show file tree

Hide file tree

Showing 34 changed files with 1,888 additions and 845 deletions.
diff --git a/alpine/distributionscanner.go b/alpine/distributionscanner.go
@@ -72,10 +72,11 @@ func (s *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*
 		return nil, err
 	}
 	defer rc.Close()
-	sys, err := tarfs.New(rc)
+	sys, err := tarfs.New(ctx, rc, -1, nil)
 	if err != nil {
 		return nil, err
 	}
+	defer sys.Close()
 	return s.scanFs(ctx, sys)
 }
 

diff --git a/alpine/packagescanner.go b/alpine/packagescanner.go
@@ -63,10 +63,11 @@ func (*Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircore.
 	if err != nil {
 		return nil, err
 	}
-	sys, err := tarfs.New(rc)
+	sys, err := tarfs.New(ctx, rc, -1, nil)
 	if err != nil {
 		return nil, err
 	}
+	defer sys.Close()
 	b, err := fs.ReadFile(sys, installedFile)
 	switch {
 	case err == nil:

diff --git a/debian/distributionscanner.go b/debian/distributionscanner.go
@@ -52,10 +52,11 @@ func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]
 		return nil, fmt.Errorf("debian: unable to open layer: %w", err)
 	}
 	defer rd.Close()
-	sys, err := tarfs.New(rd)
+	sys, err := tarfs.New(ctx, rd, -1, nil)
 	if err != nil {
 		return nil, fmt.Errorf("debian: unable to open layer: %w", err)
 	}
+	defer sys.Close()
 	d, err := findDist(ctx, sys)
 	if err != nil {
 		return nil, err

diff --git a/dpkg/distroless_scanner.go b/dpkg/distroless_scanner.go
@@ -68,10 +68,11 @@ func (ps *DistrolessScanner) Scan(ctx context.Context, layer *claircore.Layer) (
 		return nil, fmt.Errorf("opening layer failed: %w", err)
 	}
 	defer rd.Close()
-	sys, err := tarfs.New(rd)
+	sys, err := tarfs.New(ctx, rd, -1, nil)
 	if err != nil {
 		return nil, fmt.Errorf("opening layer failed: %w", err)
 	}
+	defer sys.Close()
 
 	var pkgs []*claircore.Package
 	walk := func(p string, d fs.DirEntry, err error) error {

diff --git a/dpkg/scanner.go b/dpkg/scanner.go
@@ -75,10 +75,11 @@ func (ps *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*clairco
 		return nil, fmt.Errorf("opening layer failed: %w", err)
 	}
 	defer rd.Close()
-	sys, err := tarfs.New(rd)
+	sys, err := tarfs.New(ctx, rd, -1, nil)
 	if err != nil {
 		return nil, fmt.Errorf("opening layer failed: %w", err)
 	}
+	defer sys.Close()
 
 	// This is a map keyed by directory. A "score" of 2 means this is almost
 	// certainly a dpkg database.

diff --git a/go.mod b/go.mod
@@ -25,6 +25,8 @@ require (
 	github.com/remind101/migrate v0.0.0-20170729031349-52c1edff7319
 	github.com/rs/zerolog v1.30.0
 	github.com/ulikunitz/xz v0.5.11
+	go.opentelemetry.io/otel v1.17.0
+	go.opentelemetry.io/otel/trace v1.17.0
 	golang.org/x/crypto v0.12.0
 	golang.org/x/sync v0.3.0
 	golang.org/x/text v0.12.0
@@ -38,6 +40,8 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
@@ -52,7 +56,7 @@ require (
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	go.opentelemetry.io/otel v1.11.0 // indirect
+	go.opentelemetry.io/otel/metric v1.17.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/sys v0.11.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect

diff --git a/go.sum b/go.sum
@@ -24,6 +24,11 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -187,15 +192,19 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
-go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk=
-go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk=
+go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM=
+go.opentelemetry.io/otel v1.17.0/go.mod h1:I2vmBGtFaODIVMBSTPVDlJSzBDNf93k60E6Ft0nyjo0=
+go.opentelemetry.io/otel/metric v1.17.0 h1:iG6LGVz5Gh+IuO0jmgvpTB6YVrCGngi8QGm+pMd8Pdc=
+go.opentelemetry.io/otel/metric v1.17.0/go.mod h1:h4skoxdZI17AxwITdmdZjjYJQH5nzijUUjm+wtPph5o=
+go.opentelemetry.io/otel/trace v1.17.0 h1:/SWhSRHmDPOImIAetP1QAeMnZYiQXrTy4fMMYOdSKWQ=
+go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=

diff --git a/gobin/gobin.go b/gobin/gobin.go
@@ -68,10 +68,11 @@ func (Detector) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Pack
 		return nil, err
 	}
 	defer rd.Close()
-	sys, err := tarfs.New(rd)
+	sys, err := tarfs.New(ctx, rd, -1, nil)
 	if err != nil {
 		return nil, err
 	}
+	defer sys.Close()
 
 	var out []*claircore.Package
 

diff --git a/java/packagescanner.go b/java/packagescanner.go
@@ -122,10 +122,11 @@ func (s *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircor
 		return nil, err
 	}
 	defer r.Close()
-	sys, err := tarfs.New(r)
+	sys, err := tarfs.New(ctx, r, -1, nil)
 	if err != nil {
 		return nil, err
 	}
+	defer sys.Close()
 
 	ars, err := archives(ctx, sys)
 	if err != nil {

diff --git a/java/reposcanner.go b/java/reposcanner.go
@@ -64,10 +64,11 @@ func (rs *RepoScanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*cla
 		err := errors.New("unable to coerce to io.ReaderAt")
 		return nil, fmt.Errorf("opening layer failed: %w", err)
 	}
-	sys, err := tarfs.New(ra)
+	sys, err := tarfs.New(ctx, ra, -1, nil)
 	if err != nil {
 		return nil, err
 	}
+	defer sys.Close()
 	ars, err := archives(ctx, sys)
 	if err != nil {
 		return nil, err

diff --git a/layer.go b/layer.go
@@ -2,6 +2,7 @@ package claircore
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -83,15 +84,16 @@ var ErrNotFound = errors.New("claircore: unable to find any requested files")
 //
 // Deprecated: Callers should instead use `pkg/tarfs` and the `io/fs` package.
 func (l *Layer) Files(paths ...string) (map[string]*bytes.Buffer, error) {
-	r, err := l.Reader()
+	lf, err := os.Open(l.localPath)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("claircore: unable to open tar: %w", err)
 	}
-	defer r.Close()
-	sys, err := tarfs.New(r.(*os.File))
+	defer lf.Close()
+	sys, err := tarfs.New(context.TODO(), lf, -1, nil)
 	if err != nil {
 		return nil, err
 	}
+	defer sys.Close()
 
 	// Clean the input paths.
 	want := make(map[string]struct{})

diff --git a/libindex/fetcher.go b/libindex/fetcher.go
@@ -1,6 +1,7 @@
 package libindex
 
 import (
+	"archive/tar"
 	"bufio"
 	"bytes"
 	"context"
@@ -17,13 +18,12 @@ import (
 
 	"github.com/klauspost/compress/gzip"
 	"github.com/klauspost/compress/zstd"
-	"github.com/quay/claircore/indexer"
 	"github.com/quay/zlog"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/singleflight"
 
 	"github.com/quay/claircore"
-	"github.com/quay/claircore/pkg/tarfs"
+	"github.com/quay/claircore/indexer"
 )
 
 var (
@@ -303,12 +303,14 @@ func (a *RemoteFetchArena) realizeLayer(ctx context.Context, l *claircore.Layer)
 	zlog.Debug(ctx).
 		Msg("checking if layer is a valid tar")
 	// TODO(hank) Need media types somewhere in here.
-	switch _, err := tarfs.New(fd); {
-	case errors.Is(err, nil):
-	case errors.Is(err, tarfs.ErrFormat):
-		fallthrough
-	default:
+	// This would obviously break if it were allowed for layers to be a raw disk image or similar.
+	trd := tar.NewReader(io.NewSectionReader(fd, 0, -1))
+	switch h, err := trd.Next(); {
+	case !errors.Is(err, nil):
 		return "", err
+	case h.Format == tar.FormatUnknown:
+		return "", errors.New("format error reading file (not a tar?)")
+	default:
 	}
 
 	zlog.Debug(ctx).Msg("layer fetch ok")

diff --git a/osrelease/scanner.go b/osrelease/scanner.go
@@ -70,10 +70,11 @@ func (s *Scanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Di
 		return nil, fmt.Errorf("osrelease: unable to open layer: %w", err)
 	}
 	defer r.Close()
-	sys, err := tarfs.New(r)
+	sys, err := tarfs.New(ctx, r, -1, nil)
 	if err != nil {
 		return nil, fmt.Errorf("osrelease: unable to open layer: %w", err)
 	}
+	defer sys.Close()
 
 	// Attempt to parse each os-release file encountered. On a successful parse,
 	// return the distribution.