Skip to content

Commit

Permalink
feat(debian): add digests for dpkg (#4445)
Browse files Browse the repository at this point in the history 
Co-authored-by: knqyf263 <knqyf263@gmail.com>
  • Loading branch information
@DmitriyLewen @knqyf263
DmitriyLewen and knqyf263 authored May 28, 2023
1 parent 7e99d08 commit 72e302c
Show file tree
Hide file tree
Showing 9 changed files with 427 additions and 203 deletions.
270 changes: 156 additions & 114 deletions pkg/fanal/analyzer/pkg/dpkg/dpkg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@ package dpkg
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"io/fs"
"net/textproto"
"os"
"path/filepath"
"regexp"
Expand All @@ -12,43 +16,88 @@ import (

debVersion "github.com/knqyf263/go-deb-version"
"github.com/samber/lo"
"go.uber.org/zap"
"golang.org/x/xerrors"

dio "github.com/aquasecurity/go-dep-parser/pkg/io"
"github.com/aquasecurity/trivy/pkg/digest"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/utils/fsutils"
)

func init() {
analyzer.RegisterAnalyzer(&dpkgAnalyzer{})
analyzer.RegisterPostAnalyzer(analyzer.TypeDpkg, newDpkgAnalyzer)
}

type dpkgAnalyzer struct{}

func newDpkgAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
return &dpkgAnalyzer{}, nil
}

const (
analyzerVersion = 4
analyzerVersion = 5

statusFile = "var/lib/dpkg/status"
statusDir = "var/lib/dpkg/status.d/"
infoDir = "var/lib/dpkg/info/"
statusFile = "var/lib/dpkg/status"
statusDir = "var/lib/dpkg/status.d/"
infoDir = "var/lib/dpkg/info/"
availableFile = "var/lib/dpkg/available"
)

var (
dpkgSrcCaptureRegexp = regexp.MustCompile(`Source: (?P<name>[^\s]*)( \((?P<version>.*)\))?`)
dpkgSrcCaptureRegexp = regexp.MustCompile(`(?P<name>[^\s]*)( \((?P<version>.*)\))?`)
dpkgSrcCaptureRegexpNames = dpkgSrcCaptureRegexp.SubexpNames()
)

type dpkgAnalyzer struct{}
func (a dpkgAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) {
var systemInstalledFiles []string
var packageInfos []types.PackageInfo

func (a dpkgAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) {
scanner := bufio.NewScanner(input.Content)
if a.isListFile(filepath.Split(input.FilePath)) {
return a.parseDpkgInfoList(scanner)
// parse `available` file to get digest for packages
digests, err := a.parseDpkgAvailable(input.FS)
if err != nil {
log.Logger.Debugf("Unable to parse %q file: %s", availableFile, err)
}

return a.parseDpkgStatus(input.FilePath, scanner)
required := func(path string, d fs.DirEntry) bool {
return path != availableFile
}

// parse other files
err = fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r dio.ReadSeekerAt) error {
// parse list files
if a.isListFile(filepath.Split(path)) {
scanner := bufio.NewScanner(r)
systemFiles, err := a.parseDpkgInfoList(scanner)
if err != nil {
return err
}
systemInstalledFiles = append(systemInstalledFiles, systemFiles...)
return nil
}
// parse status files
infos, err := a.parseDpkgStatus(path, r, digests)
if err != nil {
return err
}
packageInfos = append(packageInfos, infos...)
return nil
})
if err != nil {
return nil, xerrors.Errorf("dpkg walk error: %w", err)
}

return &analyzer.AnalysisResult{
PackageInfos: packageInfos,
SystemInstalledFiles: systemInstalledFiles,
}, nil

}

// parseDpkgStatus parses /var/lib/dpkg/info/*.list
func (a dpkgAnalyzer) parseDpkgInfoList(scanner *bufio.Scanner) (*analyzer.AnalysisResult, error) {
// parseDpkgInfoList parses /var/lib/dpkg/info/*.list
func (a dpkgAnalyzer) parseDpkgInfoList(scanner *bufio.Scanner) ([]string, error) {
var installedFiles []string
var previous string
for scanner.Scan() {
Expand Down Expand Up @@ -76,25 +125,55 @@ func (a dpkgAnalyzer) parseDpkgInfoList(scanner *bufio.Scanner) (*analyzer.Analy
return nil, xerrors.Errorf("scan error: %w", err)
}

return &analyzer.AnalysisResult{
SystemInstalledFiles: installedFiles,
}, nil
return installedFiles, nil
}

// parseDpkgAvailable parses /var/lib/dpkg/available
func (a dpkgAnalyzer) parseDpkgAvailable(fsys fs.FS) (map[string]digest.Digest, error) {
f, err := fsys.Open(availableFile)
if err != nil {
return nil, xerrors.Errorf("file open error: %w", err)
}
defer f.Close()

pkgs := map[string]digest.Digest{}
scanner := NewScanner(f)
for scanner.Scan() {
header, err := scanner.Header()
if !errors.Is(err, io.EOF) && err != nil {
log.Logger.Warnw("Parse error", zap.String("file", availableFile), zap.Error(err))
continue
}
name, version, checksum := header.Get("Package"), header.Get("Version"), header.Get("SHA256")
pkgID := a.pkgID(name, version)
if pkgID != "" && checksum != "" {
pkgs[pkgID] = digest.NewDigestFromString(digest.SHA256, checksum)
}
}
if err = scanner.Err(); err != nil {
return nil, xerrors.Errorf("scan error: %w", err)
}

return pkgs, nil
}

// parseDpkgStatus parses /var/lib/dpkg/status or /var/lib/dpkg/status/*
func (a dpkgAnalyzer) parseDpkgStatus(filePath string, scanner *bufio.Scanner) (*analyzer.AnalysisResult, error) {
func (a dpkgAnalyzer) parseDpkgStatus(filePath string, r dio.ReadSeekerAt, digests map[string]digest.Digest) ([]types.PackageInfo, error) {
var pkg *types.Package
pkgs := map[string]*types.Package{}
pkgIDs := map[string]string{}

scanner := NewScanner(r)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
header, err := scanner.Header()
if !errors.Is(err, io.EOF) && err != nil {
log.Logger.Warnw("Parse error", zap.String("file", filePath), zap.Error(err))
continue
}

pkg = a.parseDpkgPkg(scanner)
pkg = a.parseDpkgPkg(header)
if pkg != nil {
pkg.Digest = digests[pkg.ID]
pkgs[pkg.ID] = pkg
pkgIDs[pkg.Name] = pkg.ID
}
Expand All @@ -106,119 +185,84 @@ func (a dpkgAnalyzer) parseDpkgStatus(filePath string, scanner *bufio.Scanner) (

a.consolidateDependencies(pkgs, pkgIDs)

return &analyzer.AnalysisResult{
PackageInfos: []types.PackageInfo{
{
FilePath: filePath,
Packages: lo.MapToSlice(pkgs, func(_ string, p *types.Package) types.Package {
return *p
}),
},
return []types.PackageInfo{
{
FilePath: filePath,
Packages: lo.MapToSlice(pkgs, func(_ string, p *types.Package) types.Package {
return *p
}),
},
}, nil
}

func (a dpkgAnalyzer) parseDpkgPkg(scanner *bufio.Scanner) (pkg *types.Package) {
var (
name string
version string
sourceName string
dependencies []string
isInstalled bool
sourceVersion string
maintainer string
architecture string
)
isInstalled = true
for {
line := strings.TrimSpace(scanner.Text())
if line == "" {
break
}
switch {
case strings.HasPrefix(line, "Package: "):
name = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
case strings.HasPrefix(line, "Source: "):
// Source line (Optional)
// Gives the name of the source package
// May also specifies a version

srcCapture := dpkgSrcCaptureRegexp.FindAllStringSubmatch(line, -1)[0]
md := map[string]string{}
for i, n := range srcCapture {
md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n)
}

sourceName = md["name"]
if md["version"] != "" {
sourceVersion = md["version"]
}
case strings.HasPrefix(line, "Version: "):
version = strings.TrimPrefix(line, "Version: ")
case strings.HasPrefix(line, "Status: "):
isInstalled = a.parseStatus(line)
case strings.HasPrefix(line, "Depends: "):
dependencies = a.parseDepends(line)
case strings.HasPrefix(line, "Maintainer: "):
maintainer = strings.TrimSpace(strings.TrimPrefix(line, "Maintainer: "))
case strings.HasPrefix(line, "Architecture: "):
architecture = strings.TrimPrefix(line, "Architecture: ")
}
if !scanner.Scan() {
break
}
}

if name == "" || version == "" || !isInstalled {
func (a dpkgAnalyzer) parseDpkgPkg(header textproto.MIMEHeader) *types.Package {
if isInstalled := a.parseStatus(header.Get("Status")); !isInstalled {
return nil
}

v, err := debVersion.NewVersion(version)
if err != nil {
log.Logger.Warnf("Invalid Version: OS %s, Package %s, Version %s", "debian", name, version)
pkg := &types.Package{
Name: header.Get("Package"),
Version: header.Get("Version"), // Will be parsed later
DependsOn: a.parseDepends(header.Get("Depends")), // Will be updated later
Maintainer: header.Get("Maintainer"),
Arch: header.Get("Architecture"),
}
if pkg.Name == "" || pkg.Version == "" {
return nil
}
pkg = &types.Package{
ID: a.pkgID(name, version),
Name: name,
Epoch: v.Epoch(),
Version: v.Version(),
Release: v.Revision(),
DependsOn: dependencies, // Will be consolidated later
Maintainer: maintainer,
Arch: architecture,

// Source line (Optional)
// Gives the name of the source package
// May also specifies a version
if src := header.Get("Source"); src != "" {
srcCapture := dpkgSrcCaptureRegexp.FindAllStringSubmatch(src, -1)[0]
md := map[string]string{}
for i, n := range srcCapture {
md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n)
}
pkg.SrcName = md["name"]
pkg.SrcVersion = md["version"]
}

// Source version and names are computed from binary package names and versions
// in dpkg.
// Source version and names are computed from binary package names and versions in dpkg.
// Source package name:
// https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/pkg-format.c#n338
// Source package version:
// https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/pkg-format.c#n355
if sourceName == "" {
sourceName = name
if pkg.SrcName == "" {
pkg.SrcName = pkg.Name
}
if pkg.SrcVersion == "" {
pkg.SrcVersion = pkg.Version
}

if sourceVersion == "" {
sourceVersion = version
if v, err := debVersion.NewVersion(pkg.Version); err != nil {
log.Logger.Warnw("Invalid version", zap.String("OS", "debian"),
zap.String("package", pkg.Name), zap.String("version", pkg.Version))
return nil
} else {
pkg.ID = a.pkgID(pkg.Name, pkg.Version)
pkg.Version = v.Version()
pkg.Epoch = v.Epoch()
pkg.Release = v.Revision()
}

sv, err := debVersion.NewVersion(sourceVersion)
if err != nil {
log.Logger.Warnf("Invalid SourceVersion Found : OS %s, Package %s, Version %s", "debian", sourceName, sourceVersion)
if v, err := debVersion.NewVersion(pkg.SrcVersion); err != nil {
log.Logger.Warnw("Invalid source version", zap.String("OS", "debian"),
zap.String("package", pkg.Name), zap.String("version", pkg.SrcVersion))
return nil
} else {
pkg.SrcVersion = v.Version()
pkg.SrcEpoch = v.Epoch()
pkg.SrcRelease = v.Revision()
}
pkg.SrcName = sourceName
pkg.SrcVersion = sv.Version()
pkg.SrcEpoch = sv.Epoch()
pkg.SrcRelease = sv.Revision()

return pkg
}

func (a dpkgAnalyzer) Required(filePath string, _ os.FileInfo) bool {
dir, fileName := filepath.Split(filePath)
if a.isListFile(dir, fileName) || filePath == statusFile {
if a.isListFile(dir, fileName) || filePath == statusFile || filePath == availableFile {
return true
}

Expand All @@ -232,21 +276,19 @@ func (a dpkgAnalyzer) pkgID(name, version string) string {
return fmt.Sprintf("%s@%s", name, version)
}

func (a dpkgAnalyzer) parseStatus(line string) bool {
for _, ss := range strings.Fields(strings.TrimPrefix(line, "Status: ")) {
func (a dpkgAnalyzer) parseStatus(s string) bool {
for _, ss := range strings.Fields(s) {
if ss == "deinstall" || ss == "purge" {
return false
}
}
return true
}

func (a dpkgAnalyzer) parseDepends(line string) []string {
line = strings.TrimPrefix(line, "Depends: ")
// e.g. Depends: passwd, debconf (>= 0.5) | debconf-2.0

func (a dpkgAnalyzer) parseDepends(s string) []string {
// e.g. passwd, debconf (>= 0.5) | debconf-2.0
var dependencies []string
depends := strings.Split(line, ",")
depends := strings.Split(s, ",")
for _, dep := range depends {
// e.g. gpgv | gpgv2 | gpgv1
for _, d := range strings.Split(dep, "|") {
Expand Down
Loading

0 comments on commit 72e302c

Please sign in to comment.