Harden system/package parsing of deb size (elastic#17188)

* Harden system/package parsing of deb size Deb packages report their installed size in a field, Installed-Size, which is an integer interpreted as KiB. Some unofficial packages are adding a unit at the end of this field: Installed-Size: 65M System tools dpkg/apt ignore everything after the number. Auditbeat is currently failing to parse the list of installed packages once this mistake is reached. This updates the dataset to: - Do not fail when installed size can't be parsed. - Understand prefixes k/K, m/M and G/b. Fixes elastic#16661 (cherry picked from commit b131405)
adriansr · Mar 24, 2020 · d883f8e · d883f8e
1 parent 60b14fa
commit d883f8e
Show file tree

Hide file tree

Showing 4 changed files with 297 additions and 4 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -41,6 +41,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Auditbeat*
 
+- system/socket: Fixed compatibility issue with kernel 5.x. {pull}15771[15771]
+- system/package: Fix parsing of Installed-Size field of DEB packages. {issue}16661[16661] {pull}17188[17188]
 
 *Filebeat*
 

diff --git a/x-pack/auditbeat/module/system/package/package.go b/x-pack/auditbeat/module/system/package/package.go
@@ -462,7 +462,7 @@ func (ms *MetricSet) getPackages() (packages []*Package, err error) {
 	if err == nil {
 		foundPackageManager = true
 
-		dpkgPackages, err := listDebPackages()
+		dpkgPackages, err := ms.listDebPackages()
 		if err != nil {
 			return nil, errors.Wrap(err, "error getting DEB packages")
 		}
@@ -499,7 +499,7 @@ func (ms *MetricSet) getPackages() (packages []*Package, err error) {
 	return packages, nil
 }
 
-func listDebPackages() ([]*Package, error) {
+func (ms *MetricSet) listDebPackages() ([]*Package, error) {
 	dpkgStatusFile := filepath.Join(dpkgPath, "status")
 
 	file, err := os.Open(dpkgStatusFile)
@@ -556,9 +556,14 @@ func listDebPackages() ([]*Package, error) {
 		case "description":
 			pkg.Summary = value
 		case "installed-size":
-			pkg.Size, err = strconv.ParseUint(value, 10, 64)
+			pkg.Size, err = parseDpkgInstalledSize(value)
 			if err != nil {
-				return nil, errors.Wrapf(err, "error converting %s to int", value)
+				// If installed size is invalid, log a warning but still
+				// report the package with size=0.
+				ms.log.Warnw("Failed parsing installed size",
+					"package", pkg.Name,
+					"Installed-Size", value,
+					"Error", err)
 			}
 		case "homepage":
 			pkg.URL = value
@@ -578,3 +583,33 @@ func listDebPackages() ([]*Package, error) {
 
 	return packages, nil
 }
+
+func parseDpkgInstalledSize(value string) (size uint64, err error) {
+	// Installed-Size is an integer (KiB).
+	if size, err = strconv.ParseUint(value, 10, 64); err == nil {
+		return size, err
+	}
+
+	// Some rare third-party packages contain a unit at the end. This is ignored
+	// by dpkg tools. Try to parse to return a value as close as possible
+	// to what the package maintainer meant.
+	end := len(value)
+	for idx, chr := range value {
+		if chr < '0' || chr > '9' {
+			end = idx
+			break
+		}
+	}
+	multiplier := uint64(1)
+	if end < len(value) {
+		switch value[end] {
+		case 'm', 'M':
+			multiplier = 1024
+		case 'g', 'G':
+			multiplier = 1024 * 1024
+		}
+	}
+
+	size, err = strconv.ParseUint(value[:end], 10, 64)
+	return size * multiplier, err
+}
diff --git a/x-pack/auditbeat/module/system/package/package_test.go b/x-pack/auditbeat/module/system/package/package_test.go
@@ -71,12 +71,78 @@ func TestDpkg(t *testing.T) {
 	if assert.Len(t, events, 1) {
 		event := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent)
 		checkFieldValue(t, event, "system.audit.package.name", "test")
+		checkFieldValue(t, event, "system.audit.package.size", uint64(269))
 		checkFieldValue(t, event, "system.audit.package.summary", "Test Package")
 		checkFieldValue(t, event, "system.audit.package.url", "https://www.elastic.co/")
 		checkFieldValue(t, event, "system.audit.package.version", "8.2.0-1ubuntu2~18.04")
 	}
 }
 
+func TestDpkgInstalledSize(t *testing.T) {
+	expected := map[string]uint64{
+		"libquadmath0":      269,
+		"python-apt-common": 248,
+		"libnpth0":          32,
+		"bind9-host":        17,
+		"libpam-runtime":    300,
+		"libsepol1-dev":     1739 * 1024,
+		"libisl19":          17 * 1024 * 1024,
+		"netbase":           0,
+		"python2.7-minimal": 0,
+	}
+
+	logp.TestingSetup()
+
+	defer abtest.SetupDataDir(t)()
+
+	// Disable all except dpkg
+	rpmPathOld := rpmPath
+	dpkgPathOld := dpkgPath
+	brewPathOld := homebrewCellarPath
+	defer func() {
+		rpmPath = rpmPathOld
+		dpkgPath = dpkgPathOld
+		homebrewCellarPath = brewPathOld
+	}()
+	rpmPath = "/does/not/exist"
+	homebrewCellarPath = "/does/not/exist"
+
+	var err error
+	dpkgPath, err = filepath.Abs("testdata/dpkg-size/")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f := mbtest.NewReportingMetricSetV2(t, getConfig())
+	defer f.(*MetricSet).bucket.DeleteBucket()
+
+	events, errs := mbtest.ReportingFetchV2(f)
+	if len(errs) > 0 {
+		t.Fatalf("received error: %+v", errs[0])
+	}
+
+	got := make(map[string]uint64, len(events))
+	for _, ev := range events {
+		event := mbtest.StandardizeEvent(f, ev, core.AddDatasetToEvent)
+		name, err := event.GetValue("system.audit.package.name")
+		if err != nil {
+			t.Fatal(err)
+		}
+		size, err := event.GetValue("system.audit.package.size")
+		if err != nil {
+			size = uint64(0)
+		}
+		if !assert.IsType(t, "", name) {
+			t.Fatal("string expected")
+		}
+		if !assert.IsType(t, uint64(0), size) {
+			t.Fatal("uint64 expected")
+		}
+		got[name.(string)] = size.(uint64)
+	}
+	assert.Equal(t, expected, got)
+}
+
 func getConfig() map[string]interface{} {
 	return map[string]interface{}{
 		"module":   "system",

diff --git a/x-pack/auditbeat/module/system/package/testdata/dpkg-size/status b/x-pack/auditbeat/module/system/package/testdata/dpkg-size/status
@@ -0,0 +1,190 @@
+Package: libquadmath0
+Status: install ok installed
+Priority: optional
+Section: libs
+Installed-Size: 269B
+Maintainer: Ubuntu Core developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: same
+Source: gcc-8
+Version: 8.3.0-26ubuntu1~18.04
+Depends: gcc-8-base (= 8.3.0-26ubuntu1~18.04), libc6 (>= 2.23)
+Description: GCC Quad-Precision Math Library
+ A library, which provides quad-precision mathematical functions on targets
+ supporting the __float128 datatype. The library is used to provide on such
+ targets the REAL(16) type in the GNU Fortran compiler.
+Homepage: http://gcc.gnu.org/
+Original-Maintainer: Debian GCC Maintainers <debian-gcc@lists.debian.org>
+
+Package: python-apt-common
+Status: install ok installed
+Priority: optional
+Section: python
+Installed-Size: 248KiB
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: all
+Source: python-apt
+Version: 1.6.5ubuntu0.2
+Replaces: python-apt (<< 0.7.98+nmu1)
+Breaks: python-apt (<< 0.7.98+nmu1)
+Enhances: python-apt, python3-apt
+Description: Python interface to libapt-pkg (locales)
+ The apt_pkg Python interface will provide full access to the internal
+ libapt-pkg structures allowing Python programs to easily perform a
+ variety of functions.
+ .
+ This package contains locales.
+Original-Maintainer: APT Development Team <deity@lists.debian.org>
+
+Package: libnpth0
+Status: install ok installed
+Priority: optional
+Section: libs
+Installed-Size: 	32trash
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: same
+Source: npth
+Version: 1.5-3
+Depends: libc6 (>= 2.17)
+Description: replacement for GNU Pth using system threads
+ nPth is a non-preemptive threads implementation using an API very
+ similar to the one known from GNU Pth. It has been designed as a
+ replacement of GNU Pth for non-ancient operating systems. In
+ contrast to GNU Pth it is based on the system's standard threads
+ implementation. Thus nPth allows the use of libraries which are not
+ compatible to GNU Pth.
+Original-Maintainer: Eric Dorland <eric@debian.org>
+Homepage: https://www.gnupg.org/
+
+Package: bind9-host
+Status: install ok installed
+Priority: standard
+Section: net
+Installed-Size:    17.4
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Source: bind9
+Version: 1:9.11.3+dfsg-1ubuntu1.11
+Provides: host
+Depends: libbind9-160 (= 1:9.11.3+dfsg-1ubuntu1.11), libdns1100 (= 1:9.11.3+dfsg-1ubuntu1.11), libisc169 (= 1:9.11.3+dfsg-1ubuntu1.11), libisccfg160 (= 1:9.11.3+dfsg-1ubuntu1.11), liblwres160 (= 1:9.11.3+dfsg-1ubuntu1.11), libc6 (>= 2.4)
+Description: DNS lookup utility (deprecated)
+ This package provides /usr/bin/host, a simple utility (bundled with the
+ BIND 9.X sources) which can be used for converting domain names to IP
+ addresses and the reverse.
+ .
+ This utility is deprecated, use dig or delv from the dnsutils package.
+Homepage: https://www.isc.org/downloads/bind/
+Original-Maintainer: Debian DNS Packaging <pkg-dns-devel@lists.alioth.debian.org>
+
+Package: libpam-runtime
+Status: install ok installed
+Priority: required
+Section: admin
+Installed-Size: 300T
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: all
+Multi-Arch: foreign
+Source: pam
+Version: 1.1.8-3.6ubuntu2
+Replaces: libpam0g-dev, libpam0g-util
+Depends: debconf (>= 0.5) | debconf-2.0, debconf (>= 1.5.19) | cdebconf, libpam-modules (>= 1.0.1-6)
+Conflicts: libpam0g-util
+Conffiles:
+ /etc/pam.conf 87fc76f18e98ee7d3848f6b81b3391e5
+ /etc/pam.d/other 31aa7f2181889ffb00b87df4126d1701
+Description: Runtime support for the PAM library
+ Contains configuration files and  directories required for
+ authentication  to work on Debian systems.  This package is required
+ on almost all installations.
+Homepage: http://www.linux-pam.org/
+Original-Maintainer: Steve Langasek <vorlon@debian.org>
+
+Package: libsepol1-dev
+Status: install ok installed
+Priority: optional
+Section: libdevel
+Installed-Size: 1739m
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: same
+Source: libsepol
+Version: 2.7-1
+Provides: libsepol-dev
+Depends: libsepol1 (= 2.7-1)
+Conflicts: libsepol-dev
+Description: SELinux binary policy manipulation library and development files
+ libsepol allows programs to easily modify SELinux binary policies.  This
+ means changing the default values for booleans, or reading the policy for
+ analysis.
+ .
+ This package contains the headers and archives used for linking it into your
+ programs.
+Original-Maintainer: Debian SELinux maintainers <selinux-devel@lists.alioth.debian.org>
+Homepage: http://userspace.selinuxproject.org/
+
+Package: libisl19
+Status: install ok installed
+Priority: optional
+Section: libs
+Installed-Size: 17G
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: same
+Source: isl
+Version: 0.19-1
+Replaces: libisl-dbg (<< 0.19)
+Depends: libc6 (>= 2.14), libgmp10
+Breaks: libisl-dbg (<< 0.19)
+Description: manipulating sets and relations of integer points bounded by linear constraints
+ isl is a library for manipulating sets and relations of integer points
+ bounded by linear constraints. Supported operations on sets include
+ intersection, union, set difference, emptiness check, convex hull,
+ (integer) affine hull, integer projection, and computing the lexicographic
+ minimum using parametric integer programming. It also includes an ILP solver
+ based on generalized basis reduction.
+ .
+ This package contains the runtime library.
+Original-Maintainer: Debian GCC Maintainers <debian-gcc@lists.debian.org>
+Homepage: http://isl.gforge.inria.fr/
+
+Package: netbase
+Status: install ok installed
+Priority: important
+Section: admin
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: all
+Multi-Arch: foreign
+Version: 5.4
+Conffiles:
+ /etc/protocols bb9c019d6524e913fd72441d58b68216
+ /etc/rpc f0b6f6352bf886623adc04183120f83b
+ /etc/services 567c100888518c1163b3462993de7d47
+Description: Basic TCP/IP networking system
+ This package provides the necessary infrastructure for basic TCP/IP based
+ networking.
+Original-Maintainer: Marco d'Itri <md@linux.it>
+
+Package: python2.7-minimal
+Status: install ok installed
+Priority: optional
+Section: python
+Installed-Size: Who knows
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: allowed
+Source: python2.7
+Version: 2.7.17-1~18.04
+Replaces: python2.7 (<< 2.7.8-7~)
+Depends: libpython2.7-minimal (= 2.7.17-1~18.04)
+Pre-Depends: libc6 (>= 2.15), zlib1g (>= 1:1.2.0)
+Recommends: python2.7
+Suggests: binfmt-support
+Conflicts: binfmt-support (<< 1.1.2)
+Description: Minimal subset of the Python language (version 2.7)
+ This package contains the interpreter and some essential modules.  It can
+ be used in the boot process for some basic tasks.
+ See /usr/share/doc/python2.7-minimal/README.Debian for a list of the modules
+ contained in this package.
+Original-Maintainer: Matthias Klose <doko@debian.org>
+