Harden system/package parsing of deb size (#17188)

* Harden system/package parsing of deb size Deb packages report their installed size in a field, Installed-Size, which is an integer interpreted as KiB. Some unofficial packages are adding a unit at the end of this field: Installed-Size: 65M System tools dpkg/apt ignore everything after the number. Auditbeat is currently failing to parse the list of installed packages once this mistake is reached. This updates the dataset to: - Do not fail when installed size can't be parsed. - Understand prefixes k/K, m/M and G/b. Fixes #16661
elastic · Mar 23, 2020 · b131405 · b131405
1 parent 57e194b
commit b131405
Show file tree

Hide file tree

Showing 4 changed files with 296 additions and 4 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -75,6 +75,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Auditbeat*
 
 - system/socket: Fixed compatibility issue with kernel 5.x. {pull}15771[15771]
+- system/package: Fix parsing of Installed-Size field of DEB packages. {issue}16661[16661] {pull}17188[17188]
 
 *Filebeat*
 

diff --git a/x-pack/auditbeat/module/system/package/package.go b/x-pack/auditbeat/module/system/package/package.go
@@ -462,7 +462,7 @@ func (ms *MetricSet) getPackages() (packages []*Package, err error) {
  if err == nil {
  foundPackageManager = true
 
- dpkgPackages, err := listDebPackages()
+ dpkgPackages, err := ms.listDebPackages()
  if err != nil {
  return nil, errors.Wrap(err, "error getting DEB packages")
  }
@@ -499,7 +499,7 @@ func (ms *MetricSet) getPackages() (packages []*Package, err error) {
  return packages, nil
 }
 
-func listDebPackages() ([]*Package, error) {
+func (ms *MetricSet) listDebPackages() ([]*Package, error) {
  dpkgStatusFile := filepath.Join(dpkgPath, "status")
 
  file, err := os.Open(dpkgStatusFile)
@@ -556,9 +556,14 @@ func listDebPackages() ([]*Package, error) {
  case "description":
  pkg.Summary = value
  case "installed-size":
- pkg.Size, err = strconv.ParseUint(value, 10, 64)
+ pkg.Size, err = parseDpkgInstalledSize(value)
  if err != nil {
- return nil, errors.Wrapf(err, "error converting %s to int", value)
+ // If installed size is invalid, log a warning but still
+ // report the package with size=0.
+ ms.log.Warnw("Failed parsing installed size",
+ "package", pkg.Name,
+ "Installed-Size", value,
+ "Error", err)
  }
  case "homepage":
  pkg.URL = value
@@ -578,3 +583,33 @@ func listDebPackages() ([]*Package, error) {
 
  return packages, nil
 }
+
+func parseDpkgInstalledSize(value string) (size uint64, err error) {
+ // Installed-Size is an integer (KiB).
+ if size, err = strconv.ParseUint(value, 10, 64); err == nil {
+ return size, err
+ }
+
+ // Some rare third-party packages contain a unit at the end. This is ignored
+ // by dpkg tools. Try to parse to return a value as close as possible
+ // to what the package maintainer meant.
+ end := len(value)
+ for idx, chr := range value {
+ if chr < '0' || chr > '9' {
+ end = idx
+ break
+ }
+ }
+ multiplier := uint64(1)
+ if end < len(value) {
+ switch value[end] {
+ case 'm', 'M':
+ multiplier = 1024
+ case 'g', 'G':
+ multiplier = 1024 * 1024
+ }
+ }
+
+ size, err = strconv.ParseUint(value[:end], 10, 64)
+ return size * multiplier, err
+}
diff --git a/x-pack/auditbeat/module/system/package/package_test.go b/x-pack/auditbeat/module/system/package/package_test.go
@@ -71,12 +71,78 @@ func TestDpkg(t *testing.T) {
  if assert.Len(t, events, 1) {
  event := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent)
  checkFieldValue(t, event, "system.audit.package.name", "test")
+ checkFieldValue(t, event, "system.audit.package.size", uint64(269))
  checkFieldValue(t, event, "system.audit.package.summary", "Test Package")
  checkFieldValue(t, event, "system.audit.package.url", "https://www.elastic.co/")
  checkFieldValue(t, event, "system.audit.package.version", "8.2.0-1ubuntu2~18.04")
  }
 }
 
+func TestDpkgInstalledSize(t *testing.T) {
+ expected := map[string]uint64{
+ "libquadmath0": 269,
+ "python-apt-common": 248,
+ "libnpth0": 32,
+ "bind9-host": 17,
+ "libpam-runtime": 300,
+ "libsepol1-dev": 1739 * 1024,
+ "libisl19": 17 * 1024 * 1024,
+ "netbase": 0,
+ "python2.7-minimal": 0,
+ }
+
+ logp.TestingSetup()
+
+ defer abtest.SetupDataDir(t)()
+
+ // Disable all except dpkg
+ rpmPathOld := rpmPath
+ dpkgPathOld := dpkgPath
+ brewPathOld := homebrewCellarPath
+ defer func() {
+ rpmPath = rpmPathOld
+ dpkgPath = dpkgPathOld
+ homebrewCellarPath = brewPathOld
+ }()
+ rpmPath = "/does/not/exist"
+ homebrewCellarPath = "/does/not/exist"
+
+ var err error
+ dpkgPath, err = filepath.Abs("testdata/dpkg-size/")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ f := mbtest.NewReportingMetricSetV2(t, getConfig())
+ defer f.(*MetricSet).bucket.DeleteBucket()
+
+ events, errs := mbtest.ReportingFetchV2(f)
+ if len(errs) > 0 {
+ t.Fatalf("received error: %+v", errs[0])
+ }
+
+ got := make(map[string]uint64, len(events))
+ for _, ev := range events {
+ event := mbtest.StandardizeEvent(f, ev, core.AddDatasetToEvent)
+ name, err := event.GetValue("system.audit.package.name")
+ if err != nil {
+ t.Fatal(err)
+ }
+ size, err := event.GetValue("system.audit.package.size")
+ if err != nil {
+ size = uint64(0)
+ }
+ if !assert.IsType(t, "", name) {
+ t.Fatal("string expected")
+ }
+ if !assert.IsType(t, uint64(0), size) {
+ t.Fatal("uint64 expected")
+ }
+ got[name.(string)] = size.(uint64)
+ }
+ assert.Equal(t, expected, got)
+}
+
 func getConfig() map[string]interface{} {
  return map[string]interface{}{
  "module": "system",

diff --git a/x-pack/auditbeat/module/system/package/testdata/dpkg-size/status b/x-pack/auditbeat/module/system/package/testdata/dpkg-size/status
@@ -0,0 +1,190 @@
+Package: libquadmath0
+Status: install ok installed
+Priority: optional
+Section: libs
+Installed-Size: 269B
+Maintainer: Ubuntu Core developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: same
+Source: gcc-8
+Version: 8.3.0-26ubuntu1~18.04
+Depends: gcc-8-base (= 8.3.0-26ubuntu1~18.04), libc6 (>= 2.23)
+Description: GCC Quad-Precision Math Library
+ A library, which provides quad-precision mathematical functions on targets
+ supporting the __float128 datatype. The library is used to provide on such
+ targets the REAL(16) type in the GNU Fortran compiler.
+Homepage: http://gcc.gnu.org/
+Original-Maintainer: Debian GCC Maintainers <debian-gcc@lists.debian.org>
+
+Package: python-apt-common
+Status: install ok installed
+Priority: optional
+Section: python
+Installed-Size: 248KiB
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: all
+Source: python-apt
+Version: 1.6.5ubuntu0.2
+Replaces: python-apt (<< 0.7.98+nmu1)
+Breaks: python-apt (<< 0.7.98+nmu1)
+Enhances: python-apt, python3-apt
+Description: Python interface to libapt-pkg (locales)
+ The apt_pkg Python interface will provide full access to the internal
+ libapt-pkg structures allowing Python programs to easily perform a
+ variety of functions.
+ .
+ This package contains locales.
+Original-Maintainer: APT Development Team <deity@lists.debian.org>
+
+Package: libnpth0
+Status: install ok installed
+Priority: optional
+Section: libs
+Installed-Size: 32trash
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: same
+Source: npth
+Version: 1.5-3
+Depends: libc6 (>= 2.17)
+Description: replacement for GNU Pth using system threads
+ nPth is a non-preemptive threads implementation using an API very
+ similar to the one known from GNU Pth. It has been designed as a
+ replacement of GNU Pth for non-ancient operating systems. In
+ contrast to GNU Pth it is based on the system's standard threads
+ implementation. Thus nPth allows the use of libraries which are not
+ compatible to GNU Pth.
+Original-Maintainer: Eric Dorland <eric@debian.org>
+Homepage: https://www.gnupg.org/
+
+Package: bind9-host
+Status: install ok installed
+Priority: standard
+Section: net
+Installed-Size: 17.4
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Source: bind9
+Version: 1:9.11.3+dfsg-1ubuntu1.11
+Provides: host
+Depends: libbind9-160 (= 1:9.11.3+dfsg-1ubuntu1.11), libdns1100 (= 1:9.11.3+dfsg-1ubuntu1.11), libisc169 (= 1:9.11.3+dfsg-1ubuntu1.11), libisccfg160 (= 1:9.11.3+dfsg-1ubuntu1.11), liblwres160 (= 1:9.11.3+dfsg-1ubuntu1.11), libc6 (>= 2.4)
+Description: DNS lookup utility (deprecated)
+ This package provides /usr/bin/host, a simple utility (bundled with the
+ BIND 9.X sources) which can be used for converting domain names to IP
+ addresses and the reverse.
+ .
+ This utility is deprecated, use dig or delv from the dnsutils package.
+Homepage: https://www.isc.org/downloads/bind/
+Original-Maintainer: Debian DNS Packaging <pkg-dns-devel@lists.alioth.debian.org>
+
+Package: libpam-runtime
+Status: install ok installed
+Priority: required
+Section: admin
+Installed-Size: 300T
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: all
+Multi-Arch: foreign
+Source: pam
+Version: 1.1.8-3.6ubuntu2
+Replaces: libpam0g-dev, libpam0g-util
+Depends: debconf (>= 0.5) | debconf-2.0, debconf (>= 1.5.19) | cdebconf, libpam-modules (>= 1.0.1-6)
+Conflicts: libpam0g-util
+Conffiles:
+ /etc/pam.conf 87fc76f18e98ee7d3848f6b81b3391e5
+ /etc/pam.d/other 31aa7f2181889ffb00b87df4126d1701
+Description: Runtime support for the PAM library
+ Contains configuration files and directories required for
+ authentication to work on Debian systems. This package is required
+ on almost all installations.
+Homepage: http://www.linux-pam.org/
+Original-Maintainer: Steve Langasek <vorlon@debian.org>
+
+Package: libsepol1-dev
+Status: install ok installed
+Priority: optional
+Section: libdevel
+Installed-Size: 1739m
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: same
+Source: libsepol
+Version: 2.7-1
+Provides: libsepol-dev
+Depends: libsepol1 (= 2.7-1)
+Conflicts: libsepol-dev
+Description: SELinux binary policy manipulation library and development files
+ libsepol allows programs to easily modify SELinux binary policies. This
+ means changing the default values for booleans, or reading the policy for
+ analysis.
+ .
+ This package contains the headers and archives used for linking it into your
+ programs.
+Original-Maintainer: Debian SELinux maintainers <selinux-devel@lists.alioth.debian.org>
+Homepage: http://userspace.selinuxproject.org/
+
+Package: libisl19
+Status: install ok installed
+Priority: optional
+Section: libs
+Installed-Size: 17G
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: same
+Source: isl
+Version: 0.19-1
+Replaces: libisl-dbg (<< 0.19)
+Depends: libc6 (>= 2.14), libgmp10
+Breaks: libisl-dbg (<< 0.19)
+Description: manipulating sets and relations of integer points bounded by linear constraints
+ isl is a library for manipulating sets and relations of integer points
+ bounded by linear constraints. Supported operations on sets include
+ intersection, union, set difference, emptiness check, convex hull,
+ (integer) affine hull, integer projection, and computing the lexicographic
+ minimum using parametric integer programming. It also includes an ILP solver
+ based on generalized basis reduction.
+ .
+ This package contains the runtime library.
+Original-Maintainer: Debian GCC Maintainers <debian-gcc@lists.debian.org>
+Homepage: http://isl.gforge.inria.fr/
+
+Package: netbase
+Status: install ok installed
+Priority: important
+Section: admin
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: all
+Multi-Arch: foreign
+Version: 5.4
+Conffiles:
+ /etc/protocols bb9c019d6524e913fd72441d58b68216
+ /etc/rpc f0b6f6352bf886623adc04183120f83b
+ /etc/services 567c100888518c1163b3462993de7d47
+Description: Basic TCP/IP networking system
+ This package provides the necessary infrastructure for basic TCP/IP based
+ networking.
+Original-Maintainer: Marco d'Itri <md@linux.it>
+
+Package: python2.7-minimal
+Status: install ok installed
+Priority: optional
+Section: python
+Installed-Size: Who knows
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Architecture: amd64
+Multi-Arch: allowed
+Source: python2.7
+Version: 2.7.17-1~18.04
+Replaces: python2.7 (<< 2.7.8-7~)
+Depends: libpython2.7-minimal (= 2.7.17-1~18.04)
+Pre-Depends: libc6 (>= 2.15), zlib1g (>= 1:1.2.0)
+Recommends: python2.7
+Suggests: binfmt-support
+Conflicts: binfmt-support (<< 1.1.2)
+Description: Minimal subset of the Python language (version 2.7)
+ This package contains the interpreter and some essential modules. It can
+ be used in the boot process for some basic tasks.
+ See /usr/share/doc/python2.7-minimal/README.Debian for a list of the modules
+ contained in this package.
+Original-Maintainer: Matthias Klose <doko@debian.org>
+