High speed scan on Ubuntu/Debian

.gitignore

@@ -3,6 +3,8 @@ vuls
 *.txt
 *.json
 *.sqlite3
+*.db
+tags
 .gitmodules
 coverage.out
 issues/

Makefile

@@ -11,7 +11,7 @@
 	clean
 
 SRCS = $(shell git ls-files '*.go')
-PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands
+PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands ./cache
 
 all: test
 

README.ja.md

@@ -301,22 +301,24 @@ $ vuls tui
 # Performance Considerations
 
 - Ubuntu, Debian  
-アップデート対象のパッケージが沢山ある場合は、毎回apt-get changelogするので遅いし、スキャン対象サーバのリソースを消費する。
+`apt-get changelog`でアップデート対象のパッケージのチェンジログを取得し、含まれるCVE IDをパースする。
+アップデート対象のパッケージが沢山ある場合、チェンジログの取得に時間がかかるので、初回のスキャンは遅い。  
+ただ、２回目以降はキャッシュしたchangelogを使うので速くなる。  
 
 - CentOS  
-アップデート対象すべてのchangelogを一度で取得しパースする。スキャンスピードは高速、サーバリソース消費量は小さい。
+アップデート対象すべてのchangelogを一度で取得しパースする。スキャンスピードは速い、サーバリソース消費量は小さい。
 
 - Amazon, RHEL and FreeBSD  
 高速にスキャンし、スキャン対象サーバのリソース消費量は小さい。
 
-| Distribution|         Scan Speed | Resource Usage On Target Server |
-|:------------|:-------------------|:-------------|
-| Ubuntu      |               Slow | Heavy            |
-| Debian      |               Slow | Heavy            |
-| CentOS      |               Fast | Light            |
-| Amazon      |               Fast | Light            |
-| RHEL        |               Fast | Light            |
-| FreeBSD     |               Fast | Light            |
+| Distribution|         Scan Speed | 
+|:------------|:-------------------|
+| Ubuntu      | 初回は遅い / 2回目以降速い　|
+| Debian      | 初回は遅い / 2回目以降速い　|
+| CentOS      |               速い |
+| Amazon      |               速い | 
+| RHEL        |               速い | 
+| FreeBSD     |               速い |
 
 ----
 
@@ -340,7 +342,7 @@ web/app server in the same configuration under the load balancer
 |:------------|-------------------:|
 | Ubuntu      |          12, 14, 16|
 | Debian      |                7, 8|
-| RHEL        |          4, 5, 6, 7|
+| RHEL        |                6, 7|
 | CentOS      |             5, 6, 7|
 | Amazon Linux|                 All|
 | FreeBSD     |                  10|
@@ -595,7 +597,6 @@ prepare
 # Usage: Scan
 
 ```
-
 $ vuls scan -help
 scan:
         scan
@@ -604,6 +605,7 @@ scan:
                 [-results-dir=/path/to/results]
                 [-cve-dictionary-dbpath=/path/to/cve.sqlite3]
                 [-cve-dictionary-url=http://127.0.0.1:1323]
+                [-cache-dbpath=/path/to/cache.db]
                 [-cvss-over=7]
                 [-ignore-unscored-cves]
                 [-ssh-external]
@@ -626,7 +628,6 @@ scan:
                 [SERVER]...
 
 
-
   -ask-key-password
         Ask ssh privatekey password before scanning
   -aws-profile string
@@ -641,6 +642,8 @@ scan:
         Azure storage container name
   -azure-key string
         Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified
+  -cache-dbpath string
+        /path/to/cache.db (local cache of changelog for Ubuntu/Debian) (default "$PWD/cache.db")
   -config string
         /path/to/toml (default "$PWD/config.toml")
   -cve-dictionary-dbpath string
@@ -649,8 +652,6 @@ scan:
         http://CVE.Dictionary (default "http://127.0.0.1:1323")
   -cvss-over float
         -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))
-  -results-dir string
-        /path/to/results (default "$PWD/results")
   -debug
         debug mode
   -debug-sql
@@ -671,6 +672,8 @@ scan:
         Send report via Slack
   -report-text
         Write report to text files ($PWD/results/current)
+  -results-dir string
+        /path/to/results (default "$PWD/results")
   -ssh-external
         Use external ssh command. Default: Use the Go native implementation
 ```

README.md

@@ -297,25 +297,27 @@ see https://github.com/future-architect/vuls/tree/master/setup/docker
 ----
 # Performance Considerations
 
-- on Ubuntu and Debian  
+- On Ubuntu and Debian  
 Vuls issues `apt-get changelog` for each upgradable packages and parse the changelog.  
-`apt-get changelog` is slow and resource usage is heavy when there are many updatable packages on target server.  
+`apt-get changelog` is slow and resource usage is heavy when there are many updatable packages on target server.   
+Vuls stores these changelogs to KVS([boltdb](https://github.com/boltdb/bolt)).  
+From the second time on, the scan speed is fast by using the local cache.
 
-- on CentOS  
+- On CentOS  
 Vuls issues `yum update --changelog` to get changelogs of upgradable packages at once and parse the changelog.  
 Scan speed is fast and resource usage is light.  
 
 - On Amazon, RHEL and FreeBSD  
 High speed scan and resource usage is light because Vuls can get CVE IDs by using package manager(no need to parse a changelog).
 
-| Distribution|         Scan Speed | Resource Usage On Target Server |
+| Distribution|         Scan Speed | 
 |:------------|:-------------------|:-------------|
-| Ubuntu      |               Slow | Heavy            |
-| Debian      |               Slow | Heavy            |
-| CentOS      |               Fast | Light            |
-| Amazon      |               Fast | Light            |
-| RHEL        |               Fast | Light            |
-| FreeBSD     |               Fast | Light            |
+| Ubuntu      |  First time: Slow / From the second time: Fast |
+| Debian      |  First time: Slow / From the second time: Fast |
+| CentOS      |               Fast |
+| Amazon      |               Fast |
+| RHEL        |               Fast |
+| FreeBSD     |               Fast |
 
 ----
 
@@ -339,7 +341,7 @@ web/app server in the same configuration under the load balancer
 |:------------|-------------------:|
 | Ubuntu      |          12, 14, 16|
 | Debian      |                7, 8|
-| RHEL        |          4, 5, 6, 7|
+| RHEL        |                6, 7|
 | CentOS      |             5, 6, 7|
 | Amazon Linux|                 All|
 | FreeBSD     |                  10|
@@ -603,6 +605,7 @@ scan:
                 [-results-dir=/path/to/results]
                 [-cve-dictionary-dbpath=/path/to/cve.sqlite3]
                 [-cve-dictionary-url=http://127.0.0.1:1323]
+                [-cache-dbpath=/path/to/cache.db]
                 [-cvss-over=7]
                 [-ignore-unscored-cves]
                 [-ssh-external]
@@ -639,6 +642,8 @@ scan:
         Azure storage container name
   -azure-key string
         Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified
+  -cache-dbpath string
+        /path/to/cache.db (local cache of changelog for Ubuntu/Debian) (default "$PWD/cache.db")
   -config string
         /path/to/toml (default "$PWD/config.toml")
   -cve-dictionary-dbpath string
@@ -647,8 +652,6 @@ scan:
         http://CVE.Dictionary (default "http://127.0.0.1:1323")
   -cvss-over float
         -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))
-  -results-dir string
-        /path/to/results (default "$PWD/results")
   -debug
         debug mode
   -debug-sql
@@ -669,6 +672,8 @@ scan:
         Send report via Slack
   -report-text
         Write report to text files ($PWD/results/current)
+  -results-dir string
+        /path/to/results (default "$PWD/results")
   -ssh-external
         Use external ssh command. Default: Use the Go native implementation
 ```

cache/bolt.go

@@ -0,0 +1,173 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package cache
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/boltdb/bolt"
+	"github.com/future-architect/vuls/util"
+)
+
+// Bolt holds a pointer of bolt.DB
+// boltdb is used to store a cache of Changelogs of Ubuntu/Debian
+type Bolt struct {
+	Path string
+	Log  *logrus.Entry
+	db   *bolt.DB
+}
+
+// SetupBolt opens a boltdb and creates a meta bucket if not exists.
+func SetupBolt(path string, l *logrus.Entry) error {
+	l.Infof("Open boltDB: %s", path)
+	db, err := bolt.Open(path, 0600, nil)
+	if err != nil {
+		return err
+	}
+
+	b := Bolt{
+		Path: path,
+		Log:  l,
+		db:   db,
+	}
+	if err = b.createBucketIfNotExists(metabucket); err != nil {
+		return err
+	}
+
+	DB = b
+	return nil
+}
+
+// Close a db.
+func (b Bolt) Close() error {
+	if b.db == nil {
+		return nil
+	}
+	return b.db.Close()
+}
+
+//  CreateBucketIfNotExists creates a buket that is specified by arg.
+func (b *Bolt) createBucketIfNotExists(name string) error {
+	return b.db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(name))
+		if err != nil {
+			return fmt.Errorf("Failed to create bucket: %s", err)
+		}
+		return nil
+	})
+}
+
+// GetMeta gets a Meta Information os the servername to boltdb.
+func (b Bolt) GetMeta(serverName string) (meta Meta, found bool, err error) {
+	err = b.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket([]byte(metabucket))
+		v := bkt.Get([]byte(serverName))
+		if len(v) == 0 {
+			found = false
+			return nil
+		}
+		if e := json.Unmarshal(v, &meta); e != nil {
+			return e
+		}
+		found = true
+		return nil
+	})
+	return
+}
+
+// EnsureBuckets puts a Meta information and create a buket that holds changelogs.
+func (b Bolt) EnsureBuckets(meta Meta) error {
+	jsonBytes, err := json.Marshal(meta)
+	if err != nil {
+		return fmt.Errorf("Failed to marshal to JSON: %s", err)
+	}
+	return b.db.Update(func(tx *bolt.Tx) error {
+		b.Log.Debugf("Put to meta: %s", meta.Name)
+		bkt := tx.Bucket([]byte(metabucket))
+		if err := bkt.Put([]byte(meta.Name), jsonBytes); err != nil {
+			return err
+		}
+
+		// re-create a bucket (bucket name: servername)
+		bkt = tx.Bucket([]byte(meta.Name))
+		if bkt != nil {
+			b.Log.Debugf("Delete bucket: %s", meta.Name)
+			if err := tx.DeleteBucket([]byte(meta.Name)); err != nil {
+				return err
+			}
+			b.Log.Debugf("Bucket deleted: %s", meta.Name)
+		}
+		b.Log.Debugf("Create bucket: %s", meta.Name)
+		if _, err := tx.CreateBucket([]byte(meta.Name)); err != nil {
+			return err
+		}
+		b.Log.Debugf("Bucket created: %s", meta.Name)
+		return nil
+	})
+}
+
+// PrettyPrint is for debuging
+func (b Bolt) PrettyPrint(meta Meta) error {
+	return b.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket([]byte(metabucket))
+		v := bkt.Get([]byte(meta.Name))
+		b.Log.Debugf("key:%s, value:%s", meta.Name, v)
+
+		bkt = tx.Bucket([]byte(meta.Name))
+		c := bkt.Cursor()
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			b.Log.Debugf("key:%s, len: %d, %s...",
+				k, len(v), util.Truncate(string(v), 30))
+		}
+		return nil
+	})
+}
+
+// GetChangelog get the changelgo of specified packName from the Bucket
+func (b Bolt) GetChangelog(servername, packName string) (changelog string, err error) {
+	err = b.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket([]byte(servername))
+		if bkt == nil {
+			return fmt.Errorf("Faild to get Bucket: %s", servername)
+		}
+		v := bkt.Get([]byte(packName))
+		if v == nil {
+			changelog = ""
+			return nil
+		}
+		changelog = string(v)
+		return nil
+	})
+	return
+}
+
+// PutChangelog put the changelgo of specified packName into the Bucket
+func (b Bolt) PutChangelog(servername, packName, changelog string) error {
+	return b.db.Update(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket([]byte(servername))
+		if bkt == nil {
+			return fmt.Errorf("Faild to get Bucket: %s", servername)
+		}
+		if err := bkt.Put([]byte(packName), []byte(changelog)); err != nil {
+			return err
+		}
+		return nil
+	})
+}