diff --git a/.gitignore b/.gitignore index 5c2591bc2f..d80b0b3bdf 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ vuls *.txt *.json *.sqlite3 +*.db +tags .gitmodules coverage.out issues/ diff --git a/Makefile b/Makefile index f1e0d2ffa1..7b6d761df5 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ clean SRCS = $(shell git ls-files '*.go') -PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands +PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands ./cache all: test diff --git a/README.ja.md b/README.ja.md index 550b133013..31481e0a15 100644 --- a/README.ja.md +++ b/README.ja.md @@ -301,22 +301,24 @@ $ vuls tui # Performance Considerations - Ubuntu, Debian -アップデート対象のパッケージが沢山ある場合は、毎回apt-get changelogするので遅いし、スキャン対象サーバのリソースを消費する。 +`apt-get changelog`でアップデート対象のパッケージのチェンジログを取得し、含まれるCVE IDをパースする。 +アップデート対象のパッケージが沢山ある場合、チェンジログの取得に時間がかかるので、初回のスキャンは遅い。 +ただ、２回目以降はキャッシュしたchangelogを使うので速くなる。 - CentOS -アップデート対象すべてのchangelogを一度で取得しパースする。スキャンスピードは高速、サーバリソース消費量は小さい。 +アップデート対象すべてのchangelogを一度で取得しパースする。スキャンスピードは速い、サーバリソース消費量は小さい。 - Amazon, RHEL and FreeBSD 高速にスキャンし、スキャン対象サーバのリソース消費量は小さい。 -| Distribution| Scan Speed | Resource Usage On Target Server | -|:------------|:-------------------|:-------------| -| Ubuntu | Slow | Heavy | -| Debian | Slow | Heavy | -| CentOS | Fast | Light | -| Amazon | Fast | Light | -| RHEL | Fast | Light | -| FreeBSD | Fast | Light | +| Distribution| Scan Speed | +|:------------|:-------------------| +| Ubuntu | 初回は遅い / 2回目以降速い　| +| Debian | 初回は遅い / 2回目以降速い　| +| CentOS | 速い | +| Amazon | 速い | +| RHEL | 速い | +| FreeBSD | 速い | ---- @@ -340,7 +342,7 @@ web/app server in the same configuration under the load balancer |:------------|-------------------:| | Ubuntu | 12, 14, 16| | Debian | 7, 8| -| RHEL | 4, 5, 6, 7| +| RHEL | 6, 7| | CentOS | 5, 6, 7| | Amazon Linux| All| | FreeBSD | 10| @@ -595,7 +597,6 @@ prepare # Usage: Scan ``` - $ vuls scan -help scan: scan @@ -604,6 +605,7 @@ scan: [-results-dir=/path/to/results] [-cve-dictionary-dbpath=/path/to/cve.sqlite3] [-cve-dictionary-url=http://127.0.0.1:1323] + [-cache-dbpath=/path/to/cache.db] [-cvss-over=7] [-ignore-unscored-cves] [-ssh-external] @@ -626,7 +628,6 @@ scan: [SERVER]... - -ask-key-password Ask ssh privatekey password before scanning -aws-profile string @@ -641,6 +642,8 @@ scan: Azure storage container name -azure-key string Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified + -cache-dbpath string + /path/to/cache.db (local cache of changelog for Ubuntu/Debian) (default "$PWD/cache.db") -config string /path/to/toml (default "$PWD/config.toml") -cve-dictionary-dbpath string @@ -649,8 +652,6 @@ scan: http://CVE.Dictionary (default "http://127.0.0.1:1323") -cvss-over float -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all)) - -results-dir string - /path/to/results (default "$PWD/results") -debug debug mode -debug-sql @@ -671,6 +672,8 @@ scan: Send report via Slack -report-text Write report to text files ($PWD/results/current) + -results-dir string + /path/to/results (default "$PWD/results") -ssh-external Use external ssh command. Default: Use the Go native implementation ``` diff --git a/README.md b/README.md index b46f63fe1d..97d5ed85c2 100644 --- a/README.md +++ b/README.md @@ -297,25 +297,27 @@ see https://github.com/future-architect/vuls/tree/master/setup/docker ---- # Performance Considerations -- on Ubuntu and Debian +- On Ubuntu and Debian Vuls issues `apt-get changelog` for each upgradable packages and parse the changelog. -`apt-get changelog` is slow and resource usage is heavy when there are many updatable packages on target server. +`apt-get changelog` is slow and resource usage is heavy when there are many updatable packages on target server. +Vuls stores these changelogs to KVS([boltdb](https://github.com/boltdb/bolt)). +From the second time on, the scan speed is fast by using the local cache. -- on CentOS +- On CentOS Vuls issues `yum update --changelog` to get changelogs of upgradable packages at once and parse the changelog. Scan speed is fast and resource usage is light. - On Amazon, RHEL and FreeBSD High speed scan and resource usage is light because Vuls can get CVE IDs by using package manager(no need to parse a changelog). -| Distribution| Scan Speed | Resource Usage On Target Server | +| Distribution| Scan Speed | |:------------|:-------------------|:-------------| -| Ubuntu | Slow | Heavy | -| Debian | Slow | Heavy | -| CentOS | Fast | Light | -| Amazon | Fast | Light | -| RHEL | Fast | Light | -| FreeBSD | Fast | Light | +| Ubuntu | First time: Slow / From the second time: Fast | +| Debian | First time: Slow / From the second time: Fast | +| CentOS | Fast | +| Amazon | Fast | +| RHEL | Fast | +| FreeBSD | Fast | ---- @@ -339,7 +341,7 @@ web/app server in the same configuration under the load balancer |:------------|-------------------:| | Ubuntu | 12, 14, 16| | Debian | 7, 8| -| RHEL | 4, 5, 6, 7| +| RHEL | 6, 7| | CentOS | 5, 6, 7| | Amazon Linux| All| | FreeBSD | 10| @@ -603,6 +605,7 @@ scan: [-results-dir=/path/to/results] [-cve-dictionary-dbpath=/path/to/cve.sqlite3] [-cve-dictionary-url=http://127.0.0.1:1323] + [-cache-dbpath=/path/to/cache.db] [-cvss-over=7] [-ignore-unscored-cves] [-ssh-external] @@ -639,6 +642,8 @@ scan: Azure storage container name -azure-key string Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified + -cache-dbpath string + /path/to/cache.db (local cache of changelog for Ubuntu/Debian) (default "$PWD/cache.db") -config string /path/to/toml (default "$PWD/config.toml") -cve-dictionary-dbpath string @@ -647,8 +652,6 @@ scan: http://CVE.Dictionary (default "http://127.0.0.1:1323") -cvss-over float -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all)) - -results-dir string - /path/to/results (default "$PWD/results") -debug debug mode -debug-sql @@ -669,6 +672,8 @@ scan: Send report via Slack -report-text Write report to text files ($PWD/results/current) + -results-dir string + /path/to/results (default "$PWD/results") -ssh-external Use external ssh command. Default: Use the Go native implementation ``` diff --git a/cache/bolt.go b/cache/bolt.go new file mode 100644 index 0000000000..b085c71d8c --- /dev/null +++ b/cache/bolt.go @@ -0,0 +1,173 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package cache + +import ( + "encoding/json" + "fmt" + + "github.com/Sirupsen/logrus" + "github.com/boltdb/bolt" + "github.com/future-architect/vuls/util" +) + +// Bolt holds a pointer of bolt.DB +// boltdb is used to store a cache of Changelogs of Ubuntu/Debian +type Bolt struct { + Path string + Log *logrus.Entry + db *bolt.DB +} + +// SetupBolt opens a boltdb and creates a meta bucket if not exists. +func SetupBolt(path string, l *logrus.Entry) error { + l.Infof("Open boltDB: %s", path) + db, err := bolt.Open(path, 0600, nil) + if err != nil { + return err + } + + b := Bolt{ + Path: path, + Log: l, + db: db, + } + if err = b.createBucketIfNotExists(metabucket); err != nil { + return err + } + + DB = b + return nil +} + +// Close a db. +func (b Bolt) Close() error { + if b.db == nil { + return nil + } + return b.db.Close() +} + +// CreateBucketIfNotExists creates a buket that is specified by arg. +func (b *Bolt) createBucketIfNotExists(name string) error { + return b.db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(name)) + if err != nil { + return fmt.Errorf("Failed to create bucket: %s", err) + } + return nil + }) +} + +// GetMeta gets a Meta Information os the servername to boltdb. +func (b Bolt) GetMeta(serverName string) (meta Meta, found bool, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(metabucket)) + v := bkt.Get([]byte(serverName)) + if len(v) == 0 { + found = false + return nil + } + if e := json.Unmarshal(v, &meta); e != nil { + return e + } + found = true + return nil + }) + return +} + +// EnsureBuckets puts a Meta information and create a buket that holds changelogs. +func (b Bolt) EnsureBuckets(meta Meta) error { + jsonBytes, err := json.Marshal(meta) + if err != nil { + return fmt.Errorf("Failed to marshal to JSON: %s", err) + } + return b.db.Update(func(tx *bolt.Tx) error { + b.Log.Debugf("Put to meta: %s", meta.Name) + bkt := tx.Bucket([]byte(metabucket)) + if err := bkt.Put([]byte(meta.Name), jsonBytes); err != nil { + return err + } + + // re-create a bucket (bucket name: servername) + bkt = tx.Bucket([]byte(meta.Name)) + if bkt != nil { + b.Log.Debugf("Delete bucket: %s", meta.Name) + if err := tx.DeleteBucket([]byte(meta.Name)); err != nil { + return err + } + b.Log.Debugf("Bucket deleted: %s", meta.Name) + } + b.Log.Debugf("Create bucket: %s", meta.Name) + if _, err := tx.CreateBucket([]byte(meta.Name)); err != nil { + return err + } + b.Log.Debugf("Bucket created: %s", meta.Name) + return nil + }) +} + +// PrettyPrint is for debuging +func (b Bolt) PrettyPrint(meta Meta) error { + return b.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(metabucket)) + v := bkt.Get([]byte(meta.Name)) + b.Log.Debugf("key:%s, value:%s", meta.Name, v) + + bkt = tx.Bucket([]byte(meta.Name)) + c := bkt.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + b.Log.Debugf("key:%s, len: %d, %s...", + k, len(v), util.Truncate(string(v), 30)) + } + return nil + }) +} + +// GetChangelog get the changelgo of specified packName from the Bucket +func (b Bolt) GetChangelog(servername, packName string) (changelog string, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(servername)) + if bkt == nil { + return fmt.Errorf("Faild to get Bucket: %s", servername) + } + v := bkt.Get([]byte(packName)) + if v == nil { + changelog = "" + return nil + } + changelog = string(v) + return nil + }) + return +} + +// PutChangelog put the changelgo of specified packName into the Bucket +func (b Bolt) PutChangelog(servername, packName, changelog string) error { + return b.db.Update(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(servername)) + if bkt == nil { + return fmt.Errorf("Faild to get Bucket: %s", servername) + } + if err := bkt.Put([]byte(packName), []byte(changelog)); err != nil { + return err + } + return nil + }) +} diff --git a/cache/bolt_test.go b/cache/bolt_test.go new file mode 100644 index 0000000000..a8dad918ab --- /dev/null +++ b/cache/bolt_test.go @@ -0,0 +1,134 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package cache + +import ( + "os" + "reflect" + "testing" + + "github.com/Sirupsen/logrus" + "github.com/boltdb/bolt" + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" +) + +const path = "/tmp/vuls-test-cache-11111111.db" +const servername = "server1" + +var meta = Meta{ + Name: servername, + Distro: config.Distro{ + Family: "ubuntu", + Release: "16.04", + }, + Packs: []models.PackageInfo{ + { + Name: "apt", + Version: "1", + }, + }, +} + +func TestSetupBolt(t *testing.T) { + log := logrus.NewEntry(&logrus.Logger{}) + err := SetupBolt(path, log) + if err != nil { + t.Errorf("Failed to setup bolt: %s", err) + } + defer os.Remove(path) + + if err := DB.Close(); err != nil { + t.Errorf("Failed to close bolt: %s", err) + } + + // check if meta bucket exists + db, err := bolt.Open(path, 0600, nil) + if err != nil { + t.Errorf("Failed to open bolt: %s", err) + } + + db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(metabucket)) + if bkt == nil { + t.Errorf("Meta bucket nof found") + } + return nil + }) + +} + +func TestEnsureBuckets(t *testing.T) { + log := logrus.NewEntry(&logrus.Logger{}) + if err := SetupBolt(path, log); err != nil { + t.Errorf("Failed to setup bolt: %s", err) + } + if err := DB.EnsureBuckets(meta); err != nil { + t.Errorf("Failed to ensure buckets: %s", err) + } + defer os.Remove(path) + + m, found, err := DB.GetMeta(servername) + if err != nil { + t.Errorf("Failed to get meta: %s", err) + } + if !found { + t.Errorf("Not Found in meta") + } + if !reflect.DeepEqual(meta, m) { + t.Errorf("expected %v, actual %v", meta, m) + } + if err := DB.Close(); err != nil { + t.Errorf("Failed to close bolt: %s", err) + } + + db, err := bolt.Open(path, 0600, nil) + if err != nil { + t.Errorf("Failed to open bolt: %s", err) + } + db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(servername)) + if bkt == nil { + t.Errorf("Meta bucket nof found") + } + return nil + }) +} + +func TestPutGetChangelog(t *testing.T) { + clog := "changelog-text" + log := logrus.NewEntry(&logrus.Logger{}) + if err := SetupBolt(path, log); err != nil { + t.Errorf("Failed to setup bolt: %s", err) + } + defer os.Remove(path) + + if err := DB.EnsureBuckets(meta); err != nil { + t.Errorf("Failed to ensure buckets: %s", err) + } + if err := DB.PutChangelog(servername, "apt", clog); err != nil { + t.Errorf("Failed to put changelog: %s", err) + } + if actual, err := DB.GetChangelog(servername, "apt"); err != nil { + t.Errorf("Failed to get changelog: %s", err) + } else { + if actual != clog { + t.Errorf("changelog is not same. e: %s, a: %s", clog, actual) + } + } +} diff --git a/cache/db.go b/cache/db.go new file mode 100644 index 0000000000..607d7bf72f --- /dev/null +++ b/cache/db.go @@ -0,0 +1,56 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package cache + +import ( + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" +) + +// DB has a cache instance +var DB Cache + +const metabucket = "changelog-meta" + +// Cache is a interface of cache +type Cache interface { + Close() error + GetMeta(string) (Meta, bool, error) + EnsureBuckets(Meta) error + PrettyPrint(Meta) error + GetChangelog(string, string) (string, error) + PutChangelog(string, string, string) error +} + +// Meta holds a server name, distro information of the scanned server and +// package information that was collected at the last scan. +type Meta struct { + Name string + Distro config.Distro + Packs []models.PackageInfo +} + +// FindPack search a PackageInfo +func (m Meta) FindPack(name string) (pack models.PackageInfo, found bool) { + for _, p := range m.Packs { + if name == p.Name { + return p, true + } + } + return pack, false +} diff --git a/commands/scan.go b/commands/scan.go index 603cc70666..a7082b53eb 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -47,6 +47,7 @@ type ScanCmd struct { jsonBaseDir string cvedbpath string cveDictionaryURL string + cacheDBPath string cvssScoreOver float64 ignoreUnscoredCves bool @@ -89,6 +90,7 @@ func (*ScanCmd) Usage() string { [-results-dir=/path/to/results] [-cve-dictionary-dbpath=/path/to/cve.sqlite3] [-cve-dictionary-url=http://127.0.0.1:1323] + [-cache-dbpath=/path/to/cache.db] [-cvss-over=7] [-ignore-unscored-cves] [-ssh-external] @@ -140,6 +142,13 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { defaultURL, "http://CVE.Dictionary") + defaultCacheDBPath := filepath.Join(wd, "cache.db") + f.StringVar( + &p.cacheDBPath, + "cache-dbpath", + defaultCacheDBPath, + "/path/to/cache.db (local cache of changelog for Ubuntu/Debian)") + f.Float64Var( &p.cvssScoreOver, "cvss-over", @@ -341,6 +350,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) c.Conf.JSONBaseDir = p.jsonBaseDir c.Conf.CveDBPath = p.cvedbpath c.Conf.CveDictionaryURL = p.cveDictionaryURL + c.Conf.CacheDBPath = p.cacheDBPath c.Conf.CvssScoreOver = p.cvssScoreOver c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves c.Conf.SSHExternal = p.sshExternal diff --git a/config/config.go b/config/config.go index 35cf89dadc..a9de9bb96d 100644 --- a/config/config.go +++ b/config/config.go @@ -49,6 +49,7 @@ type Config struct { HTTPProxy string `valid:"url"` JSONBaseDir string CveDBPath string + CacheDBPath string AwsProfile string AwsRegion string @@ -69,14 +70,21 @@ func (c Config) Validate() bool { if len(c.JSONBaseDir) != 0 { if ok, _ := valid.IsFilePath(c.JSONBaseDir); !ok { errs = append(errs, fmt.Errorf( - "JSON base directory must be a *Absolute* file path. jsonBaseDir: %s", c.JSONBaseDir)) + "JSON base directory must be a *Absolute* file path. -results-dir: %s", c.JSONBaseDir)) } } if len(c.CveDBPath) != 0 { if ok, _ := valid.IsFilePath(c.CveDBPath); !ok { errs = append(errs, fmt.Errorf( - "SQLite3 DB(Cve Dictionary) path must be a *Absolute* file path. dbpath: %s", c.CveDBPath)) + "SQLite3 DB(Cve Dictionary) path must be a *Absolute* file path. -cve-dictionary-dbpath: %s", c.CveDBPath)) + } + } + + if len(c.CacheDBPath) != 0 { + if ok, _ := valid.IsFilePath(c.CacheDBPath); !ok { + errs = append(errs, fmt.Errorf( + "Cache DB path must be a *Absolute* file path. -cache-dbpath: %s", c.CacheDBPath)) } } @@ -230,7 +238,17 @@ type ServerInfo struct { // used internal LogMsgAnsiColor string // DebugLog Color Container Container - Family string + Distro Distro +} + +// Distro has distribution info +type Distro struct { + Family string + Release string +} + +func (l Distro) String() string { + return fmt.Sprintf("%s %s", l.Family, l.Release) } // IsContainer returns whether this ServerInfo is about container diff --git a/cveapi/cve_client.go b/cveapi/cve_client.go index 22a0db2907..eb33838183 100644 --- a/cveapi/cve_client.go +++ b/cveapi/cve_client.go @@ -186,41 +186,6 @@ func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errCh } } -// func (api cvedictClient) httpGet(key, url string, query map[string]string, resChan chan<- response, errChan chan<- error) { - -// var body string -// var errs []error -// var resp *http.Response -// f := func() (err error) { -// req := gorequest.New().SetDebug(true).Proxy(api.httpProxy).Get(url) -// for key := range query { -// req = req.Query(fmt.Sprintf("%s=%s", key, query[key])).Set("Content-Type", "application/x-www-form-urlencoded") -// } -// pp.Println(req) -// resp, body, errs = req.End() -// if 0 < len(errs) || resp.StatusCode != 200 { -// errChan <- fmt.Errorf("HTTP error. errs: %v, url: %s", errs, url) -// } -// return nil -// } -// notify := func(err error, t time.Duration) { -// log.Warnf("Failed to get. retrying in %s seconds. err: %s", t, err) -// } -// err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) -// if err != nil { -// errChan <- fmt.Errorf("HTTP Error %s", err) -// } -// // resChan <- body -// cveDetail := cve.CveDetail{} -// if err := json.Unmarshal([]byte(body), &cveDetail); err != nil { -// errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err) -// } -// resChan <- response{ -// key, -// cveDetail, -// } -// } - type responseGetCveDetailByCpeName struct { CpeName string CveDetails []cve.CveDetail diff --git a/img/vuls-architecture.graphml b/img/vuls-architecture.graphml index c85fe548f2..6b6f723742 100644 --- a/img/vuls-architecture.graphml +++ b/img/vuls-architecture.graphml @@ -222,14 +222,14 @@ ALAS (Amazon) - + - Vuls + Vuls - + @@ -248,7 +248,7 @@ ALAS (Amazon) - + Report @@ -296,6 +296,24 @@ ALAS (Amazon) + + + + + + + Web View +(Vulsrepo) + + + + + + + + + + @@ -319,30 +337,6 @@ ALAS (Amazon) - - - - - - - - - - - - - - SQLite3 - - - - - - - - - - @@ -356,7 +350,7 @@ ALAS (Amazon) - + @@ -379,20 +373,20 @@ ALAS (Amazon) - + - + - go-cve-dictionary + go-cve-dictionary - + @@ -407,12 +401,12 @@ ALAS (Amazon) - - + + - - + + @@ -432,10 +426,10 @@ ALAS (Amazon) - + - + HTTP server @@ -449,7 +443,7 @@ ALAS (Amazon) - + @@ -468,7 +462,7 @@ ALAS (Amazon) - + @@ -496,8 +490,8 @@ ALAS (Amazon) - - + + @@ -514,7 +508,7 @@ ALAS (Amazon) - + @@ -534,7 +528,7 @@ Container - + @@ -562,8 +556,8 @@ Container - - + + @@ -580,7 +574,7 @@ Container - + @@ -599,7 +593,89 @@ Container - + + + + + + + + + + results dir + + + + + + + + + + Folder 7 + + + + + + + + + + + + + + + + JSON + + + + + + + + + + + + + + + + + JSON + + + + + + + + + + + + + + + + + JSON + + + + + + + + + + + + + @@ -636,17 +712,17 @@ Vulnerability data - + - HTTP + HTTP or --cve-dictoianry-dbpath option - + @@ -678,7 +754,7 @@ Vulnerability data - send + send @@ -696,7 +772,7 @@ Vulnerability data - Generate + Generate @@ -714,11 +790,11 @@ Vulnerability data - Detail Information + View Detail Information - + @@ -726,7 +802,7 @@ Vulnerability data - + @@ -744,7 +820,7 @@ Vulnerability data - + @@ -762,7 +838,7 @@ Vulnerability data - + @@ -780,7 +856,7 @@ Vulnerability data - + @@ -790,7 +866,7 @@ Vulnerability data - + @@ -800,13 +876,13 @@ Vulnerability data - + - Insert + Insert @@ -818,18 +894,46 @@ Vulnerability data - + + + + + + + + + + + + + + + + + Notify + + + + + + + + + + + + - Insert -Scan Result + + - + @@ -837,13 +941,13 @@ Scan Result - + - Select + Select @@ -855,7 +959,7 @@ Scan Result - + @@ -865,58 +969,34 @@ Scan Result - + - Notify - - - - - - - - + - --cve-dictoianry-dbpath option - - - - - - - - + - Select - - - - - - - diff --git a/img/vuls-architecture.png b/img/vuls-architecture.png index ae5afebf22..eeeb7d2e01 100644 Binary files a/img/vuls-architecture.png and b/img/vuls-architecture.png differ diff --git a/img/vuls-scan-flow.graphml b/img/vuls-scan-flow.graphml index 8fd15da251..ab1269a46a 100644 --- a/img/vuls-scan-flow.graphml +++ b/img/vuls-scan-flow.graphml @@ -72,7 +72,7 @@ FreeBSD: pkg - Get upgradable packages + Check upgradable packages Debian/Ubuntu: apt-get upgrade --dry-run @@ -222,7 +222,7 @@ Reporting - Get all changelogs by using package manager + Get all changelogs of updatable packages at once CentOS: yum update --changelog @@ -397,7 +397,6 @@ FreeBSD - @@ -416,7 +415,6 @@ FreeBSD - @@ -427,7 +425,6 @@ FreeBSD - diff --git a/img/vuls-scan-flow.png b/img/vuls-scan-flow.png index a29dbee525..4bb215c22f 100644 Binary files a/img/vuls-scan-flow.png and b/img/vuls-scan-flow.png differ diff --git a/scan/base.go b/scan/base.go index 0148caff67..8353081a60 100644 --- a/scan/base.go +++ b/scan/base.go @@ -32,9 +32,8 @@ import ( type base struct { ServerInfo config.ServerInfo + Distro config.Distro - Family string - Release string Platform models.Platform osPackages @@ -54,13 +53,20 @@ func (l base) getServerInfo() config.ServerInfo { return l.ServerInfo } -func (l *base) setDistributionInfo(fam, rel string) { - l.Family = fam - l.Release = rel +func (l *base) setDistro(fam, rel string) { + d := config.Distro{ + Family: fam, + Release: rel, + } + l.Distro = d + + s := l.getServerInfo() + s.Distro = d + l.setServerInfo(s) } -func (l base) getDistributionInfo() string { - return fmt.Sprintf("%s %s", l.Family, l.Release) +func (l base) getDistro() config.Distro { + return l.Distro } func (l *base) setPlatform(p models.Platform) { @@ -250,8 +256,8 @@ func (l *base) convertToModel() (models.ScanResult, error) { return models.ScanResult{ ServerName: l.ServerInfo.ServerName, ScannedAt: time.Now(), - Family: l.Family, - Release: l.Release, + Family: l.Distro.Family, + Release: l.Distro.Release, Container: container, Platform: l.Platform, KnownCves: scoredCves, diff --git a/scan/base_test.go b/scan/base_test.go index 94cbc50377..86d5834292 100644 --- a/scan/base_test.go +++ b/scan/base_test.go @@ -70,8 +70,9 @@ func TestIsAwsInstanceID(t *testing.T) { {"no data", false}, } + r := newRedhat(config.ServerInfo{}) for _, tt := range tests { - actual := isAwsInstanceID(tt.in) + actual := r.isAwsInstanceID(tt.in) if tt.expected != actual { t.Errorf("expected %t, actual %t, str: %s", tt.expected, actual, tt.in) } diff --git a/scan/debian.go b/scan/debian.go index 8bc75aaaf6..2759bb88f1 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/future-architect/vuls/cache" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/cveapi" "github.com/future-architect/vuls/models" @@ -39,6 +40,7 @@ type debian struct { func newDebian(c config.ServerInfo) *debian { d := &debian{} d.log = util.NewCustomLogger(c) + d.setServerInfo(c) return d } @@ -46,7 +48,6 @@ func newDebian(c config.ServerInfo) *debian { // https://github.com/serverspec/specinfra/blob/master/lib/specinfra/helper/detect_os/debian.rb func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err error) { deb = newDebian(c) - deb.setServerInfo(c) if r := sshExec(c, "ls /etc/debian_version", noSudo); !r.isSuccess() { if r.Error != nil { @@ -69,12 +70,12 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err result := re.FindStringSubmatch(trim(r.Stdout)) if len(result) == 0 { - deb.setDistributionInfo("debian/ubuntu", "unknown") + deb.setDistro("debian/ubuntu", "unknown") Log.Warnf( "Unknown Debian/Ubuntu version. lsb_release -ir: %s", r) } else { distro := strings.ToLower(trim(result[1])) - deb.setDistributionInfo(distro, trim(result[2])) + deb.setDistro(distro, trim(result[2])) } return true, deb, nil } @@ -90,10 +91,10 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err if len(result) == 0 { Log.Warnf( "Unknown Debian/Ubuntu. cat /etc/lsb-release: %s", r) - deb.setDistributionInfo("debian/ubuntu", "unknown") + deb.setDistro("debian/ubuntu", "unknown") } else { distro := strings.ToLower(trim(result[1])) - deb.setDistributionInfo(distro, trim(result[2])) + deb.setDistro(distro, trim(result[2])) } return true, deb, nil } @@ -101,7 +102,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err // Debian cmd := "cat /etc/debian_version" if r := sshExec(c, cmd, noSudo); r.isSuccess() { - deb.setDistributionInfo("debian", trim(r.Stdout)) + deb.setDistro("debian", trim(r.Stdout)) return true, deb, nil } @@ -133,7 +134,7 @@ func (o *debian) install() error { return fmt.Errorf(msg) } - if o.Family == "debian" { + if o.Distro.Family == "debian" { // install aptitude cmd = util.PrependProxyEnv("apt-get install --force-yes -y aptitude") if r := o.ssh(cmd, sudo); !r.isSuccess() { @@ -208,7 +209,7 @@ func (o *debian) parseScannedPackagesLine(line string) (name, version string, er } func (o *debian) checkRequiredPackagesInstalled() error { - if o.Family == "debian" { + if o.Distro.Family == "debian" { if r := o.ssh("test -f /usr/bin/aptitude", noSudo); !r.isSuccess() { msg := fmt.Sprintf("aptitude is not installed: %s", r) o.log.Errorf(msg) @@ -218,9 +219,8 @@ func (o *debian) checkRequiredPackagesInstalled() error { return nil } -//TODO return whether already expired. func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInfo, error) { - // cmd := prependProxyEnv(conf.HTTPProxy, "apt-get update | cat; echo 1") + o.log.Infof("apt-get update...") cmd := util.PrependProxyEnv("apt-get update") if r := o.ssh(cmd, sudo); !r.isSuccess() { return nil, fmt.Errorf("Failed to SSH: %s", r) @@ -241,12 +241,21 @@ func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInf } } } - unsecurePacks, err = o.fillCandidateVersion(unsecurePacks) if err != nil { return nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err) } + current := cache.Meta{ + Name: o.getServerInfo().ServerName, + Distro: o.getServerInfo().Distro, + Packs: unsecurePacks, + } + o.log.Debugf("Ensure changelog cache: %s", current.Name) + if err := o.ensureChangelogCache(current); err != nil { + return nil, err + } + // Collect CVE information of upgradable packages cvePacksInfos, err := o.scanPackageCveInfos(unsecurePacks) if err != nil { @@ -256,63 +265,61 @@ func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInf return cvePacksInfos, nil } -func (o *debian) fillCandidateVersion(packs []models.PackageInfo) ([]models.PackageInfo, error) { - reqChan := make(chan models.PackageInfo, len(packs)) - resChan := make(chan models.PackageInfo, len(packs)) - errChan := make(chan error, len(packs)) - defer close(resChan) - defer close(errChan) - defer close(reqChan) - - go func() { - for _, pack := range packs { - reqChan <- pack +func (o *debian) ensureChangelogCache(current cache.Meta) error { + // Search from cache + old, found, err := cache.DB.GetMeta(current.Name) + if err != nil { + return fmt.Errorf("Failed to get meta. err: %s", err) + } + if !found { + o.log.Debugf("Not found in meta: %s", current.Name) + err = cache.DB.EnsureBuckets(current) + if err != nil { + return fmt.Errorf("Failed to ensure buckets. err: %s", err) } - }() - - timeout := time.After(5 * 60 * time.Second) - concurrency := 5 - tasks := util.GenWorkers(concurrency) - for range packs { - tasks <- func() { - select { - case pack := <-reqChan: - func(p models.PackageInfo) { - cmd := fmt.Sprintf("LANG=en_US.UTF-8 apt-cache policy %s", p.Name) - r := o.ssh(cmd, sudo) - if !r.isSuccess() { - errChan <- fmt.Errorf("Failed to SSH: %s.", r) - return - } - ver, err := o.parseAptCachePolicy(r.Stdout, p.Name) - if err != nil { - errChan <- fmt.Errorf("Failed to parse %s", err) - } - p.NewVersion = ver.Candidate - resChan <- p - }(pack) + } else { + if current.Distro.Family != old.Distro.Family || + current.Distro.Release != old.Distro.Release { + o.log.Debugf("Need to refesh meta: %s", current.Name) + err = cache.DB.EnsureBuckets(current) + if err != nil { + return fmt.Errorf("Failed to ensure buckets. err: %s", err) } + } else { + o.log.Debugf("Reuse meta: %s", current.Name) } } - errs := []error{} - result := []models.PackageInfo{} - for i := 0; i < len(packs); i++ { - select { - case pack := <-resChan: - result = append(result, pack) - o.log.Infof("(%d/%d) Upgradable: %s-%s -> %s", - i+1, len(packs), pack.Name, pack.Version, pack.NewVersion) - case err := <-errChan: - errs = append(errs, err) - case <-timeout: - return nil, fmt.Errorf("Timeout fillCandidateVersion") - } + if config.Conf.Debug { + cache.DB.PrettyPrint(current) } - if 0 < len(errs) { - return nil, fmt.Errorf("%v", errs) + return nil +} + +func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []models.PackageInfo, err error) { + names := []string{} + for _, p := range before { + names = append(names, p.Name) + } + cmd := fmt.Sprintf("LANG=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " ")) + r := o.ssh(cmd, sudo) + if !r.isSuccess() { + return nil, fmt.Errorf("Failed to SSH: %s.", r) } - return result, nil + packChangelog := o.splitAptCachePolicy(r.Stdout) + for k, v := range packChangelog { + ver, err := o.parseAptCachePolicy(v, k) + if err != nil { + return nil, fmt.Errorf("Failed to parse %s", err) + } + p, found := before.FindByName(k) + if !found { + return nil, fmt.Errorf("Not found: %s", k) + } + p.NewVersion = ver.Candidate + filled = append(filled, p) + } + return } func (o *debian) GetUpgradablePackNames() (packNames []string, err error) { @@ -369,9 +376,11 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er } func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePacksList CvePacksList, err error) { - - // { CVE ID: [packageInfo] } - cvePackages := make(map[string][]models.PackageInfo) + meta := cache.Meta{ + Name: o.getServerInfo().ServerName, + Distro: o.getServerInfo().Distro, + Packs: unsecurePacks, + } type strarray []string resChan := make(chan struct { @@ -398,6 +407,18 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac select { case pack := <-reqChan: func(p models.PackageInfo) { + changelog := o.getChangelogCache(meta, p) + if 0 < len(changelog) { + cveIDs := o.getCveIDFromChangelog(changelog, p.Name, p.Version) + resChan <- struct { + models.PackageInfo + strarray + }{p, cveIDs} + return + } + + // if the changelog is not in cache or failed to get from local cache, + // get the changelog of the package via internet. if cveIDs, err := o.scanPackageCveIDs(p); err != nil { errChan <- err } else { @@ -411,6 +432,8 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac } } + // { CVE ID: [packageInfo] } + cvePackages := make(map[string][]models.PackageInfo) errs := []error{} for i := 0; i < len(unsecurePacks); i++ { select { @@ -429,7 +452,6 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac return nil, fmt.Errorf("Timeout scanPackageCveIDs") } } - if 0 < len(errs) { return nil, fmt.Errorf("%v", errs) } @@ -438,7 +460,6 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac for k := range cvePackages { cveIDs = append(cveIDs, k) } - o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs) o.log.Info("Fetching CVE details...") @@ -459,9 +480,32 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac return } +func (o *debian) getChangelogCache(meta cache.Meta, pack models.PackageInfo) string { + cachedPack, found := meta.FindPack(pack.Name) + if !found { + return "" + } + if cachedPack.NewVersion != pack.NewVersion { + return "" + } + changelog, err := cache.DB.GetChangelog(meta.Name, pack.Name) + if err != nil { + o.log.Warnf("Failed to get chnagelog. bucket: %s, key:%s, err: %s", + meta.Name, pack.Name, err) + return "" + } + if len(changelog) == 0 { + return "" + } + + o.log.Debugf("Cache hit: %s, len: %d, %s...", + meta.Name, len(changelog), util.Truncate(changelog, 30)) + return changelog +} + func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) { cmd := "" - switch o.Family { + switch o.Distro.Family { case "ubuntu": cmd = fmt.Sprintf(`apt-get changelog %s | grep '\(urgency\|CVE\)'`, pack.Name) case "debian": @@ -476,36 +520,38 @@ func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) { return nil, nil } + err := cache.DB.PutChangelog(o.getServerInfo().ServerName, pack.Name, r.Stdout) + if err != nil { + return nil, fmt.Errorf("Failed to put changelog into cache") + } // No error will be returned. Only logging. - return o.getCveIDParsingChangelog(r.Stdout, pack.Name, pack.Version) + return o.getCveIDFromChangelog(r.Stdout, pack.Name, pack.Version), nil } -func (o *debian) getCveIDParsingChangelog(changelog string, - packName string, versionOrLater string) (cveIDs []string, err error) { +func (o *debian) getCveIDFromChangelog(changelog string, + packName string, versionOrLater string) []string { - cveIDs, err = o.parseChangelog(changelog, packName, versionOrLater) - if err == nil { - return + if cveIDs, err := o.parseChangelog(changelog, packName, versionOrLater); err == nil { + return cveIDs } ver := strings.Split(versionOrLater, "ubuntu")[0] - cveIDs, err = o.parseChangelog(changelog, packName, ver) - if err == nil { - return + if cveIDs, err := o.parseChangelog(changelog, packName, ver); err == nil { + return cveIDs } splittedByColon := strings.Split(versionOrLater, ":") if 1 < len(splittedByColon) { ver = splittedByColon[1] } - cveIDs, err = o.parseChangelog(changelog, packName, ver) + cveIDs, err := o.parseChangelog(changelog, packName, ver) if err == nil { - return + return cveIDs } // Only logging the error. o.log.Error(err) - return []string{}, nil + return []string{} } // Collect CVE-IDs included in the changelog. @@ -538,6 +584,29 @@ func (o *debian) parseChangelog(changelog string, return } +func (o *debian) splitAptCachePolicy(stdout string) map[string]string { + // re := regexp.MustCompile(`(?m:^[^ \t]+:$)`) + re := regexp.MustCompile(`(?m:^[^ \t]+:\r\n)`) + ii := re.FindAllStringIndex(stdout, -1) + ri := []int{} + for i := len(ii) - 1; 0 <= i; i-- { + ri = append(ri, ii[i][0]) + } + splitted := []string{} + lasti := len(stdout) + for _, i := range ri { + splitted = append(splitted, stdout[i:lasti]) + lasti = i + } + + packChangelog := map[string]string{} + for _, r := range splitted { + packName := r[:strings.Index(r, ":")] + packChangelog[packName] = r + } + return packChangelog +} + type packCandidateVer struct { Name string Installed string diff --git a/scan/debian_test.go b/scan/debian_test.go index 1656ed6294..b49f528cc0 100644 --- a/scan/debian_test.go +++ b/scan/debian_test.go @@ -18,10 +18,14 @@ along with this program. If not, see . package scan import ( + "os" "reflect" "testing" + "github.com/Sirupsen/logrus" + "github.com/future-architect/vuls/cache" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" "github.com/k0kubun/pp" ) @@ -183,7 +187,7 @@ util-linux (2.26.2-6) unstable; urgency=medium`, d := newDebian(config.ServerInfo{}) for _, tt := range tests { - actual, _ := d.getCveIDParsingChangelog(tt.in[2], tt.in[0], tt.in[1]) + actual := d.getCveIDFromChangelog(tt.in[2], tt.in[0], tt.in[1]) if len(actual) != len(tt.expected) { t.Errorf("Len of return array are'nt same. expected %#v, actual %#v", tt.expected, actual) t.Errorf(pp.Sprintf("%s", tt.in)) @@ -195,13 +199,6 @@ util-linux (2.26.2-6) unstable; urgency=medium`, } } } - - for _, tt := range tests { - _, err := d.getCveIDParsingChangelog(tt.in[2], tt.in[0], "version number do'nt match case") - if err != nil { - t.Errorf("Returning error is unexpected") - } - } } func TestGetUpdatablePackNames(t *testing.T) { @@ -520,6 +517,95 @@ Calculating upgrade... Done } } +func TestGetChangelogCache(t *testing.T) { + const servername = "server1" + pack := models.PackageInfo{ + Name: "apt", + Version: "1.0.0", + NewVersion: "1.0.1", + } + var meta = cache.Meta{ + Name: servername, + Distro: config.Distro{ + Family: "ubuntu", + Release: "16.04", + }, + Packs: []models.PackageInfo{pack}, + } + + const path = "/tmp/vuls-test-cache-11111111.db" + log := logrus.NewEntry(&logrus.Logger{}) + if err := cache.SetupBolt(path, log); err != nil { + t.Errorf("Failed to setup bolt: %s", err) + } + defer os.Remove(path) + + if err := cache.DB.EnsureBuckets(meta); err != nil { + t.Errorf("Failed to ensure buckets: %s", err) + } + + d := newDebian(config.ServerInfo{}) + actual := d.getChangelogCache(meta, pack) + if actual != "" { + t.Errorf("Failed to get empty stirng from cache:") + } + + clog := "changelog-text" + if err := cache.DB.PutChangelog(servername, "apt", clog); err != nil { + t.Errorf("Failed to put changelog: %s", err) + } + + actual = d.getChangelogCache(meta, pack) + if actual != clog { + t.Errorf("Failed to get changelog from cache: %s", actual) + } + + // increment a version of the pack + pack.NewVersion = "1.0.2" + actual = d.getChangelogCache(meta, pack) + if actual != "" { + t.Errorf("The changelog is not invalidated: %s", actual) + } + + // change a name of the pack + pack.Name = "bash" + actual = d.getChangelogCache(meta, pack) + if actual != "" { + t.Errorf("The changelog is not invalidated: %s", actual) + } +} + +func TestSplitAptCachePolicy(t *testing.T) { + var tests = []struct { + stdout string + expected map[string]string + }{ + // This function parse apt-cache policy by using Regexp multi-line mode. + // So, test data includes "\r\n" + { + "apt:\r\n Installed: 1.2.6\r\n Candidate: 1.2.12~ubuntu16.04.1\r\n Version table:\r\n 1.2.12~ubuntu16.04.1 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n 1.2.10ubuntu1 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 1.2.6 100\r\n 100 /var/lib/dpkg/status\r\napt-utils:\r\n Installed: 1.2.6\r\n Candidate: 1.2.12~ubuntu16.04.1\r\n Version table:\r\n 1.2.12~ubuntu16.04.1 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n 1.2.10ubuntu1 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 1.2.6 100\r\n 100 /var/lib/dpkg/status\r\nbase-files:\r\n Installed: 9.4ubuntu3\r\n Candidate: 9.4ubuntu4.2\r\n Version table:\r\n 9.4ubuntu4.2 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n 9.4ubuntu4 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 9.4ubuntu3 100\r\n 100 /var/lib/dpkg/status\r\n", + + map[string]string{ + "apt": "apt:\r\n Installed: 1.2.6\r\n Candidate: 1.2.12~ubuntu16.04.1\r\n Version table:\r\n 1.2.12~ubuntu16.04.1 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n 1.2.10ubuntu1 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 1.2.6 100\r\n 100 /var/lib/dpkg/status\r\n", + + "apt-utils": "apt-utils:\r\n Installed: 1.2.6\r\n Candidate: 1.2.12~ubuntu16.04.1\r\n Version table:\r\n 1.2.12~ubuntu16.04.1 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n 1.2.10ubuntu1 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 1.2.6 100\r\n 100 /var/lib/dpkg/status\r\n", + + "base-files": "base-files:\r\n Installed: 9.4ubuntu3\r\n Candidate: 9.4ubuntu4.2\r\n Version table:\r\n 9.4ubuntu4.2 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n 9.4ubuntu4 500\r\n 500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 9.4ubuntu3 100\r\n 100 /var/lib/dpkg/status\r\n", + }, + }, + } + + d := newDebian(config.ServerInfo{}) + for _, tt := range tests { + actual := d.splitAptCachePolicy(tt.stdout) + if !reflect.DeepEqual(tt.expected, actual) { + e := pp.Sprintf("%v", tt.expected) + a := pp.Sprintf("%v", actual) + t.Errorf("expected %s, actual %s", e, a) + } + } +} + func TestParseAptCachePolicy(t *testing.T) { var tests = []struct { diff --git a/scan/freebsd.go b/scan/freebsd.go index 0c6e5225bc..d3d10c3c08 100644 --- a/scan/freebsd.go +++ b/scan/freebsd.go @@ -1,3 +1,20 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + package scan import ( @@ -19,18 +36,18 @@ type bsd struct { func newBsd(c config.ServerInfo) *bsd { d := &bsd{} d.log = util.NewCustomLogger(c) + d.setServerInfo(c) return d } //https://github.com/mizzy/specinfra/blob/master/lib/specinfra/helper/detect_os/freebsd.rb func detectFreebsd(c config.ServerInfo) (itsMe bool, bsd osTypeInterface) { bsd = newBsd(c) - c.Family = "FreeBSD" if r := sshExec(c, "uname", noSudo); r.isSuccess() { if strings.Contains(r.Stdout, "FreeBSD") == true { if b := sshExec(c, "uname -r", noSudo); b.isSuccess() { - bsd.setDistributionInfo("FreeBSD", strings.TrimSpace(b.Stdout)) - bsd.setServerInfo(c) + rel := strings.TrimSpace(b.Stdout) + bsd.setDistro("FreeBSD", rel) return true, bsd } } diff --git a/scan/redhat.go b/scan/redhat.go index 13f148ed12..21411cacdb 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -42,16 +42,16 @@ type redhat struct { func newRedhat(c config.ServerInfo) *redhat { r := &redhat{} r.log = util.NewCustomLogger(c) + r.setServerInfo(c) return r } // https://github.com/serverspec/specinfra/blob/master/lib/specinfra/helper/detect_os/redhat.rb func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { red = newRedhat(c) - red.setServerInfo(c) if r := sshExec(c, "ls /etc/fedora-release", noSudo); r.isSuccess() { - red.setDistributionInfo("fedora", "unknown") + red.setDistro("fedora", "unknown") Log.Warn("Fedora not tested yet: %s", r) return true, red } @@ -72,9 +72,9 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { release := result[2] switch strings.ToLower(result[1]) { case "centos", "centos linux": - red.setDistributionInfo("centos", release) + red.setDistro("centos", release) default: - red.setDistributionInfo("rhel", release) + red.setDistro("rhel", release) } return true, red } @@ -90,7 +90,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { release = fields[4] } } - red.setDistributionInfo(family, release) + red.setDistro(family, release) return true, red } @@ -113,7 +113,7 @@ func (o *redhat) checkIfSudoNoPasswd() error { // CentOS 7 ... yum-plugin-changelog // RHEL, Amazon ... no additinal packages needed func (o *redhat) install() error { - switch o.Family { + switch o.Distro.Family { case "rhel", "amazon": o.log.Infof("Nothing to do") return nil @@ -123,14 +123,13 @@ func (o *redhat) install() error { } func (o *redhat) installYumChangelog() error { - if o.Family == "centos" { + if o.Distro.Family == "centos" { var majorVersion int - if 0 < len(o.Release) { - majorVersion, _ = strconv.Atoi(strings.Split(o.Release, ".")[0]) + if 0 < len(o.Distro.Release) { + majorVersion, _ = strconv.Atoi(strings.Split(o.Distro.Release, ".")[0]) } else { return fmt.Errorf( - "Not implemented yet. family: %s, release: %s", - o.Family, o.Release) + "Not implemented yet: %s", o.Distro) } var packName = "" @@ -157,12 +156,12 @@ func (o *redhat) installYumChangelog() error { } func (o *redhat) checkRequiredPackagesInstalled() error { - if o.Family == "centos" { + if o.Distro.Family == "centos" { var majorVersion int - if 0 < len(o.Release) { - majorVersion, _ = strconv.Atoi(strings.Split(o.Release, ".")[0]) + if 0 < len(o.Distro.Release) { + majorVersion, _ = strconv.Atoi(strings.Split(o.Distro.Release, ".")[0]) } else { - msg := fmt.Sprintf("Not implemented yet. family: %s, release: %s", o.Family, o.Release) + msg := fmt.Sprintf("Not implemented yet: %s", o.Distro) o.log.Errorf(msg) return fmt.Errorf(msg) } @@ -240,7 +239,7 @@ func (o *redhat) parseScannedPackagesLine(line string) (models.PackageInfo, erro } func (o *redhat) scanUnsecurePackages() ([]CvePacksInfo, error) { - if o.Family != "centos" { + if o.Distro.Family != "centos" { // Amazon, RHEL has yum updateinfo as default // yum updateinfo can collenct vendor advisory information. return o.scanUnsecurePackagesUsingYumPluginSecurity() @@ -460,12 +459,10 @@ func (o *redhat) getChangelogCVELines(rpm2changelog map[string]*string, packInfo func (o *redhat) parseAllChangelog(allChangelog string) (map[string]*string, error) { var majorVersion int - if 0 < len(o.Release) && o.Family == "centos" { - majorVersion, _ = strconv.Atoi(strings.Split(o.Release, ".")[0]) + if 0 < len(o.Distro.Release) && o.Distro.Family == "centos" { + majorVersion, _ = strconv.Atoi(strings.Split(o.Distro.Release, ".")[0]) } else { - return nil, fmt.Errorf( - "Not implemented yet. family: %s, release: %s", - o.Family, o.Release) + return nil, fmt.Errorf("Not implemented yet: %s", o.getDistro()) } orglines := strings.Split(allChangelog, "\n") @@ -569,7 +566,7 @@ type distroAdvisoryCveIDs struct { // Scaning unsecure packages using yum-plugin-security. // Amazon, RHEL func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (CvePacksList, error) { - if o.Family == "centos" { + if o.Distro.Family == "centos" { // CentOS has no security channel. // So use yum check-update && parse changelog return CvePacksList{}, fmt.Errorf( @@ -717,7 +714,7 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID switch sectionState { case Header: - switch o.Family { + switch o.Distro.Family { case "centos": // CentOS has no security channel. // So use yum check-update && parse changelog @@ -964,7 +961,7 @@ func (o *redhat) clone() osTypeInterface { } func (o *redhat) sudo() bool { - switch o.Family { + switch o.Distro.Family { case "amazon": return false default: diff --git a/scan/redhat_test.go b/scan/redhat_test.go index 0b968f7081..9bd87f2665 100644 --- a/scan/redhat_test.go +++ b/scan/redhat_test.go @@ -451,7 +451,7 @@ Description : The Berkeley Internet Name Domain (BIND) is an implementation of updated, _ := time.Parse("2006-01-02", "2015-09-04") r := newRedhat(config.ServerInfo{}) - r.Family = "redhat" + r.Distro = config.Distro{Family: "redhat"} var tests = []struct { in string @@ -511,7 +511,7 @@ Description : The Berkeley Internet Name Domain (BIND) is an implementation of func TestParseYumUpdateinfoAmazon(t *testing.T) { r := newRedhat(config.ServerInfo{}) - r.Family = "amazon" + r.Distro = config.Distro{Family: "redhat"} issued, _ := time.Parse("2006-01-02", "2015-12-15") updated, _ := time.Parse("2006-01-02", "2015-12-16") @@ -601,7 +601,7 @@ Description : Package updates are available for Amazon Linux AMI that fix the func TestParseYumCheckUpdateLines(t *testing.T) { r := newRedhat(config.ServerInfo{}) - r.Family = "centos" + r.Distro = config.Distro{Family: "centos"} stdout := `Loaded plugins: changelog, fastestmirror, keys, protect-packages, protectbase, security Loading mirror speeds from cached hostfile * base: mirror.fairway.ne.jp @@ -709,7 +709,7 @@ bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates func TestParseYumCheckUpdateLinesAmazon(t *testing.T) { r := newRedhat(config.ServerInfo{}) - r.Family = "amazon" + r.Distro = config.Distro{Family: "amazon"} stdout := `Loaded plugins: priorities, update-motd, upgrade-helper 34 package(s) needed for security, out of 71 available @@ -1110,8 +1110,10 @@ func TestGetChangelogCVELines(t *testing.T) { } r := newRedhat(config.ServerInfo{}) - r.Family = "centos" - r.Release = "6.7" + r.Distro = config.Distro{ + Family: "centos", + Release: "6.7", + } for _, tt := range testsCentos6 { rpm2changelog, err := r.parseAllChangelog(stdoutCentos6) if err != nil { @@ -1194,7 +1196,10 @@ func TestGetChangelogCVELines(t *testing.T) { }, } - r.Release = "5.6" + r.Distro = config.Distro{ + Family: "centos", + Release: "5.6", + } for _, tt := range testsCentos5 { rpm2changelog, err := r.parseAllChangelog(stdoutCentos5) if err != nil { diff --git a/scan/serverapi.go b/scan/serverapi.go index d338370b1e..bf4a985266 100644 --- a/scan/serverapi.go +++ b/scan/serverapi.go @@ -1,3 +1,20 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + package scan import ( @@ -5,6 +22,7 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/future-architect/vuls/cache" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" cve "github.com/kotakanbe/go-cve-dictionary/models" @@ -20,8 +38,9 @@ type osTypeInterface interface { setServerInfo(config.ServerInfo) getServerInfo() config.ServerInfo - setDistributionInfo(string, string) - getDistributionInfo() string + setDistro(string, string) + getDistro() config.Distro + // getFamily() string checkIfSudoNoPasswd() error detectPlatform() error @@ -188,7 +207,7 @@ func detectServerOSes() (sshAbleOses []osTypeInterface) { Log.Infof("(%d/%d) Detected: %s: %s", i+1, len(config.Conf.Servers), res.getServerInfo().ServerName, - res.getDistributionInfo()) + res.getDistro()) } case <-timeout: msg := "Timed out while detecting servers" @@ -248,7 +267,7 @@ func detectContainerOSes() (actives []osTypeInterface) { } oses = append(oses, res...) Log.Infof("Detected: %s@%s: %s", - sinfo.Container.Name, sinfo.ServerName, osi.getDistributionInfo()) + sinfo.Container.Name, sinfo.ServerName, osi.getDistro()) } case <-timeout: msg := "Timed out while detecting containers" @@ -417,6 +436,13 @@ func Scan() []error { return errs } + if err := setupCangelogCache(); err != nil { + return []error{err} + } + if cache.DB != nil { + defer cache.DB.Close() + } + Log.Info("Scanning vulnerable OS packages...") if errs := scanPackages(); errs != nil { return errs @@ -429,6 +455,23 @@ func Scan() []error { return nil } +func setupCangelogCache() error { + needToSetupCache := false + for _, s := range servers { + switch s.getDistro().Family { + case "ubuntu", "debian": + needToSetupCache = true + break + } + } + if needToSetupCache { + if err := cache.SetupBolt(config.Conf.CacheDBPath, Log); err != nil { + return err + } + } + return nil +} + func checkRequiredPackagesInstalled() []error { timeoutSec := 30 * 60 return parallelSSHExec(func(o osTypeInterface) error { diff --git a/scan/sshutil.go b/scan/sshutil.go index 55190fc8dd..296adcb6b6 100644 --- a/scan/sshutil.go +++ b/scan/sshutil.go @@ -302,7 +302,7 @@ func decolateCmd(c conf.ServerInfo, cmd string, sudo bool) string { cmd = strings.Replace(cmd, "|", "| sudo ", -1) } - if c.Family != "FreeBSD" { + if c.Distro.Family != "FreeBSD" { // set pipefail option. Bash only // http://unix.stackexchange.com/questions/14270/get-exit-status-of-process-thats-piped-to-another cmd = fmt.Sprintf("set -o pipefail; %s", cmd) diff --git a/util/util.go b/util/util.go index 74781e99d4..6942fb7c2d 100644 --- a/util/util.go +++ b/util/util.go @@ -124,3 +124,14 @@ func PrependProxyEnv(cmd string) string { // } // return time.Unix(i, 0), nil // } + +// Truncate truncates string to the length +func Truncate(str string, length int) string { + if length < 0 { + return str + } + if length <= len(str) { + return str[:length] + } + return str +} diff --git a/util/util_test.go b/util/util_test.go index 50035170f4..2ec538d4ed 100644 --- a/util/util_test.go +++ b/util/util_test.go @@ -131,3 +131,43 @@ func TestPrependHTTPProxyEnv(t *testing.T) { } } + +func TestTruncate(t *testing.T) { + var tests = []struct { + in string + length int + out string + }{ + { + in: "abcde", + length: 3, + out: "abc", + }, + { + in: "abcdefg", + length: 5, + out: "abcde", + }, + { + in: "abcdefg", + length: 10, + out: "abcdefg", + }, + { + in: "abcdefg", + length: 0, + out: "", + }, + { + in: "abcdefg", + length: -1, + out: "abcdefg", + }, + } + for _, tt := range tests { + actual := Truncate(tt.in, tt.length) + if actual != tt.out { + t.Errorf("\nexpected: %s\n actual: %s", tt.out, actual) + } + } +}