diff --git a/README.ja.md b/README.ja.md index 50970d21e6..c79df36d4f 100644 --- a/README.ja.md +++ b/README.ja.md @@ -198,7 +198,8 @@ go getでエラーが発生した場合は、以下の点を確認する。 ## Step6. Config -Vulの設定ファイルを作成する(TOMLフォーマット) +Vulの設定ファイルを作成する(TOMLフォーマット) +設定ファイルのチェックを行う ``` $ cat config.toml @@ -209,6 +210,8 @@ host = "172.31.4.82" port = "22" user = "ec2-user" keyPath = "/home/ec2-user/.ssh/id_rsa" + +$ vuls configtest ``` ## Step7. Setting up target servers for Vuls @@ -484,6 +487,31 @@ host = "172.31.4.82" - SSH public key authentication (with password, empty password) - Password authentication +---- + +# Usage: Configtest + +configtestサブコマンドは、config.tomlで定義されたサーバ/コンテナに対してSSH可能かどうかをチェックする。 + +``` +$ vuls configtest --help +configtest: + configtest + [-config=/path/to/config.toml] + [-ask-key-password] + [-ssh-external] + [-debug] + + [SERVER]... + -ask-key-password + Ask ssh privatekey password before scanning + -config string + /path/to/toml (default "/Users/kotakanbe/go/src/github.com/future-architect/vuls/config.toml") + -debug + debug mode + -ssh-external + Use external ssh command. Default: Use the Go native implementation +``` ---- diff --git a/README.md b/README.md index be5e694777..9657d7fe01 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,8 @@ If an error occurred while go get, check the following points. ## Step6. Config -Create a config file(TOML format). +Create a config file(TOML format). +Then check the config. ``` $ cat config.toml @@ -201,6 +202,8 @@ host = "172.31.4.82" port = "22" user = "ec2-user" keyPath = "/home/ec2-user/.ssh/id_rsa" + +$ vuls configtest ``` ## Step7. Setting up target servers for Vuls @@ -483,8 +486,31 @@ You can customize your configuration using this template. - SSH public key authentication (with password, empty password) - Password authentication - +---- + +# Usage: Configtest + +Configtest subcommand check if vuls is able to connect via ssh to servers/containers defined in the config.toml. +``` +$ vuls configtest --help +configtest: + configtest + [-config=/path/to/config.toml] + [-ask-key-password] + [-ssh-external] + [-debug] + + [SERVER]... + -ask-key-password + Ask ssh privatekey password before scanning + -config string + /path/to/toml (default "/Users/kotakanbe/go/src/github.com/future-architect/vuls/config.toml") + -debug + debug mode + -ssh-external + Use external ssh command. Default: Use the Go native implementation +``` ---- diff --git a/commands/configtest.go b/commands/configtest.go new file mode 100644 index 0000000000..765ad53855 --- /dev/null +++ b/commands/configtest.go @@ -0,0 +1,157 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package commands + +import ( + "flag" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/google/subcommands" + "github.com/labstack/gommon/log" + "golang.org/x/net/context" + + c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/scan" + "github.com/future-architect/vuls/util" +) + +// ConfigtestCmd is Subcommand +type ConfigtestCmd struct { + configPath string + askKeyPassword bool + sshExternal bool + + debug bool +} + +// Name return subcommand name +func (*ConfigtestCmd) Name() string { return "configtest" } + +// Synopsis return synopsis +func (*ConfigtestCmd) Synopsis() string { return "Test configuration" } + +// Usage return usage +func (*ConfigtestCmd) Usage() string { + return `configtest: + configtest + [-config=/path/to/config.toml] + [-ask-key-password] + [-ssh-external] + [-debug] + + [SERVER]... +` +} + +// SetFlags set flag +func (p *ConfigtestCmd) SetFlags(f *flag.FlagSet) { + wd, _ := os.Getwd() + defaultConfPath := filepath.Join(wd, "config.toml") + f.StringVar(&p.configPath, "config", defaultConfPath, "/path/to/toml") + + f.BoolVar(&p.debug, "debug", false, "debug mode") + + f.BoolVar( + &p.askKeyPassword, + "ask-key-password", + false, + "Ask ssh privatekey password before scanning", + ) + + f.BoolVar( + &p.sshExternal, + "ssh-external", + false, + "Use external ssh command. Default: Use the Go native implementation") +} + +// Execute execute +func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + + var keyPass string + var err error + if p.askKeyPassword { + prompt := "SSH key password: " + if keyPass, err = getPasswd(prompt); err != nil { + logrus.Error(err) + return subcommands.ExitFailure + } + } + + c.Conf.Debug = p.debug + + err = c.Load(p.configPath, keyPass, "") + if err != nil { + logrus.Errorf("Error loading %s, %s", p.configPath, err) + return subcommands.ExitUsageError + } + + var servernames []string + if 0 < len(f.Args()) { + servernames = f.Args() + } else { + stat, _ := os.Stdin.Stat() + if (stat.Mode() & os.ModeCharDevice) == 0 { + bytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + log.Errorf("Failed to read stdin: %s", err) + return subcommands.ExitFailure + } + fields := strings.Fields(string(bytes)) + if 0 < len(fields) { + servernames = fields + } + } + } + + target := make(map[string]c.ServerInfo) + for _, arg := range servernames { + found := false + for servername, info := range c.Conf.Servers { + if servername == arg { + target[servername] = info + found = true + break + } + } + if !found { + logrus.Errorf("%s is not in config", arg) + return subcommands.ExitUsageError + } + } + if 0 < len(servernames) { + c.Conf.Servers = target + } + + // logger + Log := util.NewCustomLogger(c.ServerInfo{}) + + Log.Info("Validating Config...") + if !c.Conf.Validate() { + return subcommands.ExitUsageError + } + + Log.Info("Detecting Server/Contianer OS... ") + scan.InitServers(Log) + + return subcommands.ExitSuccess +} diff --git a/commands/prepare.go b/commands/prepare.go index a0c7014686..e6d98df758 100644 --- a/commands/prepare.go +++ b/commands/prepare.go @@ -153,11 +153,7 @@ func (p *PrepareCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ logger := util.NewCustomLogger(c.ServerInfo{}) logger.Info("Detecting OS... ") - err = scan.InitServers(logger) - if err != nil { - logger.Errorf("Failed to init servers. err: %s", err) - return subcommands.ExitFailure - } + scan.InitServers(logger) logger.Info("Installing...") if errs := scan.Prepare(); 0 < len(errs) { diff --git a/commands/scan.go b/commands/scan.go index b39ddcec1c..1ca03c0037 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -20,8 +20,10 @@ package commands import ( "flag" "fmt" + "io/ioutil" "os" "path/filepath" + "strings" "github.com/Sirupsen/logrus" c "github.com/future-architect/vuls/config" @@ -31,6 +33,7 @@ import ( "github.com/future-architect/vuls/scan" "github.com/future-architect/vuls/util" "github.com/google/subcommands" + "github.com/labstack/gommon/log" "golang.org/x/net/context" ) @@ -260,8 +263,27 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } else { logrus.Infof("cve-dictionary: %s", p.cveDictionaryURL) } + + var servernames []string + if 0 < len(f.Args()) { + servernames = f.Args() + } else { + stat, _ := os.Stdin.Stat() + if (stat.Mode() & os.ModeCharDevice) == 0 { + bytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + log.Errorf("Failed to read stdin: %s", err) + return subcommands.ExitFailure + } + fields := strings.Fields(string(bytes)) + if 0 < len(fields) { + servernames = fields + } + } + } + target := make(map[string]c.ServerInfo) - for _, arg := range f.Args() { + for _, arg := range servernames { found := false for servername, info := range c.Conf.Servers { if servername == arg { @@ -275,7 +297,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitUsageError } } - if 0 < len(f.Args()) { + if 0 < len(servernames) { c.Conf.Servers = target } @@ -359,12 +381,11 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitFailure } - Log.Info("Detecting Server OS... ") - err = scan.InitServers(Log) - if err != nil { - Log.Errorf("Failed to init servers. Check the configuration. err: %s", err) - return subcommands.ExitFailure - } + Log.Info("Detecting Server/Contianer OS... ") + scan.InitServers(Log) + + Log.Info("Detecting Platforms... ") + scan.DetectPlatforms(Log) Log.Info("Scanning vulnerabilities... ") if errs := scan.Scan(); 0 < len(errs) { diff --git a/main.go b/main.go index b5a91614b4..35acb99b0e 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,7 @@ func main() { subcommands.Register(&commands.ScanCmd{}, "scan") subcommands.Register(&commands.PrepareCmd{}, "prepare") subcommands.Register(&commands.HistoryCmd{}, "history") + subcommands.Register(&commands.ConfigtestCmd{}, "configtest") var v = flag.Bool("v", false, "Show version") diff --git a/scan/base.go b/scan/base.go index 5730fb69ae..a8c4baac68 100644 --- a/scan/base.go +++ b/scan/base.go @@ -115,9 +115,7 @@ func (l *base) dockerPs(option string) (string, error) { cmd := fmt.Sprintf("docker ps %s", option) r := l.ssh(cmd, noSudo) if !r.isSuccess() { - return "", fmt.Errorf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return "", fmt.Errorf("Failed to SSH: %s", r) } return r.Stdout, nil } diff --git a/scan/debian.go b/scan/debian.go index 9c675f92ad..8bb2096696 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -53,13 +53,14 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err deb.setServerInfo(c) if r := sshExec(c, "ls /etc/debian_version", noSudo); !r.isSuccess() { + if r.Error != nil { + return false, deb, r.Error + } if r.ExitStatus == 255 { return false, deb, fmt.Errorf( - "Unable to connect via SSH. Check SSH settings. servername: %s, %s@%s:%s, status: %d, stdout: %s, stderr: %s", - c.ServerName, c.User, c.Host, c.Port, r.ExitStatus, r.Stdout, r.Stderr, - ) + "Unable to connect via SSH. Check SSH settings. %s", r) } - Log.Debugf("Not Debian like Linux. Host: %s:%s", c.Host, c.Port) + Log.Debugf("Not Debian like Linux. %s", r) return false, deb, nil } @@ -75,8 +76,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err if len(result) == 0 { deb.setDistributionInfo("debian/ubuntu", "unknown") Log.Warnf( - "Unknown Debian/Ubuntu version. lsb_release -ir: %s, Host: %s:%s", - r.Stdout, c.Host, c.Port) + "Unknown Debian/Ubuntu version. lsb_release -ir: %s", r) } else { distro := strings.ToLower(trim(result[1])) deb.setDistributionInfo(distro, trim(result[2])) @@ -95,8 +95,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err result := re.FindStringSubmatch(trim(r.Stdout)) if len(result) == 0 { Log.Warnf( - "Unknown Debian/Ubuntu. cat /etc/lsb-release: %s, Host: %s:%s", - r.Stdout, c.Host, c.Port) + "Unknown Debian/Ubuntu. cat /etc/lsb-release: %s", r) deb.setDistributionInfo("debian/ubuntu", "unknown") } else { distro := strings.ToLower(trim(result[1])) @@ -112,7 +111,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err return true, deb, nil } - Log.Debugf("Not Debian like Linux. Host: %s:%s", c.Host, c.Port) + Log.Debugf("Not Debian like Linux: %s", c.ServerName) return false, deb, nil } @@ -126,8 +125,7 @@ func (o *debian) install() error { o.log.Infof("apt-get update...") cmd := util.PrependProxyEnv("apt-get update") if r := o.ssh(cmd, sudo); !r.isSuccess() { - msg := fmt.Sprintf("Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + msg := fmt.Sprintf("Failed to SSH: %s", r) o.log.Errorf(msg) return fmt.Errorf(msg) } @@ -136,8 +134,7 @@ func (o *debian) install() error { // install aptitude cmd = util.PrependProxyEnv("apt-get install --force-yes -y aptitude") if r := o.ssh(cmd, sudo); !r.isSuccess() { - msg := fmt.Sprintf("Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + msg := fmt.Sprintf("Failed to SSH: %s", r) o.log.Errorf(msg) return fmt.Errorf(msg) } @@ -158,8 +155,7 @@ func (o *debian) install() error { cmd = util.PrependProxyEnv( "apt-get install --force-yes -y unattended-upgrades") if r := o.ssh(cmd, sudo); !r.isSuccess() { - msg := fmt.Sprintf("Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + msg := fmt.Sprintf("Failed to SSH: %s", r) o.log.Errorf(msg) return fmt.Errorf(msg) } @@ -189,9 +185,7 @@ func (o *debian) scanPackages() error { func (o *debian) scanInstalledPackages() (packs []models.PackageInfo, err error) { r := o.ssh("dpkg-query -W", noSudo) if !r.isSuccess() { - return packs, fmt.Errorf( - "Failed to scan packages. status: %d, stdout:%s, stderr: %s", - r.ExitStatus, r.Stdout, r.Stderr) + return packs, fmt.Errorf("Failed to SSH: %s", r) } // e.g. @@ -232,7 +226,7 @@ func (o *debian) checkRequiredPackagesInstalled() error { if o.Family == "debian" { if r := o.ssh("test -f /usr/bin/aptitude", noSudo); !r.isSuccess() { - msg := "aptitude is not installed" + msg := fmt.Sprintf("aptitude is not installed: %s", r) o.log.Errorf(msg) return fmt.Errorf(msg) } @@ -243,7 +237,7 @@ func (o *debian) checkRequiredPackagesInstalled() error { } if r := o.ssh("type unattended-upgrade", noSudo); !r.isSuccess() { - msg := "unattended-upgrade is not installed" + msg := fmt.Sprintf("unattended-upgrade is not installed: %s", r) o.log.Errorf(msg) return fmt.Errorf(msg) } @@ -255,10 +249,7 @@ func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInf // cmd := prependProxyEnv(conf.HTTPProxy, "apt-get update | cat; echo 1") cmd := util.PrependProxyEnv("apt-get update") if r := o.ssh(cmd, sudo); !r.isSuccess() { - return nil, fmt.Errorf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr, - ) + return nil, fmt.Errorf("Failed to SSH: %s", r) } var upgradablePackNames []string @@ -325,9 +316,7 @@ func (o *debian) fillCandidateVersion(packs []models.PackageInfo) ([]models.Pack cmd := fmt.Sprintf("apt-cache policy %s", p.Name) r := o.ssh(cmd, sudo) if !r.isSuccess() { - errChan <- fmt.Errorf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + errChan <- fmt.Errorf("Failed to SSH: %s.", r) return } ver, err := o.parseAptCachePolicy(r.Stdout, p.Name) @@ -473,7 +462,6 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac }() timeout := time.After(30 * 60 * time.Second) - concurrency := 10 tasks := util.GenWorkers(concurrency) for range unsecurePacks { @@ -554,9 +542,7 @@ func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) { r := o.ssh(cmd, noSudo) if !r.isSuccess() { - o.log.Warnf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + o.log.Warnf("Failed to SSH: %s", r) // Ignore this Error. return nil, nil diff --git a/scan/freebsd.go b/scan/freebsd.go index 6744194eb7..9d52500f7f 100644 --- a/scan/freebsd.go +++ b/scan/freebsd.go @@ -35,7 +35,7 @@ func detectFreebsd(c config.ServerInfo) (itsMe bool, bsd osTypeInterface) { } } } - Log.Debugf("Not FreeBSD. Host: %s:%s", c.Host, c.Port) + Log.Debugf("Not FreeBSD. servernam: %s", c.ServerName) return false, bsd } @@ -69,8 +69,7 @@ func (o *bsd) scanInstalledPackages() ([]models.PackageInfo, error) { cmd := util.PrependProxyEnv("pkg version -v") r := o.ssh(cmd, noSudo) if !r.isSuccess() { - return nil, fmt.Errorf("Failed to %s. status: %d, stdout:%s, Stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } return o.parsePkgVersion(r.Stdout), nil } @@ -80,15 +79,13 @@ func (o *bsd) scanUnsecurePackages() (cvePacksList []CvePacksInfo, err error) { cmd := "rm -f " + vulndbPath r := o.ssh(cmd, noSudo) if !r.isSuccess(0) { - return nil, fmt.Errorf("Failed to %s. status: %d, stdout:%s, Stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } cmd = util.PrependProxyEnv("pkg audit -F -r -f " + vulndbPath) r = o.ssh(cmd, noSudo) if !r.isSuccess(0, 1) { - return nil, fmt.Errorf("Failed to %s. status: %d, stdout:%s, Stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } if r.ExitStatus == 0 { // no vulnerabilities diff --git a/scan/redhat.go b/scan/redhat.go index 2a174461e4..fb833ee54f 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -56,7 +56,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { if r := sshExec(c, "ls /etc/fedora-release", noSudo); r.isSuccess() { red.setDistributionInfo("fedora", "unknown") - Log.Warn("Fedora not tested yet. Host: %s:%s", c.Host, c.Port) + Log.Warn("Fedora not tested yet: %s", r) return true, red } @@ -69,9 +69,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { re, _ := regexp.Compile(`(.*) release (\d[\d.]*)`) result := re.FindStringSubmatch(strings.TrimSpace(r.Stdout)) if len(result) != 3 { - Log.Warn( - "Failed to parse RedHat/CentOS version. stdout: %s, Host: %s:%s", - r.Stdout, c.Host, c.Port) + Log.Warn("Failed to parse RedHat/CentOS version: %s", r) return true, red } @@ -100,7 +98,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { return true, red } - Log.Debugf("Not RedHat like Linux. Host: %s:%s", c.Host, c.Port) + Log.Debugf("Not RedHat like Linux. servername: %s", c.ServerName) return false, red } @@ -132,9 +130,7 @@ func (o *redhat) installYumPluginSecurity() error { o.log.Info("Installing yum-plugin-security...") cmd := util.PrependProxyEnv("yum install -y yum-plugin-security") if r := o.ssh(cmd, sudo); !r.isSuccess() { - return fmt.Errorf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return fmt.Errorf("Failed to SSH: %s", r) } return nil } @@ -167,9 +163,7 @@ func (o *redhat) installYumChangelog() error { o.log.Infof("Installing %s...", packName) cmd = util.PrependProxyEnv("yum install -y " + packName) if r := o.ssh(cmd, sudo); !r.isSuccess() { - return fmt.Errorf( - "Failed to install %s. status: %d, stdout: %s, stderr: %s", - packName, r.ExitStatus, r.Stdout, r.Stderr) + return fmt.Errorf("Failed to SSH: %s", r) } o.log.Infof("Installed: %s", packName) } @@ -291,9 +285,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (CvePacksList, error) r := o.ssh(util.PrependProxyEnv(cmd), sudo) if !r.isSuccess(0, 100) { //returns an exit code of 100 if there are available updates. - return nil, fmt.Errorf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } // get Updateble package name, installed, candidate version. @@ -472,9 +464,7 @@ func (o *redhat) getChangelog(packageNames string) (stdout string, err error) { r := o.ssh(command, sudo) if !r.isSuccess(0, 1) { - return "", fmt.Errorf( - "Failed to get changelog. status: %d, stdout: %s, stderr: %s", - r.ExitStatus, r.Stdout, r.Stderr) + return "", fmt.Errorf("Failed to SSH: %s", r) } return r.Stdout, nil } @@ -497,18 +487,14 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (CvePacksList, err cmd := "yum --color=never repolist" r := o.ssh(util.PrependProxyEnv(cmd), sudo) if !r.isSuccess() { - return nil, fmt.Errorf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } // get advisoryID(RHSA, ALAS) - package name,version cmd = "yum --color=never updateinfo list available --security" r = o.ssh(util.PrependProxyEnv(cmd), sudo) if !r.isSuccess() { - return nil, fmt.Errorf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } advIDPackNamesList, err := o.parseYumUpdateinfoListAvailable(r.Stdout) @@ -518,9 +504,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (CvePacksList, err r = o.ssh(util.PrependProxyEnv(cmd), sudo) if !r.isSuccess(0, 100) { //returns an exit code of 100 if there are available updates. - return nil, fmt.Errorf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } updatable, err := o.parseYumCheckUpdateLines(r.Stdout) if err != nil { @@ -547,9 +531,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (CvePacksList, err cmd = "yum --color=never updateinfo --security update" r = o.ssh(util.PrependProxyEnv(cmd), sudo) if !r.isSuccess() { - return nil, fmt.Errorf( - "Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } advisoryCveIDsList, err := o.parseYumUpdateinfo(r.Stdout) if err != nil { diff --git a/scan/serverapi.go b/scan/serverapi.go index 0506faeacf..a6b1a1229c 100644 --- a/scan/serverapi.go +++ b/scan/serverapi.go @@ -109,8 +109,8 @@ func (s CvePacksList) Less(i, j int) bool { func detectOS(c config.ServerInfo) (osType osTypeInterface) { var itsMe bool var fatalErr error - itsMe, osType, fatalErr = detectDebian(c) + itsMe, osType, fatalErr = detectDebian(c) if fatalErr != nil { osType.setServerInfo(c) osType.setErrs([]error{fatalErr}) @@ -134,74 +134,61 @@ func detectOS(c config.ServerInfo) (osType osTypeInterface) { } // InitServers detect the kind of OS distribution of target servers -func InitServers(localLogger *logrus.Entry) error { +func InitServers(localLogger *logrus.Entry) { Log = localLogger + servers = detectServerOSes() - hosts, err := detectServerOSes() - if err != nil { - return fmt.Errorf("Failed to detect server OSes. err: %s", err) - } - servers = hosts - Log.Info("Detecting Container OS...") - containers, err := detectContainerOSes() - if err != nil { - return fmt.Errorf("Failed to detect Container OSes. err: %s", err) - } + containers := detectContainerOSes() servers = append(servers, containers...) - Log.Info("Detecting Platforms...") - errs := detectPlatforms() - if 0 < len(errs) { - // Only logging - Log.Errorf("Failed to detect platforms. err: %v", errs) - } - for i, s := range servers { + Log.Info("SSH-able servers are below...") + for _, s := range servers { if s.getServerInfo().IsContainer() { - Log.Infof("(%d/%d) %s on %s is running on %s", - i+1, len(servers), + fmt.Printf("%s@%s ", s.getServerInfo().Container.Name, s.getServerInfo().ServerName, - s.getPlatform().Name, ) - } else { - Log.Infof("(%d/%d) %s is running on %s", - i+1, len(servers), - s.getServerInfo().ServerName, - s.getPlatform().Name, - ) + fmt.Printf("%s ", s.getServerInfo().ServerName) } } - return nil + fmt.Printf("\n") } -func detectServerOSes() (oses []osTypeInterface, err error) { +func detectServerOSes() (sshAbleOses []osTypeInterface) { + Log.Info("Detecting OS of servers... ") osTypeChan := make(chan osTypeInterface, len(config.Conf.Servers)) defer close(osTypeChan) for _, s := range config.Conf.Servers { go func(s config.ServerInfo) { - //TODO handling Unknown OS + defer func() { + if p := recover(); p != nil { + Log.Debugf("Panic: %s on %s", p, s.ServerName) + } + }() osTypeChan <- detectOS(s) }(s) } - timeout := time.After(60 * time.Second) + var oses []osTypeInterface + timeout := time.After(30 * time.Second) for i := 0; i < len(config.Conf.Servers); i++ { select { case res := <-osTypeChan: oses = append(oses, res) if 0 < len(res.getErrs()) { - Log.Infof("(%d/%d) Failed %s", + Log.Errorf("(%d/%d) Failed: %s, err: %s", i+1, len(config.Conf.Servers), - res.getServerInfo().ServerName) + res.getServerInfo().ServerName, + res.getErrs()) } else { - Log.Infof("(%d/%d) Detected %s: %s", + Log.Infof("(%d/%d) Detected: %s: %s", i+1, len(config.Conf.Servers), res.getServerInfo().ServerName, res.getDistributionInfo()) } case <-timeout: - msg := "Timeout occurred while detecting" + msg := "Timed out while detecting servers" Log.Error(msg) for servername := range config.Conf.Servers { found := false @@ -212,52 +199,56 @@ func detectServerOSes() (oses []osTypeInterface, err error) { } } if !found { - Log.Errorf("Failed to detect. servername: %s", servername) + Log.Errorf("(%d/%d) Timed out: %s", + i+1, len(config.Conf.Servers), + servername) + i++ } } - return oses, fmt.Errorf(msg) } } - errs := []error{} - for _, osi := range oses { - if 0 < len(osi.getErrs()) { - errs = append(errs, fmt.Errorf( - "Error occurred on %s. errs: %s", - osi.getServerInfo().ServerName, osi.getErrs())) + for _, o := range oses { + if len(o.getErrs()) == 0 { + sshAbleOses = append(sshAbleOses, o) } } - if 0 < len(errs) { - return oses, fmt.Errorf("%s", errs) - } return } -func detectContainerOSes() (oses []osTypeInterface, err error) { +func detectContainerOSes() (actives []osTypeInterface) { + Log.Info("Detecting OS of containers... ") osTypesChan := make(chan []osTypeInterface, len(servers)) defer close(osTypesChan) for _, s := range servers { go func(s osTypeInterface) { + defer func() { + if p := recover(); p != nil { + Log.Debugf("Panic: %s on %s", + p, s.getServerInfo().ServerName) + } + }() osTypesChan <- detectContainerOSesOnServer(s) }(s) } - timeout := time.After(60 * time.Second) - for i := 0; i < len(config.Conf.Servers); i++ { + var oses []osTypeInterface + timeout := time.After(30 * time.Second) + for i := 0; i < len(servers); i++ { select { case res := <-osTypesChan: - oses = append(oses, res...) for _, osi := range res { + sinfo := osi.getServerInfo() if 0 < len(osi.getErrs()) { + Log.Errorf("Failed: %s err: %s", sinfo.ServerName, osi.getErrs()) continue } - sinfo := osi.getServerInfo() - Log.Infof("Detected %s/%s on %s: %s", - sinfo.Container.ContainerID, sinfo.Container.Name, - sinfo.ServerName, osi.getDistributionInfo()) + oses = append(oses, res...) + Log.Infof("Detected: %s@%s: %s", + sinfo.Container.Name, sinfo.ServerName, osi.getDistributionInfo()) } case <-timeout: - msg := "Timeout occurred while detecting" + msg := "Timed out while detecting containers" Log.Error(msg) for servername := range config.Conf.Servers { found := false @@ -268,24 +259,17 @@ func detectContainerOSes() (oses []osTypeInterface, err error) { } } if !found { - Log.Errorf("Failed to detect. servername: %s", servername) + Log.Errorf("Timed out: %s", servername) + } } - return oses, fmt.Errorf(msg) } } - - errs := []error{} - for _, osi := range oses { - if 0 < len(osi.getErrs()) { - errs = append(errs, fmt.Errorf( - "Error occurred on %s. errs: %s", - osi.getServerInfo().ServerName, osi.getErrs())) + for _, o := range oses { + if len(o.getErrs()) == 0 { + actives = append(actives, o) } } - if 0 < len(errs) { - return oses, fmt.Errorf("%s", errs) - } return } @@ -361,6 +345,33 @@ func detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeIn return oses } +// DetectPlatforms detects the platform of each servers. +func DetectPlatforms(localLogger *logrus.Entry) { + errs := detectPlatforms() + if 0 < len(errs) { + // Only logging + Log.Warnf("Failed to detect platforms. err: %v", errs) + } + for i, s := range servers { + if s.getServerInfo().IsContainer() { + Log.Infof("(%d/%d) %s on %s is running on %s", + i+1, len(servers), + s.getServerInfo().Container.Name, + s.getServerInfo().ServerName, + s.getPlatform().Name, + ) + + } else { + Log.Infof("(%d/%d) %s is running on %s", + i+1, len(servers), + s.getServerInfo().ServerName, + s.getPlatform().Name, + ) + } + } + return +} + func detectPlatforms() []error { timeoutSec := 1 * 60 return parallelSSHExec(func(o osTypeInterface) error { @@ -410,7 +421,7 @@ func checkRequiredPackagesInstalled() []error { } func scanPackages() []error { - timeoutSec := 30 * 60 + timeoutSec := 120 * 60 return parallelSSHExec(func(o osTypeInterface) error { return o.scanPackages() }, timeoutSec) diff --git a/scan/sshutil.go b/scan/sshutil.go index 88ea7bab35..1c30e5eda2 100644 --- a/scan/sshutil.go +++ b/scan/sshutil.go @@ -37,18 +37,30 @@ import ( "github.com/Sirupsen/logrus" "github.com/cenkalti/backoff" conf "github.com/future-architect/vuls/config" - "github.com/k0kubun/pp" + "github.com/future-architect/vuls/util" ) type sshResult struct { + Servername string Host string Port string + Cmd string Stdout string Stderr string ExitStatus int + Error error +} + +func (s sshResult) String() string { + return fmt.Sprintf( + "SSHResult: servername: %s, cmd: %s, exitstatus: %d, stdout: %s, stderr: %s, err: %s", + s.Servername, s.Cmd, s.ExitStatus, s.Stdout, s.Stderr, s.Error) } func (s sshResult) isSuccess(expectedStatusCodes ...int) bool { + if s.Error != nil { + return false + } if len(expectedStatusCodes) == 0 { return s.ExitStatus == 0 } @@ -71,6 +83,12 @@ func parallelSSHExec(fn func(osTypeInterface) error, timeoutSec ...int) (errs [] defer close(errChan) for _, s := range servers { go func(s osTypeInterface) { + defer func() { + if p := recover(); p != nil { + logrus.Debugf("Panic: %s on %s", + p, s.getServerInfo().ServerName) + } + }() if err := fn(s); err != nil { errChan <- fmt.Errorf("%s@%s:%s: %s", s.getServerInfo().User, @@ -109,28 +127,35 @@ func parallelSSHExec(fn func(osTypeInterface) error, timeoutSec ...int) (errs [] func sshExec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (result sshResult) { if runtime.GOOS == "windows" || !conf.Conf.SSHExternal { - return sshExecNative(c, cmd, sudo, log...) + result = sshExecNative(c, cmd, sudo) + } else { + result = sshExecExternal(c, cmd, sudo) } - return sshExecExternal(c, cmd, sudo, log...) -} -func sshExecNative(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (result sshResult) { logger := getSSHLogger(log...) + logger.Debug(result) + return +} - cmd = decolateCmd(c, cmd, sudo) - logger.Debugf("Command: %s", - strings.Replace(maskPassword(cmd, c.Password), "\n", "", -1)) +func sshExecNative(c conf.ServerInfo, cmd string, sudo bool) (result sshResult) { + result.Servername = c.ServerName + result.Host = c.Host + result.Port = c.Port var client *ssh.Client var err error - client, err = sshConnect(c) + if client, err = sshConnect(c); err != nil { + result.Error = err + result.ExitStatus = 999 + return + } defer client.Close() var session *ssh.Session if session, err = client.NewSession(); err != nil { - logger.Errorf("Failed to new session. err: %s, c: %s", - err, - pp.Sprintf("%v", c)) + result.Error = fmt.Errorf( + "Failed to create a new session. servername: %s, err: %s", + c.ServerName, err) result.ExitStatus = 999 return } @@ -143,10 +168,9 @@ func sshExecNative(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entr ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud } if err = session.RequestPty("xterm", 400, 256, modes); err != nil { - logger.Errorf("Failed to request for pseudo terminal. err: %s, c: %s", - err, - pp.Sprintf("%v", c)) - + result.Error = fmt.Errorf( + "Failed to request for pseudo terminal. servername: %s, err: %s", + c.ServerName, err) result.ExitStatus = 999 return } @@ -155,6 +179,7 @@ func sshExecNative(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entr session.Stdout = &stdoutBuf session.Stderr = &stderrBuf + cmd = decolateCmd(c, cmd, sudo) if err := session.Run(cmd); err != nil { if exitErr, ok := err.(*ssh.ExitError); ok { result.ExitStatus = exitErr.ExitStatus() @@ -167,23 +192,14 @@ func sshExecNative(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entr result.Stdout = stdoutBuf.String() result.Stderr = stderrBuf.String() - result.Host = c.Host - result.Port = c.Port - - logger.Debugf( - "SSH executed. cmd: %s, err: %#v, status: %d\nstdout: \n%s\nstderr: \n%s", - maskPassword(cmd, c.Password), err, result.ExitStatus, result.Stdout, result.Stderr) - + result.Cmd = strings.Replace(maskPassword(cmd, c.Password), "\n", "", -1) return } -func sshExecExternal(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (result sshResult) { - logger := getSSHLogger(log...) - +func sshExecExternal(c conf.ServerInfo, cmd string, sudo bool) (result sshResult) { sshBinaryPath, err := exec.LookPath("ssh") if err != nil { - logger.Debug("Failed to find SSH binary, using native Go implementation") - return sshExecNative(c, cmd, sudo, log...) + return sshExecNative(c, cmd, sudo) } defaultSSHArgs := []string{ @@ -235,31 +251,17 @@ func sshExecExternal(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.En result.Stdout = stdoutBuf.String() result.Stderr = stderrBuf.String() + result.Servername = c.ServerName result.Host = c.Host result.Port = c.Port - - logger.Debugf( - "SSH executed. cmd: %s %s, err: %#v, status: %d\nstdout: \n%s\nstderr: \n%s", - sshBinaryPath, - maskPassword(strings.Join(args, " "), c.Password), - err, result.ExitStatus, result.Stdout, result.Stderr) - + result.Cmd = fmt.Sprintf("%s %s", + sshBinaryPath, maskPassword(strings.Join(args, " "), c.Password)) return } func getSSHLogger(log ...*logrus.Entry) *logrus.Entry { if len(log) == 0 { - level := logrus.InfoLevel - if conf.Conf.Debug == true { - level = logrus.DebugLevel - } - l := &logrus.Logger{ - Out: os.Stderr, - Formatter: new(logrus.TextFormatter), - Hooks: make(logrus.LevelHooks), - Level: level, - } - return logrus.NewEntry(l) + return util.NewCustomLogger(conf.ServerInfo{}) } return log[0] } @@ -314,15 +316,13 @@ func tryAgentConnect(c conf.ServerInfo) *ssh.Client { } func sshConnect(c conf.ServerInfo) (client *ssh.Client, err error) { - if client = tryAgentConnect(c); client != nil { return client, nil } var auths = []ssh.AuthMethod{} if auths, err = addKeyAuth(auths, c.KeyPath, c.KeyPassword); err != nil { - logrus.Fatalf("Failed to add keyAuth. %s@%s:%s err: %s", - c.User, c.Host, c.Port, err) + return nil, err } if c.Password != "" { @@ -336,8 +336,9 @@ func sshConnect(c conf.ServerInfo) (client *ssh.Client, err error) { } notifyFunc := func(e error, t time.Duration) { - logrus.Warnf("Failed to ssh %s@%s:%s err: %s, Retrying in %s...", - c.User, c.Host, c.Port, e, t) + logger := getSSHLogger() + logger.Debugf("Failed to Dial to %s, err: %s, Retrying in %s...", + c.ServerName, e, t) } err = backoff.RetryNotify(func() error { if client, err = ssh.Dial("tcp", c.Host+":"+c.Port, config); err != nil { diff --git a/util/util.go b/util/util.go index 21f9b218c7..ecda23e3cd 100644 --- a/util/util.go +++ b/util/util.go @@ -31,6 +31,12 @@ func GenWorkers(num int) chan<- func() { tasks := make(chan func()) for i := 0; i < num; i++ { go func() { + defer func() { + if p := recover(); p != nil { + log := NewCustomLogger(config.ServerInfo{}) + log.Debugf("Panic: %s") + } + }() for f := range tasks { f() }