feat!(cmd): add vulncheck

README.md

@@ -32,7 +32,7 @@ Flags:
 Use "go-kev [command] --help" for more information about a command.
 ```
 
-# Fetch Known Exploited Vulnerabilities
+# Fetch CISA Known Exploited Vulnerabilities
 ```console
 $ go-kev fetch kevuln
 INFO[11-16|04:39:00] Fetching Known Exploited Vulnerabilities 
@@ -43,6 +43,18 @@ INFO[11-16|04:39:00] Inserting Known Exploited Vulnerabilities...
 INFO[11-16|04:39:00] CveID Count                              count=291
 ```
 
+# Fetch VulnCheck Known Exploited Vulnerabilities (https://vulncheck.com/kev)
+Before you use this data from VulnCheck, you MUST read https://docs.vulncheck.com/community/vulncheck-kev/attribution and make sure it's satisfied.
+
+```console
+$ go-kev fetch vulncheck
+INFO[08-23|02:34:55] Fetching VulnCheck Known Exploited Vulnerabilities 
+INFO[08-23|02:34:56] Insert VulnCheck Known Exploited Vulnerabilities into go-kev. db=sqlite3
+INFO[08-23|02:34:56] Inserting VulnCheck Known Exploited Vulnerabilities... 
+2832 / 2832 [------------------------------------------------------------------------------] 100.00% 2931 p/s
+INFO[08-23|02:34:57] CveID Count                              count=2832
+```
+
 # Server mode
 ```console
 $ go-kev server 
@@ -61,19 +73,107 @@ ____________________________________O/_______
 {"time":"2021-11-16T04:40:30.511368993+09:00","id":"","remote_ip":"127.0.0.1","host":"127.0.0.1:1328","method":"GET","uri":"/cves/CVE-2021-27104​","user_agent":"curl/7.68.0","status":200,"error":"","latency":5870905,"latency_human":"5.870905ms","bytes_in":0,"bytes_out":397}
 
 $ curl http://127.0.0.1:1328/cves/CVE-2021-27104 | jq
-[
-  {
-    "CveID": "CVE-2021-27104",
-    "Source": "Accellion",
-    "Product": "FTA",
-    "Title": "Accellion FTA OS Command Injection Vulnerability",
-    "AddedDate": "2021-11-03T00:00:00Z",
-    "Description": "Accellion FTA 9_12_370 and earlier is affected by OS command execution via a crafted POST request to various admin endpoints.",
-    "Action": "Apply updates per vendor instructions.",
-    "DueDate": "2021-11-17T00:00:00Z",
-    "Notes": ""
-  }
-]
+{
+  "cisa": [
+    {
+      "cveID": "CVE-2021-27104",
+      "vendorProject": "Accellion",
+      "product": "FTA",
+      "vulnerabilityName": "Accellion FTA OS Command Injection Vulnerability",
+      "dateAdded": "2021-11-03T00:00:00Z",
+      "shortDescription": "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.",
+      "requiredAction": "Apply updates per vendor instructions.",
+      "dueDate": "2021-11-17T00:00:00Z",
+      "knownRansomwareCampaignUse": "Known",
+      "notes": ""
+    }
+  ],
+  "vulncheck": [
+    {
+      "vendorProject": "Accellion",
+      "product": "FTA",
+      "shortDescription": "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.",
+      "vulnerabilityName": "Accellion FTA OS Command Injection Vulnerability",
+      "required_action": "Apply updates per vendor instructions.",
+      "knownRansomwareCampaignUse": "Known",
+      "cve": [
+        {
+          "cveID": "CVE-2021-27104"
+        }
+      ],
+      "vulncheck_xdb": [],
+      "vulncheck_reported_exploitation": [
+        {
+          "url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
+          "date_added": "2021-11-03T00:00:00Z"
+        },
+        {
+          "url": "https://unit42.paloaltonetworks.com/clop-ransomware/",
+          "date_added": "2021-04-13T00:00:00Z"
+        },
+        {
+          "url": "https://www.trendmicro.com/vinfo/us/security/news/cybercrime-and-digital-threats/ransomware-double-extortion-and-beyond-revil-clop-and-conti",
+          "date_added": "2021-06-15T00:00:00Z"
+        },
+        {
+          "url": "https://cybersecurityworks.com/howdymanage/uploads/file/ransomware-_-2022-spotlight-report_compressed.pdf",
+          "date_added": "2022-01-26T00:00:00Z"
+        },
+        {
+          "url": "https://www.paloaltonetworks.com/content/dam/pan/en_US/assets/pdf/reports/2022-unit42-ransomware-threat-report-final.pdf",
+          "date_added": "2022-03-24T00:00:00Z"
+        },
+        {
+          "url": "https://static.tenable.com/marketing/whitepapers/Whitepaper-Ransomware_Ecosystem.pdf",
+          "date_added": "2022-06-22T00:00:00Z"
+        },
+        {
+          "url": "https://www.group-ib.com/resources/research-hub/hi-tech-crime-trends-2022/",
+          "date_added": "2023-01-17T00:00:00Z"
+        },
+        {
+          "url": "https://fourcore.io/blogs/clop-ransomware-history-adversary-simulation",
+          "date_added": "2023-06-03T00:00:00Z"
+        },
+        {
+          "url": "https://blog.talosintelligence.com/talos-ir-q2-2023-quarterly-recap/",
+          "date_added": "2023-07-26T00:00:00Z"
+        },
+        {
+          "url": "https://www.sentinelone.com/resources/watchtower-end-of-year-report-2023/",
+          "date_added": "2021-11-03T00:00:00Z"
+        },
+        {
+          "url": "https://www.trustwave.com/en-us/resources/blogs/trustwave-blog/defending-the-energy-sector-against-cyber-threats-insights-from-trustwave-spiderlabs/",
+          "date_added": "2024-05-15T00:00:00Z"
+        },
+        {
+          "url": "https://cisa.gov/news-events/cybersecurity-advisories/aa21-055a",
+          "date_added": "2021-06-17T00:00:00Z"
+        },
+        {
+          "url": "https://www.cisa.gov/news-events/cybersecurity-advisories/aa21-209a",
+          "date_added": "2021-08-20T00:00:00Z"
+        },
+        {
+          "url": "https://cisa.gov/news-events/alerts/2022/04/27/2021-top-routinely-exploited-vulnerabilities",
+          "date_added": "2022-04-28T00:00:00Z"
+        },
+        {
+          "url": "https://cisa.gov/news-events/cybersecurity-advisories/aa22-117a",
+          "date_added": "2022-04-28T00:00:00Z"
+        },
+        {
+          "url": "https://www.hhs.gov/sites/default/files/threat-profile-june-2023.pdf",
+          "date_added": "2023-06-13T00:00:00Z"
+        }
+      ],
+      "dueDate": "2021-11-17T00:00:00Z",
+      "cisa_date_added": "2021-11-03T00:00:00Z",
+      "date_added": "2021-04-13T00:00:00Z"
+    }
+  ]
+}
 ```
 
 # License

commands/fetch-kevuln.go

@@ -1,4 +1,4 @@
 package commands
 
 import (
 	"time"
@@ -9,7 +9,7 @@
 	"golang.org/x/xerrors"
 
 	"github.com/vulsio/go-kev/db"
-	"github.com/vulsio/go-kev/fetcher"
+	fetcher "github.com/vulsio/go-kev/fetcher/kevuln"
 	"github.com/vulsio/go-kev/models"
 	"github.com/vulsio/go-kev/utils"
 )
@@ -52,7 +52,7 @@
 
 	log15.Info("Fetching Known Exploited Vulnerabilities")
 	var vulns []models.KEVuln
-	if vulns, err = fetcher.FetchKEVuln(); err != nil {
+	if vulns, err = fetcher.Fetch(); err != nil {
 		return xerrors.Errorf("Failed to fetch Known Exploited Vulnerabilities. err: %w", err)
 	}
 

commands/fetch-vulncheck.go

@@ -0,0 +1,71 @@
+package commands
+
+import (
+	"time"
+
+	"github.com/inconshreveable/log15"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"golang.org/x/xerrors"
+
+	"github.com/vulsio/go-kev/db"
+	fetcher "github.com/vulsio/go-kev/fetcher/vulncheck"
+	"github.com/vulsio/go-kev/models"
+	"github.com/vulsio/go-kev/utils"
+)
+
+var fetchVulnCheckCmd = &cobra.Command{
+	Use:   "vulncheck <vuls-data-raw-vulncheck-kev directory>",
+	Short: "Fetch the data of known exploited vulnerabilities catalog by VulnCheck (https://vulncheck.com/kev)",
+	Long:  `Fetch the data of known exploited vulnerabilities catalog by VulnCheck (https://vulncheck.com/kev)`,
+	Args:  cobra.NoArgs,
+	RunE:  fetchVulnCheck,
+}
+
+func init() {
+	fetchCmd.AddCommand(fetchVulnCheckCmd)
+}
+
+func fetchVulnCheck(_ *cobra.Command, _ []string) (err error) {
+	if err := utils.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
+	driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
+	if err != nil {
+		if xerrors.Is(err, db.ErrDBLocked) {
+			return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err)
+		}
+		return xerrors.Errorf("Failed to open DB. err: %w", err)
+	}
+
+	fetchMeta, err := driver.GetFetchMeta()
+	if err != nil {
+		return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
+	}
+	if fetchMeta.OutDated() {
+		return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
+	}
+	// If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	log15.Info("Fetching VulnCheck Known Exploited Vulnerabilities")
+	var vulns []models.VulnCheck
+	if vulns, err = fetcher.Fetch(); err != nil {
+		return xerrors.Errorf("Failed to fetch VulnCheck Known Exploited Vulnerabilities. err: %w", err)
+	}
+
+	log15.Info("Insert VulnCheck Known Exploited Vulnerabilities into go-kev.", "db", driver.Name())
+	if err := driver.InsertVulnCheck(vulns); err != nil {
+		return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	fetchMeta.LastFetchedAt = time.Now()
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	return nil
+}

db/db.go

@@ -1,4 +1,4 @@
 package db
 
 import (
 	"time"
@@ -20,15 +20,22 @@
 	UpsertFetchMeta(*models.FetchMeta) error
 
 	InsertKEVulns([]models.KEVuln) error
-	GetKEVulnByCveID(string) ([]models.KEVuln, error)
-	GetKEVulnByMultiCveID([]string) (map[string][]models.KEVuln, error)
+	InsertVulnCheck([]models.VulnCheck) error
+	GetKEVByCveID(string) (Response, error)
+	GetKEVByMultiCveID([]string) (map[string]Response, error)
 }
 
 // Option :
 type Option struct {
 	RedisTimeout time.Duration
 }
 
+// Response :
+type Response struct {
+	CISA      []models.KEVuln    `json:"cisa,omitempty"`
+	VulnCheck []models.VulnCheck `json:"vulncheck,omitempty"`
+}
+
 // NewDB :
 func NewDB(dbType string, dbPath string, debugSQL bool, option Option) (driver DB, err error) {
 	if driver, err = newDB(dbType); err != nil {

db/rdb.go

@@ -141,6 +141,11 @@ func (r *RDBDriver) MigrateDB() error {
 		&models.FetchMeta{},
 
 		&models.KEVuln{},
+
+		&models.VulnCheck{},
+		&models.VulnCheckCVE{},
+		&models.VulnCheckXDB{},
+		&models.VulnCheckReportedExploitation{},
 	); err != nil {
 		switch r.name {
 		case dialectSqlite3:
@@ -163,9 +168,7 @@ func (r *RDBDriver) MigrateDB() error {
 				}
 			}
 		case dialectMysql, dialectPostgreSQL:
-			if err != nil {
-				return xerrors.Errorf("Failed to migrate. err: %w", err)
-			}
+			return xerrors.Errorf("Failed to migrate. err: %w", err)
 		default:
 			return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name)
 		}
@@ -261,24 +264,77 @@ func (r *RDBDriver) deleteAndInsertKEVulns(records []models.KEVuln) (err error)
 	return nil
 }
 
-// GetKEVulnByCveID :
-func (r *RDBDriver) GetKEVulnByCveID(cveID string) ([]models.KEVuln, error) {
-	vuln := []models.KEVuln{}
-	if err := r.conn.Where(&models.KEVuln{CveID: cveID}).Find(&vuln).Error; err != nil {
-		return nil, xerrors.Errorf("Failed to get info by CVE-ID. err: %w", err)
+// InsertVulnCheck :
+func (r *RDBDriver) InsertVulnCheck(records []models.VulnCheck) (err error) {
+	log15.Info("Inserting VulnCheck Known Exploited Vulnerabilities...")
+	return r.deleteAndInsertVulnCheck(records)
+}
+
+func (r *RDBDriver) deleteAndInsertVulnCheck(records []models.VulnCheck) (err error) {
+	bar := pb.StartNew(len(records)).SetWriter(func() io.Writer {
+		if viper.GetBool("log-json") {
+			return io.Discard
+		}
+		return os.Stderr
+	}())
+	tx := r.conn.Begin()
+	defer func() {
+		if err != nil {
+			tx.Rollback()
+			return
+		}
+		tx.Commit()
+	}()
+
+	// Delete all old records
+	for _, table := range []interface{}{models.VulnCheck{}, models.VulnCheckCVE{}, models.VulnCheckXDB{}, models.VulnCheckReportedExploitation{}} {
+		if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(table).Error; err != nil {
+			return xerrors.Errorf("Failed to delete old records. err: %w", err)
+		}
 	}
-	return vuln, nil
+
+	batchSize := viper.GetInt("batch-size")
+	if batchSize < 1 {
+		return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly")
+	}
+
+	for idx := range chunkSlice(len(records), batchSize) {
+		if err = tx.Create(records[idx.From:idx.To]).Error; err != nil {
+			return xerrors.Errorf("Failed to insert. err: %w", err)
+		}
+		bar.Add(idx.To - idx.From)
+	}
+	bar.Finish()
+	log15.Info("CveID Count", "count", len(records))
+	return nil
 }
 
-// GetKEVulnByMultiCveID :
-func (r *RDBDriver) GetKEVulnByMultiCveID(cveIDs []string) (map[string][]models.KEVuln, error) {
-	vuln := map[string][]models.KEVuln{}
+// GetKEVByCveID :
+func (r *RDBDriver) GetKEVByCveID(cveID string) (Response, error) {
+	var res Response
+	if err := r.conn.Where(&models.KEVuln{CveID: cveID}).Find(&res.CISA).Error; err != nil {
+		return Response{}, xerrors.Errorf("Failed to get CISA info by CVE-ID. err: %w", err)
+	}
+	if err := r.conn.
+		Joins("JOIN vuln_check_cves ON vuln_check_cves.vuln_check_id = vuln_checks.id AND vuln_check_cves.cve_id = ?", cveID).
+		Preload("CVE").
+		Preload("VulnCheckXDB").
+		Preload("VulnCheckReportedExploitation").
+		Find(&res.VulnCheck).Error; err != nil {
+		return Response{}, xerrors.Errorf("Failed to get VulnCheck info by CVE-ID. err: %w", err)
+	}
+	return res, nil
+}
+
+// GetKEVByMultiCveID :
+func (r *RDBDriver) GetKEVByMultiCveID(cveIDs []string) (map[string]Response, error) {
+	m := make(map[string]Response)
 	for _, cveID := range cveIDs {
-		v, err := r.GetKEVulnByCveID(cveID)
+		res, err := r.GetKEVByCveID(cveID)
 		if err != nil {
-			return nil, err
+			return nil, xerrors.Errorf("Failed to get KEV by %s. err: %w", cveID, err)
 		}
-		vuln[cveID] = v
+		m[cveID] = res
 	}
-	return vuln, nil
+	return m, nil
 }