Skip to content

Commit

Permalink
feat(cmd): add vulncheck
Browse files Browse the repository at this point in the history
  • Loading branch information
MaineK00n committed Aug 29, 2024
1 parent bd5ef68 commit 98dd60e
Show file tree
Hide file tree
Showing 12 changed files with 662 additions and 93 deletions.
126 changes: 112 additions & 14 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ Flags:
Use "go-kev [command] --help" for more information about a command.
```

# Fetch Known Exploited Vulnerabilities
# Fetch CISA Known Exploited Vulnerabilities
```console
$ go-kev fetch kevuln
INFO[11-16|04:39:00] Fetching Known Exploited Vulnerabilities
Expand All @@ -43,6 +43,16 @@ INFO[11-16|04:39:00] Inserting Known Exploited Vulnerabilities...
INFO[11-16|04:39:00] CveID Count count=291
```

# Fetch VulnCheck Known Exploited Vulnerabilities (https://vulncheck.com/kev)
```console
$ go-kev fetch vulncheck
INFO[08-23|02:34:55] Fetching VulnCheck Known Exploited Vulnerabilities
INFO[08-23|02:34:56] Insert VulnCheck Known Exploited Vulnerabilities into go-kev. db=sqlite3
INFO[08-23|02:34:56] Inserting VulnCheck Known Exploited Vulnerabilities...
2832 / 2832 [------------------------------------------------------------------------------] 100.00% 2931 p/s
INFO[08-23|02:34:57] CveID Count count=2832
```

# Server mode
```console
$ go-kev server
Expand All @@ -61,19 +71,107 @@ ____________________________________O/_______
{"time":"2021-11-16T04:40:30.511368993+09:00","id":"","remote_ip":"127.0.0.1","host":"127.0.0.1:1328","method":"GET","uri":"/cves/CVE-2021-27104​","user_agent":"curl/7.68.0","status":200,"error":"","latency":5870905,"latency_human":"5.870905ms","bytes_in":0,"bytes_out":397}

$ curl http://127.0.0.1:1328/cves/CVE-2021-27104 | jq
[
{
"CveID": "CVE-2021-27104",
"Source": "Accellion",
"Product": "FTA",
"Title": "Accellion FTA OS Command Injection Vulnerability",
"AddedDate": "2021-11-03T00:00:00Z",
"Description": "Accellion FTA 9_12_370 and earlier is affected by OS command execution via a crafted POST request to various admin endpoints.",
"Action": "Apply updates per vendor instructions.",
"DueDate": "2021-11-17T00:00:00Z",
"Notes": ""
}
]
{
"cisa": [
{
"cveID": "CVE-2021-27104",
"vendorProject": "Accellion",
"product": "FTA",
"vulnerabilityName": "Accellion FTA OS Command Injection Vulnerability",
"dateAdded": "2021-11-03T00:00:00Z",
"shortDescription": "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.",
"requiredAction": "Apply updates per vendor instructions.",
"dueDate": "2021-11-17T00:00:00Z",
"knownRansomwareCampaignUse": "Known",
"notes": ""
}
],
"vulncheck": [
{
"vendorProject": "Accellion",
"product": "FTA",
"shortDescription": "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.",
"vulnerabilityName": "Accellion FTA OS Command Injection Vulnerability",
"required_action": "Apply updates per vendor instructions.",
"knownRansomwareCampaignUse": "Known",
"cve": [
{
"cveID": "CVE-2021-27104"
}
],
"vulncheck_xdb": [],
"vulncheck_reported_exploitation": [
{
"url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
"date_added": "2021-11-03T00:00:00Z"
},
{
"url": "https://unit42.paloaltonetworks.com/clop-ransomware/",
"date_added": "2021-04-13T00:00:00Z"
},
{
"url": "https://www.trendmicro.com/vinfo/us/security/news/cybercrime-and-digital-threats/ransomware-double-extortion-and-beyond-revil-clop-and-conti",
"date_added": "2021-06-15T00:00:00Z"
},
{
"url": "https://cybersecurityworks.com/howdymanage/uploads/file/ransomware-_-2022-spotlight-report_compressed.pdf",
"date_added": "2022-01-26T00:00:00Z"
},
{
"url": "https://www.paloaltonetworks.com/content/dam/pan/en_US/assets/pdf/reports/2022-unit42-ransomware-threat-report-final.pdf",
"date_added": "2022-03-24T00:00:00Z"
},
{
"url": "https://static.tenable.com/marketing/whitepapers/Whitepaper-Ransomware_Ecosystem.pdf",
"date_added": "2022-06-22T00:00:00Z"
},
{
"url": "https://www.group-ib.com/resources/research-hub/hi-tech-crime-trends-2022/",
"date_added": "2023-01-17T00:00:00Z"
},
{
"url": "https://fourcore.io/blogs/clop-ransomware-history-adversary-simulation",
"date_added": "2023-06-03T00:00:00Z"
},
{
"url": "https://blog.talosintelligence.com/talos-ir-q2-2023-quarterly-recap/",
"date_added": "2023-07-26T00:00:00Z"
},
{
"url": "https://www.sentinelone.com/resources/watchtower-end-of-year-report-2023/",
"date_added": "2021-11-03T00:00:00Z"
},
{
"url": "https://www.trustwave.com/en-us/resources/blogs/trustwave-blog/defending-the-energy-sector-against-cyber-threats-insights-from-trustwave-spiderlabs/",
"date_added": "2024-05-15T00:00:00Z"
},
{
"url": "https://cisa.gov/news-events/cybersecurity-advisories/aa21-055a",
"date_added": "2021-06-17T00:00:00Z"
},
{
"url": "https://www.cisa.gov/news-events/cybersecurity-advisories/aa21-209a",
"date_added": "2021-08-20T00:00:00Z"
},
{
"url": "https://cisa.gov/news-events/alerts/2022/04/27/2021-top-routinely-exploited-vulnerabilities",
"date_added": "2022-04-28T00:00:00Z"
},
{
"url": "https://cisa.gov/news-events/cybersecurity-advisories/aa22-117a",
"date_added": "2022-04-28T00:00:00Z"
},
{
"url": "https://www.hhs.gov/sites/default/files/threat-profile-june-2023.pdf",
"date_added": "2023-06-13T00:00:00Z"
}
],
"dueDate": "2021-11-17T00:00:00Z",
"cisa_date_added": "2021-11-03T00:00:00Z",
"date_added": "2021-04-13T00:00:00Z"
}
]
}
```

# License
Expand Down
4 changes: 2 additions & 2 deletions commands/fetch-kevuln.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"golang.org/x/xerrors"

"github.com/vulsio/go-kev/db"
"github.com/vulsio/go-kev/fetcher"
fetcher "github.com/vulsio/go-kev/fetcher/kevuln"
"github.com/vulsio/go-kev/models"
"github.com/vulsio/go-kev/utils"
)
Expand Down Expand Up @@ -52,7 +52,7 @@ func fetchKEVuln(_ *cobra.Command, _ []string) (err error) {

log15.Info("Fetching Known Exploited Vulnerabilities")
var vulns []models.KEVuln
if vulns, err = fetcher.FetchKEVuln(); err != nil {
if vulns, err = fetcher.Fetch(); err != nil {
return xerrors.Errorf("Failed to fetch Known Exploited Vulnerabilities. err: %w", err)
}

Expand Down
71 changes: 71 additions & 0 deletions commands/fetch-vulncheck.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package commands

import (
"time"

"github.com/inconshreveable/log15"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"golang.org/x/xerrors"

"github.com/vulsio/go-kev/db"
fetcher "github.com/vulsio/go-kev/fetcher/vulncheck"
"github.com/vulsio/go-kev/models"
"github.com/vulsio/go-kev/utils"
)

var fetchVulnCheckCmd = &cobra.Command{
Use: "vulncheck <vuls-data-raw-vulncheck-kev directory>",
Short: "Fetch the data of known exploited vulnerabilities catalog by VulnCheck (https://vulncheck.com/kev)",
Long: `Fetch the data of known exploited vulnerabilities catalog by VulnCheck (https://vulncheck.com/kev)`,
Args: cobra.NoArgs,
RunE: fetchVulnCheck,
}

func init() {
fetchCmd.AddCommand(fetchVulnCheckCmd)
}

func fetchVulnCheck(_ *cobra.Command, _ []string) (err error) {
if err := utils.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
return xerrors.Errorf("Failed to SetLogger. err: %w", err)
}

driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
if err != nil {
if xerrors.Is(err, db.ErrDBLocked) {
return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err)
}
return xerrors.Errorf("Failed to open DB. err: %w", err)
}

fetchMeta, err := driver.GetFetchMeta()
if err != nil {
return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
}
if fetchMeta.OutDated() {
return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
}
// If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
}

log15.Info("Fetching VulnCheck Known Exploited Vulnerabilities")
var vulns []models.VulnCheck
if vulns, err = fetcher.Fetch(); err != nil {
return xerrors.Errorf("Failed to fetch VulnCheck Known Exploited Vulnerabilities. err: %w", err)
}

log15.Info("Insert VulnCheck Known Exploited Vulnerabilities into go-kev.", "db", driver.Name())
if err := driver.InsertVulnCheck(vulns); err != nil {
return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
}

fetchMeta.LastFetchedAt = time.Now()
if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
}

return nil
}
11 changes: 9 additions & 2 deletions db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,22 @@ type DB interface {
UpsertFetchMeta(*models.FetchMeta) error

InsertKEVulns([]models.KEVuln) error
GetKEVulnByCveID(string) ([]models.KEVuln, error)
GetKEVulnByMultiCveID([]string) (map[string][]models.KEVuln, error)
InsertVulnCheck([]models.VulnCheck) error
GetKEVByCveID(string) (Response, error)
GetKEVByMultiCveID([]string) (map[string]Response, error)
}

// Option :
type Option struct {
RedisTimeout time.Duration
}

// Response :
type Response struct {
CISA []models.KEVuln `json:"cisa,omitempty"`
VulnCheck []models.VulnCheck `json:"vulncheck,omitempty"`
}

// NewDB :
func NewDB(dbType string, dbPath string, debugSQL bool, option Option) (driver DB, err error) {
if driver, err = newDB(dbType); err != nil {
Expand Down
88 changes: 72 additions & 16 deletions db/rdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,11 @@ func (r *RDBDriver) MigrateDB() error {
&models.FetchMeta{},

&models.KEVuln{},

&models.VulnCheck{},
&models.VulnCheckCVE{},
&models.VulnCheckXDB{},
&models.VulnCheckReportedExploitation{},
); err != nil {
switch r.name {
case dialectSqlite3:
Expand All @@ -163,9 +168,7 @@ func (r *RDBDriver) MigrateDB() error {
}
}
case dialectMysql, dialectPostgreSQL:
if err != nil {
return xerrors.Errorf("Failed to migrate. err: %w", err)
}
return xerrors.Errorf("Failed to migrate. err: %w", err)
default:
return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name)
}
Expand Down Expand Up @@ -261,24 +264,77 @@ func (r *RDBDriver) deleteAndInsertKEVulns(records []models.KEVuln) (err error)
return nil
}

// GetKEVulnByCveID :
func (r *RDBDriver) GetKEVulnByCveID(cveID string) ([]models.KEVuln, error) {
vuln := []models.KEVuln{}
if err := r.conn.Where(&models.KEVuln{CveID: cveID}).Find(&vuln).Error; err != nil {
return nil, xerrors.Errorf("Failed to get info by CVE-ID. err: %w", err)
// InsertVulnCheck :
func (r *RDBDriver) InsertVulnCheck(records []models.VulnCheck) (err error) {
log15.Info("Inserting VulnCheck Known Exploited Vulnerabilities...")
return r.deleteAndInsertVulnCheck(records)
}

func (r *RDBDriver) deleteAndInsertVulnCheck(records []models.VulnCheck) (err error) {
bar := pb.StartNew(len(records)).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
tx := r.conn.Begin()
defer func() {
if err != nil {
tx.Rollback()
return
}
tx.Commit()
}()

// Delete all old records
for _, table := range []interface{}{models.VulnCheck{}, models.VulnCheckCVE{}, models.VulnCheckXDB{}, models.VulnCheckReportedExploitation{}} {
if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(table).Error; err != nil {
return xerrors.Errorf("Failed to delete old records. err: %w", err)
}
}
return vuln, nil

batchSize := viper.GetInt("batch-size")
if batchSize < 1 {
return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly")
}

for idx := range chunkSlice(len(records), batchSize) {
if err = tx.Create(records[idx.From:idx.To]).Error; err != nil {
return xerrors.Errorf("Failed to insert. err: %w", err)
}
bar.Add(idx.To - idx.From)
}
bar.Finish()
log15.Info("CveID Count", "count", len(records))
return nil
}

// GetKEVulnByMultiCveID :
func (r *RDBDriver) GetKEVulnByMultiCveID(cveIDs []string) (map[string][]models.KEVuln, error) {
vuln := map[string][]models.KEVuln{}
// GetKEVByCveID :
func (r *RDBDriver) GetKEVByCveID(cveID string) (Response, error) {
var res Response
if err := r.conn.Where(&models.KEVuln{CveID: cveID}).Find(&res.CISA).Error; err != nil {
return Response{}, xerrors.Errorf("Failed to get CISA info by CVE-ID. err: %w", err)
}
if err := r.conn.
Joins("JOIN vuln_check_cves ON vuln_check_cves.vuln_check_id = vuln_checks.id AND vuln_check_cves.cve_id = ?", cveID).
Preload("CVE").
Preload("VulnCheckXDB").
Preload("VulnCheckReportedExploitation").
Find(&res.VulnCheck).Error; err != nil {
return Response{}, xerrors.Errorf("Failed to get VulnCheck info by CVE-ID. err: %w", err)
}
return res, nil
}

// GetKEVByMultiCveID :
func (r *RDBDriver) GetKEVByMultiCveID(cveIDs []string) (map[string]Response, error) {
m := make(map[string]Response)
for _, cveID := range cveIDs {
v, err := r.GetKEVulnByCveID(cveID)
res, err := r.GetKEVByCveID(cveID)
if err != nil {
return nil, err
return nil, xerrors.Errorf("Failed to get KEV by %s. err: %w", cveID, err)
}
vuln[cveID] = v
m[cveID] = res
}
return vuln, nil
return m, nil
}
Loading

0 comments on commit 98dd60e

Please sign in to comment.