diff --git a/README.md b/README.md index c6d46a2..6aea06e 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Flags: Use "go-kev [command] --help" for more information about a command. ``` -# Fetch Known Exploited Vulnerabilities +# Fetch CISA Known Exploited Vulnerabilities ```console $ go-kev fetch kevuln INFO[11-16|04:39:00] Fetching Known Exploited Vulnerabilities @@ -43,6 +43,16 @@ INFO[11-16|04:39:00] Inserting Known Exploited Vulnerabilities... INFO[11-16|04:39:00] CveID Count count=291 ``` +# Fetch VulnCheck Known Exploited Vulnerabilities (https://vulncheck.com/kev) +```console +$ go-kev fetch vulncheck +INFO[08-23|02:34:55] Fetching VulnCheck Known Exploited Vulnerabilities +INFO[08-23|02:34:56] Insert VulnCheck Known Exploited Vulnerabilities into go-kev. db=sqlite3 +INFO[08-23|02:34:56] Inserting VulnCheck Known Exploited Vulnerabilities... +2832 / 2832 [------------------------------------------------------------------------------] 100.00% 2931 p/s +INFO[08-23|02:34:57] CveID Count count=2832 +``` + # Server mode ```console $ go-kev server @@ -61,19 +71,107 @@ ____________________________________O/_______ {"time":"2021-11-16T04:40:30.511368993+09:00","id":"","remote_ip":"127.0.0.1","host":"127.0.0.1:1328","method":"GET","uri":"/cves/CVE-2021-27104​","user_agent":"curl/7.68.0","status":200,"error":"","latency":5870905,"latency_human":"5.870905ms","bytes_in":0,"bytes_out":397} $ curl http://127.0.0.1:1328/cves/CVE-2021-27104 | jq -[ - { - "CveID": "CVE-2021-27104", - "Source": "Accellion", - "Product": "FTA", - "Title": "Accellion FTA OS Command Injection Vulnerability", - "AddedDate": "2021-11-03T00:00:00Z", - "Description": "Accellion FTA 9_12_370 and earlier is affected by OS command execution via a crafted POST request to various admin endpoints.", - "Action": "Apply updates per vendor instructions.", - "DueDate": "2021-11-17T00:00:00Z", - "Notes": "" - } -] +{ + "cisa": [ + { + "cveID": "CVE-2021-27104", + "vendorProject": "Accellion", + "product": "FTA", + "vulnerabilityName": "Accellion FTA OS Command Injection Vulnerability", + "dateAdded": "2021-11-03T00:00:00Z", + "shortDescription": "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", + "requiredAction": "Apply updates per vendor instructions.", + "dueDate": "2021-11-17T00:00:00Z", + "knownRansomwareCampaignUse": "Known", + "notes": "" + } + ], + "vulncheck": [ + { + "vendorProject": "Accellion", + "product": "FTA", + "shortDescription": "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", + "vulnerabilityName": "Accellion FTA OS Command Injection Vulnerability", + "required_action": "Apply updates per vendor instructions.", + "knownRansomwareCampaignUse": "Known", + "cve": [ + { + "cveID": "CVE-2021-27104" + } + ], + "vulncheck_xdb": [], + "vulncheck_reported_exploitation": [ + { + "url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", + "date_added": "2021-11-03T00:00:00Z" + }, + { + "url": "https://unit42.paloaltonetworks.com/clop-ransomware/", + "date_added": "2021-04-13T00:00:00Z" + }, + { + "url": "https://www.trendmicro.com/vinfo/us/security/news/cybercrime-and-digital-threats/ransomware-double-extortion-and-beyond-revil-clop-and-conti", + "date_added": "2021-06-15T00:00:00Z" + }, + { + "url": "https://cybersecurityworks.com/howdymanage/uploads/file/ransomware-_-2022-spotlight-report_compressed.pdf", + "date_added": "2022-01-26T00:00:00Z" + }, + { + "url": "https://www.paloaltonetworks.com/content/dam/pan/en_US/assets/pdf/reports/2022-unit42-ransomware-threat-report-final.pdf", + "date_added": "2022-03-24T00:00:00Z" + }, + { + "url": "https://static.tenable.com/marketing/whitepapers/Whitepaper-Ransomware_Ecosystem.pdf", + "date_added": "2022-06-22T00:00:00Z" + }, + { + "url": "https://www.group-ib.com/resources/research-hub/hi-tech-crime-trends-2022/", + "date_added": "2023-01-17T00:00:00Z" + }, + { + "url": "https://fourcore.io/blogs/clop-ransomware-history-adversary-simulation", + "date_added": "2023-06-03T00:00:00Z" + }, + { + "url": "https://blog.talosintelligence.com/talos-ir-q2-2023-quarterly-recap/", + "date_added": "2023-07-26T00:00:00Z" + }, + { + "url": "https://www.sentinelone.com/resources/watchtower-end-of-year-report-2023/", + "date_added": "2021-11-03T00:00:00Z" + }, + { + "url": "https://www.trustwave.com/en-us/resources/blogs/trustwave-blog/defending-the-energy-sector-against-cyber-threats-insights-from-trustwave-spiderlabs/", + "date_added": "2024-05-15T00:00:00Z" + }, + { + "url": "https://cisa.gov/news-events/cybersecurity-advisories/aa21-055a", + "date_added": "2021-06-17T00:00:00Z" + }, + { + "url": "https://www.cisa.gov/news-events/cybersecurity-advisories/aa21-209a", + "date_added": "2021-08-20T00:00:00Z" + }, + { + "url": "https://cisa.gov/news-events/alerts/2022/04/27/2021-top-routinely-exploited-vulnerabilities", + "date_added": "2022-04-28T00:00:00Z" + }, + { + "url": "https://cisa.gov/news-events/cybersecurity-advisories/aa22-117a", + "date_added": "2022-04-28T00:00:00Z" + }, + { + "url": "https://www.hhs.gov/sites/default/files/threat-profile-june-2023.pdf", + "date_added": "2023-06-13T00:00:00Z" + } + ], + "dueDate": "2021-11-17T00:00:00Z", + "cisa_date_added": "2021-11-03T00:00:00Z", + "date_added": "2021-04-13T00:00:00Z" + } + ] +} ``` # License diff --git a/commands/fetch-kevuln.go b/commands/fetch-kevuln.go index b93ec9e..1c4cdc4 100644 --- a/commands/fetch-kevuln.go +++ b/commands/fetch-kevuln.go @@ -9,7 +9,7 @@ import ( "golang.org/x/xerrors" "github.com/vulsio/go-kev/db" - "github.com/vulsio/go-kev/fetcher" + fetcher "github.com/vulsio/go-kev/fetcher/kevuln" "github.com/vulsio/go-kev/models" "github.com/vulsio/go-kev/utils" ) @@ -52,7 +52,7 @@ func fetchKEVuln(_ *cobra.Command, _ []string) (err error) { log15.Info("Fetching Known Exploited Vulnerabilities") var vulns []models.KEVuln - if vulns, err = fetcher.FetchKEVuln(); err != nil { + if vulns, err = fetcher.Fetch(); err != nil { return xerrors.Errorf("Failed to fetch Known Exploited Vulnerabilities. err: %w", err) } diff --git a/commands/fetch-vulncheck.go b/commands/fetch-vulncheck.go new file mode 100644 index 0000000..e8c179a --- /dev/null +++ b/commands/fetch-vulncheck.go @@ -0,0 +1,71 @@ +package commands + +import ( + "time" + + "github.com/inconshreveable/log15" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "golang.org/x/xerrors" + + "github.com/vulsio/go-kev/db" + fetcher "github.com/vulsio/go-kev/fetcher/vulncheck" + "github.com/vulsio/go-kev/models" + "github.com/vulsio/go-kev/utils" +) + +var fetchVulnCheckCmd = &cobra.Command{ + Use: "vulncheck ", + Short: "Fetch the data of known exploited vulnerabilities catalog by VulnCheck (https://vulncheck.com/kev)", + Long: `Fetch the data of known exploited vulnerabilities catalog by VulnCheck (https://vulncheck.com/kev)`, + Args: cobra.NoArgs, + RunE: fetchVulnCheck, +} + +func init() { + fetchCmd.AddCommand(fetchVulnCheckCmd) +} + +func fetchVulnCheck(_ *cobra.Command, _ []string) (err error) { + if err := utils.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { + return xerrors.Errorf("Failed to SetLogger. err: %w", err) + } + + driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) + if err != nil { + if xerrors.Is(err, db.ErrDBLocked) { + return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) + } + return xerrors.Errorf("Failed to open DB. err: %w", err) + } + + fetchMeta, err := driver.GetFetchMeta() + if err != nil { + return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) + } + if fetchMeta.OutDated() { + return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) + } + // If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion. + if err := driver.UpsertFetchMeta(fetchMeta); err != nil { + return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) + } + + log15.Info("Fetching VulnCheck Known Exploited Vulnerabilities") + var vulns []models.VulnCheck + if vulns, err = fetcher.Fetch(); err != nil { + return xerrors.Errorf("Failed to fetch VulnCheck Known Exploited Vulnerabilities. err: %w", err) + } + + log15.Info("Insert VulnCheck Known Exploited Vulnerabilities into go-kev.", "db", driver.Name()) + if err := driver.InsertVulnCheck(vulns); err != nil { + return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err) + } + + fetchMeta.LastFetchedAt = time.Now() + if err := driver.UpsertFetchMeta(fetchMeta); err != nil { + return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) + } + + return nil +} diff --git a/db/db.go b/db/db.go index 8e77bf0..61be83f 100644 --- a/db/db.go +++ b/db/db.go @@ -20,8 +20,9 @@ type DB interface { UpsertFetchMeta(*models.FetchMeta) error InsertKEVulns([]models.KEVuln) error - GetKEVulnByCveID(string) ([]models.KEVuln, error) - GetKEVulnByMultiCveID([]string) (map[string][]models.KEVuln, error) + InsertVulnCheck([]models.VulnCheck) error + GetKEVByCveID(string) (Response, error) + GetKEVByMultiCveID([]string) (map[string]Response, error) } // Option : @@ -29,6 +30,12 @@ type Option struct { RedisTimeout time.Duration } +// Response : +type Response struct { + CISA []models.KEVuln `json:"cisa,omitempty"` + VulnCheck []models.VulnCheck `json:"vulncheck,omitempty"` +} + // NewDB : func NewDB(dbType string, dbPath string, debugSQL bool, option Option) (driver DB, err error) { if driver, err = newDB(dbType); err != nil { diff --git a/db/rdb.go b/db/rdb.go index d116632..d1edf13 100644 --- a/db/rdb.go +++ b/db/rdb.go @@ -141,6 +141,11 @@ func (r *RDBDriver) MigrateDB() error { &models.FetchMeta{}, &models.KEVuln{}, + + &models.VulnCheck{}, + &models.VulnCheckCVE{}, + &models.VulnCheckXDB{}, + &models.VulnCheckReportedExploitation{}, ); err != nil { switch r.name { case dialectSqlite3: @@ -163,9 +168,7 @@ func (r *RDBDriver) MigrateDB() error { } } case dialectMysql, dialectPostgreSQL: - if err != nil { - return xerrors.Errorf("Failed to migrate. err: %w", err) - } + return xerrors.Errorf("Failed to migrate. err: %w", err) default: return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name) } @@ -261,24 +264,77 @@ func (r *RDBDriver) deleteAndInsertKEVulns(records []models.KEVuln) (err error) return nil } -// GetKEVulnByCveID : -func (r *RDBDriver) GetKEVulnByCveID(cveID string) ([]models.KEVuln, error) { - vuln := []models.KEVuln{} - if err := r.conn.Where(&models.KEVuln{CveID: cveID}).Find(&vuln).Error; err != nil { - return nil, xerrors.Errorf("Failed to get info by CVE-ID. err: %w", err) +// InsertVulnCheck : +func (r *RDBDriver) InsertVulnCheck(records []models.VulnCheck) (err error) { + log15.Info("Inserting VulnCheck Known Exploited Vulnerabilities...") + return r.deleteAndInsertVulnCheck(records) +} + +func (r *RDBDriver) deleteAndInsertVulnCheck(records []models.VulnCheck) (err error) { + bar := pb.StartNew(len(records)).SetWriter(func() io.Writer { + if viper.GetBool("log-json") { + return io.Discard + } + return os.Stderr + }()) + tx := r.conn.Begin() + defer func() { + if err != nil { + tx.Rollback() + return + } + tx.Commit() + }() + + // Delete all old records + for _, table := range []interface{}{models.VulnCheck{}, models.VulnCheckCVE{}, models.VulnCheckXDB{}, models.VulnCheckReportedExploitation{}} { + if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(table).Error; err != nil { + return xerrors.Errorf("Failed to delete old records. err: %w", err) + } } - return vuln, nil + + batchSize := viper.GetInt("batch-size") + if batchSize < 1 { + return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") + } + + for idx := range chunkSlice(len(records), batchSize) { + if err = tx.Create(records[idx.From:idx.To]).Error; err != nil { + return xerrors.Errorf("Failed to insert. err: %w", err) + } + bar.Add(idx.To - idx.From) + } + bar.Finish() + log15.Info("CveID Count", "count", len(records)) + return nil } -// GetKEVulnByMultiCveID : -func (r *RDBDriver) GetKEVulnByMultiCveID(cveIDs []string) (map[string][]models.KEVuln, error) { - vuln := map[string][]models.KEVuln{} +// GetKEVByCveID : +func (r *RDBDriver) GetKEVByCveID(cveID string) (Response, error) { + var res Response + if err := r.conn.Where(&models.KEVuln{CveID: cveID}).Find(&res.CISA).Error; err != nil { + return Response{}, xerrors.Errorf("Failed to get CISA info by CVE-ID. err: %w", err) + } + if err := r.conn. + Joins("JOIN vuln_check_cves ON vuln_check_cves.vuln_check_id = vuln_checks.id AND vuln_check_cves.cve_id = ?", cveID). + Preload("CVE"). + Preload("VulnCheckXDB"). + Preload("VulnCheckReportedExploitation"). + Find(&res.VulnCheck).Error; err != nil { + return Response{}, xerrors.Errorf("Failed to get VulnCheck info by CVE-ID. err: %w", err) + } + return res, nil +} + +// GetKEVByMultiCveID : +func (r *RDBDriver) GetKEVByMultiCveID(cveIDs []string) (map[string]Response, error) { + m := make(map[string]Response) for _, cveID := range cveIDs { - v, err := r.GetKEVulnByCveID(cveID) + res, err := r.GetKEVByCveID(cveID) if err != nil { - return nil, err + return nil, xerrors.Errorf("Failed to get KEV by %s. err: %w", cveID, err) } - vuln[cveID] = v + m[cveID] = res } - return vuln, nil + return m, nil } diff --git a/db/redis.go b/db/redis.go index 48f8e2c..5ed2692 100644 --- a/db/redis.go +++ b/db/redis.go @@ -9,6 +9,7 @@ import ( "io" "os" "strconv" + "strings" "time" "github.com/cheggaaa/pb/v3" @@ -23,26 +24,21 @@ import ( /** # Redis Data Structure -- Strings - ┌───┬─────────┬────────┬──────────────────────────────────────────────────┐ - │NO │ KEY │ MEMBER │ PURPOSE │ - └───┴─────────┴────────┴──────────────────────────────────────────────────┘ - ┌───┬─────────┬────────┬──────────────────────────────────────────────────┐ - │ 1 │ KEV#DEP │ JSON │ TO DELETE OUTDATED AND UNNEEDED FIELD AND MEMBER │ - └───┴─────────┴────────┴──────────────────────────────────────────────────┘ - Hash - ┌───┬────────────────┬───────────────┬──────────────┬──────────────────────────────┐ - │NO │ KEY │ FIELD │ VALUE │ PURPOSE │ - └───┴────────────────┴───────────────┴──────────────┴──────────────────────────────┘ - ┌───┬────────────────┬───────────────┬──────────────┬──────────────────────────────┐ - │ 1 │ KEV#CVE#$CVEID │ MD5SUM │ JSON │ TO GET VULN FROM CVEID │ - ├───┼────────────────┼───────────────┼──────────────┼──────────────────────────────┤ - │ 2 │ KEV#FETCHMETA │ Revision │ string │ GET Go-KEV Binary Revision │ - ├───┼────────────────┼───────────────┼──────────────┼──────────────────────────────┤ - │ 3 │ KEV#FETCHMETA │ SchemaVersion │ uint │ GET Go-KEV Schema Version │ - ├───┼────────────────┼───────────────┼──────────────┼──────────────────────────────┤ - │ 4 │ KEV#FETCHMETA │ LastFetchedAt │ time.Time │ GET Go-KEV Last Fetched Time │ - └───┴────────────────┴───────────────┴──────────────┴──────────────────────────────┘ + ┌───┬────────────────┬─────────────────────┬───────────┬──────────────────────────────────────────────────┐ + │NO │ KEY │ FIELD │ VALUE │ PURPOSE │ + └───┴────────────────┴─────────────────────┴───────────┴──────────────────────────────────────────────────┘ + ┌───┬────────────────┬─────────────────────┬───────────┬──────────────────────────────────────────────────┐ + │ 1 │ KEV#CVE#$CVEID │ :MD5SUM │ JSON │ TO GET VULN FROM CVEID │ + ├───┼────────────────┼─────────────────────┼───────────┼──────────────────────────────────────────────────┤ + │ 2 │ KEV#DEP │ CISA/VulnCheck │ JSON │ TO DELETE OUTDATED AND UNNEEDED FIELD AND MEMBER │ + ├───┼────────────────┼─────────────────────┼───────────┼──────────────────────────────────────────────────┤ + │ 3 │ KEV#FETCHMETA │ Revision │ string │ GET Go-KEV Binary Revision │ + ├───┼────────────────┼─────────────────────┼───────────┼──────────────────────────────────────────────────┤ + │ 4 │ KEV#FETCHMETA │ SchemaVersion │ uint │ GET Go-KEV Schema Version │ + ├───┼────────────────┼─────────────────────┼───────────┼──────────────────────────────────────────────────┤ + │ 5 │ KEV#FETCHMETA │ LastFetchedAt │ time.Time │ GET Go-KEV Last Fetched Time │ + └───┴────────────────┴─────────────────────┴───────────┴──────────────────────────────────────────────────┘ **/ const ( @@ -50,6 +46,9 @@ const ( cveIDKeyFormat = "KEV#CVE#%s" depKey = "KEV#DEP" fetchMetaKey = "KEV#FETCHMETA" + + kevulnType = "CISA" + vulncheckType = "VulnCheck" ) // RedisDriver is Driver for Redis @@ -177,14 +176,33 @@ func (r *RedisDriver) InsertKEVulns(records []models.KEVuln) (err error) { return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") } - // newDeps, oldDeps: {"CVEID": {"HashSum(CVEJSON)": {}}} + // newDeps, oldDeps: {"CVEID": {"CISA:HashSum(CVEJSON)": {}}} newDeps := map[string]map[string]struct{}{} - oldDepsStr, err := r.conn.Get(ctx, depKey).Result() + oldDepsStr := "{}" + t, err := r.conn.Type(ctx, depKey).Result() if err != nil { - if !errors.Is(err, redis.Nil) { + return xerrors.Errorf("Failed to Type key: %s. err: %w", depKey, err) + } + switch t { + case "string": + oldDepsStr, err = r.conn.Get(ctx, depKey).Result() + if err != nil { return xerrors.Errorf("Failed to Get key: %s. err: %w", depKey, err) } - oldDepsStr = "{}" + if _, err := r.conn.Del(ctx, depKey).Result(); err != nil { + return xerrors.Errorf("Failed to Del key: %s. err: %w", depKey, err) + } + case "hash": + oldDepsStr, err = r.conn.HGet(ctx, depKey, kevulnType).Result() + if err != nil { + if !errors.Is(err, redis.Nil) { + return xerrors.Errorf("Failed to Get key: %s, field: %s. err: %w", depKey, kevulnType, err) + } + oldDepsStr = "{}" + } + case "none": + default: + return xerrors.Errorf("unexpected %s key type. expected: %q, actual: %q", depKey, []string{"string", "hash", "none"}, t) } var oldDeps map[string]map[string]struct{} if err := json.Unmarshal([]byte(oldDepsStr), &oldDeps); err != nil { @@ -206,7 +224,7 @@ func (r *RedisDriver) InsertKEVulns(records []models.KEVuln) (err error) { return xerrors.Errorf("Failed to marshal json. err: %w", err) } - hash := fmt.Sprintf("%x", md5.Sum(j)) + hash := fmt.Sprintf("%s:%x", kevulnType, md5.Sum(j)) _ = pipe.HSet(ctx, fmt.Sprintf(cveIDKeyFormat, record.CveID), hash, string(j)) if _, ok := newDeps[record.CveID]; !ok { @@ -239,7 +257,7 @@ func (r *RedisDriver) InsertKEVulns(records []models.KEVuln) (err error) { if err != nil { return xerrors.Errorf("Failed to Marshal JSON. err: %w", err) } - _ = pipe.Set(ctx, depKey, string(newDepsJSON), 0) + _ = pipe.HSet(ctx, depKey, kevulnType, string(newDepsJSON)) if _, err := pipe.Exec(ctx); err != nil { return xerrors.Errorf("Failed to exec pipeline. err: %w", err) } @@ -248,30 +266,151 @@ func (r *RedisDriver) InsertKEVulns(records []models.KEVuln) (err error) { return nil } -// GetKEVulnByCveID : -func (r *RedisDriver) GetKEVulnByCveID(cveID string) ([]models.KEVuln, error) { +// InsertVulnCheck : +func (r *RedisDriver) InsertVulnCheck(records []models.VulnCheck) (err error) { + ctx := context.Background() + batchSize := viper.GetInt("batch-size") + if batchSize < 1 { + return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") + } + + // newDeps, oldDeps: {"CVEID": {"VulnCheck:HashSum(CVEJSON)": {}}} + newDeps := map[string]map[string]struct{}{} + oldDepsStr := "{}" + t, err := r.conn.Type(ctx, depKey).Result() + if err != nil { + return xerrors.Errorf("Failed to Type key: %s. err: %w", depKey, err) + } + switch t { + case "string": + depsStr, err := r.conn.Get(ctx, depKey).Result() + if err != nil { + return xerrors.Errorf("Failed to Get key: %s. err: %w", depKey, err) + } + if _, err := r.conn.Del(ctx, depKey).Result(); err != nil { + return xerrors.Errorf("Failed to Del key: %s. err: %w", depKey, err) + } + if _, err := r.conn.HSet(ctx, depKey, kevulnType, depsStr).Result(); err != nil { + return xerrors.Errorf("Failed to HSet key: %s, field: %s. err: %w", depKey, kevulnType, err) + } + case "hash": + oldDepsStr, err = r.conn.HGet(ctx, depKey, vulncheckType).Result() + if err != nil { + if !errors.Is(err, redis.Nil) { + return xerrors.Errorf("Failed to Get key: %s, field: %s. err: %w", depKey, vulncheckType, err) + } + oldDepsStr = "{}" + } + case "none": + default: + return xerrors.Errorf("unexpected %s key type. expected: %q, actual: %q", depKey, []string{"string", "hash", "none"}, t) + } + var oldDeps map[string]map[string]struct{} + if err := json.Unmarshal([]byte(oldDepsStr), &oldDeps); err != nil { + return xerrors.Errorf("Failed to unmarshal JSON. err: %w", err) + } + + log15.Info("Inserting VulnCheck Known Exploited Vulnerabilities...") + bar := pb.StartNew(len(records)).SetWriter(func() io.Writer { + if viper.GetBool("log-json") { + return io.Discard + } + return os.Stderr + }()) + for idx := range chunkSlice(len(records), batchSize) { + pipe := r.conn.Pipeline() + for _, record := range records[idx.From:idx.To] { + j, err := json.Marshal(record) + if err != nil { + return xerrors.Errorf("Failed to marshal json. err: %w", err) + } + + hash := fmt.Sprintf("%s:%x", vulncheckType, md5.Sum(j)) + for _, c := range record.CVE { + _ = pipe.HSet(ctx, fmt.Sprintf(cveIDKeyFormat, c.CveID), hash, string(j)) + + if _, ok := newDeps[c.CveID]; !ok { + newDeps[c.CveID] = map[string]struct{}{} + } + if _, ok := newDeps[c.CveID][hash]; !ok { + newDeps[c.CveID][hash] = struct{}{} + } + if _, ok := oldDeps[c.CveID]; ok { + delete(oldDeps[c.CveID], hash) + if len(oldDeps[c.CveID]) == 0 { + delete(oldDeps, c.CveID) + } + } + } + } + if _, err := pipe.Exec(ctx); err != nil { + return xerrors.Errorf("Failed to exec pipeline. err: %w", err) + } + bar.Add(idx.To - idx.From) + } + bar.Finish() + + pipe := r.conn.Pipeline() + for cveID, hashes := range oldDeps { + for hash := range hashes { + _ = pipe.HDel(ctx, fmt.Sprintf(cveIDKeyFormat, cveID), hash) + } + } + newDepsJSON, err := json.Marshal(newDeps) + if err != nil { + return xerrors.Errorf("Failed to Marshal JSON. err: %w", err) + } + _ = pipe.HSet(ctx, depKey, vulncheckType, string(newDepsJSON)) + if _, err := pipe.Exec(ctx); err != nil { + return xerrors.Errorf("Failed to exec pipeline. err: %w", err) + } + + log15.Info("CveID Count", "count", len(records)) + return nil +} + +// GetKEVByCveID : +func (r *RedisDriver) GetKEVByCveID(cveID string) (Response, error) { results, err := r.conn.HGetAll(context.Background(), fmt.Sprintf(cveIDKeyFormat, cveID)).Result() if err != nil { - return nil, err + return Response{}, xerrors.Errorf("Failed to HGetAll key: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), err) } - vulns := []models.KEVuln{} - for _, s := range results { - var vuln models.KEVuln - if err := json.Unmarshal([]byte(s), &vuln); err != nil { - return nil, err + var res Response + for f, s := range results { + switch { + case strings.HasPrefix(f, kevulnType): + var v models.KEVuln + if err := json.Unmarshal([]byte(s), &v); err != nil { + return Response{}, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) + } + res.CISA = append(res.CISA, v) + case strings.HasPrefix(f, vulncheckType): + var v models.VulnCheck + if err := json.Unmarshal([]byte(s), &v); err != nil { + return Response{}, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) + } + res.VulnCheck = append(res.VulnCheck, v) + default: + if f != fmt.Sprintf("%x", md5.Sum([]byte(s))) { + return Response{}, xerrors.Errorf("unexpected %s field. expected: %q, actual: %q", fmt.Sprintf(cveIDKeyFormat, cveID), []string{fmt.Sprintf("%s:", kevulnType), fmt.Sprintf("%s:", vulncheckType)}, f) + } + var v models.KEVuln + if err := json.Unmarshal([]byte(s), &v); err != nil { + return Response{}, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) + } + res.CISA = append(res.CISA, v) } - vulns = append(vulns, vuln) } - return vulns, nil + return res, nil } -// GetKEVulnByMultiCveID : -func (r *RedisDriver) GetKEVulnByMultiCveID(cveIDs []string) (map[string][]models.KEVuln, error) { +// GetKEVByMultiCveID : +func (r *RedisDriver) GetKEVByMultiCveID(cveIDs []string) (map[string]Response, error) { ctx := context.Background() if len(cveIDs) == 0 { - return map[string][]models.KEVuln{}, nil + return map[string]Response{}, nil } m := map[string]*redis.StringStringMapCmd{} @@ -280,25 +419,43 @@ func (r *RedisDriver) GetKEVulnByMultiCveID(cveIDs []string) (map[string][]model m[cveID] = pipe.HGetAll(ctx, fmt.Sprintf(cveIDKeyFormat, cveID)) } if _, err := pipe.Exec(ctx); err != nil { - return nil, err + return nil, xerrors.Errorf("Failed to exec pipeline. err: %w", err) } - vulns := map[string][]models.KEVuln{} + rm := make(map[string]Response) for cveID, cmd := range m { results, err := cmd.Result() if err != nil { - return nil, err + return nil, xerrors.Errorf("Failed to HGetAll. err: %w", err) } - var vs []models.KEVuln - for _, s := range results { - var v models.KEVuln - if err := json.Unmarshal([]byte(s), &v); err != nil { - return nil, err + var res Response + for f, s := range results { + switch { + case strings.HasPrefix(f, kevulnType): + var v models.KEVuln + if err := json.Unmarshal([]byte(s), &v); err != nil { + return nil, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) + } + res.CISA = append(res.CISA, v) + case strings.HasPrefix(f, vulncheckType): + var v models.VulnCheck + if err := json.Unmarshal([]byte(s), &v); err != nil { + return nil, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) + } + res.VulnCheck = append(res.VulnCheck, v) + default: + if f != fmt.Sprintf("%x", md5.Sum([]byte(s))) { + return nil, xerrors.Errorf("unexpected %s field. expected: %q, actual: %q", fmt.Sprintf(cveIDKeyFormat, cveID), []string{fmt.Sprintf("%s:", kevulnType), fmt.Sprintf("%s:", vulncheckType)}, f) + } + var v models.KEVuln + if err := json.Unmarshal([]byte(s), &v); err != nil { + return nil, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) + } + res.CISA = append(res.CISA, v) } - vs = append(vs, v) } - vulns[cveID] = vs + rm[cveID] = res } - return vulns, nil + return rm, nil } diff --git a/fetcher/kevuln.go b/fetcher/kevuln/kevuln.go similarity index 95% rename from fetcher/kevuln.go rename to fetcher/kevuln/kevuln.go index a5abe1a..359be23 100644 --- a/fetcher/kevuln.go +++ b/fetcher/kevuln/kevuln.go @@ -1,4 +1,4 @@ -package fetcher +package kevuln import ( "encoding/json" @@ -11,8 +11,8 @@ import ( "github.com/vulsio/go-kev/utils" ) -// FetchKEVuln : -func FetchKEVuln() ([]models.KEVuln, error) { +// Fetch : +func Fetch() ([]models.KEVuln, error) { url := "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" log15.Info("Fetching", "URL", url) vulnJSON, err := utils.FetchURL(url) diff --git a/fetcher/types.go b/fetcher/kevuln/types.go similarity index 98% rename from fetcher/types.go rename to fetcher/kevuln/types.go index 012a386..f1e11de 100644 --- a/fetcher/types.go +++ b/fetcher/kevuln/types.go @@ -1,4 +1,4 @@ -package fetcher +package kevuln import "time" diff --git a/fetcher/vulncheck/types.go b/fetcher/vulncheck/types.go new file mode 100644 index 0000000..5a8b99a --- /dev/null +++ b/fetcher/vulncheck/types.go @@ -0,0 +1,35 @@ +package vulncheck + +import "time" + +// https://docs.vulncheck.com/community/vulncheck-kev/schema +type vulncheck struct { + VendorProject string `json:"vendorProject"` + Product string `json:"product"` + Description string `json:"shortDescription"` + Name string `json:"vulnerabilityName"` + RequiredAction string `json:"required_action"` + KnownRansomwareCampaignUse string `json:"knownRansomwareCampaignUse"` + + CVE []string `json:"cve"` + + VulnCheckXDB []xdb `json:"vulncheck_xdb"` + VulnCheckReportedExploitation []reportedExploit `json:"vulncheck_reported_exploitation"` + + DueDate *time.Time `json:"dueDate,omitempty"` + CisaDateAdded *time.Time `json:"cisa_date_added,omitempty"` + DateAdded time.Time `json:"date_added"` +} + +type reportedExploit struct { + URL string `json:"url"` + DateAdded time.Time `json:"date_added"` +} + +type xdb struct { + XDBID string `json:"xdb_id"` + XDBURL string `json:"xdb_url"` + DateAdded time.Time `json:"date_added"` + ExploitType string `json:"exploit_type"` + CloneSSHURL string `json:"clone_ssh_url"` +} diff --git a/fetcher/vulncheck/vulncheck.go b/fetcher/vulncheck/vulncheck.go new file mode 100644 index 0000000..190ab87 --- /dev/null +++ b/fetcher/vulncheck/vulncheck.go @@ -0,0 +1,99 @@ +package vulncheck + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "io" + "path/filepath" + + "golang.org/x/xerrors" + + "github.com/vulsio/go-kev/models" + "github.com/vulsio/go-kev/utils" +) + +// Fetch : +func Fetch() ([]models.VulnCheck, error) { + bs, err := utils.FetchURL("https://github.com/vulsio/vuls-data-raw-vulncheck-kev/archive/refs/heads/main.tar.gz") + if err != nil { + return nil, xerrors.Errorf("Failed to fetch vulsio/vuls-data-raw-vulncheck-kev. err: %w", err) + } + + var vs []models.VulnCheck + + gr, err := gzip.NewReader(bytes.NewReader(bs)) + if err != nil { + return nil, xerrors.Errorf("Failed to new gzip reader. err: %w", err) + } + defer gr.Close() + + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, xerrors.Errorf("Failed to next tar reader. err: %w", err) + } + + if hdr.FileInfo().IsDir() || filepath.Ext(hdr.Name) != ".json" { + continue + } + + var v vulncheck + if err := json.NewDecoder(tr).Decode(&v); err != nil { + return nil, xerrors.Errorf("Failed to decode %s", hdr.Name) + } + + vs = append(vs, models.VulnCheck{ + VendorProject: v.VendorProject, + Product: v.Product, + Description: v.Description, + Name: v.Name, + RequiredAction: v.RequiredAction, + KnownRansomwareCampaignUse: v.KnownRansomwareCampaignUse, + + CVE: func() []models.VulnCheckCVE { + cs := make([]models.VulnCheckCVE, 0, len(v.CVE)) + for _, c := range v.CVE { + cs = append(cs, models.VulnCheckCVE{ + CveID: c, + }) + } + return cs + }(), + + VulnCheckXDB: func() []models.VulnCheckXDB { + xs := make([]models.VulnCheckXDB, 0, len(v.VulnCheckXDB)) + for _, x := range v.VulnCheckXDB { + xs = append(xs, models.VulnCheckXDB{ + XDBID: x.XDBID, + XDBURL: x.XDBURL, + DateAdded: x.DateAdded, + ExploitType: x.ExploitType, + CloneSSHURL: x.CloneSSHURL, + }) + } + return xs + }(), + VulnCheckReportedExploitation: func() []models.VulnCheckReportedExploitation { + es := make([]models.VulnCheckReportedExploitation, 0, len(v.VulnCheckReportedExploitation)) + for _, e := range v.VulnCheckReportedExploitation { + es = append(es, models.VulnCheckReportedExploitation{ + URL: e.URL, + DateAdded: e.DateAdded, + }) + } + return es + }(), + + DueDate: v.DueDate, + CisaDateAdded: v.CisaDateAdded, + DateAdded: v.DateAdded, + }) + } + return vs, nil +} diff --git a/models/models.go b/models/models.go index 91a85c1..d773e42 100644 --- a/models/models.go +++ b/models/models.go @@ -36,3 +36,49 @@ type KEVuln struct { KnownRansomwareCampaignUse string `gorm:"type:varchar(255)" json:"knownRansomwareCampaignUse"` Notes string `gorm:"type:text" json:"notes"` } + +// VulnCheck : https://docs.vulncheck.com/community/vulncheck-kev/schema +type VulnCheck struct { + ID int64 `json:"-"` + VendorProject string `gorm:"type:varchar(255)" json:"vendorProject"` + Product string `gorm:"type:varchar(255)" json:"product"` + Description string `gorm:"type:text" json:"shortDescription"` + Name string `gorm:"type:varchar(255)" json:"vulnerabilityName"` + RequiredAction string `gorm:"type:text" json:"required_action"` + KnownRansomwareCampaignUse string `gorm:"type:varchar(255)" json:"knownRansomwareCampaignUse"` + + CVE []VulnCheckCVE `json:"cve"` + + VulnCheckXDB []VulnCheckXDB `json:"vulncheck_xdb"` + VulnCheckReportedExploitation []VulnCheckReportedExploitation `json:"vulncheck_reported_exploitation"` + + DueDate *time.Time `json:"dueDate,omitempty"` + CisaDateAdded *time.Time `json:"cisa_date_added,omitempty"` + DateAdded time.Time `json:"date_added"` +} + +// VulnCheckCVE : +type VulnCheckCVE struct { + ID int64 `json:"-"` + VulnCheckID uint `json:"-" gorm:"index:idx_vulncheck_cve"` + CveID string `gorm:"type:varchar(255);index:idx_vulncheck_cve_cve_id" json:"cveID"` +} + +// VulnCheckXDB : +type VulnCheckXDB struct { + ID int64 `json:"-"` + VulnCheckID uint `json:"-" gorm:"index:idx_vulncheck_xdb"` + XDBID string `gorm:"type:varchar(255)" json:"xdb_id"` + XDBURL string `gorm:"type:varchar(255)" json:"xdb_url"` + DateAdded time.Time `json:"date_added"` + ExploitType string `gorm:"type:varchar(255)" json:"exploit_type"` + CloneSSHURL string `gorm:"type:text" json:"clone_ssh_url"` +} + +// VulnCheckReportedExploitation : +type VulnCheckReportedExploitation struct { + ID int64 `json:"-"` + VulnCheckID uint `json:"-" gorm:"index:idx_vulncheck_reported_exploitation"` + URL string `gorm:"type:text" json:"url"` + DateAdded time.Time `json:"date_added"` +} diff --git a/server/server.go b/server/server.go index 3d1c744..8c16503 100644 --- a/server/server.go +++ b/server/server.go @@ -57,11 +57,11 @@ func getVulnByCveID(driver db.DB) echo.HandlerFunc { cve := context.Param("cve") log15.Debug("Params", "CVE", cve) - vuln, err := driver.GetKEVulnByCveID(cve) + r, err := driver.GetKEVByCveID(cve) if err != nil { - return xerrors.Errorf("Failed to get Vuln Info by CVE. err: %w", err) + return xerrors.Errorf("Failed to get KEV Info by CVE. err: %w", err) } - return context.JSON(http.StatusOK, vuln) + return context.JSON(http.StatusOK, r) } } @@ -77,10 +77,10 @@ func getVulnByMultiCveID(driver db.DB) echo.HandlerFunc { } log15.Debug("Params", "CVEIDs", cveIDs.Args) - vulns, err := driver.GetKEVulnByMultiCveID(cveIDs.Args) + r, err := driver.GetKEVByMultiCveID(cveIDs.Args) if err != nil { - return xerrors.Errorf("Failed to get Vuln Info by CVE. err: %w", err) + return xerrors.Errorf("Failed to get KEV Info by CVE. err: %w", err) } - return context.JSON(http.StatusOK, vulns) + return context.JSON(http.StatusOK, r) } }