breaking-change(fetch): change from CSV to JSON (#9)

* feat(fetch): change from CSV to JSON

* feat(fetch): parse time when fetching

* chore: remove unwanted diff in PR

Co-authored-by: Kota Kanbe <kotakanbe@gmail.com>

fetcher/kevuln.go

@@ -1,63 +1,28 @@
 package fetcher
 
 import (
- "bytes"
+ "encoding/json"
 
- "github.com/gocarina/gocsv"
  "github.com/inconshreveable/log15"
  "golang.org/x/xerrors"
 
  "github.com/vulsio/go-kev/models"
  "github.com/vulsio/go-kev/utils"
 )
 
-var utf8Bom = []byte{239, 187, 191}
-
-func hasBOM(in []byte) bool {
- return bytes.HasPrefix(in, utf8Bom)
-}
-
-func stripBOM(in []byte) []byte {
- return bytes.TrimPrefix(in, utf8Bom)
-}
-
-var zeroWidthSpace = []byte{226, 128, 139}
-
-func hasZeroWidthSpace(in []byte) bool {
- return bytes.HasPrefix(in, zeroWidthSpace) || bytes.HasSuffix(in, zeroWidthSpace)
-}
-
-func stripZeroWidthSpace(in []byte) []byte {
- return bytes.ReplaceAll(in, zeroWidthSpace, []byte{})
-}
-
 // FetchKEVuln :
 func FetchKEVuln() ([]models.KEVuln, error) {
- url := "https://www.cisa.gov/sites/default/files/csv/known_exploited_vulnerabilities.csv"
+ url := "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
  log15.Info("Fetching", "URL", url)
- vulnCsv, err := utils.FetchURL(url)
+ vulnJSON, err := utils.FetchURL(url)
  if err != nil {
  return nil, err
  }
 
- if hasBOM(vulnCsv) {
- vulnCsv = stripBOM(vulnCsv)
- }
-
- var vulns []models.KEVuln
- if err := gocsv.UnmarshalBytes(vulnCsv, &vulns); err != nil {
- return nil, err
- }
-
- for i := range vulns {
- if hasZeroWidthSpace([]byte(vulns[i].CveID)) {
- vulns[i].CveID = string(stripZeroWidthSpace([]byte(vulns[i].CveID)))
- }
-
- if vulns[i].CveID == "" {
- return nil, xerrors.New("Failed to fetch vulnerability info. err: CVE-ID is empty. CSV format may have been changed.")
- }
+ kevCatalog := models.KEVCatalog{}
+ if err := json.Unmarshal(vulnJSON, &kevCatalog); err != nil {
+ return nil, xerrors.Errorf("failed to decode CISA Known Exploited Vulnerabilities JSON: %w", err)
  }
 
- return vulns, nil
+ return kevCatalog.Vulnerabilities, nil
 }

models/models.go

@@ -8,7 +8,7 @@ import (
 )
 
 // LatestSchemaVersion manages the Schema version used in the latest go-kev.
-const LatestSchemaVersion = 1
+const LatestSchemaVersion = 2
 
 // FetchMeta has meta information
 type FetchMeta struct {
@@ -23,18 +23,26 @@ func (f FetchMeta) OutDated() bool {
  return f.SchemaVersion != LatestSchemaVersion
 }
 
+// KEVCatalog : CISA Catalog of Known Exploited Vulnerabilities
+type KEVCatalog struct {
+ Title string `json:"title"`
+ CatalogVersion string `json:"catalogVersion"`
+ DateReleased time.Time `json:"dateReleased"`
+ Count int `json:"count"`
+ Vulnerabilities []KEVuln `json:"vulnerabilities"`
+}
+
 // KEVuln : Known Exploited Vulnerabilities
 type KEVuln struct {
- ID int64 `json:"-"`
- CveID string `gorm:"type:varchar(255);index:idx_kev_cve_id" csv:"cveID"`
- Source string `gorm:"type:varchar(255)" csv:"vendorProject"`
- Product string `gorm:"type:varchar(255)" csv:"product"`
- Title string `gorm:"type:varchar(255)" csv:"vulnerabilityName"`
- AddedDate KEVulnTime `gorm:"type:time" csv:"dateAdded"`
- Description string `gorm:"type:text" csv:"shortDescription"`
- Action string `gorm:"type:varchar(255)" csv:"requiredAction"`
- DueDate KEVulnTime `gorm:"type:time" csv:"dueDate"`
- Notes string `gorm:"type:text" csv:"notes"`
+ ID int64 `json:"-"`
+ CveID string `gorm:"type:varchar(255);index:idx_kev_cve_id" json:"cveID"`
+ VendorProject string `gorm:"type:varchar(255)" json:"vendorProject"`
+ Product string `gorm:"type:varchar(255)" json:"product"`
+ VulnerabilityName string `gorm:"type:varchar(255)" json:"vulnerabilityName"`
+ DateAdded KEVulnTime `gorm:"type:time" json:"dateAdded"`
+ ShortDescription string `gorm:"type:text" json:"shortDescription"`
+ RequiredAction string `gorm:"type:varchar(255)" json:"requiredAction"`
+ DueDate KEVulnTime `gorm:"type:time" json:"dueDate"`
 }
 
 // KEVulnTime :
@@ -44,9 +52,15 @@ type KEVulnTime struct {
 
 const kevDateFormat = "2006-01-02"
 
-// UnmarshalCSV :
-func (date *KEVulnTime) UnmarshalCSV(csv string) (err error) {
- date.Time, err = time.Parse(kevDateFormat, csv)
+// UnmarshalJSON :
+func (date *KEVulnTime) UnmarshalJSON(b []byte) error {
+ if string(b) == "null" {
+ date.Time = time.Time{}
+ return nil
+ }
+
+ var err error
+ date.Time, err = time.Parse(`"`+kevDateFormat+`"`, string(b))
  return err
 }