feat(mitre): add new datasource: mitre (#392)

* feat(mitre): add new datasource: mitre

* chore: adopt reviewer comments

.github/workflows/fetch.yml

@@ -184,4 +184,65 @@ jobs:
         run: ./go-cve-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" fortinet
       - name: fetch redis
         if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
-        run: ./go-cve-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" fortinet
+        run: ./go-cve-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" fortinet
+
+  fetch-mitre:
+    name: fetch-mitre
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mysql
+        ports:
+          - 3306:3306
+        env:
+          MYSQL_ROOT_PASSWORD: password
+          MYSQL_DATABASE: test
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+      - name: build
+        id: build
+        run: make build
+      - name: fetch sqlite3
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./go-cve-dictionary fetch --dbtype sqlite3 mitre
+      - name: fetch mysql
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./go-cve-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" mitre
+      - name: fetch postgres
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./go-cve-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" mitre
+      - name: fetch redis
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./go-cve-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" mitre
+

README.md

@@ -245,6 +245,7 @@ Usage:
 Available Commands:
   fortinet    Fetch Vulnerability dictionary from Fortinet Advisories
   jvn         Fetch Vulnerability dictionary from JVN
+  mitre       Fetch Vulnerability dictionary from MITRE
   nvd         Fetch Vulnerability dictionary from NVD
 
 Flags:
@@ -292,6 +293,17 @@ $ go-cve-dictionary fetch jvn 2021
 $ go-cve-dictionary fetch fortinet
 ```
 
+#### Fetch MITRE data
+- to fetch all years
+```bash
+$ go-cve-dictionary fetch mitre
+```
+
+- to fetch specific years
+```bash
+$ go-cve-dictionary fetch mitre 2021
+```
+
 ----
 
 ### Usage: Run HTTP Server
@@ -480,6 +492,14 @@ $ go-cve-dictionary search cpe --cveid-only "cpe:/a:fortinet:fortiportal"
           --dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true"
     ```
 
+- fetch mitre
+
+    ```bash
+    $ go-cve-dictionary fetch mitre \
+          --dbtype mysql \
+          --dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true"
+    ```
+
 - server
 
     ```bash
@@ -514,6 +534,14 @@ $ go-cve-dictionary search cpe --cveid-only "cpe:/a:fortinet:fortiportal"
           --dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password"
     ```
 
+- fetch mitre
+
+    ```bash
+    $ go-cve-dictionary fetch mitre \
+          --dbtype postgres \
+          --dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password"
+    ```
+
 - server
 
     ```bash
@@ -548,6 +576,14 @@ $ go-cve-dictionary search cpe --cveid-only "cpe:/a:fortinet:fortiportal"
           --dbpath "redis://localhost/0"
     ```
 
+- fetch mitre
+
+    ```bash
+    $ go-cve-dictionary fetch mitre \
+          --dbtype redis \
+          --dbpath "redis://localhost/0"
+    ```
+
 - server
 
     ```bash

commands/fetchfortinet.go

@@ -7,9 +7,9 @@ import (
 	"github.com/spf13/viper"
 	"golang.org/x/xerrors"
 
-	db "github.com/vulsio/go-cve-dictionary/db"
+	"github.com/vulsio/go-cve-dictionary/db"
 	"github.com/vulsio/go-cve-dictionary/fetcher/fortinet"
-	log "github.com/vulsio/go-cve-dictionary/log"
+	"github.com/vulsio/go-cve-dictionary/log"
 	"github.com/vulsio/go-cve-dictionary/models"
 )
 

commands/fetchjvn.go

@@ -7,8 +7,8 @@ import (
 	"github.com/spf13/viper"
 	"golang.org/x/xerrors"
 
-	db "github.com/vulsio/go-cve-dictionary/db"
-	log "github.com/vulsio/go-cve-dictionary/log"
+	"github.com/vulsio/go-cve-dictionary/db"
+	"github.com/vulsio/go-cve-dictionary/log"
 	"github.com/vulsio/go-cve-dictionary/models"
 )
 

commands/fetchmitre.go

@@ -0,0 +1,63 @@
+package commands
+
+import (
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"golang.org/x/xerrors"
+
+	"github.com/vulsio/go-cve-dictionary/db"
+	"github.com/vulsio/go-cve-dictionary/log"
+	"github.com/vulsio/go-cve-dictionary/models"
+)
+
+var fetchMitreCmd = &cobra.Command{
+	Use:   "mitre",
+	Short: "Fetch Vulnerability dictionary from MITRE",
+	Long:  "Fetch Vulnerability dictionary from MITRE",
+	RunE:  fetchMitre,
+}
+
+func init() {
+	fetchCmd.AddCommand(fetchMitreCmd)
+}
+
+func fetchMitre(_ *cobra.Command, args []string) (err error) {
+	if err := log.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
+	driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
+	if err != nil {
+		if xerrors.Is(err, db.ErrDBLocked) {
+			return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err)
+		}
+		return xerrors.Errorf("Failed to open DB. err: %w", err)
+	}
+
+	fetchMeta, err := driver.GetFetchMeta()
+	if err != nil {
+		return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
+	}
+	if fetchMeta.OutDated() {
+		return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
+	}
+	// If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	log.Infof("Inserting MITRE into DB (%s).", driver.Name())
+	if err := driver.InsertMitre(args); err != nil {
+		return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	fetchMeta.LastFetchedAt = time.Now()
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	log.Infof("Finished fetching MITRE.")
+	return nil
+}

commands/fetchnvd.go

@@ -5,10 +5,11 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"golang.org/x/xerrors"
+
 	"github.com/vulsio/go-cve-dictionary/db"
-	log "github.com/vulsio/go-cve-dictionary/log"
+	"github.com/vulsio/go-cve-dictionary/log"
 	"github.com/vulsio/go-cve-dictionary/models"
-	"golang.org/x/xerrors"
 )
 
 var fetchNvdCmd = &cobra.Command{

commands/root.go

@@ -8,7 +8,8 @@ import (
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	log "github.com/vulsio/go-cve-dictionary/log"
+
+	"github.com/vulsio/go-cve-dictionary/log"
 )
 
 var cfgFile string

commands/search.go

@@ -77,12 +77,20 @@ func searchCVE(_ *cobra.Command, args []string) error {
 	}
 	count += fortinetCount
 
+	mitreCount, err := driver.CountMitre()
+	if err != nil {
+		log.Errorf("Failed to count MITRE table: %s", err)
+		return err
+	}
+	count += mitreCount
+
 	if count == 0 {
-		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet")
+		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet, MITRE")
 		log.Infof("")
 		log.Infof(" go-cve-dictionary fetch nvd")
 		log.Infof(" go-cve-dictionary fetch jvn")
 		log.Infof(" go-cve-dictionary fetch fortinet")
+		log.Infof(" go-cve-dictionary fetch mitre")
 		log.Infof("")
 		return nil
 	}
@@ -162,12 +170,20 @@ func searchCPE(_ *cobra.Command, args []string) error {
 	}
 	count += fortinetCount
 
+	mitreCount, err := driver.CountMitre()
+	if err != nil {
+		log.Errorf("Failed to count MITRE table: %s", err)
+		return err
+	}
+	count += mitreCount
+
 	if count == 0 {
-		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet")
+		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet, MITRE")
 		log.Infof("")
 		log.Infof(" go-cve-dictionary fetch nvd")
 		log.Infof(" go-cve-dictionary fetch jvn")
 		log.Infof(" go-cve-dictionary fetch fortinet")
+		log.Infof(" go-cve-dictionary fetch mitre")
 		log.Infof("")
 		return nil
 	}

commands/server.go

@@ -3,11 +3,12 @@ package commands
 import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"golang.org/x/xerrors"
+
 	"github.com/vulsio/go-cve-dictionary/db"
-	log "github.com/vulsio/go-cve-dictionary/log"
+	"github.com/vulsio/go-cve-dictionary/log"
 	"github.com/vulsio/go-cve-dictionary/models"
-	server "github.com/vulsio/go-cve-dictionary/server"
-	"golang.org/x/xerrors"
+	"github.com/vulsio/go-cve-dictionary/server"
 )
 
 // serverCmd is Subcommand for CVE dictionary HTTP Server
@@ -71,12 +72,20 @@ func executeServer(_ *cobra.Command, _ []string) (err error) {
 	}
 	count += fortinetCount
 
+	mitreCount, err := driver.CountMitre()
+	if err != nil {
+		log.Errorf("Failed to count MITRE table: %s", err)
+		return err
+	}
+	count += mitreCount
+
 	if count == 0 {
-		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet")
+		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet, MITRE")
 		log.Infof("")
 		log.Infof(" go-cve-dictionary fetch nvd")
 		log.Infof(" go-cve-dictionary fetch jvn")
 		log.Infof(" go-cve-dictionary fetch fortinet")
+		log.Infof(" go-cve-dictionary fetch mitre")
 		log.Infof("")
 		return nil
 	}

commands/version.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+
 	"github.com/vulsio/go-cve-dictionary/config"
 )
 

db/db.go

@@ -35,9 +35,11 @@ type DB interface {
 	InsertJvn([]string) error
 	InsertNvd([]string) error
 	InsertFortinet([]models.Fortinet) error
+	InsertMitre([]string) error
 	CountNvd() (int, error)
 	CountJvn() (int, error)
 	CountFortinet() (int, error)
+	CountMitre() (int, error)
 }
 
 // Option :