breaking-change(redis): change Redis Architecture (#205)

* feat(cmd): add version command

* feat(redis): change redis architecture and fix unimplemented parts

* fix(cmd): restore expire option

* chore(rdb): remove batchSize in RDBDriver

* feat(redis): change redis architecture

* feat(redis): change Redis Architecture

* feat(redis): update redis architecture

* fix(redis): Get by CPEURI

* chore: expand in argument

* chore(integration): set filename to cveid when mode is cves

* fix(redis): fix order of field, value

* style: fix message

* chore(integration): check the communication with the server

* chore(integration): update test data

* style: do file checks in close proximity

* chore(integration): move to integration/.gitignore

* chore: fmt.Errorf to xerrors.Errors

Co-authored-by: Kota Kanbe <kotakanbe@gmail.com>

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -30,7 +30,7 @@ Please re-run the command using ```-debug``` and provide the output below.
 Hash : ____
 
 To check the commit hash of HEAD
-$ go-cve-dictionary -v
+$ go-cve-dictionary version
 
 or
 

.gitignore

@@ -6,5 +6,4 @@ go-cve-dictionary
 *.sqlite3-shm
 *.sqlite3-wal
 tags
-/dist/
-integration/diff
+/dist/

README.md

@@ -214,6 +214,7 @@ Available Commands:
   fetch       Fetch Vulnerability dictionary
   help        Help about any command
   server      Start CVE dictionary HTTP Server
+  version     Show version
 
 Flags:
       --config string       config file (default is $HOME/.go-cve-dictionary.yaml)
@@ -229,9 +230,6 @@ Flags:
 
 Use "go-cve-dictionary [command] --help" for more information about a command.
 ```
-
-![screen shot 2018-03-29 at 14 21 56](https://user-images.githubusercontent.com/534611/38073437-fb2a54b2-3365-11e8-88b5-165f5954f0c9.png)
-
 ----
 
 ### Usage: Fetch Command

commands/fetch.go

@@ -14,6 +14,9 @@ var fetchCmd = &cobra.Command{
 func init() {
 	RootCmd.AddCommand(fetchCmd)
 
-	fetchCmd.PersistentFlags().Int("batch-size", 5, "The number of batch size to insert. NOTE: This Option does not work for dbtype: redis.")
+	fetchCmd.PersistentFlags().Int("batch-size", 5, "The number of batch size to insert.")
 	_ = viper.BindPFlag("batch-size", fetchCmd.PersistentFlags().Lookup("batch-size"))
+
+	fetchCmd.PersistentFlags().Uint("expire", 0, "timeout to set for Redis keys in seconds. If set to 0, the key is persistent.")
+	_ = viper.BindPFlag("expire", fetchCmd.PersistentFlags().Lookup("expire"))
 }

commands/server.go

@@ -67,10 +67,10 @@ func executeServer(cmd *cobra.Command, args []string) (err error) {
 	count += jvnCount
 
 	if count == 0 {
-		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD")
+		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN")
 		log.Infof("")
-		log.Infof(" for i in `seq 2002 $(date +\"%%Y\")`; do go-cve-dictionary fetch nvd -years $i ; done")
-		log.Infof(" for i in `seq 1998 $(date +\"%%Y\")`; do go-cve-dictionary fetch jvn -years $i ; done")
+		log.Infof(" go-cve-dictionary fetch nvd")
+		log.Infof(" go-cve-dictionary fetch jvn")
 		log.Infof("")
 		return nil
 	}

commands/version.go

@@ -0,0 +1,21 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/vulsio/go-cve-dictionary/config"
+)
+
+func init() {
+	RootCmd.AddCommand(versionCmd)
+}
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Show version",
+	Long:  `Show version`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Printf("go-cve-dictionary %s %s\n", config.Version, config.Revision)
+	},
+}

db/db.go

@@ -6,7 +6,6 @@ import (
 
 	version "github.com/hashicorp/go-version"
 	rpmver "github.com/knqyf263/go-rpm-version"
-	"github.com/spf13/viper"
 	log "github.com/vulsio/go-cve-dictionary/log"
 	"github.com/vulsio/go-cve-dictionary/models"
 	"golang.org/x/xerrors"
@@ -74,11 +73,11 @@ func NewDB(dbType, dbpath string, debugSQL bool) (driver DB, locked bool, err er
 func newDB(dbType string) (DB, error) {
 	switch dbType {
 	case dialectSqlite3, dialectMysql, dialectPostgreSQL:
-		return &RDBDriver{name: dbType, batchSize: viper.GetInt("batch-size")}, nil
+		return &RDBDriver{name: dbType}, nil
 	case dialectRedis:
 		return &RedisDriver{name: dbType}, nil
 	}
-	return nil, fmt.Errorf("Invalid database dialect. err: %s", dbType)
+	return nil, fmt.Errorf("Invalid database dialect: %s", dbType)
 }
 
 // IndexChunk has a starting point and an ending point for Chunk

db/rdb.go

@@ -12,6 +12,7 @@ import (
 	"github.com/knqyf263/go-cpe/common"
 	"github.com/knqyf263/go-cpe/naming"
 	sqlite3 "github.com/mattn/go-sqlite3"
+	"github.com/spf13/viper"
 	"github.com/vulsio/go-cve-dictionary/config"
 	"github.com/vulsio/go-cve-dictionary/fetcher/jvn"
 	"github.com/vulsio/go-cve-dictionary/fetcher/nvd"
@@ -34,9 +35,8 @@ const (
 
 // RDBDriver is Driver for RDB
 type RDBDriver struct {
-	name      string
-	conn      *gorm.DB
-	batchSize int
+	name string
+	conn *gorm.DB
 }
 
 // Name return db name
@@ -79,14 +79,13 @@ func (r *RDBDriver) OpenDB(dbType, dbPath string, debugSQL bool) (locked bool, e
 	}
 
 	if err != nil {
-		msg := fmt.Sprintf("Failed to open DB. dbtype: %s, dbpath: %s, err: %s", dbType, dbPath, err)
 		if r.name == dialectSqlite3 {
 			switch err.(sqlite3.Error).Code {
 			case sqlite3.ErrLocked, sqlite3.ErrBusy:
-				return true, fmt.Errorf(msg)
+				return true, xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err)
 			}
 		}
-		return false, fmt.Errorf(msg)
+		return false, xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err)
 	}
 
 	if r.name == dialectSqlite3 {
@@ -134,7 +133,7 @@ func (r *RDBDriver) MigrateDB() error {
 		&models.JvnReference{},
 		&models.JvnCert{},
 	); err != nil {
-		return fmt.Errorf("Failed to migrate. err: %s", err)
+		return xerrors.Errorf("Failed to migrate. err: %w", err)
 	}
 
 	return nil
@@ -347,8 +346,10 @@ func (r *RDBDriver) GetByCpeURI(uri string) ([]models.CveDetail, error) {
 // CountJvn count jvn table
 func (r *RDBDriver) CountJvn() (int, error) {
 	var count int64
-	err := r.conn.Model(&models.Jvn{}).Count(&count).Error
-	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+	if err := r.conn.Model(&models.Jvn{}).Count(&count).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return 0, nil
+		}
 		return 0, err
 	}
 	return int(count), nil
@@ -366,6 +367,11 @@ func (r *RDBDriver) InsertJvn(feedMetas map[string]models.FeedMeta) error {
 		return err
 	}
 
+	batchSize := viper.GetInt("batch-size")
+	if batchSize < 1 {
+		return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly")
+	}
+
 	logger.Infof("Deleting JVN tables...")
 	if err := deleteJvn(tx); err != nil {
 		tx.Rollback()
@@ -378,7 +384,7 @@ func (r *RDBDriver) InsertJvn(feedMetas map[string]models.FeedMeta) error {
 	logger.Infof("Fetching CVE information from JVN(recent, modified).")
 	if err := jvn.FetchConvert(uniqCves, []models.FeedMeta{feedMetas["recent"], feedMetas["modified"]}); err != nil {
 		tx.Rollback()
-		return fmt.Errorf("Failed to FetchConvert. err: %s", err)
+		return xerrors.Errorf("Failed to FetchConvert. err: %w", err)
 	}
 	delete(feedMetas, "recent")
 	delete(feedMetas, "modified")
@@ -387,7 +393,7 @@ func (r *RDBDriver) InsertJvn(feedMetas map[string]models.FeedMeta) error {
 		logger.Infof("Fetching CVE information from JVN(%s).", meta.Year)
 		if err := jvn.FetchConvert(uniqCves, []models.FeedMeta{meta}); err != nil {
 			tx.Rollback()
-			return fmt.Errorf("Failed to FetchConvert. err: %s", err)
+			return xerrors.Errorf("Failed to FetchConvert. err: %w", err)
 		}
 		delete(feedMetas, meta.Year)
 
@@ -398,7 +404,7 @@ func (r *RDBDriver) InsertJvn(feedMetas map[string]models.FeedMeta) error {
 		delete(uniqCves, meta.Year)
 
 		logger.Infof("Inserting fetched CVEs(%s)...", meta.Year)
-		if err := insertJvn(tx, cves, r.batchSize); err != nil {
+		if err := insertJvn(tx, cves, batchSize); err != nil {
 			tx.Rollback()
 			return xerrors.Errorf("Failed to insertJvn. err: %w", err)
 		}
@@ -442,7 +448,7 @@ func insertJvn(tx *gorm.DB, cves []models.Jvn, batchSize int) error {
 	bar := pb.StartNew(len(cves))
 	for _, cve := range cves {
 		if err := tx.Omit("Cpes").Create(&cve).Error; err != nil {
-			return fmt.Errorf("Failed to insert. err: %s", err)
+			return xerrors.Errorf("Failed to insert. err: %w", err)
 		}
 
 		for i := range cve.Cpes {
@@ -451,7 +457,7 @@ func insertJvn(tx *gorm.DB, cves []models.Jvn, batchSize int) error {
 
 		for idx := range chunkSlice(len(cve.Cpes), batchSize) {
 			if err := tx.Create(cve.Cpes[idx.From:idx.To]).Error; err != nil {
-				return fmt.Errorf("Failed to insert. err: %s", err)
+				return xerrors.Errorf("Failed to insert. err: %w", err)
 			}
 		}
 		bar.Increment()
@@ -464,8 +470,10 @@ func insertJvn(tx *gorm.DB, cves []models.Jvn, batchSize int) error {
 // CountNvd count nvd table
 func (r *RDBDriver) CountNvd() (int, error) {
 	var count int64
-	err := r.conn.Model(&models.Nvd{}).Count(&count).Error
-	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+	if err := r.conn.Model(&models.Nvd{}).Count(&count).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return 0, nil
+		}
 		return 0, err
 	}
 	return int(count), nil
@@ -483,6 +491,11 @@ func (r *RDBDriver) InsertNvd(feedMetas map[string]models.FeedMeta) (err error)
 		return err
 	}
 
+	batchSize := viper.GetInt("batch-size")
+	if batchSize < 1 {
+		return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly")
+	}
+
 	logger.Infof("Deleting NVD tables...")
 	if err := deleteNvd(tx); err != nil {
 		tx.Rollback()
@@ -495,7 +508,7 @@ func (r *RDBDriver) InsertNvd(feedMetas map[string]models.FeedMeta) (err error)
 	logger.Infof("Fetching CVE information from NVD(recent, modified).")
 	if err := nvd.FetchConvert(uniqCves, []models.FeedMeta{feedMetas["recent"], feedMetas["modified"]}); err != nil {
 		tx.Rollback()
-		return fmt.Errorf("Failed to FetchConvert. err: %s", err)
+		return xerrors.Errorf("Failed to FetchConvert. err: %w", err)
 	}
 	delete(feedMetas, "recent")
 	delete(feedMetas, "modified")
@@ -504,7 +517,7 @@ func (r *RDBDriver) InsertNvd(feedMetas map[string]models.FeedMeta) (err error)
 		logger.Infof("Fetching CVE information from NVD(%s).", meta.Year)
 		if err := nvd.FetchConvert(uniqCves, []models.FeedMeta{meta}); err != nil {
 			tx.Rollback()
-			return fmt.Errorf("Failed to FetchConvert. err: %s", err)
+			return xerrors.Errorf("Failed to FetchConvert. err: %w", err)
 		}
 		delete(feedMetas, meta.Year)
 
@@ -515,7 +528,7 @@ func (r *RDBDriver) InsertNvd(feedMetas map[string]models.FeedMeta) (err error)
 		delete(uniqCves, meta.Year)
 
 		logger.Infof("Inserting fetched CVEs(%s)...", meta.Year)
-		if err := insertNvd(tx, cves, r.batchSize); err != nil {
+		if err := insertNvd(tx, cves, batchSize); err != nil {
 			tx.Rollback()
 			return xerrors.Errorf("Failed to insertNvd. err: %w", err)
 		}
@@ -568,7 +581,7 @@ func insertNvd(tx *gorm.DB, cves []models.Nvd, batchSize int) error {
 	bar := pb.StartNew(len(cves))
 	for _, cve := range cves {
 		if err := tx.Omit("Cpes").Create(&cve).Error; err != nil {
-			return fmt.Errorf("Failed to insert. err: %s", err)
+			return xerrors.Errorf("Failed to insert. err: %w", err)
 		}
 
 		for i := range cve.Cpes {
@@ -577,7 +590,7 @@ func insertNvd(tx *gorm.DB, cves []models.Nvd, batchSize int) error {
 
 		for idx := range chunkSlice(len(cve.Cpes), batchSize) {
 			if err := tx.Create(cve.Cpes[idx.From:idx.To]).Error; err != nil {
-				return fmt.Errorf("Failed to insert. err: %s", err)
+				return xerrors.Errorf("Failed to insert. err: %w", err)
 			}
 		}
 		bar.Increment()
@@ -590,7 +603,7 @@ func insertNvd(tx *gorm.DB, cves []models.Nvd, batchSize int) error {
 func (r *RDBDriver) upsertFeedMetas(feedMetas map[string]models.FeedMeta) error {
 	for _, meta := range feedMetas {
 		if err := r.conn.Where(&models.FeedMeta{Source: meta.Source, Year: meta.Year}).Take(&meta).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
-			return fmt.Errorf("Failed to select. err: %s", err)
+			return xerrors.Errorf("Failed to select. err: %w", err)
 		}
 
 		meta.Hash = meta.LatestHash