feat(db): add LastFetchedDate in fetchmeta (#235)

* feat(db): add LastFetchedDate in fetchmeta * fix(fetch/redis): fetchmeta does not have LastFetchedDate * chore: rename from LastFetchedDate to LastFetchedAt
vulsio · Jan 6, 2022 · 221f5bb · 221f5bb
1 parent aa64b3b
commit 221f5bb
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 34 deletions.
diff --git a/commands/fetchjvn.go b/commands/fetchjvn.go
@@ -1,6 +1,8 @@
 package commands
 
 import (
+	"time"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	db "github.com/vulsio/go-cve-dictionary/db"
@@ -28,30 +30,31 @@ func fetchJvn(_ *cobra.Command, args []string) (err error) {
 	driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
 	if err != nil {
 		if locked {
-			log.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %s", err)
+			return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err)
 		}
-		return err
+		return xerrors.Errorf("Failed to open DB. err: %w", err)
 	}
 
 	fetchMeta, err := driver.GetFetchMeta()
 	if err != nil {
-		log.Errorf("Failed to get FetchMeta from DB. err: %s", err)
-		return err
+		return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
 	}
 	if fetchMeta.OutDated() {
-		log.Errorf("Failed to Insert CVEs into DB. SchemaVersion is old. SchemaVersion: %v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
-		return xerrors.New("Failed to Insert CVEs into DB. SchemaVersion is old")
+		return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
 	}
-
+	// If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
 	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
-		log.Errorf("Failed to upsert FetchMeta to DB. err: %s", err)
-		return err
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
 	}
 
 	log.Infof("Inserting JVN into DB (%s).", driver.Name())
 	if err := driver.InsertJvn(args); err != nil {
-		log.Fatalf("Failed to insert. dbpath: %s, err: %s", viper.GetString("dbpath"), err)
-		return err
+		return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	fetchMeta.LastFetchedAt = time.Now()
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
 	}
 
 	log.Infof("Finished fetching JVN.")

diff --git a/commands/fetchnvd.go b/commands/fetchnvd.go
@@ -1,6 +1,8 @@
 package commands
 
 import (
+	"time"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/vulsio/go-cve-dictionary/db"
@@ -31,30 +33,31 @@ func fetchNvd(_ *cobra.Command, args []string) (err error) {
 	driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
 	if err != nil {
 		if locked {
-			log.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %s", err)
+			return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err)
 		}
-		return err
+		return xerrors.Errorf("Failed to open DB. err: %w", err)
 	}
 
 	fetchMeta, err := driver.GetFetchMeta()
 	if err != nil {
-		log.Errorf("Failed to get FetchMeta from DB. err: %s", err)
-		return err
+		return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
 	}
 	if fetchMeta.OutDated() {
-		log.Errorf("Failed to Insert CVEs into DB. SchemaVersion is old. SchemaVersion: %v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
-		return xerrors.New("Failed to Insert CVEs into DB. SchemaVersion is old")
+		return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
 	}
-
+	// If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
 	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
-		log.Errorf("Failed to upsert FetchMeta to DB. err: %s", err)
-		return err
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
 	}
 
 	log.Infof("Inserting NVD into DB (%s).", driver.Name())
 	if err := driver.InsertNvd(args); err != nil {
-		log.Errorf("Failed to insert. dbpath: %s, err: %s", viper.GetString("dbpath"), err)
-		return err
+		return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	fetchMeta.LastFetchedAt = time.Now()
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
 	}
 
 	log.Infof("Finished fetching NVD.")

diff --git a/commands/server.go b/commands/server.go
@@ -36,19 +36,17 @@ func executeServer(_ *cobra.Command, _ []string) (err error) {
 	driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
 	if err != nil {
 		if locked {
-			log.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %s", err)
+			return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err)
 		}
-		return err
+		return xerrors.Errorf("Failed to open DB. err: %w", err)
 	}
 
 	fetchMeta, err := driver.GetFetchMeta()
 	if err != nil {
-		log.Errorf("Failed to get FetchMeta from DB. err: %s", err)
-		return err
+		return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
 	}
 	if fetchMeta.OutDated() {
-		log.Errorf("Failed to Insert CVEs into DB. SchemaVersion is old. SchemaVersion: %v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
-		return xerrors.New("Failed to Insert CVEs into DB. SchemaVersion is old")
+		return xerrors.Errorf("Failed to start server. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
 	}
 
 	count := 0
@@ -77,8 +75,7 @@ func executeServer(_ *cobra.Command, _ []string) (err error) {
 
 	log.Infof("Starting HTTP Server...")
 	if err = server.Start(viper.GetBool("log-to-file"), viper.GetString("log-dir"), driver); err != nil {
-		log.Errorf("%s", err)
-		return err
+		return xerrors.Errorf("Failed to start server. err: %w", err)
 	}
 	return nil
 }
diff --git a/db/rdb.go b/db/rdb.go
@@ -169,7 +169,7 @@ func (r *RDBDriver) GetFetchMeta() (fetchMeta *models.FetchMeta, err error) {
 		if !errors.Is(err, gorm.ErrRecordNotFound) {
 			return nil, err
 		}
-		return &models.FetchMeta{GoCVEDictRevision: config.Revision, SchemaVersion: models.LatestSchemaVersion}, nil
+		return &models.FetchMeta{GoCVEDictRevision: config.Revision, SchemaVersion: models.LatestSchemaVersion, LastFetchedAt: time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC)}, nil
 	}
 
 	return fetchMeta, nil

diff --git a/db/redis.go b/db/redis.go
@@ -45,6 +45,8 @@ import (
   │ CVE#FETCHMETA    │ Revision      │ string      │ Get Go-Cve-Dictionary Binary Revision            │
   ├──────────────────┼───────────────┼─────────────┼──────────────────────────────────────────────────┤
   │ CVE#FETCHMETA    │ SchemaVersion │ uint        │ Get Go-Cve-Dictionary Schema Version             │
+  ├──────────────────┼───────────────┼─────────────┼──────────────────────────────────────────────────┤
+  │ CVE#FETCHMETA    │ LastFetchedAt │ time.Time   │ Get Go-Cve-Dictionary Last Fetched Time          │
   └──────────────────┴───────────────┴─────────────┴──────────────────────────────────────────────────┘
 
 **/
@@ -137,7 +139,7 @@ func (r *RedisDriver) GetFetchMeta() (*models.FetchMeta, error) {
 		return nil, xerrors.Errorf("Failed to Exists. err: %w", err)
 	}
 	if exists == 0 {
-		return &models.FetchMeta{GoCVEDictRevision: config.Revision, SchemaVersion: models.LatestSchemaVersion}, nil
+		return &models.FetchMeta{GoCVEDictRevision: config.Revision, SchemaVersion: models.LatestSchemaVersion, LastFetchedAt: time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC)}, nil
 	}
 
 	revision, err := r.conn.HGet(ctx, fetchMetaKey, "Revision").Result()
@@ -154,12 +156,24 @@ func (r *RedisDriver) GetFetchMeta() (*models.FetchMeta, error) {
 		return nil, xerrors.Errorf("Failed to ParseUint. err: %w", err)
 	}
 
-	return &models.FetchMeta{GoCVEDictRevision: revision, SchemaVersion: uint(version)}, nil
+	datestr, err := r.conn.HGet(ctx, fetchMetaKey, "LastFetchedAt").Result()
+	if err != nil {
+		if !errors.Is(err, redis.Nil) {
+			return nil, xerrors.Errorf("Failed to HGet LastFetchedAt. err: %w", err)
+		}
+		datestr = time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339)
+	}
+	date, err := time.Parse(time.RFC3339, datestr)
+	if err != nil {
+		return nil, xerrors.Errorf("Failed to Parse date. err: %w", err)
+	}
+
+	return &models.FetchMeta{GoCVEDictRevision: revision, SchemaVersion: uint(version), LastFetchedAt: date}, nil
 }
 
 // UpsertFetchMeta upsert FetchMeta to Database
 func (r *RedisDriver) UpsertFetchMeta(fetchMeta *models.FetchMeta) error {
-	return r.conn.HSet(context.Background(), fetchMetaKey, map[string]interface{}{"Revision": fetchMeta.GoCVEDictRevision, "SchemaVersion": fetchMeta.SchemaVersion}).Err()
+	return r.conn.HSet(context.Background(), fetchMetaKey, map[string]interface{}{"Revision": config.Revision, "SchemaVersion": models.LatestSchemaVersion, "LastFetchedAt": fetchMeta.LastFetchedAt}).Err()
 }
 
 // Get Select Cve information from DB.

diff --git a/models/models.go b/models/models.go
@@ -50,6 +50,7 @@ type FetchMeta struct {
 	gorm.Model        `json:"-"`
 	GoCVEDictRevision string
 	SchemaVersion     uint
+	LastFetchedAt     time.Time
 }
 
 // OutDated checks whether last fetched feed is out dated