feat(models): remove FeedMeta (#218)

* feat(models): remove FeedMeta

* chore: remove unneeded function

* chore: add testcase

commands/fetchjvn.go

@@ -1,13 +1,9 @@
 package commands
 
 import (
-	"time"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	c "github.com/vulsio/go-cve-dictionary/config"
 	db "github.com/vulsio/go-cve-dictionary/db"
-	"github.com/vulsio/go-cve-dictionary/fetcher/jvn"
 	log "github.com/vulsio/go-cve-dictionary/log"
 	"github.com/vulsio/go-cve-dictionary/models"
 	"golang.org/x/xerrors"
@@ -47,17 +43,8 @@ func fetchJvn(cmd *cobra.Command, args []string) (err error) {
 		return xerrors.New("Failed to Insert CVEs into DB. SchemaVersion is old")
 	}
 
-	years := []int{c.Latest}
-	for y := 1998; y <= time.Now().Year(); y++ {
-		years = append(years, y)
-	}
-	feedMetas, err := jvn.FetchLatestFeedMeta(years)
-	if err != nil {
-		return err
-	}
-
 	log.Infof("Inserting JVN into DB (%s).", driver.Name())
-	if err := driver.InsertJvn(feedMetas); err != nil {
+	if err := driver.InsertJvn(); err != nil {
 		log.Fatalf("Failed to insert. dbpath: %s, err: %s", viper.GetString("dbpath"), err)
 		return err
 	}

commands/fetchnvd.go

@@ -1,13 +1,9 @@
 package commands
 
 import (
-	"time"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	c "github.com/vulsio/go-cve-dictionary/config"
 	"github.com/vulsio/go-cve-dictionary/db"
-	"github.com/vulsio/go-cve-dictionary/fetcher/nvd"
 	log "github.com/vulsio/go-cve-dictionary/log"
 	"github.com/vulsio/go-cve-dictionary/models"
 	"golang.org/x/xerrors"
@@ -50,18 +46,8 @@ func fetchNvd(cmd *cobra.Command, args []string) (err error) {
 		return xerrors.New("Failed to Insert CVEs into DB. SchemaVersion is old")
 	}
 
-	years := []int{c.Latest}
-	for y := 2002; y <= time.Now().Year(); y++ {
-		years = append(years, y)
-	}
-	feedMetas, err := nvd.FetchLatestFeedMeta(years)
-	if err != nil {
-		log.Errorf("%s", err)
-		return err
-	}
-
 	log.Infof("Inserting NVD into DB (%s).", driver.Name())
-	if err := driver.InsertNvd(feedMetas); err != nil {
+	if err := driver.InsertNvd(); err != nil {
 		log.Errorf("Failed to insert. dbpath: %s, err: %s", viper.GetString("dbpath"), err)
 		return err
 	}

db/db.go

@@ -22,8 +22,6 @@ type DB interface {
 	CloseDB() error
 	MigrateDB() error
 
-	GetFetchedFeedMeta(string) (*models.FeedMeta, error)
-	GetFetchedFeedMetas() ([]models.FeedMeta, error)
 	IsGoCVEDictModelV1() (bool, error)
 	GetFetchMeta() (*models.FetchMeta, error)
 	UpsertFetchMeta(*models.FetchMeta) error
@@ -32,8 +30,8 @@ type DB interface {
 	GetMulti([]string) (map[string]models.CveDetail, error)
 	GetCveIDsByCpeURI(string) ([]string, []string, error)
 	GetByCpeURI(string) ([]models.CveDetail, error)
-	InsertJvn(map[string]models.FeedMeta) error
-	InsertNvd(map[string]models.FeedMeta) error
+	InsertJvn() error
+	InsertNvd() error
 	CountNvd() (int, error)
 	CountJvn() (int, error)
 }

db/rdb.go

@@ -113,7 +113,6 @@ func (r *RDBDriver) CloseDB() (err error) {
 // MigrateDB migrates Database
 func (r *RDBDriver) MigrateDB() error {
 	if err := r.conn.AutoMigrate(
-		&models.FeedMeta{},
 		&models.FetchMeta{},
 
 		&models.Nvd{},
@@ -356,7 +355,7 @@ func (r *RDBDriver) CountJvn() (int, error) {
 }
 
 // InsertJvn inserts Cve Information into DB
-func (r *RDBDriver) InsertJvn(feedMetas map[string]models.FeedMeta) error {
+func (r *RDBDriver) InsertJvn() error {
 	tx := r.conn.Begin()
 	defer func() {
 		if re := recover(); re != nil {
@@ -382,28 +381,26 @@ func (r *RDBDriver) InsertJvn(feedMetas map[string]models.FeedMeta) error {
 	uniqCves := map[string]map[string]models.Jvn{}
 
 	logger.Infof("Fetching CVE information from JVN(recent, modified).")
-	if err := jvn.FetchConvert(uniqCves, []models.FeedMeta{feedMetas["recent"], feedMetas["modified"]}); err != nil {
+	if err := jvn.FetchConvert(uniqCves, []string{"recent", "modified"}); err != nil {
 		tx.Rollback()
 		return xerrors.Errorf("Failed to FetchConvert. err: %w", err)
 	}
-	delete(feedMetas, "recent")
-	delete(feedMetas, "modified")
 
-	for _, meta := range feedMetas {
-		logger.Infof("Fetching CVE information from JVN(%s).", meta.Year)
-		if err := jvn.FetchConvert(uniqCves, []models.FeedMeta{meta}); err != nil {
+	for y := 1998; y <= time.Now().Year(); y++ {
+		year := fmt.Sprintf("%d", y)
+		logger.Infof("Fetching CVE information from JVN(%s).", year)
+		if err := jvn.FetchConvert(uniqCves, []string{year}); err != nil {
 			tx.Rollback()
 			return xerrors.Errorf("Failed to FetchConvert. err: %w", err)
 		}
-		delete(feedMetas, meta.Year)
 
 		cves := []models.Jvn{}
-		for _, cve := range uniqCves[meta.Year] {
+		for _, cve := range uniqCves[year] {
 			cves = append(cves, cve)
 		}
-		delete(uniqCves, meta.Year)
+		delete(uniqCves, year)
 
-		logger.Infof("Inserting fetched CVEs(%s)...", meta.Year)
+		logger.Infof("Inserting fetched CVEs(%s)...", year)
 		if err := insertJvn(tx, cves, batchSize); err != nil {
 			tx.Rollback()
 			return xerrors.Errorf("Failed to insertJvn. err: %w", err)
@@ -414,11 +411,6 @@ func (r *RDBDriver) InsertJvn(feedMetas map[string]models.FeedMeta) error {
 	if err := tx.Commit().Error; err != nil {
 		return xerrors.Errorf("Failed to Commit Transaction. err: %w", err)
 	}
-
-	logger.Infof("Updating FeedMetas...")
-	if err := r.upsertFeedMetas(feedMetas); err != nil {
-		return xerrors.Errorf("Failed to upsertFeedMetas. err: %w", err)
-	}
 	return nil
 }
 
@@ -480,7 +472,7 @@ func (r *RDBDriver) CountNvd() (int, error) {
 }
 
 // InsertNvd Cve information from DB.
-func (r *RDBDriver) InsertNvd(feedMetas map[string]models.FeedMeta) (err error) {
+func (r *RDBDriver) InsertNvd() (err error) {
 	tx := r.conn.Begin()
 	defer func() {
 		if re := recover(); re != nil {
@@ -506,28 +498,26 @@ func (r *RDBDriver) InsertNvd(feedMetas map[string]models.FeedMeta) (err error)
 	uniqCves := map[string]map[string]models.Nvd{}
 
 	logger.Infof("Fetching CVE information from NVD(recent, modified).")
-	if err := nvd.FetchConvert(uniqCves, []models.FeedMeta{feedMetas["recent"], feedMetas["modified"]}); err != nil {
+	if err := nvd.FetchConvert(uniqCves, []string{"recent", "modified"}); err != nil {
 		tx.Rollback()
 		return xerrors.Errorf("Failed to FetchConvert. err: %w", err)
 	}
-	delete(feedMetas, "recent")
-	delete(feedMetas, "modified")
 
-	for _, meta := range feedMetas {
-		logger.Infof("Fetching CVE information from NVD(%s).", meta.Year)
-		if err := nvd.FetchConvert(uniqCves, []models.FeedMeta{meta}); err != nil {
+	for y := 2002; y <= time.Now().Year(); y++ {
+		year := fmt.Sprintf("%d", y)
+		logger.Infof("Fetching CVE information from NVD(%s).", year)
+		if err := nvd.FetchConvert(uniqCves, []string{year}); err != nil {
 			tx.Rollback()
 			return xerrors.Errorf("Failed to FetchConvert. err: %w", err)
 		}
-		delete(feedMetas, meta.Year)
 
 		cves := []models.Nvd{}
-		for _, cve := range uniqCves[meta.Year] {
+		for _, cve := range uniqCves[year] {
 			cves = append(cves, cve)
 		}
-		delete(uniqCves, meta.Year)
+		delete(uniqCves, year)
 
-		logger.Infof("Inserting fetched CVEs(%s)...", meta.Year)
+		logger.Infof("Inserting fetched CVEs(%s)...", year)
 		if err := insertNvd(tx, cves, batchSize); err != nil {
 			tx.Rollback()
 			return xerrors.Errorf("Failed to insertNvd. err: %w", err)
@@ -538,11 +528,6 @@ func (r *RDBDriver) InsertNvd(feedMetas map[string]models.FeedMeta) (err error)
 	if err := tx.Commit().Error; err != nil {
 		return xerrors.Errorf("Failed to Commit Transaction. err: %w", err)
 	}
-
-	logger.Infof("Updating FeedMetas...")
-	if err := r.upsertFeedMetas(feedMetas); err != nil {
-		return xerrors.Errorf("Failed to upsertFeedMetas. err: %w", err)
-	}
 	return nil
 }
 
@@ -599,41 +584,3 @@ func insertNvd(tx *gorm.DB, cves []models.Nvd, batchSize int) error {
 
 	return nil
 }
-
-func (r *RDBDriver) upsertFeedMetas(feedMetas map[string]models.FeedMeta) error {
-	for _, meta := range feedMetas {
-		if err := r.conn.Where(&models.FeedMeta{Source: meta.Source, Year: meta.Year}).Take(&meta).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
-			return xerrors.Errorf("Failed to select. err: %w", err)
-		}
-
-		meta.Hash = meta.LatestHash
-		meta.LastModifiedDate = meta.LatestLastModifiedDate
-		if err := r.conn.Save(&meta).Error; err != nil {
-			return xerrors.Errorf("Failed to insert. err: %w", err)
-		}
-	}
-	return nil
-}
-
-// GetFetchedFeedMeta selects fetchmeta of the year
-func (r *RDBDriver) GetFetchedFeedMeta(url string) (*models.FeedMeta, error) {
-	meta := models.FeedMeta{}
-	m := &models.FeedMeta{
-		URL: url,
-	}
-	err := r.conn.Where(m).First(&meta).Error
-	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
-		return nil, err
-	}
-	return &meta, nil
-}
-
-// GetFetchedFeedMetas selects a list of FeedMeta
-func (r *RDBDriver) GetFetchedFeedMetas() ([]models.FeedMeta, error) {
-	metas := []models.FeedMeta{}
-	err := r.conn.Find(&metas).Error
-	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
-		return nil, err
-	}
-	return metas, nil
-}