Add Pagination Support for Retrieving All CVEs API

db/db.go

@@ -28,6 +28,7 @@ type DB interface {
 	UpsertFetchMeta(*models.FetchMeta) error
 
 	Get(string) (*models.CveDetail, error)
+	GetAll(limit, pageNum int) ([]models.CveDetail, int, error)
 	GetMulti([]string) (map[string]models.CveDetail, error)
 	GetCveIDsByCpeURI(string) ([]string, []string, []string, error)
 	GetByCpeURI(string) ([]models.CveDetail, error)

db/rdb.go

@@ -308,6 +308,80 @@ func (r *RDBDriver) Get(cveID string) (*models.CveDetail, error) {
 	return &detail, nil
 }
 
+func (r *RDBDriver) GetAll(limit, pageNum int) ([]models.CveDetail, int, error) {
+	var details []models.CveDetail
+
+	// Calculate offset based on page number and limit
+	offset := (pageNum - 1) * limit
+
+	var totalCount int64
+	if err := r.conn.Model(&models.Nvd{}).Count(&totalCount).Error; err != nil {
+		return nil, 0, xerrors.Errorf("Failed to count Nvd records: %w", err)
+	}
+
+	// Get all Nvd records
+	var nvds []models.Nvd
+	if err := r.conn.Limit(limit).Offset(offset).
+		Preload("Descriptions").
+		Preload("Cvss2").
+		Preload("Cvss3").
+		Preload("Cwes").
+		Preload("Cpes").
+		Preload("References").
+		Preload("Certs").
+		Find(&nvds).Error; err != nil {
+		return nil, 0, xerrors.Errorf("Failed to get Nvd records: %w", err)
+	}
+
+	// Map Nvd records to CveDetail
+	for _, nvd := range nvds {
+		detail := models.CveDetail{
+			CveID: nvd.CveID,
+			Nvds:  []models.Nvd{nvd},
+		}
+
+		// Populate EnvCpes for each Cpe in Nvd
+		for i := range nvd.Cpes {
+			if err := r.conn.
+				Where(&models.NvdEnvCpe{NvdCpeID: uint(nvd.Cpes[i].ID)}).
+				Find(&nvd.Cpes[i].EnvCpes).Error; err != nil {
+				return nil, 0, xerrors.Errorf("Failed to fill Nvd EnvCpes for Cpe ID: %d, err: %w", uint(nvd.Cpes[i].ID), err)
+			}
+		}
+
+		// Get corresponding JVN records
+		var jvns []models.Jvn
+		if err := r.conn.
+			Where(&models.Jvn{CveID: nvd.CveID}).
+			Preload("Cvss2").
+			Preload("Cvss3").
+			Preload("Cpes").
+			Preload("References").
+			Preload("Certs").
+			Find(&jvns).Error; err != nil {
+			return nil, 0, xerrors.Errorf("Failed to get Jvn records for CVE ID: %s, err: %w", nvd.CveID, err)
+		}
+		detail.Jvns = jvns
+
+		// Get corresponding Fortinet records
+		var fortinets []models.Fortinet
+		if err := r.conn.
+			Where(&models.Fortinet{CveID: nvd.CveID}).
+			Preload("Cvss3").
+			Preload("Cwes").
+			Preload("Cpes").
+			Preload("References").
+			Find(&fortinets).Error; err != nil {
+			return nil, 0, xerrors.Errorf("Failed to get Fortinet records for CVE ID: %s, err: %w", nvd.CveID, err)
+		}
+		detail.Fortinets = fortinets
+
+		details = append(details, detail)
+	}
+
+	return details, int(totalCount), nil
+}
+
 func (r *RDBDriver) getCveIDsByPartVendorProduct(uri string) ([]string, error) {
 	specified, err := naming.UnbindURI(uri)
 	if err != nil {

db/redis.go

@@ -191,6 +191,47 @@ func (r *RedisDriver) Get(cveID string) (*models.CveDetail, error) {
 	return &detail, nil
 }
 
+// GetAll retrieves all CVE details from the database
+func (r *RedisDriver) GetAll(limit, pageNum int) ([]models.CveDetail, int, error) {
+	ctx := context.Background()
+
+	// Get all keys for CVE details
+	keys, err := r.conn.Keys(ctx, fmt.Sprintf(cveKeyFormat, "*")).Result()
+	if err != nil {
+		return nil, 0, xerrors.Errorf("Failed to get keys from Redis: %w", err)
+	}
+
+	// Total count is the length of keys
+	totalCount := len(keys)
+
+	// Calculate offset based on page number and limit
+	offset := (pageNum - 1) * limit
+	end := offset + limit
+	if end > totalCount {
+		end = totalCount
+	}
+
+	// Select the keys for the current page
+	paginatedKeys := keys[offset:end]
+
+	// Extract CVE IDs from keys and fetch details
+	var cveDetails []models.CveDetail
+	for _, key := range paginatedKeys {
+		cveID := strings.TrimPrefix(key, fmt.Sprintf(cveKeyFormat, ""))
+		results, err := r.conn.HGetAll(ctx, key).Result()
+		if err != nil {
+			return nil, 0, xerrors.Errorf("Failed to HGetAll for key %s: %w", key, err)
+		}
+		detail, err := convertToCveDetail(cveID, results)
+		if err != nil {
+			return nil, 0, xerrors.Errorf("Failed to convertToCveDetail for CVE ID %s: %w", cveID, err)
+		}
+		cveDetails = append(cveDetails, detail)
+	}
+
+	return cveDetails, totalCount, nil
+}
+
 func convertToCveDetail(cveID string, results map[string]string) (models.CveDetail, error) {
 	detail := models.CveDetail{
 		CveID:     cveID,

server/server.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
@@ -38,6 +39,7 @@ func Start(logToFile bool, logDir string, driver db.DB) error {
 	// Routes
 	e.GET("/health", health())
 	e.GET("/cves/:id", getCve(driver))
+	e.GET("/cves", getAllCves(driver))
 	e.POST("/cpes", getCveByCpeName(driver))
 	e.POST("/cpes/ids", getCveIDsByCpeName(driver))
 
@@ -67,6 +69,26 @@ func getCve(driver db.DB) echo.HandlerFunc {
 	}
 }
 
+func getAllCves(driver db.DB) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		cveDetails, err := driver.GetAll()
+		limit, _ := strconv.Atoi(c.QueryParam("limit"))
+		pageNum, _ := strconv.Atoi(c.QueryParam("page_num"))
+
+		// Fetch paginated data
+		cveDetails, totalCount, err := driver.GetAll(limit, pageNum)
+		if err != nil {
+			log.Errorf("Failed to get all CVEs: %s", err)
+			return c.JSON(http.StatusInternalServerError, map[string]string{"error": "failed to retrieve CVEs"})
+		}
+
+		return c.JSON(http.StatusOK, echo.Map{
+			"total": totalCount,
+			"data":  cveDetails,
+		})
+	}
+}
+
 type cpeName struct {
 	Name string `form:"name"`
 }