Skip to content

Commit

Permalink
refactor: use pure go sqlite driver
Browse files Browse the repository at this point in the history
  • Loading branch information
MaineK00n committed Apr 27, 2023
1 parent 30c20b8 commit 67ea039
Show file tree
Hide file tree
Showing 9 changed files with 126 additions and 56 deletions.
6 changes: 3 additions & 3 deletions commands/fetchjvn.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ func fetchJvn(_ *cobra.Command, args []string) (err error) {
return xerrors.Errorf("Failed to SetLogger. err: %w", err)
}

driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
if err != nil {
if locked {
return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err)
if xerrors.Is(err, db.ErrDBLocked) {
return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err)
}
return xerrors.Errorf("Failed to open DB. err: %w", err)
}
Expand Down
6 changes: 3 additions & 3 deletions commands/fetchnvd.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ func fetchNvd(_ *cobra.Command, args []string) (err error) {
return xerrors.Errorf("Failed to SetLogger. err: %w", err)
}

driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
if err != nil {
if locked {
return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err)
if xerrors.Is(err, db.ErrDBLocked) {
return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err)
}
return xerrors.Errorf("Failed to open DB. err: %w", err)
}
Expand Down
6 changes: 3 additions & 3 deletions commands/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@ func executeServer(_ *cobra.Command, _ []string) (err error) {
return xerrors.Errorf("Failed to SetLogger. err: %w", err)
}

driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
if err != nil {
if locked {
return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err)
if xerrors.Is(err, db.ErrDBLocked) {
return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err)
}
return xerrors.Errorf("Failed to open DB. err: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion commands/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ var versionCmd = &cobra.Command{
Use: "version",
Short: "Show version",
Long: `Show version`,
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, _ []string) {
fmt.Printf("go-cve-dictionary %s %s\n", config.Version, config.Revision)
},
}
24 changes: 9 additions & 15 deletions db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
// DB is interface for a database driver
type DB interface {
Name() string
OpenDB(string, string, bool, Option) (bool, error)
OpenDB(string, string, bool, Option) error
CloseDB() error
MigrateDB() error

Expand All @@ -43,35 +43,29 @@ type Option struct {
}

// NewDB return DB accessor.
func NewDB(dbType, dbpath string, debugSQL bool, option Option) (driver DB, locked bool, err error) {
func NewDB(dbType, dbPath string, debugSQL bool, option Option) (driver DB, err error) {
if driver, err = newDB(dbType); err != nil {
log.Errorf("Failed to new db. err: %s", err)
return driver, false, err
return driver, xerrors.Errorf("Failed to new db. err: %w", err)
}

if locked, err := driver.OpenDB(dbType, dbpath, debugSQL, option); err != nil {
if locked {
return nil, true, err
}
return nil, false, err
if err := driver.OpenDB(dbType, dbPath, debugSQL, option); err != nil {
return nil, xerrors.Errorf("Failed to open db. err: %w", err)
}

isV1, err := driver.IsGoCVEDictModelV1()
if err != nil {
log.Errorf("Failed to IsGoCVEDictModelV1. err: %s", err)
return nil, false, err
return nil, xerrors.Errorf("Failed to IsGoCVEDictModelV1. err: %w", err)
}
if isV1 {
log.Errorf("Failed to NewDB. Since SchemaVersion is incompatible, delete Database and fetch again")
return nil, false, xerrors.New("Failed to NewDB. Since SchemaVersion is incompatible, delete Database and fetch again.")
return nil, xerrors.New("Failed to NewDB. Since SchemaVersion is incompatible, delete Database and fetch again.")
}

if err := driver.MigrateDB(); err != nil {
log.Errorf("Failed to migrate db. err: %s", err)
return driver, false, err
return driver, xerrors.Errorf("Failed to migrate db. err: %w", err)
}

return driver, false, nil
return driver, nil
}

func newDB(dbType string) (DB, error) {
Expand Down
94 changes: 74 additions & 20 deletions db/rdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,17 @@ package db

import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"time"

"github.com/cheggaaa/pb/v3"
"github.com/glebarez/sqlite"
"github.com/knqyf263/go-cpe/common"
"github.com/knqyf263/go-cpe/naming"
sqlite3 "github.com/mattn/go-sqlite3"
"github.com/spf13/viper"
"github.com/vulsio/go-cve-dictionary/config"
"github.com/vulsio/go-cve-dictionary/fetcher/jvn"
Expand All @@ -21,7 +22,6 @@ import (
"golang.org/x/xerrors"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
gormLogger "gorm.io/gorm/logger"
)
Expand All @@ -39,13 +39,29 @@ type RDBDriver struct {
conn *gorm.DB
}

// https://github.com/mattn/go-sqlite3/blob/edc3bb69551dcfff02651f083b21f3366ea2f5ab/error.go#L18-L66
type errNo int

type sqliteError struct {
Code errNo /* The error code returned by SQLite */
}

// result codes from http://www.sqlite.org/c3ref/c_abort.html
var (
errBusy = errNo(5) /* The database file is locked */
errLocked = errNo(6) /* A table in the database is locked */
)

// ErrDBLocked :
var ErrDBLocked = xerrors.New("database is locked")

// Name return db name
func (r *RDBDriver) Name() string {
return r.name
}

// OpenDB opens Database
func (r *RDBDriver) OpenDB(dbType, dbPath string, debugSQL bool, _ Option) (locked bool, err error) {
func (r *RDBDriver) OpenDB(dbType, dbPath string, debugSQL bool, _ Option) (err error) {
gormConfig := gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: gormLogger.New(
Expand All @@ -70,28 +86,40 @@ func (r *RDBDriver) OpenDB(dbType, dbPath string, debugSQL bool, _ Option) (lock
switch r.name {
case dialectSqlite3:
r.conn, err = gorm.Open(sqlite.Open(dbPath), &gormConfig)
case dialectMysql:
r.conn, err = gorm.Open(mysql.Open(dbPath), &gormConfig)
case dialectPostgreSQL:
r.conn, err = gorm.Open(postgres.Open(dbPath), &gormConfig)
default:
err = xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name)
}
if err != nil {
parsedErr, marshalErr := json.Marshal(err)
if marshalErr != nil {
return xerrors.Errorf("Failed to marshal err. err: %w", marshalErr)
}

if err != nil {
if r.name == dialectSqlite3 {
switch err.(sqlite3.Error).Code {
case sqlite3.ErrLocked, sqlite3.ErrBusy:
return true, xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err)
var errMsg sqliteError
if unmarshalErr := json.Unmarshal(parsedErr, &errMsg); unmarshalErr != nil {
return xerrors.Errorf("Failed to unmarshal. err: %w", unmarshalErr)
}

switch errMsg.Code {
case errBusy, errLocked:
return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, ErrDBLocked)
default:
return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err)
}
}
return false, xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err)
}

if r.name == dialectSqlite3 {
r.conn.Exec("PRAGMA foreign_keys = ON")
case dialectMysql:
r.conn, err = gorm.Open(mysql.Open(dbPath), &gormConfig)
if err != nil {
return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err)
}
case dialectPostgreSQL:
r.conn, err = gorm.Open(postgres.Open(dbPath), &gormConfig)
if err != nil {
return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err)
}
default:
return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name)
}
return false, nil
return nil
}

// CloseDB close Database
Expand Down Expand Up @@ -132,7 +160,33 @@ func (r *RDBDriver) MigrateDB() error {
&models.JvnReference{},
&models.JvnCert{},
); err != nil {
return xerrors.Errorf("Failed to migrate. err: %w", err)
switch r.name {
case dialectSqlite3:
if r.name == dialectSqlite3 {
parsedErr, marshalErr := json.Marshal(err)
if marshalErr != nil {
return xerrors.Errorf("Failed to marshal err. err: %w", marshalErr)
}

var errMsg sqliteError
if unmarshalErr := json.Unmarshal(parsedErr, &errMsg); unmarshalErr != nil {
return xerrors.Errorf("Failed to unmarshal. err: %w", unmarshalErr)
}

switch errMsg.Code {
case errBusy, errLocked:
return xerrors.Errorf("Failed to migrate. err: %w", ErrDBLocked)
default:
return xerrors.Errorf("Failed to migrate. err: %w", err)
}
}
case dialectMysql, dialectPostgreSQL:
if err != nil {
return xerrors.Errorf("Failed to migrate. err: %w", err)
}
default:
return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name)
}
}

return nil
Expand Down
8 changes: 4 additions & 4 deletions db/redis.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,11 @@ func (r *RedisDriver) Name() string {
}

// OpenDB opens Database
func (r *RedisDriver) OpenDB(dbType, dbPath string, _ bool, option Option) (locked bool, err error) {
if err = r.connectRedis(dbPath, option); err != nil {
err = xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err)
func (r *RedisDriver) OpenDB(_, dbPath string, _ bool, option Option) error {
if err := r.connectRedis(dbPath, option); err != nil {
return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dialectRedis, dbPath, err)
}
return
return nil
}

func (r *RedisDriver) connectRedis(dbPath string, option Option) error {
Expand Down
11 changes: 9 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,20 @@ require (
github.com/PuerkitoBio/goquery v1.8.1
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/cheggaaa/pb/v3 v3.1.2
github.com/glebarez/sqlite v1.8.1-0.20230417114740-1accfe103bf2
github.com/go-redis/redis/v8 v8.11.5
github.com/hashicorp/go-version v1.6.0
github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible
github.com/k0kubun/pp v3.0.1+incompatible
github.com/knqyf263/go-cpe v0.0.0-20201213041631-54f6ab28673f
github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936
github.com/labstack/echo/v4 v4.10.2
github.com/mattn/go-sqlite3 v1.14.16
github.com/mitchellh/go-homedir v1.1.0
github.com/spf13/cobra v1.7.0
github.com/spf13/viper v1.15.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
gorm.io/driver/mysql v1.5.0
gorm.io/driver/postgres v1.5.0
gorm.io/driver/sqlite v1.5.0
gorm.io/gorm v1.25.0
)

Expand All @@ -29,11 +28,14 @@ require (
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.14.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/glebarez/go-sqlite v1.21.1 // indirect
github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand All @@ -50,6 +52,7 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/spf13/afero v1.9.3 // indirect
github.com/spf13/cast v1.5.0 // indirect
Expand All @@ -66,4 +69,8 @@ require (
golang.org/x/time v0.3.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.22.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/sqlite v1.22.0 // indirect
)
Loading

0 comments on commit 67ea039

Please sign in to comment.