From f3b88f32e7d96b8e8c4f4aca697989a5f1962479 Mon Sep 17 00:00:00 2001 From: "m.gavrilenko" Date: Fri, 27 Aug 2021 17:13:06 +0300 Subject: [PATCH] Init commit --- .gitignore | 2 + README.md | 6 +- dbpool.go | 212 +++++++++++++++++++++++++++++++++++++++++++++++++ dbpool_test.go | 130 ++++++++++++++++++++++++++++++ getter.go | 45 +++++++++++ go.mod | 12 +++ go.sum | 31 ++++++++ 7 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 dbpool.go create mode 100644 dbpool_test.go create mode 100644 getter.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/.gitignore b/.gitignore index 66fd13c..1c21222 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.git +*.idea # Binaries for programs and plugins *.exe *.exe~ diff --git a/README.md b/README.md index 7246c68..eba6a0e 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ -# ngr-dbpool \ No newline at end of file +# ngr-dbpool + +``` +import "github.com/NGRsoftlab/ngr-dbpool" +``` \ No newline at end of file diff --git a/dbpool.go b/dbpool.go new file mode 100644 index 0000000..f967534 --- /dev/null +++ b/dbpool.go @@ -0,0 +1,212 @@ +package dbpool + +import ( + logging "github.com/NGRsoftlab/ngr-logging" + + "errors" + "github.com/jmoiron/sqlx" + "sync" + "time" +) + +///////Safe db pool map with string in key/////////// + +type PoolItem struct { + Expiration int64 + Duration time.Duration + Created time.Time + + Db *sqlx.DB +} + +type SafeDbMapCache struct { + sync.RWMutex + + pool map[string]PoolItem + defaultExpiration time.Duration + cleanupInterval time.Duration +} + +// New. Initializing a new memory cache +func New(defaultExpiration, cleanupInterval time.Duration) *SafeDbMapCache { + items := make(map[string]PoolItem) + + // cache item + cache := SafeDbMapCache{ + pool: items, + defaultExpiration: defaultExpiration, + cleanupInterval: cleanupInterval, + } + + if cleanupInterval > 0 { + cache.StartGC() + } + + return &cache +} + +// Set setting a cache by key +func (c *SafeDbMapCache) Set(key string, value *sqlx.DB, duration time.Duration) { + var expiration int64 + + if duration == 0 { + duration = c.defaultExpiration + } + + if duration > 0 { + expiration = time.Now().Add(duration).UnixNano() + } + + c.Lock() + + defer c.Unlock() + + c.pool[key] = PoolItem{ + Db: value, + Expiration: expiration, + Duration: duration, + Created: time.Now(), + } +} + +// Get getting a cache by key +func (c *SafeDbMapCache) Get(key string) (*sqlx.DB, bool) { + c.RLock() + defer c.RUnlock() + + item, found := c.pool[key] + + // cache not found + if !found { + return nil, false + } + + if item.Expiration > 0 { + + // cache expired + if time.Now().UnixNano() > item.Expiration { + return nil, false + } + } + + ////TODO: set new timeout (?????? - think about it) + var newExpiration int64 + if item.Duration > 0 { + newExpiration = time.Now().Add(item.Duration).UnixNano() + } + + c.pool[key] = PoolItem{ + Db: item.Db, + Expiration: newExpiration, + Duration: item.Duration, + Created: time.Now(), + } + + return item.Db, true +} + +// Delete cache by key +// Return false if key not found +func (c *SafeDbMapCache) Delete(key string) error { + c.Lock() + defer c.Unlock() + + connector, found := c.pool[key] + + if !found { + return errors.New("key not found") + } + + err := connector.Db.Close() + if err != nil { + logging.Logger.Warning("db connection close error: ", err) + } + + delete(c.pool, key) + + return nil +} + +// StartGC start Garbage Collection +func (c *SafeDbMapCache) StartGC() { + go c.GC() +} + +// GC Garbage Collection +func (c *SafeDbMapCache) GC() { + for { + <-time.After(c.cleanupInterval) + + if c.pool == nil { + return + } + + if keys := c.ExpiredKeys(); len(keys) != 0 { + c.clearItems(keys) + } + } +} + +// expiredKeys returns key list which are expired. +func (c *SafeDbMapCache) GetItems() (items []string) { + c.RLock() + defer c.RUnlock() + + for k, _ := range c.pool { + items = append(items, k) + } + + return +} + +// expiredKeys returns key list which are expired. +func (c *SafeDbMapCache) ExpiredKeys() (keys []string) { + c.RLock() + defer c.RUnlock() + + for k, i := range c.pool { + if time.Now().UnixNano() > i.Expiration && i.Expiration > 0 { + keys = append(keys, k) + } + } + + return +} + +// clearItems removes all the items which key in keys. +func (c *SafeDbMapCache) clearItems(keys []string) { + c.Lock() + defer c.Unlock() + + for _, k := range keys { + connector, ok := c.pool[k] + + if ok { + err := connector.Db.Close() + if err != nil { + logging.Logger.Warning("db connection close error: ", err) + } + } + + delete(c.pool, k) + } +} + +// ClearAll removes all the items which key in keys. +func (c *SafeDbMapCache) ClearAll() { + c.Lock() + defer c.Unlock() + + for k := range c.pool { + connector, ok := c.pool[k] + + if ok { + err := connector.Db.Close() + if err != nil { + logging.Logger.Warning("db connection close error: ", err) + } + } + + delete(c.pool, k) + } +} diff --git a/dbpool_test.go b/dbpool_test.go new file mode 100644 index 0000000..dbf3a6a --- /dev/null +++ b/dbpool_test.go @@ -0,0 +1,130 @@ +package dbpool + +import ( + _ "github.com/lib/pq" + _ "github.com/mailru/go-clickhouse" + + logging "github.com/NGRsoftlab/ngr-logging" + + "context" + "fmt" + "github.com/jmoiron/sqlx" + "testing" + "time" +) + +var okTimeout = time.Second * 60 + +func TestDbPoolCache(t *testing.T) { + LocalCache := New(30*time.Second, 5*time.Second) + + defer LocalCache.ClearAll() + + ////CH + //driver := "clickhouse" + //connStr := fmt.Sprintf("http://%s:%s@%s:%s/%s?read_timeout=20s&write_timeout=30s", + // "", + // "", + // "", + // "8123", + // "") + // + //Ctx, cancel := context.WithTimeout(context.Background(), okTimeout) + //defer cancel() + // + //db, err := sqlx.ConnectContext(Ctx, driver, connStr) + //if err != nil { + // logging.Logger.Fatal(err) + //} + // + //LocalCache.Set(connStr, db, 10*time.Second) + // + //time.Sleep(5 * time.Second) + // + //cachedRes, ok := LocalCache.Get(connStr) + //if ok { + // logging.Logger.Debug("cached db is here: ", cachedRes) + // + // // use cachedRes + // var name string + // err = cachedRes.GetContext(Ctx, &name,"SELECT name FROM test WHERE id=1") + // if err != nil { + // logging.Logger.Fatal(err) + // } + // + //} else { + // logging.Logger.Debug("no res: ", connStr) + //} + // + //time.Sleep(6 * time.Second) + // + //cachedRes, ok = LocalCache.Get(connStr) + //if ok { + // logging.Logger.Debug("cached db is here: ", cachedRes) + // + // // use cachedRes + // var name string + // err = cachedRes.GetContext(Ctx, &name,"SELECT name FROM test WHERE id=1") + // if err != nil { + // logging.Logger.Fatal(err) + // } + // + //} else { + // logging.Logger.Debug("no res: ", connStr) + //} + + //////////////////////////////////////////////////////////////////////// + + //// POSTGRES + driver := "postgres" + connStr := fmt.Sprintf("%s://%s:%s@%s/%s", + "postgres", + "", // username + "", // userpass + "", // ip + "") // dbname + + Ctx, cancel := context.WithTimeout(context.Background(), okTimeout) + defer cancel() + + db, err := sqlx.ConnectContext(Ctx, driver, connStr) + if err != nil { + logging.Logger.Fatal(err) + } + + LocalCache.Set(connStr, db, 10*time.Second) + + //time.Sleep(5 * time.Second) + + cachedRes, ok := LocalCache.Get(connStr) + if ok { + logging.Logger.Debug("cached db is here: ", cachedRes) + + // use cachedRes + var name string + err = cachedRes.GetContext(Ctx, &name, "SELECT name FROM test WHERE id=1") + if err != nil { + logging.Logger.Fatal(err) + } + + } else { + logging.Logger.Debug("no res: ", connStr) + } + + time.Sleep(5 * time.Second) + + cachedRes, ok = LocalCache.Get(connStr) + if ok { + logging.Logger.Debug("cached db is here: ", cachedRes) + + // use cachedRes + var name string + err = cachedRes.GetContext(Ctx, &name, "SELECT name FROM test WHERE id=1") + if err != nil { + logging.Logger.Fatal(err) + } + + } else { + logging.Logger.Debug("no res: ", connStr) + } +} diff --git a/getter.go b/getter.go new file mode 100644 index 0000000..13a0b3a --- /dev/null +++ b/getter.go @@ -0,0 +1,45 @@ +package dbpool + +import ( + errorCustom "github.com/NGRsoftlab/error-lib" + + "context" + "github.com/jmoiron/sqlx" + "github.com/jmoiron/sqlx/reflectx" + "time" +) + +func GetConnectionByParams(Ctx context.Context, connCache *SafeDbMapCache, + duration time.Duration, driver, connString string) (*sqlx.DB, error) { + + conn, ok := connCache.Get(connString) + if ok && conn != nil { + // ping to check + err := conn.PingContext(Ctx) + if err != nil { + return nil, err + } + + return conn, nil + } + + //create conn + db, err := sqlx.ConnectContext(Ctx, driver, connString) + if err != nil { + return nil, errorCustom.GlobalErrors.ErrBadDbConn() + } + + //db.SetMaxIdleConns(10) + db.SetConnMaxLifetime(duration) + db.Mapper = reflectx.NewMapperFunc("json", func(s string) string { return s }) + + //set conn to connCache + connCache.Set(connString, db, duration) + + conn, ok = connCache.Get(connString) + if !ok && conn == nil { + return nil, errorCustom.GlobalErrors.ErrBadDbConn() + } + + return conn, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2679b8f --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module github.com/NGRsoftlab/ngr-dbpool + +go 1.13 + +require ( + github.com/NGRsoftlab/error-lib v1.0.2 + github.com/NGRsoftlab/ngr-logging v1.0.0 + github.com/jmoiron/sqlx v1.3.4 + github.com/lib/pq v1.10.2 + github.com/mailru/go-clickhouse v1.7.0 + github.com/sirupsen/logrus v1.8.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..6c0078b --- /dev/null +++ b/go.sum @@ -0,0 +1,31 @@ +github.com/NGRsoftlab/error-lib v1.0.2 h1:uOmhFNWRptfZso4g+Nk1S6o9HFwnk5AAFgjFLptWrgY= +github.com/NGRsoftlab/error-lib v1.0.2/go.mod h1:RbbAZ5CPZziD++EecZFG2fIbh+K1VIWpxNrg/TAziL4= +github.com/NGRsoftlab/ngr-logging v1.0.0 h1:Yp42kvw/bofZ6xXC5jPlPx1HNabZQY9cvzGnB0earJY= +github.com/NGRsoftlab/ngr-logging v1.0.0/go.mod h1:99kZ+XwSK7rKRitmhZvqOdYnPf9Qepywt3zjJJcJDME= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= +github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/go-clickhouse v1.7.0 h1:okmbyRMbRu1Xpev8YnwhvZfHX3V1iKbpce8vPW4zH0M= +github.com/mailru/go-clickhouse v1.7.0/go.mod h1:crHi+yrqslIClnYPm8IOxYVX6GmYVYymJ601I4jDqvo= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=