TT-7953 use storage library

.github/workflows/release.yml

@@ -38,9 +38,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        golang_cross: [ 1.15 ]
+        golang_cross: [ 1.16 ]
         include:
-          - golang_cross: 1.15
+          - golang_cross: 1.16
             goreleaser: 'ci/goreleaser/goreleaser.yml'
             rpmvers: 'el/7 el/8'
             debvers: 'ubuntu/xenial ubuntu/bionic debian/jessie ubuntu/focal debian/buster debian/bullseye'

.gitignore

@@ -15,3 +15,4 @@ tyk.io.signing.key
 *.swp
 *~
 .terraform**
+.DS_Store

backends/mongo.go

@@ -1,48 +1,45 @@
 package backends
 
 import (
+	"context"
 	"encoding/json"
+	"github.com/TykTechnologies/storage/persistent"
+	"github.com/TykTechnologies/storage/persistent/dbm"
 
 	"github.com/TykTechnologies/tyk-identity-broker/log"
 	"github.com/TykTechnologies/tyk-identity-broker/tap"
-	mgo "gopkg.in/mgo.v2"
-	"gopkg.in/mgo.v2/bson"
 )
 
 var mongoPrefix = "mongo-backend"
 var mongoLogger = log.Get().WithField("prefix", mongoPrefix).Logger
 
 type MongoBackend struct {
-	Db         *mgo.Database
+	Store      persistent.PersistentStorage
 	Collection string
 }
 
 func (m MongoBackend) Init(interface{}) {
 
 }
 
-func (m *MongoBackend) getCollection() *mgo.Collection {
-	session := m.Db.Session.Copy()
-	return session.DB("").C(m.Collection)
-}
-
 func (m MongoBackend) SetKey(key string, orgId string, value interface{}) error {
-	profilesCollection := m.getCollection()
-	defer profilesCollection.Database.Session.Close()
 
-	filter := bson.M{"ID": key}
+	profile := value.(tap.Profile)
+	filter := dbm.DBM{}
+	filter["ID"] = key
 	if orgId != "" {
 		filter["OrgID"] = orgId
 	}
-	// delete if exist, where matches the profile ID and org
-	err := profilesCollection.Remove(filter)
+
+	// delete if exists, where matches the profile ID and org
+	err := m.Store.Delete(context.Background(), profile, filter)
 	if err != nil {
 		if err.Error() != "not found" {
 			mongoLogger.WithError(err).Error("setting profile in mongo")
 		}
 	}
 
-	err = profilesCollection.Insert(value)
+	err = m.Store.Insert(context.Background(), profile)
 	if err != nil {
 		mongoLogger.WithError(err).Error("inserting profile in mongo")
 	}
@@ -51,16 +48,15 @@ func (m MongoBackend) SetKey(key string, orgId string, value interface{}) error
 }
 
 func (m MongoBackend) GetKey(key string, orgId string, val interface{}) error {
-	profilesCollection := m.getCollection()
-	defer profilesCollection.Database.Session.Close()
 
-	filter := bson.M{"ID": key}
+	filter := dbm.DBM{}
+	filter["ID"] = key
 	if orgId != "" {
 		filter["OrgID"] = orgId
 	}
 
 	p := tap.Profile{}
-	err := profilesCollection.Find(filter).One(&p)
+	err := m.Store.Query(context.Background(), p, &p, filter)
 	if err != nil {
 		if err.Error() != "not found" {
 			mongoLogger.WithError(err).Error("error reading profile from mongo, key:", key)
@@ -86,14 +82,12 @@ func (m MongoBackend) GetKey(key string, orgId string, val interface{}) error {
 func (m MongoBackend) GetAll(orgId string) []interface{} {
 	var profiles []tap.Profile
 
-	filter := bson.M{}
+	filter := dbm.DBM{}
 	if orgId != "" {
 		filter["OrgID"] = orgId
 	}
 
-	profilesCollection := m.getCollection()
-	defer profilesCollection.Database.Session.Close()
-	err := profilesCollection.Find(filter).All(&profiles)
+	err := m.Store.Query(context.Background(), tap.Profile{}, &profiles, filter)
 	if err != nil {
 		mongoLogger.Error("error reading profiles from mongo: " + err.Error())
 	}
@@ -107,15 +101,13 @@ func (m MongoBackend) GetAll(orgId string) []interface{} {
 }
 
 func (m MongoBackend) DeleteKey(key string, orgId string) error {
-	profilesCollection := m.getCollection()
-	defer profilesCollection.Database.Session.Close()
-
-	filter := bson.M{"ID": key}
+	filter := dbm.DBM{}
+	filter["ID"] = key
 	if orgId != "" {
 		filter["OrgID"] = orgId
 	}
 
-	err := profilesCollection.Remove(filter)
+	err := m.Store.Delete(context.Background(), tap.Profile{}, filter)
 	if err != nil {
 		mongoLogger.WithError(err).Error("removing profile")
 	}

configuration/config.go

@@ -48,6 +48,7 @@ type MongoConf struct {
 	MaxDocumentSizeBytes       int    `json:"max_document_size_bytes" mapstructure:"max_document_size_bytes"`
 	CollectionCapMaxSizeBytes  int    `json:"collection_cap_max_size_bytes" mapstructure:"collection_cap_max_size_bytes"`
 	CollectionCapEnable        bool   `json:"collection_cap_enable" mapstructure:"collection_cap_enable"`
+	SessionConsistency         string `json:"session_consistency" mapstructure:"session_consistency"`
 }
 
 // Storage object to configure the storage where the profiles lives in
@@ -71,8 +72,8 @@ type Backend struct {
 
 // Configuration holds all configuration settings for TAP
 type Configuration struct {
-	Secret string
-	Port   int
+	Secret            string
+	Port              int
 	ProfileDir        string
 	BackEnd           Backend
 	TykAPISettings    tyk.TykAPI
@@ -113,7 +114,6 @@ func LoadConfig(filePath string, conf *Configuration) {
 		*conf = Configuration{}
 	}
 
-
 	if err = envconfig.Process(tothic.EnvPrefix, conf); err != nil {
 		mainLogger.Errorf("Failed to process config env vars: %v", err)
 	}

data_loader/data_loader.go

@@ -1,12 +1,11 @@
 package data_loader
 
 import (
-	"github.com/sirupsen/logrus"
-	"gopkg.in/mgo.v2"
-
+	"github.com/TykTechnologies/storage/persistent"
 	"github.com/TykTechnologies/tyk-identity-broker/configuration"
 	logger "github.com/TykTechnologies/tyk-identity-broker/log"
 	"github.com/TykTechnologies/tyk-identity-broker/tap"
+	"github.com/sirupsen/logrus"
 )
 
 var log = logger.Get()
@@ -26,16 +25,6 @@ func reloadDataLoaderLogger() {
 	dataLogger = dataLogger.Logger.WithField("prefix", dataLoaderLoggerTag)
 }
 
-func CreateMongoLoaderFromConnection(db *mgo.Database) DataLoader {
-	var dataLoader DataLoader
-
-	reloadDataLoaderLogger()
-	dataLogger.Info("Set mongo loader for TIB")
-	dataLoader = &MongoLoader{Db: db}
-
-	return dataLoader
-}
-
 func CreateDataLoader(config configuration.Configuration, ProfileFilename string) (DataLoader, error) {
 	var dataLoader DataLoader
 	var loaderConf interface{}
@@ -53,13 +42,17 @@ func CreateDataLoader(config configuration.Configuration, ProfileFilename string
 		dataLoader = &MongoLoader{}
 
 		mongoConf := config.Storage.MongoConf
-		dialInfo, err := MongoDialInfo(mongoConf.MongoURL, mongoConf.MongoUseSSL, mongoConf.MongoSSLInsecureSkipVerify)
-		if err != nil {
-			dataLogger.Error("Error getting mongo settings: " + err.Error())
-			return nil, err
+		// map from tib mongo conf structure to persistent.ClientOpts
+		connectionConf := persistent.ClientOpts{
+			ConnectionString:      mongoConf.MongoURL,
+			UseSSL:                mongoConf.MongoUseSSL,
+			SSLInsecureSkipVerify: mongoConf.MongoSSLInsecureSkipVerify,
+			SessionConsistency:    mongoConf.SessionConsistency,
+			Type:                  persistent.OfficialMongo,
 		}
+
 		loaderConf = MongoLoaderConf{
-			DialInfo: dialInfo,
+			ClientOpts: &connectionConf,
 		}
 	default:
 		//default: FILE

data_loader/mongo_loader.go

@@ -1,13 +1,11 @@
 package data_loader
 
 import (
-	"crypto/tls"
+	"context"
 	"encoding/json"
-	"net"
+	"github.com/TykTechnologies/storage/persistent"
 	"time"
 
-	"gopkg.in/mgo.v2"
-
 	"github.com/TykTechnologies/tyk-identity-broker/tap"
 )
 
@@ -17,13 +15,13 @@ var (
 
 // MongoLoaderConf is the configuration struct for a MongoLoader
 type MongoLoaderConf struct {
-	DialInfo *mgo.DialInfo
+	ClientOpts *persistent.ClientOpts
 }
 
 // MongoLoader implements DataLoader and will load TAP Profiles from a file
 type MongoLoader struct {
 	config    MongoLoaderConf
-	Db        *mgo.Database
+	store     persistent.PersistentStorage
 	SkipFlush bool
 }
 
@@ -34,25 +32,25 @@ type ProfilesBackup struct {
 
 // Init initialises the mongo loader
 func (m *MongoLoader) Init(conf interface{}) error {
-	m.config = conf.(MongoLoaderConf)
+	mongoConfig := conf.(MongoLoaderConf)
 
-	var err error
-	session, err := mgo.DialWithInfo(m.config.DialInfo)
+	store, err := persistent.NewPersistentStorage(mongoConfig.ClientOpts)
 	if err != nil {
 		dataLogger.WithError(err).WithField("prefix", mongoPrefix).Error("failed to init MongoDB connection")
 		time.Sleep(5 * time.Second)
 		m.Init(conf)
 	}
 
-	m.Db = session.DB("")
+	m.store = store
 	return err
 }
 
 // LoadIntoStore will load, unmarshal and copy profiles into a an AuthRegisterBackend
 func (m *MongoLoader) LoadIntoStore(store tap.AuthRegisterBackend) error {
 	var profiles []tap.Profile
 
-	err := m.Db.C(tap.ProfilesCollectionName).Find(nil).All(&profiles)
+	err := m.store.Query(context.Background(), tap.Profile{}, &profiles, nil)
+
 	if err != nil {
 		dataLogger.Error("error reading profiles from mongo: " + err.Error())
 		return err
@@ -72,18 +70,17 @@ func (m *MongoLoader) LoadIntoStore(store tap.AuthRegisterBackend) error {
 // Flush creates a backup of the current loaded config
 func (m *MongoLoader) Flush(store tap.AuthRegisterBackend) error {
 	//read all
-	//save the changes in the main profiles collection, so empty and store as we dont know what was removed, updated or added
+	//save the changes in the main profile's collection, so empty and store as we don't know what was removed, updated or added
 	updatedSet := store.GetAll("")
-	profilesCollection := m.Db.C(tap.ProfilesCollectionName)
 
 	//empty to store new changes
-	_, err := profilesCollection.RemoveAll(nil)
+	err := m.store.Delete(context.Background(), tap.Profile{}, nil)
 	if err != nil {
 		dataLogger.WithError(err).Error("emptying profiles collection")
 		return err
 	}
 
-	for i, p := range updatedSet {
+	for _, p := range updatedSet {
 		profile := tap.Profile{}
 		switch p := p.(type) {
 		case string:
@@ -92,14 +89,11 @@ func (m *MongoLoader) Flush(store tap.AuthRegisterBackend) error {
 				dataLogger.WithError(err).Error("un-marshaling interface for mongo flushing")
 				return err
 			}
-			updatedSet[i] = profile
 		default:
-			updatedSet[i] = p
+			profile = p.(tap.Profile)
 		}
-	}
 
-	if len(updatedSet) > 0 {
-		err = profilesCollection.Insert(updatedSet...)
+		m.store.Insert(context.Background(), profile)
 		if err != nil {
 			dataLogger.WithError(err).Error("error refreshing profiles records in mongo")
 			return err
@@ -108,21 +102,3 @@ func (m *MongoLoader) Flush(store tap.AuthRegisterBackend) error {
 
 	return nil
 }
-
-func MongoDialInfo(mongoURL string, useSSL bool, SSLInsecureSkipVerify bool) (dialInfo *mgo.DialInfo, err error) {
-	if dialInfo, err = mgo.ParseURL(mongoURL); err != nil {
-		return dialInfo, err
-	}
-
-	if useSSL {
-		dialInfo.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) {
-			tlsConfig := &tls.Config{}
-			if SSLInsecureSkipVerify {
-				tlsConfig.InsecureSkipVerify = true
-			}
-			return tls.Dial("tcp", addr.String(), tlsConfig)
-		}
-	}
-
-	return dialInfo, err
-}

go.mod

@@ -1,9 +1,10 @@
 module github.com/TykTechnologies/tyk-identity-broker
 
-go 1.13
+go 1.16
 
 require (
 	github.com/Jeffail/gabs v1.4.0
+	github.com/TykTechnologies/storage v0.0.0-20230308174156-ed14b745c68b
 	github.com/TykTechnologies/tyk v1.9.2-0.20211217130848-b04d51712be7
 	github.com/crewjam/saml v0.4.6
 	github.com/go-ldap/ldap/v3 v3.2.3
@@ -13,15 +14,17 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/markbates/goth v1.64.2
 	github.com/matryer/is v1.4.0
-	github.com/mattn/go-colorable v0.1.7 // indirect
-	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.4.3-0.20191026113918-67a7fdcf741f
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.8.1
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
-	golang.org/x/text v0.3.6
-	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
+	golang.org/x/text v0.3.7
+)
+
+require (
+	github.com/mattn/go-colorable v0.1.7 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 )
 
 replace github.com/crewjam/saml => github.com/TykTechnologies/saml v0.4.6-0.20211129150050-bf5ed7a9f748