Skip to content

Commit

Permalink
[tests] Improve db test performance (#605)
Browse files Browse the repository at this point in the history 
* [tests] Introduce db client coll() function to determine the collection to use

Goal: isolate runs in separate databases

This doesn't yield any performance improvements, yet.
But the next steps (commits) will...

* [tests] Improve sessions_test.go performance

Use shared mongodb testcontainer.

* [tests] Improve db_test.go performance

Use shared mongodb testcontainer.

* [tests] Reduce time.Sleep in tests from 2sec to 550ms

* [tests] Reduce time.Sleep in tests from 2sec to 1sec

* Fix SA1029 linter error

SA1029: should not use built-in type string as key for value; define your own type to avoid collisions (staticcheck)

see: https://stackoverflow.com/questions/40891345/fix-should-not-use-basic-type-string-as-key-in-context-withvalue-golint

* Fix goimports linter error
  • Loading branch information
@scthi
scthi authored Nov 16, 2023
1 parent 8f6b250 commit 0130d2b
Show file tree
Hide file tree
Showing 4 changed files with 794 additions and 853 deletions.
70 changes: 43 additions & 27 deletions pkg/hub/db/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ type Config struct {
URI string `json:"uri" env:"URI" default:"mongodb://localhost:27017" help:"The connection uri for MongoDB"`
}

type key int

const (
kobsDbName key = iota
)

// Client is the interface with all the methods to interact with the db.
type Client interface {
DB() *mongo.Client
Expand Down Expand Up @@ -104,8 +110,18 @@ func NewClient(config Config) (Client, error) {
}, nil
}

// Returns the collection with given name
// uses specified "kobs.db.name" value from cxt if given as db or "kobs" as default
func (c *client) coll(ctx context.Context, collection string) *mongo.Collection {
dbName := ctx.Value(kobsDbName)
if dbName == nil {
dbName = "kobs"
}
return c.db.Database(dbName.(string)).Collection(collection)
}

func (c *client) save(ctx context.Context, collection string, models []mongo.WriteModel, cluster string, updatedAt int64) error {
_, err := c.db.Database("kobs").Collection(collection).BulkWrite(ctx, models)
_, err := c.coll(ctx, collection).BulkWrite(ctx, models)
if err != nil {
return err
}
Expand All @@ -115,7 +131,7 @@ func (c *client) save(ctx context.Context, collection string, models []mongo.Wri
filter = bson.D{{Key: "$and", Value: bson.A{bson.D{{Key: "cluster", Value: bson.D{{Key: "$eq", Value: cluster}}}}, filter}}}
}

_, err = c.db.Database("kobs").Collection(collection).DeleteMany(ctx, filter)
_, err = c.coll(ctx, collection).DeleteMany(ctx, filter)
if err != nil {
return err
}
Expand All @@ -132,7 +148,7 @@ func (c *client) CreateIndexes(ctx context.Context) error {
defer span.End()

// Create TTL index for sessions collection, which will delete all inactive sessions which are older than 7 days (168h).
_, err := c.db.Database("kobs").Collection("sessions").Indexes().CreateOne(
_, err := c.coll(ctx, "sessions").Indexes().CreateOne(
ctx,
mongo.IndexModel{
Keys: bson.D{{Key: "updatedAt", Value: 1}},
Expand Down Expand Up @@ -271,7 +287,7 @@ func (c *client) SaveApplication(ctx context.Context, application *applicationv1
upsert := true
application.UpdatedAt = time.Now().Unix()

_, err := c.db.Database("kobs").Collection("applications").ReplaceOne(ctx, bson.D{{Key: "_id", Value: application.ID}}, application, &options.ReplaceOptions{Upsert: &upsert})
_, err := c.coll(ctx, "applications").ReplaceOne(ctx, bson.D{{Key: "_id", Value: application.ID}}, application, &options.ReplaceOptions{Upsert: &upsert})
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -342,7 +358,7 @@ func (c *client) SaveTeam(ctx context.Context, team *teamv1.TeamSpec) error {
upsert := true
team.UpdatedAt = time.Now().Unix()

_, err := c.db.Database("kobs").Collection("teams").ReplaceOne(ctx, bson.D{{Key: "_id", Value: team.ID}}, team, &options.ReplaceOptions{Upsert: &upsert})
_, err := c.coll(ctx, "teams").ReplaceOne(ctx, bson.D{{Key: "_id", Value: team.ID}}, team, &options.ReplaceOptions{Upsert: &upsert})
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -386,7 +402,7 @@ func (c *client) SaveUser(ctx context.Context, user *userv1.UserSpec) error {
upsert := true
user.UpdatedAt = time.Now().Unix()

_, err := c.db.Database("kobs").Collection("users").ReplaceOne(ctx, bson.D{{Key: "_id", Value: user.ID}}, user, &options.ReplaceOptions{Upsert: &upsert})
_, err := c.coll(ctx, "users").ReplaceOne(ctx, bson.D{{Key: "_id", Value: user.ID}}, user, &options.ReplaceOptions{Upsert: &upsert})
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -466,14 +482,14 @@ func (c *client) SaveTopology(ctx context.Context, cluster string, applications
return nil
}

_, err := c.db.Database("kobs").Collection("topology").BulkWrite(ctx, models)
_, err := c.coll(ctx, "topology").BulkWrite(ctx, models)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}

_, err = c.db.Database("kobs").Collection("topology").DeleteMany(ctx, bson.D{{Key: "$and", Value: bson.A{bson.D{{Key: "sourceCluster", Value: bson.D{{Key: "$eq", Value: cluster}}}}, bson.D{{Key: "updatedAt", Value: bson.D{{Key: "$lt", Value: updatedAt}}}}}}})
_, err = c.coll(ctx, "topology").DeleteMany(ctx, bson.D{{Key: "$and", Value: bson.A{bson.D{{Key: "sourceCluster", Value: bson.D{{Key: "$eq", Value: cluster}}}}, bson.D{{Key: "updatedAt", Value: bson.D{{Key: "$lt", Value: updatedAt}}}}}}})
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -489,7 +505,7 @@ func (c *client) GetPlugins(ctx context.Context) ([]plugin.Instance, error) {

var plugins []plugin.Instance

cursor, err := c.db.Database("kobs").Collection("plugins").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
cursor, err := c.coll(ctx, "plugins").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -512,7 +528,7 @@ func (c *client) GetNamespaces(ctx context.Context) ([]Namespace, error) {

var namespaces []Namespace

cursor, err := c.db.Database("kobs").Collection("namespaces").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
cursor, err := c.coll(ctx, "namespaces").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -542,7 +558,7 @@ func (c *client) GetNamespacesByClusters(ctx context.Context, clusters []string)

var namespaces []Namespace

cursor, err := c.db.Database("kobs").Collection("namespaces").Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
cursor, err := c.coll(ctx, "namespaces").Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -565,7 +581,7 @@ func (c *client) GetCRDs(ctx context.Context) ([]kubernetes.CRD, error) {

var crds []kubernetes.CRD

cursor, err := c.db.Database("kobs").Collection("crds").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
cursor, err := c.coll(ctx, "crds").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -589,7 +605,7 @@ func (c *client) GetCRDByID(ctx context.Context, id string) (*kubernetes.CRD, er

var crd kubernetes.CRD

result := c.db.Database("kobs").Collection("crds").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
result := c.coll(ctx, "crds").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
if err := result.Err(); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -612,7 +628,7 @@ func (c *client) GetApplications(ctx context.Context) ([]applicationv1.Applicati

var applications []applicationv1.ApplicationSpec

cursor, err := c.db.Database("kobs").Collection("applications").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
cursor, err := c.coll(ctx, "applications").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -664,7 +680,7 @@ func (c *client) GetApplicationsByFilter(ctx context.Context, teams, clusters, n

var applications []applicationv1.ApplicationSpec

cursor, err := c.db.Database("kobs").Collection("applications").Find(ctx, filter, options.Find().SetSort(bson.M{"name": 1}).SetLimit(int64(limit)).SetSkip(int64(offset)))
cursor, err := c.coll(ctx, "applications").Find(ctx, filter, options.Find().SetSort(bson.M{"name": 1}).SetLimit(int64(limit)).SetSkip(int64(offset)))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -712,7 +728,7 @@ func (c *client) GetApplicationsByFilterCount(ctx context.Context, teams, cluste
filter["tags"] = bson.M{"$in": tags}
}

count, err := c.db.Database("kobs").Collection("applications").CountDocuments(ctx, filter)
count, err := c.coll(ctx, "applications").CountDocuments(ctx, filter)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -739,7 +755,7 @@ func (c *client) GetApplicationsByGroup(ctx context.Context, teams, groups []str

var applications []ApplicationGroup

cursor, err := c.db.Database("kobs").Collection("applications").Aggregate(ctx, mongo.Pipeline{
cursor, err := c.coll(ctx, "applications").Aggregate(ctx, mongo.Pipeline{
bson.D{{Key: "$match", Value: bson.D{{Key: "teams", Value: bson.D{{Key: "$in", Value: teams}}}}}},
bson.D{
{Key: "$group",
Expand Down Expand Up @@ -780,7 +796,7 @@ func (c *client) GetApplicationByID(ctx context.Context, id string) (*applicatio

var application applicationv1.ApplicationSpec

result := c.db.Database("kobs").Collection("applications").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
result := c.coll(ctx, "applications").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
if err := result.Err(); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -813,7 +829,7 @@ func (c *client) GetDashboards(ctx context.Context, clusters, namespaces []strin

var dashboards []dashboardv1.DashboardSpec

cursor, err := c.db.Database("kobs").Collection("dashboards").Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
cursor, err := c.coll(ctx, "dashboards").Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -837,7 +853,7 @@ func (c *client) GetDashboardByID(ctx context.Context, id string) (*dashboardv1.

var dashboard dashboardv1.DashboardSpec

result := c.db.Database("kobs").Collection("dashboards").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
result := c.coll(ctx, "dashboards").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
if err := result.Err(); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -865,7 +881,7 @@ func (c *client) GetTeams(ctx context.Context, searchTerm string) ([]teamv1.Team
filter = bson.D{{Key: "_id", Value: bson.M{"$regex": primitive.Regex{Pattern: searchTerm, Options: "i"}}}}
}

cursor, err := c.db.Database("kobs").Collection("teams").Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
cursor, err := c.coll(ctx, "teams").Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -898,7 +914,7 @@ func (c *client) GetTeamsByIDs(ctx context.Context, ids []string, searchTerm str
filter = bson.D{{Key: "$and", Value: bson.A{bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: ids}}}}, bson.D{{Key: "_id", Value: bson.M{"$regex": primitive.Regex{Pattern: searchTerm, Options: "i"}}}}}}}
}

cursor, err := c.db.Database("kobs").Collection("teams").Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
cursor, err := c.coll(ctx, "teams").Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -922,7 +938,7 @@ func (c *client) GetTeamByID(ctx context.Context, id string) (*teamv1.TeamSpec,

var team teamv1.TeamSpec

result := c.db.Database("kobs").Collection("teams").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
result := c.coll(ctx, "teams").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
if err := result.Err(); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -945,7 +961,7 @@ func (c *client) GetUsers(ctx context.Context) ([]userv1.UserSpec, error) {

var users []userv1.UserSpec

cursor, err := c.db.Database("kobs").Collection("users").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
cursor, err := c.coll(ctx, "users").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "_id", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand All @@ -969,7 +985,7 @@ func (c *client) GetUserByID(ctx context.Context, id string) (*userv1.UserSpec,

var user userv1.UserSpec

result := c.db.Database("kobs").Collection("users").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
result := c.coll(ctx, "users").FindOne(ctx, bson.D{{Key: "_id", Value: id}})
if err := result.Err(); err != nil {
if errors.Is(err, mongo.ErrNoDocuments) {
return nil, nil
Expand All @@ -995,7 +1011,7 @@ func (c *client) GetTags(ctx context.Context) ([]Tag, error) {

var tags []Tag

cursor, err := c.db.Database("kobs").Collection("tags").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "tag", Value: 1}}))
cursor, err := c.coll(ctx, "tags").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "tag", Value: 1}}))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down Expand Up @@ -1024,7 +1040,7 @@ func (c *client) GetTopologyByIDs(ctx context.Context, field string, ids []strin

var topology []Topology

cursor, err := c.db.Database("kobs").Collection("topology").Find(ctx, bson.D{{Key: field, Value: bson.D{{Key: "$in", Value: ids}}}})
cursor, err := c.coll(ctx, "topology").Find(ctx, bson.D{{Key: field, Value: bson.D{{Key: "$in", Value: ids}}}})
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
Expand Down
Loading

0 comments on commit 0130d2b

Please sign in to comment.